keep latest record based on multiple grouping - mongodb

Question

I wrote a multi-stage pipeline to arrive at this set of documents:
{'_id': '1234'),
'info': [{'type': 'patient',
'patient_id': 'p1'},
{'type': 'doc',
'doc_id': 'd1'},
{'type': 'ldlc',
'dt': datetime.datetime(2018, 10, 29, 12, 7, 23),
'val': 136},
{'type': 'bp',
'dt': datetime.datetime(2014, 8, 25, 4, 2, 27),
'val': [{'dias': 74}, {'sys': 105}]}]},
{'_id': '1235'),
'info': [{'type': 'patient',
'patient_id': 'p2'},
{'type': 'doc',
'doc_id': 'd1'},
{'type': 'ldlc',
'dt': datetime.datetime(2016, 3, 31, 21, 30, 34),
'val': 153},
{'type': 'bp',
'dt': datetime.datetime(2013, 7, 3, 18, 3, 12),
'val': [{'dias': 86}, {'sys': 101}]},
{'type': 'bp',
'dt': datetime.datetime(2016, 3, 15, 18, 35, 25),
'val': [{'dias': 85}, {'sys': 108}]},
{'type': 'ldlc',
'dt': datetime.datetime(2018, 10, 1, 12, 7, 23),
'val': 144}]}
I am using pymongo, hence the datetime objects.
Now in each document I only want to keep the last recorded values (sort by dt) for 'ldlc' and 'bp'.
I would prefer it to be as:
{
"_id": '1234',
"patient_id": "p1",
"doc_id": "d1".
"sys": 105,
"dias": 74,
"ldlc": 136
},
{
"_id": '1235',
"patient_id": "p2",
"doc_id": "d1".
"sys": 108,
"dias": 85,
"ldlc": 144
}
since the source documents are generated in an aggregation pipeline, i want to add $project and $group stages after that in order to product the desired result.
Thanks for your help!

Answer 1

There are different approaches to achieve this use case.
I started with $sort to sort based on dates. And then used $facet for parallel grouping.Since you need to keep only the latest record, $last is used to get required values.
Your aggregation can look like below:
db.collection.aggregate([
{
$unwind: "$info"
},
{
$sort: {
"info.dt": 1
}
},
{
"$facet": {
"ldlc": [
{
"$match": {
"info.type": "ldlc"
}
},
{
"$group": {
"_id": "$_id",
"ldlc": {
$last: "$info.val"
}
}
}
],
"bp": [
{
"$match": {
"info.type": "bp"
}
},
{
"$group": {
"_id": "$_id",
"bp": {
$last: "$info.val"
}
}
},
{
$unwind: "$bp"
}
],
"others": [
{
$match: {
$or: [
{
"info.type": "patient"
},
{
"info.type": "doc"
}
]
}
},
{
"$group": {
"_id": "$_id",
"ids": {
$push: {
p: "$info.patient_id",
d: "$info.doc_id"
}
}
}
},
{
$unwind: "$ids"
}
],
}
},
{
$project: {
data: {
$concatArrays: [
"$others",
"$ldlc",
"$bp"
]
}
}
},
{
$unwind: "$data"
},
{
"$group": {
"_id": "$data._id",
"val": {
$push: {
patient_id: "$data.ids.p",
doc_id: "$data.ids.d",
ldlc: "$data.ldlc",
dias: "$data.bp.dias",
sys: "$data.bp.sys"
}
}
}
},
{
"$project": {
_id: 1,
"v": {
"$reduce": {
"input": "$val",
"initialValue": {},
"in": {
"$mergeObjects": [
"$$value",
"$$this"
]
}
}
}
}
},
{
"$project": {
_id: 1,
patient_id: "$v.patient_id",
doc_id: "$v.doc_id",
ldlc: "$v.ldlc",
dias: "$v.dias",
sys: "$v.sys"
}
}
])
Check out the query result here: Mongo Playground
PS: This may not be the best approach