Group and Merge array of objects - mongodb

Question

I am struggling around with the aggregation pipeline feature from MongoDB.
So far the output for one result looks like this:
{
"type": "inbound",
"sender": "postAG",
"receiver": "maxMusterMan",
"datetime": "20191125",
"info": [
{
"q": "A",
"value": "5",
"name": null,
"plz": 1234
},
{
"q": "B",
"value": "AS",
"name": "ABS",
"plz": null
},
{
"q": "A",
"value": "5",
"name": "aa",
"plz": null
},
... more objects
]
}
The final result should look like:
{
"type": "inbound",
"sender": "postAG",
"receiver": "maxMusterMan",
"datetime": "20191125",
"info": [
{
"q": "A",
"value": "0",
"name": "aa",
"plz": 1234
},
{
"q": "B",
"value": "AS",
"name": "ABS"
}
]
}
So in a nutshell, I want to group the values from the array field info by the "q" field and merge the objects (newer one overwrites the old value).
Further I would like to remove all the values with value "" or null;
There are more fields in the real payload, so I would like to avoid to add a $cond for each field of the object.
Some approaches so far from my side:
for the cleanup, use a UDF, but this is not possible in the pipeline.
use map-reduce for the group and merge, not available in the pipeline.
Please consider that the input file is the output from the several pipeline steps.
So I can not just use map-reduce alone, first I need the pipeline too.
My idea was to create two views, first will do the pipeline stuff and second map-reduce, is this a good solution?
Thx
Andreas

Answer 1

I didn't really understand from your explanation if you can or cannot use map-reduce.
However assuming you can't and you have to 'concat' the pipelines there is no 'generic' workaround for multiple fields - you have to create a condition for each in the pipeline.
With that said here is a working pipeline:
db.collection.aggregate(
[
{
"$unwind" : "$info"
},
{
"$group" : {
"_id" : "$info.q",
"type" : {
"$first" : "$type"
},
"sender" : {
"$first" : "$sender"
},
"receiver" : {
"$first" : "$receiver"
},
"datetime" : {
"$first" : "$datetime"
},
"values" : {
"$push" : "$info.value"
},
"names" : {
"$push" : "$info.name"
},
"plz" : {
"$push" : "$info.plz"
}
}
},
{
"$project" : {
"_id" : 1.0,
"type" : 1.0,
"sender" : 1.0,
"receiver" : 1.0,
"datetime" : 1.0,
"values" : {
"$filter" : {
"input" : "$values",
"as" : "curr",
"cond" : {
"$or" : [
{
"$ne" : [
"$$curr",
null
]
},
{
"$ne" : [
"$$curr",
""
]
}
]
}
}
},
"names" : {
"$filter" : {
"input" : "$names",
"as" : "curr",
"cond" : {
"$or" : [
{
"$ne" : [
"$$curr",
null
]
},
{
"$ne" : [
"$$curr",
""
]
}
]
}
}
},
"plz" : {
"$filter" : {
"input" : "$plz",
"as" : "curr",
"cond" : {
"$or" : [
{
"$ne" : [
"$$curr",
null
]
},
{
"$ne" : [
"$$curr",
""
]
}
]
}
}
}
}
},
{
"$project" : {
"sender" : 1.0,
"receiver" : 1.0,
"datetime" : 1.0,
"type" : 1.0,
"_id" : 1.0,
"value" : {
"$cond" : {
"if" : {
"$gt" : [
{
"$size" : "$values"
},
0.0
]
},
"then" : {
"$arrayElemAt" : [
"$values",
-1.0
]
},
"else" : null
}
},
"name" : {
"$cond" : {
"if" : {
"$gt" : [
{
"$size" : "$names"
},
0.0
]
},
"then" : {
"$arrayElemAt" : [
"$names",
-1.0
]
},
"else" : null
}
},
"plz" : {
"$cond" : {
"if" : {
"$gt" : [
{
"$size" : "$plz"
},
0.0
]
},
"then" : {
"$arrayElemAt" : [
"$plz",
-1.0
]
},
"else" : null
}
}
}
},
{
"$addFields" : {
"infoArray" : [
{
"k" : "type",
"v" : "$_id"
},
{
"k" : "value",
"v" : "$value"
},
{
"k" : "name",
"v" : "$name"
},
{
"k" : "plz",
"v" : "$plz"
}
]
}
},
{
"$addFields" : {
"info" : {
"$arrayToObject" : {
"$filter" : {
"input" : "$infoArray",
"as" : "curr",
"cond" : {
"$ne" : [
"$$curr.v",
null
]
}
}
}
}
}
},
{
"$group" : {
"_id" : null,
"type" : {
"$first" : "$type"
},
"sender" : {
"$first" : "$sender"
},
"receiver" : {
"$first" : "$receiver"
},
"datetime" : {
"$first" : "$datetime"
},
"info" : {
"$push" : "$info"
}
}
}
]
)