How to group an array by same value from different collections in Mongo aggregation - mongodb

Question

I need to group at an array named location by the same coordinates field from both engagements and broadcastreports. Also if one coordinate exist in one but not on the other, It should be included anyway.
In that array I need to include a scans which is the number of engagements related to a campaign_ad_group and a impressions field which is the sum of broadcastreports.analytics.impressions related also to a campaign_ad_group.
I have the following aggregation:
.aggregate([
{
$match: {
'_id': ObjectId("60520e1d09ba3a769ab85f68"),
}
},
{
$project: {
'_id': 1,
'broadcasts_to': 1
}
},
{
$lookup: {
from: 'campaignadgroups',
localField: '_id',
foreignField: 'campaign_id',
as:'campaign_ad_group'
}
},
{
$unwind: '$campaign_ad_group'
},
{
$lookup: {
from: 'broadcastplans',
let: { 'campaign_ad_group_id': '$campaign_ad_group._id' },
as: 'broadcastplans',
pipeline: [
{
$match: {
$expr: {
$eq: ['$campaign_ad_group_id','$$campaign_ad_group_id' ]
}
}
},
{ $project: { '_id': 1 } }
]
}
},
// Above it's only the necessary data gathering to get the needed engagements and broadcastreports.
{
$lookup: {
from: 'engagements',
let: { 'broadcastplan_ids': '$broadcastplans._id' },
as: 'engagements',
pipeline: [
{
$match: {
$expr: {
$and: [
{ $in: ['$broadcast_plan_id', '$$broadcastplan_ids'] },
{ $gte: ['$reported_at', ISODate('2019-12-29T15:02:47.305Z')] },
{ $lte: ['$reported_at', ISODate('2021-12-29T15:02:47.305Z')] }
]
}
}
}
]
}
},
{
$unwind: '$engagements'
},
{
$group: {
'_id': {
'adgroup_id': '$campaign_ad_group._id',
'coordinates': '$engagements.location.coordinates',
},
'adgroup_name': {$first: '$campaign_ad_group.name'},
'scans': {$sum: 1}
}
},
{
$group: {
'_id': '$_id.adgroup_id',
'name': {$first: '$adgroup_name'},
'location': {
$push: {
'scans': '$scans',
'coordinates': '$_id.coordinates'
}
}
}
}
])
The output I get is the following:
{
"_id" : ObjectId("5fe9f89f603eaa22b3d374f5"),
"name" : "Sick name",
"location" : [
{
"scans" : 1.0,
"coordinates" : [
-8.63294964028813,
56.1520268933931
]
},
{
"scans" : 1.0,
"coordinates" : [
-9.63294964028813,
58.1520268933931
]
}
]
}
Now it comes the part that I'm stuck at. I need to look up for the collection broadcastreports and group by its coordinates with the engagements coordinates and include a field impressions which is the sum of broadcastreports.analytics.impressions.
By removing the group pipeline and lookup from the broadcastreports it will look like so:
.aggregate([
.
. // ABOVE IS THE SAME DATA GATHERING AS BEFORE
.
{
$lookup: {
from: 'engagements',
let: { 'broadcastplan_ids': '$broadcastplans._id' },
as: 'engagements',
pipeline: [
{
$match: {
$expr: {
$and: [
{ $in: ['$broadcast_plan_id', '$$broadcastplan_ids'] },
{ $gte: ['$reported_at', ISODate('2019-12-29T15:02:47.305Z')] },
{ $lte: ['$reported_at', ISODate('2021-12-29T15:02:47.305Z')] }
]
}
}
}
]
}
},
{
$lookup: {
from: 'broadcastreports',
let: { 'broadcastplan_ids': '$broadcastplans._id' },
as: 'broadcastreports',
pipeline: [
{
$match: {
$expr: {
$and: [
{ $in: ['$broadcast_plan_id', '$$broadcastplan_ids'] },
{ $gte: ['$reported_at', ISODate('2019-12-29T15:02:47.305Z')] },
{ $lte: ['$reported_at', ISODate('2021-12-29T15:02:47.305Z')] }
]
}
}
}
]
}
},
])
This gets me all the data I need which is something like:
{
_id: ObjectId("60520e1d09ba3a769ab85f68"),
campaign_ad_group: {
_id: ObjectId("5fc1215a039674c600c0f533"),
name: 'Sick name'
},
broadcastplans: [_ids],
engagements: [
{
_id,
coordinates: [-9.63294964028813,58.1520268933931],
reported_at: Date
},
{
_id,
coordinates: [-8.63294964028813,56.1520268933931],
reported_at: Date
}
],
broadcastreports: [
{
_id,
coordinates: [-9.63294964028813,58.1520268933931],
reported_at: Date,
analytics: {
impressions: 10
}
},
{
_id,
coordinates: [-5.20923334343244,40.4874833434234],
reported_at: Date,
analytics: {
impressions: 5
}
}
]
}
I need to group this in a way that I get the following output:
[
{
_id: campaign_ad_group._id,
name: campaign_ad_group.name,
location: [
{
scans: 1,
impressions: 10,
coordinates: [-9.63294964028813,58.1520268933931]
},
{
scans: 0, // This coordinate does not exist on the engagements so its 0 but needs to be
// included anyway.
impressions: 5,
coordinates: [-5.20923334343244,40.4874833434234]
},
{
scans: 1,
impressions: 0, // Same here
coordinates: [-8.63294964028813,56.1520268933931]
},
]
}
]

Answer 1

Use $unionWith.
Documentation: https://docs.mongodb.com/manual/reference/operator/aggregation/unionWith/

Answer 2

The solution was to make use of unionWith. I did it like so:
.aggregate([
{
$match: {
'_id': ObjectId("60520e1d09ba3a769ab85f68"),
}
},
{
$project: {
'_id': 1,
'broadcasts_to': 1
}
},
{
$lookup: {
from: 'campaignadgroups',
localField: '_id',
foreignField: 'campaign_id',
as:'campaign_ad_group'
}
},
{
$unwind: '$campaign_ad_group'
},
{
$lookup: {
from: 'broadcastplans',
let: { 'campaign_ad_group_id': '$campaign_ad_group._id' },
as: 'broadcastplans',
pipeline: [
{
$match: {
$expr: {
$eq: ['$campaign_ad_group_id','$$campaign_ad_group_id' ]
}
}
},
{ $project: { '_id': 1 } }
]
}
},
// Needed data gathering for the next lookups
{
$lookup: {
from: 'engagements',
let: { 'broadcastplan_ids': '$broadcastplans._id' },
as: 'locations',
pipeline: [
{
$match: {
$expr: {
$and: [
{ $in: ['$broadcast_plan_id', '$$broadcastplan_ids'] },
{ $gte: ['$reported_at', ISODate('2019-12-29T15:02:47.305Z')] },
{ $lte: ['$reported_at', ISODate('2021-12-29T15:02:47.305Z')] }
]
}
}
},
{
$group: {
'_id': '$location.coordinates',
'location': {$first:'$location'},
'scans': {$sum: 1}
}
},
{
$unionWith: {
coll: 'broadcastreports',
pipeline: [
{
$match: {
$expr: {
$and: [
{ $in: ['$broadcast_plan_id', '$$broadcastplan_ids'] },
{ $gte: ['$reported_at', ISODate('2019-12-29T15:02:47.305Z')] },
{ $lte: ['$reported_at', ISODate('2021-12-29T15:02:47.305Z')] }
]
}
}
},
]
}
},
{
$group: {
'_id': '$location.coordinates',
'impressions': {$sum: '$analytics.impressions'},
'scans': {$first: '$scans'}
}
},
{
$project: {
'_id': 0,
'coordinates': '$_id',
'impressions': {$ifNull: ['$impressions', 0]},
'scans': {$ifNull: ['$scans',0]}
}
}
]
}
},
{
$group: {
'_id': '$campaign_ad_group._id',
'name': {$first:'$campaign_ad_group.name'},
'locations': {$first: '$locations'}
}
}
])
The output:
{
"_id" : ObjectId("5fe9f89f603eaa22b3d374f5"),
"name" : "Sick name",
"locations" : [
{
"coordinates" : [
-9.63294964028813,
42.1520268933931
],
"impressions" : 0,
"scans" : 1.0
},
{
"coordinates" : [
-8.63294964028813,
41.1520268933931
],
"impressions" : 0,
"scans" : 1.0
},
{
"coordinates" : [
0,
0
],
"impressions" : 1,
"scans" : 0.0
}
]
}