Reducing an array of values into object with their count using aggregation framework - mongodb

Question

We are using MongoDB to record statistics. The approach is to record each action for an object in its own document and later aggregate them on hourly basis and store them in different collection. Sample documents are below:
[{
"_id" : ObjectId("5e05de1e86029610dc2c6f9c"),
"object_type" : 1,
"object_id" : 1003,
"browser" : "chrome",
"os" : "osx",
"device" : "android",
"category" : 3,
"country" : "gb",
"action" : "impression",
"date_added" : ISODate("2019-12-26T19:00:00.000Z")
},{
"_id" : ObjectId("5e06226586029610db417b7a"),
"object_type" : 1,
"object_id" : 1003,
"browser" : "firefox",
"os" : "osx",
"device" : "lg_tv",
"category" : 1,
"country" : "pe",
"action" : "impression",
"date_added" : ISODate("2019-12-25T19:00:00.000Z")
},{
"_id" : ObjectId("5e06226586029610db417b7b"),
"object_type" : 1,
"object_id" : 1009,
"browser" : "uc_browser",
"os" : "osx",
"device" : "android",
"category" : 4,
"country" : "ru",
"action" : "view",
"date_added" : ISODate("2019-12-25T19:00:00.000Z")
}]
Output should be:
[{
"object_id": 1003,
"object_type": 1,
"action": "impression",
"total": 2,
"date": "2019-12-26 19:00:00",
"browsers": { "firefox": 1, "chrome": 1 },
"systems": { "osx": 2 },
"countries": { "gb": 1, "pe": 1 },
"devices": { "android": 1, "lg_tv": 1 },
"categories": { "3": 1, "1": 1 }
},
{
"object_id": 1009,
"object_type": 1,
"action": "view",
"total": 1,
"date": "2019-12-26 19:00:00",
"browsers": { "uc_browser": 1 },
"systems": { "osx": 1 },
"countries": { "ru": 1 },
"devices": { "android": 1 },
"categories": { "4": 1 }
}]
Aggregation pipeline:
[
{
"$match": {
"date_added": {
"$gte": {
"$date": {
"$numberLong": "1576820825000"
}
}
}
}
},
{
"$group": {
"_id": {
"object_id": "$object_id",
"object_type": "$object_type",
"action": "$action",
"date": {
"$dateToString": {
"format": "%Y-%m-%d %H-00-00",
"date": "$date_added"
}
}
},
"total": {
"$sum": 1
},
"countries": {
"$push": "$country"
}
}
},
{
"$project": {
"action": "$_id.action",
"object_id": "$_id.object_id",
"object_type": "$_id.object_type",
"date": "$_id.date",
"total": 1,
"countries": 1,
"systems": 1,
"devices": 1,
"categories": 1,
"_id": 0
}
},
{
"$sort": {
"total": -1
}
}
]
This pipeline provides total of an object for a certain action on given hour and push each country into countries array - for readability removed other indexes from $group.
I’m stuck at transforming countries array into desired object. Two question popped in my mind.
Is it possible with single aggregation pipeline?
Should I just return documents using above pipeline and process rest of the indexes with scripting?

Answer 1

It's possible, but a bit tedious...
You need to $group each new field in the next stage and acumulate previous fields.
ASSUMPTION
Your expected result for "object_id": 1003 with total:2, but date_added is 2019-12-26 and 2019-12-25. So, I've changed to 2019-12-26 both documents.
db.collection.aggregate([
{
"$match": {
"date_added": {
"$gte": {
"$date": {
"$numberLong": "1576820825000"
}
}
}
}
},
{
$group: {
_id: {
"object_id": "$object_id",
"object_type": "$object_type",
"action": "$action",
"date": {
"$dateToString": {
"format": "%Y-%m-%d %H-00-00",
"date": "$date_added",
timezone: "GMT"
}
}
},
data: {
"$push": "$$ROOT"
},
total: {
$sum: 1
}
}
},
{
$unwind: "$data"
},
{
$group: {
_id: {
_id: "$_id",
"tmp": "$data.category"
},
data: {
$push: "$data"
},
total: {
$first: "$total"
},
count: {
$sum: 1
}
}
},
{
$group: {
_id: "$_id._id",
data: {
$push: "$data"
},
total: {
$first: "$total"
},
categories: {
$push: {
k: {
$toString: "$_id.tmp"
},
v: "$count"
}
}
}
},
{
$unwind: "$data"
},
{
$unwind: "$data"
},
{
$group: {
_id: {
_id: "$_id",
"tmp": "$data.device"
},
categories: {
$first: "$categories"
},
data: {
$push: "$data"
},
total: {
$first: "$total"
},
count: {
$sum: 1
}
}
},
{
$group: {
_id: "$_id._id",
data: {
$push: "$data"
},
total: {
$first: "$total"
},
categories: {
$first: "$categories"
},
devices: {
$push: {
k: "$_id.tmp",
v: "$count"
}
}
}
},
{
$unwind: "$data"
},
{
$unwind: "$data"
},
{
$group: {
_id: {
_id: "$_id",
"tmp": "$data.country"
},
devices: {
$first: "$devices"
},
categories: {
$first: "$categories"
},
data: {
$push: "$data"
},
total: {
$first: "$total"
},
count: {
$sum: 1
}
}
},
{
$group: {
_id: "$_id._id",
data: {
$push: "$data"
},
total: {
$first: "$total"
},
devices: {
$first: "$devices"
},
categories: {
$first: "$categories"
},
countries: {
$push: {
k: "$_id.tmp",
v: "$count"
}
}
}
},
{
$unwind: "$data"
},
{
$unwind: "$data"
},
{
$group: {
_id: {
_id: "$_id",
"tmp": "$data.os"
},
countries: {
$first: "$countries"
},
devices: {
$first: "$devices"
},
categories: {
$first: "$categories"
},
data: {
$push: "$data"
},
total: {
$first: "$total"
},
count: {
$sum: 1
}
}
},
{
$group: {
_id: "$_id._id",
data: {
$push: "$data"
},
total: {
$first: "$total"
},
countries: {
$first: "$countries"
},
devices: {
$first: "$devices"
},
categories: {
$first: "$categories"
},
systems: {
$push: {
k: "$_id.tmp",
v: "$count"
}
}
}
},
{
$unwind: "$data"
},
{
$unwind: "$data"
},
{
$group: {
_id: {
_id: "$_id",
"tmp": "$data.browser"
},
systems: {
$first: "$systems"
},
countries: {
$first: "$countries"
},
devices: {
$first: "$devices"
},
categories: {
$first: "$categories"
},
data: {
$push: "$data"
},
total: {
$first: "$total"
},
count: {
$sum: 1
}
}
},
{
$group: {
_id: "$_id._id",
data: {
$push: "$data"
},
total: {
$first: "$total"
},
systems: {
$first: "$systems"
},
countries: {
$first: "$countries"
},
devices: {
$first: "$devices"
},
categories: {
$first: "$categories"
},
browsers: {
$push: {
k: "$_id.tmp",
v: "$count"
}
}
}
},
{
$project: {
_id: 0,
action: "$_id.action",
date: "$_id.date",
object_id: "$_id.object_id",
object_type: "$_id.object_type",
total: 1,
categories: {
$arrayToObject: "$categories"
},
countries: {
$arrayToObject: "$countries"
},
devices: {
$arrayToObject: "$devices"
},
systems: {
$arrayToObject: "$systems"
},
browsers: {
$arrayToObject: "$browsers"
}
}
},
{
$sort: {
object_id: 1,
date: 1
}
}
])
MongoPlayground
Note: Other approach was to use $facet and create fields separately and then merge them into final object, but MongoPlayground sometimes worked buggy (click Run button several times and you get different result)