Mongodb group by and sum and get media with Map - mongodb

I have these collections in my database:
Items:
{ "IdUser" : "1", "IdItem" : "1" },
{ "IdUser" : "1", "IdItem" : "2" },
{ "IdUser" : "1", "IdItem" : "3" },
{ "IdUser" : "2", "IdItem" : "4" },
{ "IdUser" : "2", "IdItem" : "5" },
{ "IdUser" : "4", "IdItem" : "6" },
{ "IdUser" : "5", "IdItem" : "7" }
Users
{ "_id" : "1", "DateRegister" : ISODate("2016-03-29T22:00:38.764+0000") },
{ "_id" : "2", "DateRegister" : ISODate("2014-03-29T22:00:38.764+0000") },
{ "_id" : "2", "DateRegister" : ISODate("2015-02-29T22:00:38.764+0000") },
{ "_id" : "4", "DateRegister" : ISODate("2013-01-29T22:00:38.764+0000") },
{ "_id" : "5", "DateRegister" : ISODate("2016-04-29T22:00:38.764+0000") }
How can I obtain this result but FILTERED with users registered after 2015:
Users with one item: 2
Users with two items: 1
Users with three items: 1
I have tried with that, but I don't know how to filter... Thanks!
db.collection.aggregate([
{
"$group": {
"_id": "$IdUser",
"count": {
"$sum": { "$cond": [{ "$gt": [ "$IdItem", null ] }, 1, 0 ] }
}
}
},
{
"$group": {
"_id": "$count",
"users": { "$push": "$_id" }
}
},
{
"$project": {
"_id": 0,
"number_of_items": "$_id",
"number_of_users": { "$size": "$users" }
}
}
])

You may want to utilize the $lookup operator to perform a join of the items collection with the users collection and then do a $match filter on the DateRegistered field before piping the main grouping operations.
Following this example + the links herein to the documentation will give you an idea:
db.items.aggregate([
{
"$lookup": {
"from": "users",
"localField": "IdUser",
"foreignField": "_id",
"as": "user"
}
},
{ "$match": { "user.DateRegister": { "$gt": new Date(2015, 11, 31) } } },
{
"$group": {
"_id": "$IdUser",
"count": {
"$sum": { "$cond": [{ "$gt": [ "$IdItem", null ] }, 1, 0 ] }
}
}
},
{
"$group": {
"_id": "$count",
"users": { "$push": "$_id" }
}
},
{
"$project": {
"_id": 0,
"number_of_items": "$_id",
"number_of_users": { "$size": "$users" }
}
}
])
In the event that your MongoDB server does not support the $lookup operator, you will then need a workaround where you split the operations on the different collections i.e.
get a list of user id's that match the given date range criteria, this could be done with the distinct() method on the users collection with the date query option.
use that list in the items collection aggregation pipeline within the $match operator initial step.
The following demonstrates this:
// use distinct to get the user id's list
var userIds = db.users.distinct("_id", { "DateRegister": { "$gt": new Date(2015, 11, 31) } })
// perform your aggregation with a filtered collection using the list from the above operations
db.items.aggregate([
{ "$match": { "IdUser": { "$in": userIds } } },
{
"$group": {
"_id": "$IdUser",
"count": {
"$sum": { "$cond": [{ "$gt": [ "$IdItem", null ] }, 1, 0 ] }
}
}
},
{
"$group": {
"_id": "$count",
"users": { "$push": "$_id" }
}
},
{
"$project": {
"_id": 0,
"number_of_items": "$_id",
"number_of_users": { "$size": "$users" }
}
}
])

Related

Counting the two value in a attribute using aggregate in mongodb

I have some documents in a collection which looks like this
{
"_id" : "5a2e50b32d43ba00010041e5",
account_id:"23232323"
status:"accepted",
keyname:"java"
},
{
"_id" : "5a2e54332d43ba00010041e5",
account_id:"2323233"
status:"pending",
keyname:"java"
},
{
"_id" : "5a2e54332d43ba00010041e5",
account_id:"23232sdsd3"
status:"pending",
keyname:"Nodejs"
}
I need to get the counts of the pending and accepted status for each keyname for a particular account_id
eg: should give a result like this.
{
keyname:"java",
pending:10,
accepted:10
}
This is the code that I have tried out
db.getCollection("programs").aggregate([
{ "$match": { "account_id": "1" } },
{ "$group": { "_id": "$keyname", "count": { "$sum": 1 } } },
{ "$match": { "_id": { "$ne": null } } }
])
which gives a result like this
{
"_id" : "java",
"count" : 3.0
},
{
"_id" : "nodejs",
"count" : 3.0
},
{
"_id" : "C#",
"count" : 3.0
}
You can use below aggregation
db.collection.aggregate([
{ "$match": { "account_id": "1" } },
{ "$group": {
"_id": "$keyname",
"accepted": {
"$sum": {
"$cond": [
{ "$eq": ["$status", "accepted"] },
0,
1
]
}
},
"pending": {
"$sum": {
"$cond": [
{ "$eq": ["$status", "pending"] },
0,
1
]
}
}
}}
])

MongoDB aggregate nested grouping

I have Asset collection which has data like
{
"_id" : ObjectId("5bfb962ee2a301554915"),
"users" : [
"abc.abc#abc.com",
"abc.xyz#xyz.com"
],
"remote" : {
"source" : "dropbox",
"bytes" : 1234
}
{
"_id" : ObjectId("5bfb962ee2a301554915"),
"users" : [
"pqr.pqr#pqr.com",
],
"remote" : {
"source" : "google_drive",
"bytes" : 785
}
{
"_id" : ObjectId("5bfb962ee2a301554915"),
"users" : [
"abc.abc#abc.com",
"abc.xyz#xyz.com"
],
"remote" : {
"source" : "gmail",
"bytes" : 5647
}
What I am looking for is group by users and get the total of bytes according to its source like
{
"_id" : "abc.abc#abc.com",
"bytes" : {
"google_drive": 1458,
"dropbox" : 1254
}
}
I am not getting how to get the nested output using grouping.
I have tried with the query
db.asset.aggregate(
[
{$unwind : '$users'},
{$group:{
_id:
{'username': "$users",
'source': "$remote.source",
'total': {$sum: "$remote.bytes"}} }
}
]
)
This way I am getting the result with the repeated username.
With MongoDb 3.6 and newer, you can leverage the use of $arrayToObject operator within a $mergeObjects expression and a $replaceRoot pipeline to get the desired result.
You would need to run the following aggregate pipeline though:
db.asset.aggregate([
{ "$unwind": "$users" },
{ "$group": {
"_id": {
"users": "$users",
"source": "$remote.source"
},
"totalBytes": { "$sum": "$remote.bytes" }
} },
{ "$group": {
"_id": "$_id.users",
"counts": {
"$push": {
"k": "$_id.source",
"v": "$totalBytes"
}
}
} },
{ "$replaceRoot": {
"newRoot": {
"$mergeObjects": [
{ "bytes": { "$arrayToObject": "$counts" } },
"$$ROOT"
]
}
} },
{ "$project": { "counts": 0 } }
])
which yields
/* 1 */
{
"bytes" : {
"gmail" : 5647.0,
"dropbox" : 1234.0
},
"_id" : "abc.abc#abc.com"
}
/* 2 */
{
"bytes" : {
"google_drive" : 785.0
},
"_id" : "pqr.pqr#pqr.com"
}
/* 3 */
{
"bytes" : {
"gmail" : 5647.0,
"dropbox" : 1234.0
},
"_id" : "abc.xyz#xyz.com"
}
using the above sample documents.
You have to use $group couple of times here. First with the users and the source and count the total number of bytes using $sum.
And second with the users and $push the source and the bytes into an array
db.collection.aggregate([
{ "$unwind": "$users" },
{ "$group": {
"_id": {
"users": "$users",
"source": "$remote.source"
},
"bytes": { "$sum": "$remote.bytes" }
}},
{ "$group": {
"_id": "$_id.users",
"data": {
"$push": {
"source": "$_id.source",
"bytes": "$bytes"
}
}
}}
])
And even if you want to convert the source and the bytes into key value format then replace the last $group stage with the below two stages.
{ "$group": {
"_id": "$_id.users",
"data": {
"$push": {
"k": "$_id.source",
"v": "$bytes"
}
}
}},
{ "$project": {
"_id": 0,
"username": "$_id",
"bytes": { "$arrayToObject": "$data" }
}}

$match and $cond query doesn't give the same result in MongoDB aggregate

Considering this two different MongoDB queries:
startDate query in $match
db.myCollection.aggregate([{
"$match": {
"code": "2",
"startDate": {
"$lt": ISODate("2017-01-31T23:59:59.999Z")
},
}
},
{
"$group": {
"_id": {
"code": "$code"
},
"count": {
"$sum": 1
}
}
}
]);
Result:
{
"_id" : {
"code" : "2"
},
"count" : 4844.0
}
startDate query in $cond
db.myCollection.aggregate([{
"$match": {
"code": "2"
}
},
{
"$project": {
"code": "$code",
"count": {
"$cond": [{
"$lt": ["$startDate", ISODate('2017-01-31T23:59:59.999Z')]
}, 1, 0]
}
}
}
}, {
"$group": {
"_id": {
"code": "$code"
},
"count": {
"$sum": "$count"
}
}
}])
Result:
{
"_id" : {
"code" : "2"
},
"count" : 4935.0
}
I don't understand why the second query give me more documents. It seems to me that this queries must give me a identical result... Am I using the "$cond" in a wrong way? What may causes this difference?

Using the aggregation framework to compare array element overlap

I have a collections with documents structured like below:
{
carrier: "abc",
flightNumber: 123,
dates: [
ISODate("2015-01-01T00:00:00Z"),
ISODate("2015-01-02T00:00:00Z"),
ISODate("2015-01-03T00:00:00Z")
]
}
I would like to search the collection to see if there are any documents with the same carrier and flightNumber that also have dates in the dates array that over lap. For example:
{
carrier: "abc",
flightNumber: 123,
dates: [
ISODate("2015-01-01T00:00:00Z"),
ISODate("2015-01-02T00:00:00Z"),
ISODate("2015-01-03T00:00:00Z")
]
},
{
carrier: "abc",
flightNumber: 123,
dates: [
ISODate("2015-01-03T00:00:00Z"),
ISODate("2015-01-04T00:00:00Z"),
ISODate("2015-01-05T00:00:00Z")
]
}
If the above records were present in the collection I would like to return them because they both have carrier: abc, flightNumber: 123 and they also have the date ISODate("2015-01-03T00:00:00Z") in the dates array. If this date were not present in the second document then neither should be returned.
Typically I would do this by grouping and counting like below:
db.flights.aggregate([
{
$group: {
_id: { carrier: "$carrier", flightNumber: "$flightNumber" },
uniqueIds: { $addToSet: "$_id" },
count: { $sum: 1 }
}
},
{
$match: {
count: { $gt: 1 }
}
}
])
But I'm not sure how I could modify this to look for array overlap. Can anyone suggest how to achieve this?
You $unwind the array if you want to look at the contents as "grouped" within them:
db.flights.aggregate([
{ "$unwind": "$dates" },
{ "$group": {
"_id": { "carrier": "$carrier", "flightnumber": "$flightnumber", "date": "$dates" },
"count": { "$sum": 1 },
"_ids": { "$addToSet": "$_id" }
}},
{ "$match": { "count": { "$gt": 1 } } },
{ "$unwind": "$_ids" },
{ "$group": { "_id": "$_ids" } }
])
That does in fact tell you which documents where the "overlap" resides, because the "same dates" along with the other same grouping key values that you are concerned about have a "count" which occurs more than once. Indicating the overlap.
Anything after the $match is really just for "presentation" as there is no point reporting the same _id value for multiple overlaps if you just want to see the overlaps. In fact if you want to see them together it would probably be best to leave the "grouped set" alone.
Now you could add a $lookup to that if retrieving the actual documents was important to you:
db.flights.aggregate([
{ "$unwind": "$dates" },
{ "$group": {
"_id": { "carrier": "$carrier", "flightnumber": "$flightnumber", "date": "$dates" },
"count": { "$sum": 1 },
"_ids": { "$addToSet": "$_id" }
}},
{ "$match": { "count": { "$gt": 1 } } },
{ "$unwind": "$_ids" },
{ "$group": { "_id": "$_ids" } },
}},
{ "$lookup": {
"from": "flights",
"localField": "_id",
"foreignField": "_id",
"as": "_ids"
}},
{ "$unwind": "$_ids" },
{ "$replaceRoot": {
"newRoot": "$_ids"
}}
])
And even do a $replaceRoot or $project to make it return the whole document. Or you could have even done $addToSet with $$ROOT if it was not a problem for size.
But the overall point is covered in the first three pipeline stages, or mostly in just the "first". If you want to work with arrays "across documents", then the primary operator is still $unwind.
Alternately for a more "reporting" like format:
db.flights.aggregate([
{ "$addFields": { "copy": "$$ROOT" } },
{ "$unwind": "$dates" },
{ "$group": {
"_id": {
"carrier": "$carrier",
"flightNumber": "$flightNumber",
"dates": "$dates"
},
"count": { "$sum": 1 },
"_docs": { "$addToSet": "$copy" }
}},
{ "$match": { "count": { "$gt": 1 } } },
{ "$group": {
"_id": {
"carrier": "$_id.carrier",
"flightNumber": "$_id.flightNumber",
},
"overlaps": {
"$push": {
"date": "$_id.dates",
"_docs": "$_docs"
}
}
}}
])
Which would report the overlapped dates within each group and tell you which documents contained the overlap:
{
"_id" : {
"carrier" : "abc",
"flightNumber" : 123.0
},
"overlaps" : [
{
"date" : ISODate("2015-01-03T00:00:00.000Z"),
"_docs" : [
{
"_id" : ObjectId("5977f9187dcd6a5f6a9b4b97"),
"carrier" : "abc",
"flightNumber" : 123.0,
"dates" : [
ISODate("2015-01-03T00:00:00.000Z"),
ISODate("2015-01-04T00:00:00.000Z"),
ISODate("2015-01-05T00:00:00.000Z")
]
},
{
"_id" : ObjectId("5977f9187dcd6a5f6a9b4b96"),
"carrier" : "abc",
"flightNumber" : 123.0,
"dates" : [
ISODate("2015-01-01T00:00:00.000Z"),
ISODate("2015-01-02T00:00:00.000Z"),
ISODate("2015-01-03T00:00:00.000Z")
]
}
]
}
]
}

MongoDB aggregate/grouping by key-value pairs

My data looks something like this:
{
"_id" : "9aa072e4-b706-47e6-9607-1a39e904a05a",
"customerId" : "2164289-4",
"channelStatuses" : {
"FOO" : {
"status" : "done"
},
"BAR" : {
"status" : "error"
}
},
"channel" : "BAR",
}
My aggregate/group looks like this:
{
"_id" : {
"customerId" : "$customerId",
"channel" : "$channel",
"status" : "$channelStatuses[$channel].status"
},
"count" : {
"$sum" : 1
}
}
So basically with the example data the group should give me a group grouped by:
{"customerId": "2164289-4", "channel": "BAR", "status": "error"}
But I cannot use []-indexing in a aggregate/group. What should I do instead?
You cannot get the result you want with the current structure using .aggregate(). You "could" change the structure to use an array rather than named keys, and the operation is actually quite simple.
So with a document like:
{
"_id" : "9aa072e4-b706-47e6-9607-1a39e904a05a",
"customerId" : "2164289-4",
"channelStatuses" : [
{
"channel": "FOO",
"status" : "done"
},
{
"channel": "BAR",
"status" : "error"
}
],
"channel" : "BAR",
}
You can then do in modern releases with $filter, $map and $arrayElemAt:
{ "$group": {
"_id": {
"customerId" : "$customerId",
"channel" : "$channel",
"status": {
"$arrayElemAt": [
{ "$map": {
"input": { "$filter": {
"input": "$chanelStatuses",
"as": "el",
"cond": { "$eq": [ "$$el.channel", "$channel" ] }
}},
"as": "el",
"in": "$$el.status"
}},
0
]
}
},
"count": { "$sum": 1 }
}}
Older versions of MongoDB are going to going to require $unwind to access the matched array element.
In MongoDB 2.6 then you can still "pre-filter" the array before unwind:
[
{ "$project": {
"customerId": 1,
"channel": 1,
"status": {
"$setDifference": [
{ "$map": {
"input": "$channelStatuses",
"as": "el",
"in": {
"$cond": [
{ "$eq": [ "$$el.channel", "$channel" ] },
"$$el.status",
false
]
}
}},
[false]
]
}
}},
{ "$unwind": "$status" },
{ "$group": {
"_id": {
"customerId": "$customerId",
"channel": "$channel",
"status": "$status"
},
"count": { "$sum": 1 }
}}
]
And anything prior to that you "filter" after $unwind instead:
[
{ "$unwind": "$channelStatuses" },
{ "$project": {
"customerId": 1,
"channel": 1,
"status": "$channelStatuses.status",
"same": { "$eq": [ "$channelStatuses.status", "$channel" ] }
}},
{ "$match": { "same": true } },
{ "$group": {
"_id": "$_id",
"customerId": { "$first": "$customerId" },
"channel": { "$first": "$channel" },
"status": { "$first": "$status" }
}},
{ "$group": {
"_id": {
"customerId": "$customerId",
"channel": "$channel",
"status": "$status"
},
"count": { "$sum": 1 }
}}
]
In a lesser version than MongoDB 2.6 you also need to $project the result of the equality test between the two fields and then $match on the result in a seperate stage. You might also note the "two" $group stages, since the first one removes any possible duplicates of the "channel" values after the filter via the $first accumulators. The following $group is exactly the same as in the previous listing.
But if you cannot change the structure and need "flexible" matching of keys where you cannot supply every name, then you must use mapReduce:
db.collection.mapReduce(
function() {
emit({
"customerId": this.customerId,
"channel": this.channel,
"status": this.channelStatuses[this.channel].status
},1);
},
function(key,values) {
return Array.sum(values);
},
{ "out": { "inline": 1 } }
)
Where of course you can use that sort of notation