Related
Im a MongoDB noob so please dont judge me if my question is stupid:P
Im trying to get some results from MongoDB to create a table that would show percentage statistics of how much do i play a certain game per day of week (all games together per day = 100%). This is my JSON import for the database:
[
{"title":"GTA","date":"2017-11-13"},
{"title":"GTA","date":"2017-11-13"},
{"title":"BattleField","date":"2017-11-13"},
{"title":"BattleField","date":"2017-11-13"},
{"title":"BattleField","date":"2017-11-14"}
]
Ive written an aggregation that grouped the results by days and counted the total amount of times a game has been played per each day…:
db.games.aggregate([
{ $project: { _id: 0, date : { $dayOfWeek: "$date" }, "title":1} },
{ $group: { _id: {title: "$title", date: "$date"}, total: {$sum: 1} } },
{ $group: { _id: "$_id.date", types: {$addToSet: {title:"$_id.title", total: "$total"} } } }
])
…and this is what i got from MongoDB now:
/* 1 */
{
"_id" : 3,
"types" : [
{
"title" : "BattleField",
"total" : 1.0
}
]
},
/* 2 */
{
"_id" : 2,
"types" : [
{
"title" : "GTA",
"total" : 2.0
},
{
"title" : "BattleField",
"total" : 2.0
}
]
}
what i need to get is a table that would look like this:
Monday Tuesday
GTA 50,00% 0%
BattleField 50,00% 100%
Could you please advise me how can i get such percentage results from Mongo?
Your attempt was pretty close to a solution! The following should point you in the right direction:
aggregate([
{ $project: { "_id": 0, "date" : { $dayOfWeek: "$date" }, "title": 1 } }, // get the day of the week from the "date" field
{ $group: { "_id": { "title": "$title", "date": "$date" }, "total": { $sum: 1 } } }, // group by title and date to get the total per title and date
{ $group: { "_id": "$_id.date", "types": { $push: { "title": "$_id.title", total: "$total" } }, "grandTotal": { $sum: "$total" } } }, // group by date only to get the grand total
{ $unwind: "$types" }, // flatten grouped items
{ $project: { "_id": 0, "title": "$types.title", "percentage": { $divide: [ "$types.total", "$grandTotal" ] }, "day": { $arrayElemAt: [ [ "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" ], "$_id" ] } } }, // calculate percentage and beautify output for "day"
])
Results:
{
"title" : "BattleField",
"percentage" : 0.5,
"day" : "Tue"
}
{
"title" : "GTA",
"percentage" : 0.5,
"day" : "Tue"
}
{
"title" : "BattleField",
"percentage" : 1.0,
"day" : "Wed"
}
I'm attempting to group the items in a collection by year/month/day. The grouping should be based on the pubDate and pubTimezoneOffset.
I've got an aggregate pipeline that:
- $project - adds the timezoneOffset to the pubDate
- $group - groups by the modified pubDate
- $project - removes the timezoneOffset
- $sort - sorts by pubDate
I tested each stage on it's own and it seems to be some issue with the second $project. In the final output the pubDate is null.
I've been going over it for a few hours now and can't see where I've gone wrong. What am I missing?
The aggregate pipeline:
db.messages.aggregate([
{
$project: {
_id: 1,
pubTimezoneOffset: 1,
pubDate: {
$add: [
'$pubDate', {
$add: [
{ $multiply: [ '$pubTimezoneOffset.hours', 60, 60, 1000 ] },
{ $multiply: [ '$pubTimezoneOffset.minutes', 60, 1000 ] }
]
}
]
}
}
},
{
$group: {
_id: {
year: { $year: '$pubDate' },
month: { $month: '$pubDate' },
day: { $dayOfMonth: '$pubDate' }
},
count: { $sum: 1 },
messages: {
$push: {
_id: '$_id',
pubTimezoneOffset: '$pubTimezoneOffset',
pubDate: '$pubDate'
}
}
}
},
{
$project: {
_id: 1,
messages: {
_id: 1,
pubTimezoneOffset: 1,
pubDate: {
$subtract: [
'$pubDate', {
$add: [
{ $multiply: [ '$pubTimezoneOffset.hours', 60, 60, 1000 ] },
{ $multiply: [ '$pubTimezoneOffset.minutes', 60, 1000 ] }
]
}
]
}
},
count: 1
}
},
{
$sort: {
'_id.year': -1,
'_id.month': -1,
'_id.day': -1
}
}
]).pretty();
To recreate the source data:
db.messages.insertOne({
pubDate: ISODate('2017-10-25T10:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
db.messages.insertOne({
pubDate: ISODate('2017-10-25T11:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
db.messages.insertOne({
pubDate: ISODate('2017-10-24: 10:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
db.messages.insertOne({
pubDate: ISODate('2017-10-24: 11:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
Running it in mongo shell outputs:
{
"_id" : {
"year" : 2017,
"month" : 10,
"day" : 25
},
"count" : 2,
"messages" : [
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b3"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
},
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b4"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
}
]
}
{
"_id" : {
"year" : 2017,
"month" : 10,
"day" : 23
},
"count" : 2,
"messages" : [
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b5"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
},
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b6"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
}
]
}
Kudos for the attempt but, you actually have quite a few things conceptually incorrect here, with the basic error you are seeing is because your premise of "array projection" is incorrect. You are trying to refer to variables "inside the array" by simply notating the "property name".
What you actually need to do here is apply $map in order to apply the functions to "transform" each element:
db.messages.aggregate([
{ "$project": {
"pubTimezoneOffset": 1,
"pubDate": {
"$add": [
"$pubDate",
{ "$add": [
{ "$multiply": [ '$pubTimezoneOffset.hours', 60 * 60 * 1000 ] },
{ "$multiply": [ '$pubTimezoneOffset.minutes', 60 * 1000 ] }
]}
]
}
}},
{ "$group": {
"_id": {
"year": { "$year": "$pubDate" },
"month": { "$month": "$pubDate" },
"day": { "$dayOfMonth": "$pubDate" }
},
"count": { "$sum": 1 },
"messages": {
"$push": {
"_id": "$_id",
"pubTimezoneOffset": "$pubTimezoneOffset",
"pubDate": "$pubDate"
}
}
}},
{ "$project": {
"messages": {
"$map": {
"input": "$messages",
"as": "m",
"in": {
"_id": "$$m._id",
"pubTimezoneOffset": "$$m.pubTimezoneOffset",
"pubDate": {
"$subtract": [
"$$m.pubDate",
{ "$add": [
{ "$multiply": [ "$$m.pubTimezoneOffset.hours", 60 * 60 * 1000 ] },
{ "$multiply": [ "$$m.pubTimezoneOffset.minutes", 60 * 1000 ] }
]}
]
}
}
}
},
"count": 1
}},
{ "$sort": { "_id": -1 } }
]).pretty();
Noting here that you are doing a lot of unnecessary work in "tranforming" the dates kept in the array, and then trying to "tranform" them back to the original state. Instead, you should have simply supplied a "variable" with $let to the _id of $group and left the original document state "as is" using $$ROOT instead of naming all the fields:
db.messages.aggregate([
{ "$group": {
"_id": {
"$let": {
"vars": {
"pubDate": {
"$add": [
"$pubDate",
{ "$add": [
{ "$multiply": [ '$pubTimezoneOffset.hours', 60 * 60 * 1000 ] },
{ "$multiply": [ '$pubTimezoneOffset.minutes', 60 * 1000 ] }
]}
]
}
},
"in": {
"year": { "$year": "$$pubDate" },
"month": { "$month": "$$pubDate" },
"day": { "$dayOfMonth": "$$pubDate" }
}
}
},
"docs": { "$push": "$$ROOT" }
}},
{ "$sort": { "_id": -1 } }
])
Also note that $sort simply does actually consider all the "sub-keys" anyway, so there is no need to name them explicitly.
Back to your error, the point of $map is essentially because whilst you can notate array "field inclusion" with MongoDB 3.2 and above like this:
"messages": {
"_id": 1,
"pubTimeZoneOffset": 1
}
The thing you cannot do is actually "calculate values" on the elements themselves. You tried "$pubDate" which actually looks in the "ROOT" space for a property of that name, which does not exist and is null. If you then tried:
"messages": {
"_id": 1,
"pubTimeZoneOffset": 1,
"pubDate": "$messages.pubDate"
}
Then you would get "a result", but not the result you might think. Because what would actually be included in "every element" is the value of that property in each array element as a "new array" itself.
So the short and sweet is use $map instead, which iterates the array elements with a local variable referring to the current element for you to notate values for in expressions.
MongoDB 3.6
MongoDB date operators are all timezone aware. So instead of all the juggling then all you need do is supply the additional "timezone" parameter to any option and the conversion will be done for you.
As a sample:
db.messages.aggregate([
{ "$group": {
"_id": {
"$dateToString": {
"date": "$pubDate",
"format": "%Y-%m-%d",
"timezone": {
"$concat": [
{ "$cond": {
"if": { "$gt": [ "$pubTimezoneOffset", 0 ] },
"then": "+",
"else": "-"
}},
{ "$let": {
"vars": {
"hours": { "$substr": [{ "$abs": "$pubTimezoneOffset.hours" },0,2] },
"minutes": { "$substr": [{ "$abs": "$pubTimezoneOffset.minutes" },0,2] }
},
"in": {
"$concat": [
{ "$cond": {
"if": { "$eq": [{ "$strLenCP": "$$hours" }, 1 ] },
"then": { "$concat": [ "0", "$$hours" ] },
"else": "$$hours"
}},
":",
{ "$cond": {
"if": { "$eq": [{ "$strLenCP": "$$minutes" }, 1 ] },
"then": { "$concat": [ "0", "$$minutes" ] },
"else": "$$minutes"
}}
]
}
}}
]
}
}
},
"docs": { "$push": "$$ROOT" }
}},
{ "$sort": { "_id": -1 } }
])
Note that most of the "juggling" in there is to convert your own "offset" to the "string" format required by the new operators. If you simply stored this as "offset": "-07:00" then you can instead simply write:
db.messages.aggregate([
{ "$group": {
"_id": {
"$dateToString": {
"date": "$pubDate",
"format": "%Y-%m-%d",
"timezone": "$offset"
}
},
"docs": { "$push": "$$ROOT" }
}},
{ "$sort": { "_id": -1 } }
])
Please Reconsider
I can't let this pass without making a note that your general approach here is conceptually incorrect. Storing "offset" or "local time string" within the database is just intrinsically wrong.
The date information should be stored as UTC and should be returned as UTC. Sure you can and "should" covert when aggregating, but the general premise is that you always convert back to UTC. And "conversion" comes from the "locale of the observer" and not a "stored" adjustment. Because dates are always relative to the "observer" point of view, and are not from the "point of origin" as you seem to have interpreted it.
I put some lengthy detail on this on Group by Date with Local Time Zone in MongoDB about why you store this way and why "locale" conversion from the "observer" is necessary. That also details "Daylight savings considerations" from the observer point of view.
The basic premise there still remains the same when MongoDB becomes "timezone aware" in that you :
Store in UTC
Query with local time converted to UTC
Aggregate converted from the "observer" offset
Convert the "offset" back to UTC
Because at the end of the day it's the "clients" job to supply that "locale" conversion, since that's the part that "knows where it is".
If I have data in the following format:
[
{
_id: 1,
startDate: ISODate("2017-01-1T00:00:00.000Z"),
endDate: ISODate("2017-02-25T00:00:00.000Z"),
type: 'CAR'
},
{
_id: 2,
startDate: ISODate("2017-02-17T00:00:00.000Z"),
endDate: ISODate("2017-03-22T00:00:00.000Z"),
type: 'HGV'
}
]
Is it possible to retrieve data grouped by 'type', but also with a count of the type for each of month in a given date range e.g. between 2017/1/1 to 2017/4/1 would return:
[
{
_id: 'CAR',
monthCounts: [
/*January*/
{
from: ISODate("2017-01-1T00:00:00.000Z"),
to: ISODate("2017-01-31T23:59:59.999Z"),
count: 1
},
/*February*/
{
from: ISODate("2017-02-1T00:00:00.000Z"),
to: ISODate("2017-02-28T23:59:59.999Z"),
count: 1
},
/*March*/
{
from: ISODate("2017-03-1T00:00:00.000Z"),
to: ISODate("2017-03-31T23:59:59.999Z"),
count: 0
},
]
},
{
_id: 'HGV',
monthCounts: [
{
from: ISODate("2017-01-1T00:00:00.000Z"),
to: ISODate("2017-01-31T23:59:59.999Z"),
count: 0
},
{
from: ISODate("2017-02-1T00:00:00.000Z"),
to: ISODate("2017-02-28T23:59:59.999Z"),
count: 1
},
{
from: ISODate("2017-03-1T00:00:00.000Z"),
to: ISODate("2017-03-31T23:59:59.999Z"),
count: 1
},
]
}
]
The returned format is not really important, but what I am trying to achieve is in a single query to retrieve a number of counts for the same grouping (one per month). The input could be simply a start and end date to report from or more likely it could be an array of the date ranges to group by.
The algorithm for this is to basically "iterate" values between the interval of the two values. MongoDB has a couple of ways to deal with this, being what has always been present with mapReduce() and with new features available to the aggregate() method.
I'm going expand on your selection to deliberately show an overlapping month since your examples did not have one. This will result in the "HGV" values appearing in "three" months of output.
{
"_id" : 1,
"startDate" : ISODate("2017-01-01T00:00:00Z"),
"endDate" : ISODate("2017-02-25T00:00:00Z"),
"type" : "CAR"
}
{
"_id" : 2,
"startDate" : ISODate("2017-02-17T00:00:00Z"),
"endDate" : ISODate("2017-03-22T00:00:00Z"),
"type" : "HGV"
}
{
"_id" : 3,
"startDate" : ISODate("2017-02-17T00:00:00Z"),
"endDate" : ISODate("2017-04-22T00:00:00Z"),
"type" : "HGV"
}
Aggregate - Requires MongoDB 3.4
db.cars.aggregate([
{ "$addFields": {
"range": {
"$reduce": {
"input": { "$map": {
"input": { "$range": [
{ "$trunc": {
"$divide": [
{ "$subtract": [ "$startDate", new Date(0) ] },
1000
]
}},
{ "$trunc": {
"$divide": [
{ "$subtract": [ "$endDate", new Date(0) ] },
1000
]
}},
60 * 60 * 24
]},
"as": "el",
"in": {
"$let": {
"vars": {
"date": {
"$add": [
{ "$multiply": [ "$$el", 1000 ] },
new Date(0)
]
},
"month": {
}
},
"in": {
"$add": [
{ "$multiply": [ { "$year": "$$date" }, 100 ] },
{ "$month": "$$date" }
]
}
}
}
}},
"initialValue": [],
"in": {
"$cond": {
"if": { "$in": [ "$$this", "$$value" ] },
"then": "$$value",
"else": { "$concatArrays": [ "$$value", ["$$this"] ] }
}
}
}
}
}},
{ "$unwind": "$range" },
{ "$group": {
"_id": {
"type": "$type",
"month": "$range"
},
"count": { "$sum": 1 }
}},
{ "$sort": { "_id": 1 } },
{ "$group": {
"_id": "$_id.type",
"monthCounts": {
"$push": { "month": "$_id.month", "count": "$count" }
}
}}
])
The key to making this work is the $range operator which takes values for a "start" and and "end" as well as an "interval" to apply. The result is an array of values taken from the "start" and incremented until the "end" is reached.
We use this with startDate and endDate to generate the possible dates in between those values. You will note that we need to do some math here since the $range only takes a 32-bit integer, but we can take the milliseconds away from the timestamp values so that is okay.
Because we want "months", the operations applied extract the month and year values from the generated range. We actually generate the range as the "days" in between since "months" are difficult to deal with in math. The subsequent $reduce operation takes only the "distinct months" from the date range.
The result therefore of the first aggregation pipeline stage is a new field in the document which is an "array" of all the distinct months covered between startDate and endDate. This gives an "iterator" for the rest of the operation.
By "iterator" I mean than when we apply $unwind we get a copy of the original document for every distinct month covered in the interval. This then allows the following two $group stages to first apply a grouping to the common key of "month" and "type" in order to "total" the counts via $sum, and next $group makes the key just the "type" and puts the results in an array via $push.
This gives the result on the above data:
{
"_id" : "HGV",
"monthCounts" : [
{
"month" : 201702,
"count" : 2
},
{
"month" : 201703,
"count" : 2
},
{
"month" : 201704,
"count" : 1
}
]
}
{
"_id" : "CAR",
"monthCounts" : [
{
"month" : 201701,
"count" : 1
},
{
"month" : 201702,
"count" : 1
}
]
}
Note that the coverage of "months" is only present where there is actual data. Whilst possible to produce zero values over a range, it requires quite a bit of wrangling to do so and is not very practical. If you want zero values then it is better to add that in post processing in the client once the results have been retrieved.
If you really have your heart set on the zero values, then you should separately query for $min and $max values, and pass these in to "brute force" the pipeline into generating the copies for each supplied possible range value.
So this time the "range" is made externally to all documents, and you then use a $cond statement into the accumulator to see if the current data is within the grouped range produced. Also since the generation is "external", we really don't need the MongoDB 3.4 operator of $range, so this can be applied to earlier versions as well:
// Get min and max separately
var ranges = db.cars.aggregate(
{ "$group": {
"_id": null,
"startRange": { "$min": "$startDate" },
"endRange": { "$max": "$endDate" }
}}
).toArray()[0]
// Make the range array externally from all possible values
var range = [];
for ( var d = new Date(ranges.startRange.valueOf()); d <= ranges.endRange; d.setUTCMonth(d.getUTCMonth()+1)) {
var v = ( d.getUTCFullYear() * 100 ) + d.getUTCMonth()+1;
range.push(v);
}
// Run conditional aggregation
db.cars.aggregate([
{ "$addFields": { "range": range } },
{ "$unwind": "$range" },
{ "$group": {
"_id": {
"type": "$type",
"month": "$range"
},
"count": {
"$sum": {
"$cond": {
"if": {
"$and": [
{ "$gte": [
"$range",
{ "$add": [
{ "$multiply": [ { "$year": "$startDate" }, 100 ] },
{ "$month": "$startDate" }
]}
]},
{ "$lte": [
"$range",
{ "$add": [
{ "$multiply": [ { "$year": "$endDate" }, 100 ] },
{ "$month": "$endDate" }
]}
]}
]
},
"then": 1,
"else": 0
}
}
}
}},
{ "$sort": { "_id": 1 } },
{ "$group": {
"_id": "$_id.type",
"monthCounts": {
"$push": { "month": "$_id.month", "count": "$count" }
}
}}
])
Which produces the consistent zero fills for all possible months on all groupings:
{
"_id" : "HGV",
"monthCounts" : [
{
"month" : 201701,
"count" : 0
},
{
"month" : 201702,
"count" : 2
},
{
"month" : 201703,
"count" : 2
},
{
"month" : 201704,
"count" : 1
}
]
}
{
"_id" : "CAR",
"monthCounts" : [
{
"month" : 201701,
"count" : 1
},
{
"month" : 201702,
"count" : 1
},
{
"month" : 201703,
"count" : 0
},
{
"month" : 201704,
"count" : 0
}
]
}
MapReduce
All versions of MongoDB support mapReduce, and the simple case of the "iterator" as mentioned above is handled by a for loop in the mapper. We can get output as generated up to the first $group from above by simply doing:
db.cars.mapReduce(
function () {
for ( var d = this.startDate; d <= this.endDate;
d.setUTCMonth(d.getUTCMonth()+1) )
{
var m = new Date(0);
m.setUTCFullYear(d.getUTCFullYear());
m.setUTCMonth(d.getUTCMonth());
emit({ id: this.type, date: m},1);
}
},
function(key,values) {
return Array.sum(values);
},
{ "out": { "inline": 1 } }
)
Which produces:
{
"_id" : {
"id" : "CAR",
"date" : ISODate("2017-01-01T00:00:00Z")
},
"value" : 1
},
{
"_id" : {
"id" : "CAR",
"date" : ISODate("2017-02-01T00:00:00Z")
},
"value" : 1
},
{
"_id" : {
"id" : "HGV",
"date" : ISODate("2017-02-01T00:00:00Z")
},
"value" : 2
},
{
"_id" : {
"id" : "HGV",
"date" : ISODate("2017-03-01T00:00:00Z")
},
"value" : 2
},
{
"_id" : {
"id" : "HGV",
"date" : ISODate("2017-04-01T00:00:00Z")
},
"value" : 1
}
So it does not have the second grouping to compound to arrays, but we did produce the same basic aggregated output.
I required records with the output of gender, count, and updated hour for two days.
db.FaceData.aggregate([ {$match: { 'Timestamp' : { $gte : 1448121600000, $lt : 1448294399000 }, 'DID' : "ABFR001" }}, {$group: { _id: {'Gen': '$Gen'}, count : { $sum : 1 } }} ]);
output:
------
{ "_id" : { "Gen" : 1 }, "count" : 3055 }
{ "_id" : { "Gen" : 0 }, "count" : 2866 }
In the above output I have to group by hour for two days, For Example, Every hour I need Gender, Count for 2days.
Timestamp is in millisecond.
You would need a mechanism to get the actual date object from the unix timestamp, one way is to add the timestamp to a zero-milliseconds Date() object, using the $add operator in the $project stage before the actual grouping aggregation pipeline.
Once you get the date, extract the hour part by using the $hour operator, something like the following:
db.FaceData.aggregate([
{
"$match": {
"Timestamp" : { $gte : 1448121600000, $lt : 1448294399000 },
"DID" : "ABFR001"
}
},
{
$project : {
"hourPart" : {
"$hour": { "$add": [ new Date(0), "$Timestamp" ] }
},
"Gen": 1
}
},
{
"$group": {
"_id": "$hourPart",
"Gen_0_count" : {
"$sum": {
"$cond": [ { "$eq": [ "$Gen", 0 ] }, 1, 0 ]
}
},
"Gen_1_count" : {
"$sum": {
"$cond": [ { "$eq": [ "$Gen", 1 ] }, 1, 0 ]
}
}
}
}
]);
{"$match": {
"Timestamp" : { $gte : 1448121600000, $lt : 1448294399000 },
"DID" : "ABFR001"
}} ,
{ "$group" : {
"_id" : {
"$divide" : [{ "$subtract" : [{"$divide" : ["$Timestamp", 1000]}, { "$mod" : [{"$divide" : ["$Tstmp", 1000]}, 3600] }] }, 3600 ]
},
"Male" : {
"$sum": {
"$cond": [ { "$eq": [ "$Gen", 0 ] }, 1, 0 ]
}
},
"Female" : {
"$sum": {
"$cond": [ { "$eq": [ "$Gen", 1 ] }, 1, 0 ]
}
}
} }
I have collection in my mongoDB which stores service given to customer along with their email address something like below
{
"_id" : ObjectId("56a84627f8fd4a136c0e944a"),
"Vehicle" : "Honda",
"ServiceSelected" : "FULL SERVICE",
"FullName" : "xyz",
"Email" : "xyz#xyz.com",
"BookingTime" : ISODate("2015-12-27T06:00:00.000Z")
},
{
"_id" : ObjectId("56a84627f8fd4a136c0e944b"),
"Vehicle" : "AUDI",
"ServiceSelected" : "FLAT TYRE",
"FullName" : "abc",
"Email" : "abc#abc.com",
"BookingTime" : ISODate("2015-12-26T06:00:00.000Z")
},
{
"_id" : ObjectId("56a84627f8fd4a136c0e944c"),
"Vehicle" : "BMW",
"ServiceSelected" : "OTHERS",
"FullName" : "def",
"Email" : "def#def.com",
"BookingTime" : ISODate("2015-12-25T06:00:00.000Z")
},
{
"_id" : ObjectId("56a84627f8fd4a136c0e944d"),
"Vehicle" : "BMW",
"ServiceSelected" : "OTHERS",
"FullName" : "def",
"Email" : "def#def.com",
"BookingTime" : ISODate("2015-12-30T06:00:00.000Z")
},
{
"_id" : ObjectId("56a84627f8fd4a136c0e944a"),
"Vehicle" : "Honda",
"ServiceSelected" : "FULL SERVICE",
"FullName" : "xyz",
"Email" : "xyz#xyz.com",
"BookingTime" : ISODate("2016-01-27T06:00:00.000Z")
}
From the above collection I want to fetch all the documents that have taken our service with a gap of at-least 30 days i.e. from the above collection "Email" : "xyz#xyz.com" should be returned but not "Email" : "def#def.com" as the second service was taken with in 5 days.
I know there is flaw in the design and an additional flag can be set while inserting the record from the application but I need to fetch the data for the existing records.
You need to use the $min and $max operators which respectively return the minimum and maximum value for "BookingTime" in your $group stage. The last stage in the pipeline is the $redact stage where you use a simple "date" math using the $divide and $subtract arithmetic operators.to return those documents where the number of days between first "service" and last "service" is greater than 30
db.collection.aggregate( [
{ "$group": {
"_id": "$Email",
"date1": { "$min": "$BookingTime" },
"date2": { "$max": "$BookingTime" }
}},
{ "$redact": {
"$cond": [
{ "$gte": [
{ "$divide": [
{ "$subtract": [ "$date2", "$date1" ] },
1000 * 60 * 60 * 24
]},
30
]},
"$$KEEP",
"$$PRUNE"
]
}}
])
Which returns:
{
"_id" : "xyz#xyz.com",
"date1" : ISODate("2015-12-27T06:00:00Z"),
"date2" : ISODate("2016-01-27T06:00:00Z")
}
Another way to do this is by using the $cond operator in a $project stage to avoid a collection scan.
db.collection.aggregate( [
{ "$group": {
"_id": "$Email",
"date1": { "$min": "$BookingTime" },
"date2": { "$max": "$BookingTime" },
"count": { "$sum": 1 }
}},
{ "$match": { "count": { "$gte": 2 } } },
{ "$project": {
"emails": {
"$cond": [
{ "$gte": [
{ "$divide": [
{ "$subtract": [ "$date2", "$date1" ] },
1000 * 60 * 60 * 24
]},
30
] },
"$_id",
false
]
}
}},
{ "$match": { "emails": { "$ne": false } } }
])
You can get first sales date and last sales date by $min and $max:
db.services.aggregate({
$group: {
"_id" :"$Email",
lastSalesDate: { $max: "$BookingTime" },
firstSalesDate: { $min: "$BookingTime" }
}
}
)
After that you can add filter based on lastSalesDate. You can calculate ISO date which 30 days before. ex. ISODate("2015-12-28T00:00:00.000Z"). By $lt , you will get customers of 30 days before.
db.services.aggregate(
{
$group: {
"_id" :"$Email",
lastSalesDate: { $max: "$BookingTime" },
firstSalesDate: { $min: "$BookingTime" }
}
},
{
$match : {
"lastSalesDate" : { $lt: ISODate("2015-12-28T00:00:00.000Z") }
}
}
)
Results like:
{
"_id" : "abc#abc.com",
"lastSalesDate" : ISODate("2015-12-26T06:00:00.000+0000"),
"firstSalesDate" : ISODate("2015-12-26T06:00:00.000+0000")
}
This is what I used finally
db.services.aggregate(
{$group: {
"_id" :"$Email",
count:{$sum:1},
lastSalesDate: { $max: "$BookingTime" },
firstSalesDate: { $min: "$BookingTime" }
},
{$project:{
_id:1,count:1,dateDifference: { $divide:[ {$subtract: [ "$lastSalesDate", "$firstSalesDate" ]},86400000] }
}
},
{$match:{
count:{$gt:1},dateDifference:{$gt:20}
}
}
}
)
Count > 1 helped to filter the records which never repeated and datedifferentce > 20 is for days as I already converted milliseconds to days using division operation.