Getting first day of week from week number in mongodb - mongodb

I have collection containing date field. I'm Grouping records by week and other related fields.
This is my aggregation query:
db.raw.aggregate([
{ "$match" : {
"Timestamp":{
"$gte": new Date("2012-05-30"),
"$lt": new Date("2014-07-31")
}
}},
{ "$group" : {
"_id":{
"ApplicationId": "$ApplicationId",
"Country": "$Country",
"week":{ "$week": "$Timestamp" }
},
"Date":{ "$first": "$Timestamp" },
"Visits": { "$sum": 1 }
}}
])
I want to Project : Visits and Start Date of week from week number.

For mongo >= v3.4, look at weekStart.
The idea is to substruct milliseconds from given Timestamp
db.raw.aggregate([
// stage 1
{ "$match" : {
"Timestamp":{
"$gte": ISODate("2012-05-30"),
"$lt": ISODate("2014-07-31")
}
}},
// stage 2
{ "$project" : {
ApplicationId: 1,
Country: 1,
week: {$isoWeek: "$Timestamp"},
// [TRICK IS HERE] Timestamp - dayOfWeek * msInOneDay
weekStart: { $dateToString: { format: "%Y-%m-%d", date: { // convert date
$subtract: ["$Timestamp", {$multiply: [ {$subtract:[{$isoDayOfWeek: "$Timestamp"},1]}, 86400000]}]
}}},
// stage 3
{ "$group" : {
"_id":{
"ApplicationId": "$ApplicationId",
"Country": "$Country",
"week": "$week"
},
"Date":{ "$first": "$weekStart" },
"Visits": { "$sum": 1 }
}}
])

You seem to want a "date value" representing the date at the start of the week. Your best approach is "date math" with a little help from the aggregation operator $dayOfWeek:
db.raw.aggregate([
{ "$match" : {
"Timestamp":{
"$gte": new Date("2012-05-30"),
"$lt": new Date("2014-07-31")
}
}},
{ "$group" : {
"_id":{
"ApplicationId": "$ApplicationId",
"Country": "$Country",
"weekStart":{
"$subtract": [
{ "$subtract": [
{ "$subtract": [ "$Timestamp", new Date("1970-01-01") ] },
{ "$cond": [
{ "$eq": [{ "$dayOfWeek": "$Timestamp" }, 1 ] },
0,
{ "$multiply": [
1000 * 60 * 60 * 24,
{ "$subtract": [{ "$dayOfWeek": "$Timestamp" }, 1 ] }
]}
]}
]},
{ "$mod": [
{ "$subtract": [
{ "$subtract": [ "$Timestamp", new Date("1970-01-01") ] },
{ "$cond": [
{ "$eq": [{ "$dayOfWeek": "$Timestamp" }, 1 ] },
0,
{ "$multiply": [
1000 * 60 * 60 * 24,
{ "$subtract": [{ "$dayOfWeek": "$Timestamp" }, 1 ] }
]}
]}
]},
1000 * 60 * 60 * 24
]}
]
}
},
"Date":{ "$first": "$Timestamp" },
"Visits": { "$sum": 1 }
}}
])
Or a little cleaner with $let from MongoDB 2.6 and upwards:
db.raw.aggregate([
{ "$match" : {
"Timestamp":{
"$gte": new Date("2012-05-30"),
"$lt": new Date("2014-07-31")
}
}},
{ "$group" : {
"_id":{
"ApplicationId": "$ApplicationId",
"Country": "$Country",
"weekStart":{
"$let": {
"vars": {
"dayMillis": 1000 * 60 * 60 * 24,
"beginWeek": {
"$subtract": [
{ "$subtract": [ "$Timestamp", new Date("1970-01-01") ] },
{ "$cond": [
{ "$eq": [{ "$dayOfWeek": "$Timestamp" }, 1 ] },
0,
{ "$multiply": [
1000 * 60 * 60 * 24,
{ "$subtract": [{ "$dayOfWeek": "$Timestamp" }, 1 ] }
]}
]}
]
}
},
"in": {
"$subtract": [
"$$beginWeek",
{ "$mod": [ "$$beginWeek", "$$dayMillis" ]}
]
}
}
}
},
"Date":{ "$first": "$Timestamp" },
"Visits": { "$sum": 1 }
}}
])
The resulting value in the "grouping" is the epoch milliseconds that represents the start of the day at the start of the week. The "start of the week" is generally considered to be "Sunday", so if you intend another day then you would need to adjust by the appropriate amount. The $add operator with the $dayMillis variable value can be used here to apply "Monday" for example.
It's not a date object, but something that you can easily feed to another method to construct a date object in post processing.
Also note that other things you are using such as $first usually require that the documents are sorted in a particular order, or generally by your "Timestamp" values. If those documents are not already ordered then you either $sort first or use an operator such as $min to get the first actual timestamp in the range.

With MongoDB 3.6
{
'$project' : {
'firstDateOfWeek': {
'$dateFromString': {
'dateString': {
'$concat': [
{
'$toString': '$_id.year'
},
'-',
{
'$toString': '$_id.week'
}
]
},
'format': "%G-%V"
}
}
}
}

From mongo 3.6
https://docs.mongodb.com/manual/reference/operator/aggregation/dateFromParts/
db.raw.aggregate([
{
"$match": {
"Timestamp": {
"$gte": new Date("2012-05-30"),
"$lt": new Date("2014-07-31")
}
}
},
{
"$group": {
"_id": {
"ApplicationId": "$ApplicationId",
"Country": "$Country",
"week": {
"$isoWeek": "$Timestamp"
},
"year": {
"$year": "$Timestamp"
}
},
"Visits": {
"$sum": 1
}
}
},
{
"$addFields": {
"Date": {
$dateFromParts: {
isoWeekYear: '$_id.year',
isoWeek: '$_id.week',
isoDayOfWeek: 1
}
}
}
}
])

For MongoDB >= v5.0 there is an even easier option now with the $dateTrunc operator, e.g.
$project: {
weekStart: {
$dateTrunc: {
date: "$Timestamp",
unit: "week",
startOfWeek: "Monday",
}
},
}

Related

Groupby Array elements and categorize them based on date in Mongodb Query

I have an array element in my DB and I want to group By and calculate the number of repetitions of different elements in this array based on different datetime. assume following collection:
{
_id: ObjectId(7df78ad8902c)
title: 'MongoDB Overview',
tags: ['SQL', 'database', 'NoSQL'],
created_at: 2021-10-03 10:05:51.755Z
},
{
_id: ObjectId(7df78ad8902d)
title: 'NoSQL Overview',
tags: ['mongodb', 'database', 'PHP'],
created_at: 2021-10-03 14:05:51.755Z
},
{
_id: ObjectId(7df78ad8902d)
title: 'Developing',
tags: ['java', 'btc/usdt', 'PHP'],
created_at: 2021-10-03 14:05:51.755Z
}
,
{
_id: ObjectId(7df78ad8902d)
title: 'databases for search',
tags: ['elasticsearch', 'database', 'PHP'],
created_at: 2021-10-03 12:05:51.755Z
}
I want to calculate the number of repetitions of different elements in tags field such as mongodb, database, noSQL based on datetime (for example hot hashtags in last hour, today or this month) in this collection. How can I solve this problem in mongo?
expected answer like .
1 - hot hashtags in last hour ['a' , 'b' , 'c']
2 - hot hastags in last 5 hours : ...
3 - today : ...
4 - this month : ...
Query
not calendar based, based on difference on milliseconds
keep only last 30 days data
facet and 4 group by
its exactly the same code 4x
The only difference is the multiply
last 30days : (now_date-created_at) <= (* 30 24 60 60 1000)
last 24h : (now_date-created_at) <= (* 24 60 60 1000)
last 120 hours(5days) : (now_date-created_at) <= (* 5 60 60 1000)
last 60 min : (now_date-created_at) <= (* 60 60 1000)
*subtraction works on dates also, and returns milliseconds
filter the date depending on what we want its 4 different filters
unwind
group by tag and count occurences
sort by count
limit 2 to keep like the 2 top hotttest tags, you can change it to any value, like limit 1 to keep only the hottest tag
*if you want calendar based like 3 October(3 days data only), query must be changed, the same is for day(query is for 24 hours) etc (in those cases we should use $hour $month etc)
Its not big change in query.
//same month
{"$eq" : [ {"$month" : "$$NOW"}, {"$month" : "$created_at"} ]
//same day(assuming same month from previous filter)
{"$eq" : [ {"$dayOfMonth" : "$$NOW"}, {"$dayOfMonth" : "$created_at"} ]
//same hour
*we could also use the new $dateTrunc to check for same month etc.
Test code here
(Query is big but its the same thing 4x)
db.collection.aggregate([
{
"$set": {
"created_at": {
"$dateFromString": {
"dateString": "$created_at"
}
}
}
},
{
"$unwind": {
"path": "$tags"
}
},
{
"$match": {
"$expr": {
"$lte": [
{
"$subtract": [
"$$NOW",
"$created_at"
]
},
{
"$multiply": [
30,
24,
60,
60,
1000
]
}
]
}
}
},
{
"$facet": {
"month-tag": [
{
"$match": {
"$expr": {
"$lte": [
{
"$subtract": [
"$$NOW",
"$created_at"
]
},
{
"$multiply": [
30,
24,
60,
60,
1000
]
}
]
}
}
},
{
"$group": {
"_id": "$tags",
"count": {
"$sum": 1
}
}
},
{
"$sort": {
"count": -1
}
},
{
"$limit": 2
},
{
"$project": {
"_id": 0,
"tag": "$_id"
}
}
],
"day-tag": [
{
"$match": {
"$expr": {
"$lte": [
{
"$subtract": [
"$$NOW",
"$created_at"
]
},
{
"$multiply": [
24,
60,
60,
1000
]
}
]
}
}
},
{
"$group": {
"_id": "$tags",
"count": {
"$sum": 1
}
}
},
{
"$sort": {
"count": -1
}
},
{
"$limit": 2
},
{
"$project": {
"_id": 0,
"tag": "$_id"
}
}
],
"5hour-tag": [
{
"$match": {
"$expr": {
"$lte": [
{
"$subtract": [
"$$NOW",
"$created_at"
]
},
{
"$multiply": [
5,
60,
60,
1000
]
}
]
}
}
},
{
"$group": {
"_id": "$tags",
"count": {
"$sum": 1
}
}
},
{
"$sort": {
"count": -1
}
},
{
"$limit": 2
},
{
"$project": {
"_id": 0,
"tag": "$_id"
}
}
],
"hour-tag": [
{
"$match": {
"$expr": {
"$lte": [
{
"$subtract": [
"$$NOW",
"$created_at"
]
},
{
"$multiply": [
60,
60,
1000
]
}
]
}
}
},
{
"$group": {
"_id": "$tags",
"count": {
"$sum": 1
}
}
},
{
"$sort": {
"count": -1
}
},
{
"$limit": 2
},
{
"$project": {
"_id": 0,
"tag": "$_id"
}
}
]
}
}
])

Aggregate Pipeline groups by day but projects a null date

I'm attempting to group the items in a collection by year/month/day. The grouping should be based on the pubDate and pubTimezoneOffset.
I've got an aggregate pipeline that:
- $project - adds the timezoneOffset to the pubDate
- $group - groups by the modified pubDate
- $project - removes the timezoneOffset
- $sort - sorts by pubDate
I tested each stage on it's own and it seems to be some issue with the second $project. In the final output the pubDate is null.
I've been going over it for a few hours now and can't see where I've gone wrong. What am I missing?
The aggregate pipeline:
db.messages.aggregate([
{
$project: {
_id: 1,
pubTimezoneOffset: 1,
pubDate: {
$add: [
'$pubDate', {
$add: [
{ $multiply: [ '$pubTimezoneOffset.hours', 60, 60, 1000 ] },
{ $multiply: [ '$pubTimezoneOffset.minutes', 60, 1000 ] }
]
}
]
}
}
},
{
$group: {
_id: {
year: { $year: '$pubDate' },
month: { $month: '$pubDate' },
day: { $dayOfMonth: '$pubDate' }
},
count: { $sum: 1 },
messages: {
$push: {
_id: '$_id',
pubTimezoneOffset: '$pubTimezoneOffset',
pubDate: '$pubDate'
}
}
}
},
{
$project: {
_id: 1,
messages: {
_id: 1,
pubTimezoneOffset: 1,
pubDate: {
$subtract: [
'$pubDate', {
$add: [
{ $multiply: [ '$pubTimezoneOffset.hours', 60, 60, 1000 ] },
{ $multiply: [ '$pubTimezoneOffset.minutes', 60, 1000 ] }
]
}
]
}
},
count: 1
}
},
{
$sort: {
'_id.year': -1,
'_id.month': -1,
'_id.day': -1
}
}
]).pretty();
To recreate the source data:
db.messages.insertOne({
pubDate: ISODate('2017-10-25T10:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
db.messages.insertOne({
pubDate: ISODate('2017-10-25T11:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
db.messages.insertOne({
pubDate: ISODate('2017-10-24: 10:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
db.messages.insertOne({
pubDate: ISODate('2017-10-24: 11:00:00:000Z'),
pubTimezoneOffset: {
hours: -7,
minutes: 0
}
});
Running it in mongo shell outputs:
{
"_id" : {
"year" : 2017,
"month" : 10,
"day" : 25
},
"count" : 2,
"messages" : [
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b3"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
},
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b4"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
}
]
}
{
"_id" : {
"year" : 2017,
"month" : 10,
"day" : 23
},
"count" : 2,
"messages" : [
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b5"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
},
{
"_id" : ObjectId("59f0e8b47d0a206bdfde87b6"),
"pubTimezoneOffset" : {
"hours" : -7,
"minutes" : 0
},
"pubDate" : null
}
]
}
Kudos for the attempt but, you actually have quite a few things conceptually incorrect here, with the basic error you are seeing is because your premise of "array projection" is incorrect. You are trying to refer to variables "inside the array" by simply notating the "property name".
What you actually need to do here is apply $map in order to apply the functions to "transform" each element:
db.messages.aggregate([
{ "$project": {
"pubTimezoneOffset": 1,
"pubDate": {
"$add": [
"$pubDate",
{ "$add": [
{ "$multiply": [ '$pubTimezoneOffset.hours', 60 * 60 * 1000 ] },
{ "$multiply": [ '$pubTimezoneOffset.minutes', 60 * 1000 ] }
]}
]
}
}},
{ "$group": {
"_id": {
"year": { "$year": "$pubDate" },
"month": { "$month": "$pubDate" },
"day": { "$dayOfMonth": "$pubDate" }
},
"count": { "$sum": 1 },
"messages": {
"$push": {
"_id": "$_id",
"pubTimezoneOffset": "$pubTimezoneOffset",
"pubDate": "$pubDate"
}
}
}},
{ "$project": {
"messages": {
"$map": {
"input": "$messages",
"as": "m",
"in": {
"_id": "$$m._id",
"pubTimezoneOffset": "$$m.pubTimezoneOffset",
"pubDate": {
"$subtract": [
"$$m.pubDate",
{ "$add": [
{ "$multiply": [ "$$m.pubTimezoneOffset.hours", 60 * 60 * 1000 ] },
{ "$multiply": [ "$$m.pubTimezoneOffset.minutes", 60 * 1000 ] }
]}
]
}
}
}
},
"count": 1
}},
{ "$sort": { "_id": -1 } }
]).pretty();
Noting here that you are doing a lot of unnecessary work in "tranforming" the dates kept in the array, and then trying to "tranform" them back to the original state. Instead, you should have simply supplied a "variable" with $let to the _id of $group and left the original document state "as is" using $$ROOT instead of naming all the fields:
db.messages.aggregate([
{ "$group": {
"_id": {
"$let": {
"vars": {
"pubDate": {
"$add": [
"$pubDate",
{ "$add": [
{ "$multiply": [ '$pubTimezoneOffset.hours', 60 * 60 * 1000 ] },
{ "$multiply": [ '$pubTimezoneOffset.minutes', 60 * 1000 ] }
]}
]
}
},
"in": {
"year": { "$year": "$$pubDate" },
"month": { "$month": "$$pubDate" },
"day": { "$dayOfMonth": "$$pubDate" }
}
}
},
"docs": { "$push": "$$ROOT" }
}},
{ "$sort": { "_id": -1 } }
])
Also note that $sort simply does actually consider all the "sub-keys" anyway, so there is no need to name them explicitly.
Back to your error, the point of $map is essentially because whilst you can notate array "field inclusion" with MongoDB 3.2 and above like this:
"messages": {
"_id": 1,
"pubTimeZoneOffset": 1
}
The thing you cannot do is actually "calculate values" on the elements themselves. You tried "$pubDate" which actually looks in the "ROOT" space for a property of that name, which does not exist and is null. If you then tried:
"messages": {
"_id": 1,
"pubTimeZoneOffset": 1,
"pubDate": "$messages.pubDate"
}
Then you would get "a result", but not the result you might think. Because what would actually be included in "every element" is the value of that property in each array element as a "new array" itself.
So the short and sweet is use $map instead, which iterates the array elements with a local variable referring to the current element for you to notate values for in expressions.
MongoDB 3.6
MongoDB date operators are all timezone aware. So instead of all the juggling then all you need do is supply the additional "timezone" parameter to any option and the conversion will be done for you.
As a sample:
db.messages.aggregate([
{ "$group": {
"_id": {
"$dateToString": {
"date": "$pubDate",
"format": "%Y-%m-%d",
"timezone": {
"$concat": [
{ "$cond": {
"if": { "$gt": [ "$pubTimezoneOffset", 0 ] },
"then": "+",
"else": "-"
}},
{ "$let": {
"vars": {
"hours": { "$substr": [{ "$abs": "$pubTimezoneOffset.hours" },0,2] },
"minutes": { "$substr": [{ "$abs": "$pubTimezoneOffset.minutes" },0,2] }
},
"in": {
"$concat": [
{ "$cond": {
"if": { "$eq": [{ "$strLenCP": "$$hours" }, 1 ] },
"then": { "$concat": [ "0", "$$hours" ] },
"else": "$$hours"
}},
":",
{ "$cond": {
"if": { "$eq": [{ "$strLenCP": "$$minutes" }, 1 ] },
"then": { "$concat": [ "0", "$$minutes" ] },
"else": "$$minutes"
}}
]
}
}}
]
}
}
},
"docs": { "$push": "$$ROOT" }
}},
{ "$sort": { "_id": -1 } }
])
Note that most of the "juggling" in there is to convert your own "offset" to the "string" format required by the new operators. If you simply stored this as "offset": "-07:00" then you can instead simply write:
db.messages.aggregate([
{ "$group": {
"_id": {
"$dateToString": {
"date": "$pubDate",
"format": "%Y-%m-%d",
"timezone": "$offset"
}
},
"docs": { "$push": "$$ROOT" }
}},
{ "$sort": { "_id": -1 } }
])
Please Reconsider
I can't let this pass without making a note that your general approach here is conceptually incorrect. Storing "offset" or "local time string" within the database is just intrinsically wrong.
The date information should be stored as UTC and should be returned as UTC. Sure you can and "should" covert when aggregating, but the general premise is that you always convert back to UTC. And "conversion" comes from the "locale of the observer" and not a "stored" adjustment. Because dates are always relative to the "observer" point of view, and are not from the "point of origin" as you seem to have interpreted it.
I put some lengthy detail on this on Group by Date with Local Time Zone in MongoDB about why you store this way and why "locale" conversion from the "observer" is necessary. That also details "Daylight savings considerations" from the observer point of view.
The basic premise there still remains the same when MongoDB becomes "timezone aware" in that you :
Store in UTC
Query with local time converted to UTC
Aggregate converted from the "observer" offset
Convert the "offset" back to UTC
Because at the end of the day it's the "clients" job to supply that "locale" conversion, since that's the part that "knows where it is".

Mongodb Muliple averages based on a different column

Using mongodb aggregate, is there a way to have the query return Weight average on all Scale , an average of Scale 1 , an average of Scale 2 all all returned in the same query?
This is an example of an entry in my data set
{
"Profile" : "P1",
"AvgWeight" : 639,
"Time" : "2017-04-14T05:17:42.000Z",
"Scale" : 1,
"Weight" : 1504,
"Target" : 680
}
My query that I am currently running that is averaging the weight accrost all scales ( Might not help, but better to have more info )
[{
"$match": {
"Time": {
"$gt": moment(start).format("YYYY-MM-DD HH:mm:ss"),
"$lt": moment(end).format("YYYY-MM-DD HH:mm:ss")
}
}
},
{
"$group": {
"_id": {
"hour": {
"$hour": "$Time"
},
"day": {
"$dayOfYear": "$Time"
},
"interval": {
"$add": [{
"$multiply": [{
"$minute": "$Time"
}]
},
{
"$multiply": [{
"$hour": "$Time"
},
100
]
},
{
"$multiply": [{
"$dayOfYear": "$Time"
},
10000
]
},
{
"$multiply": [{
"$year": "$Time"
},
10000000
]
}
]
}
},
"time": {
"$first": "$Time"
},
"avgW": {
"$avg": "$AvgWeight"
},
"avgWe": {
"$avg": "$Weight"
},
"avgTarget": {
"$avg": "$Target"
}
}
}, {
"$sort": {
"Time": -1
}
}
]
Adding Expected Response SOmething like
[
{
"_id : {"hour":1,"day":105,"interval":20971050122},
"time":"2017-04-15T01:22:58.000Z",
"avgW":646,
"avgWe":1577,
"avgTarget":680 ,
"Scale1" : 100 ,
"Scale2" : 120
} ,
{ "_id":{"hour":1,"day":105,"interval":20771050122},
"time":"2017-04-15T01:22:55.000Z",
"avgW":646,
"avgWe":1335,
"avgTarget":680 ,
"Scale1" : 100 ,
"Scale2" : 120 }
]
But if it is a little different I can handle it, as long all the scales are in the same parent object ( it would be to cpu intensive to post process them to link up the matching groups )
You can split the first group into two groups.
First group to calculate weight avg for all scales and second group to do rest of avgs.
Something like:
[{
"$match": {
"Time": {
"$gt": moment(start).format("YYYY-MM-DD HH:mm:ss"),
"$lt": moment(end).format("YYYY-MM-DD HH:mm:ss")
}
}
}, {
"$group": {
"_id": {
"scale": "$Scale",
"hour": {
"$hour": "$Time"
},
"day": {
"$dayOfYear": "$Time"
},
"interval": {
"$add": [{
"$multiply": [{
"$minute": "$Time"
}]
},
{
"$multiply": [{
"$hour": "$Time"
},
100
]
},
{
"$multiply": [{
"$dayOfYear": "$Time"
},
10000
]
},
{
"$multiply": [{
"$year": "$Time"
},
10000000
]
}
]
}
},
"time": {
"$first": "$Time"
},
"scaleAvg: {
"$avg": "$Weight"
}
}
}, {
"$group": {
"_id": {
"hour": "$_id.hour",
"day": "$_id.day",
"interval": "$_id.interval"
},
"time": {
"$first": "$time"
},
"avgW": {
"$avg": "$AvgWeight"
},
"avgWe": {
"$avg": "$Weight"
},
"avgTarget": {
"$avg": "$Target"
},
"scaleAvgs": {
"$push": {
"scale": "$_id.scale",
"scaleAvg": "$scaleAvg"
}
}
}
}, {
"$sort": {
"time": -1
}
}]

Mongodb aggregation by day based on unix timestamp

I have googled alot, but not found any helpful solution... I want to find total number of daily users.
I have a collection named session_log having documents like following
{
"_id" : ObjectId("52c690955d3cdd831504ce30"),
"SORTID" : NumberLong(1388744853),
"PLAYERID" : 3,
"LASTLOGIN" : NumberLong(1388744461),
"ISLOGIN" : 1,
"LOGOUT" : NumberLong(1388744853)
}
I want to aggregate from LASTLOGIN...
This is my query:
db.session_log.aggregate(
{ $group : {
_id: {
LASTLOGIN : "$LASTLOGIN"
},
count: { $sum: 1 }
}}
);
But it is aggregating by each login time, not by each day. Any help would be appreciated
MongoDB 4.0 and newer
Use $toDate
db.session_log.aggregate([
{ "$group": {
"_id": {
"$dateToString": {
"format": "%Y-%m-%d",
"date": {
"$toDate": {
"$multiply": [1000, "$LASTLOGIN"]
}
}
}
},
"count": { "$sum": 1 }
} }
])
or $convert
db.session_log.aggregate([
{ "$group": {
"_id": {
"$dateToString": {
"format": "%Y-%m-%d",
"date": {
"$convert": {
"input": {
"$multiply": [1000, "$LASTLOGIN"]
},
"to": "date"
}
}
}
},
"count": { "$sum": 1 }
} }
])
MongoDB >= 3.0 and < 4.0:
db.session_log.aggregate([
{ "$group": {
"_id": {
"$dateToString": {
"format": "%Y-%m-%d",
"date": {
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
}
}
},
"count": { "$sum": 1 }
} }
])
You would need to convert the LASTLOGIN field to a millisecond timestamp through multiplying the value by 1000
{ "$multiply": [1000, "$LASTLOGIN"] }
, then convert to a date
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
and this can be done in the $project pipeline by adding your milliseconds time to a zero-milliseconds Date(0) object, then extract $year, $month, $dayOfMonth parts from the converted date which you can then use in your $group pipeline to group the documents by the day.
You should thus change your aggregation pipeline to this:
var project = {
"$project":{
"_id": 0,
"y": {
"$year": {
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
}
},
"m": {
"$month": {
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
}
},
"d": {
"$dayOfMonth": {
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
}
}
}
},
group = {
"$group": {
"_id": {
"year": "$y",
"month": "$m",
"day": "$d"
},
"count" : { "$sum" : 1 }
}
};
Running the aggregation pipeline:
db.session_log.aggregate([ project, group ])
would give the following results (based on the sample document):
{ "_id" : { "year" : 2014, "month" : 1, "day" : 3 }, "count" : 1 }
An improvement would be to run the above in a single pipeline as
var group = {
"$group": {
"_id": {
"year": {
"$year": {
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
}
},
"mmonth": {
"$month": {
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
}
},
"day": {
"$dayOfMonth": {
"$add": [
new Date(0),
{ "$multiply": [1000, "$LASTLOGIN"] }
]
}
}
},
"count" : { "$sum" : 1 }
}
};
Running the aggregation pipeline:
db.session_log.aggregate([ group ])
First thing is your date is stored in timestamp so you need to first convert timestamp to ISODate using adding new Date(0) and multiply timestamp to 1000 then you will get the ISODate like this :
{"$add":[new Date(0),{"$multiply":[1000,"$LASTLOGIN"]}]} this convert to timestamp to ISODate.
Now using date aggregation you need to convert ISODate in required format using $concat and then group by final formatting date so aggregation query will be :
db.session_log.aggregate({
$project: {
date: {
$concat: [{
$substr: [{
$year: {
"$add": [new Date(0), {
"$multiply": [1000, "$LASTLOGIN"]
}]
}
}, 0, 4]
}, "/", {
$substr: [{
$month: {
"$add": [new Date(0), {
"$multiply": [1000, "$LASTLOGIN"]
}]
}
}, 0, 4]
}, "/", {
$substr: [{
$dayOfMonth: {
"$add": [new Date(0), {
"$multiply": [1000, "$LASTLOGIN"]
}]
}
}, 0, 4]
}]
}
}
}, {
"$group": {
"_id": "$date",
"count": {
"$sum": 1
}
}
})
If you will used mongo version 3.0 and above then use dateToString operator to convert ISODate to predefined format, and aggregation query is :
db.session_log.aggregate({
"$project": {
"ISODate": {
"$add": [new Date(0), {
"$multiply": [1000, "$LASTLOGIN"]
}]
}
}
}, {
"$project": {
"yearMonthDay": {
"$dateToString": {
"format": "%Y-%m-%d",
"date": "$ISODate"
}
}
}
}, {
"$group": {
"_id": "$yearMonthDay",
"count": {
"$sum": 1
}
}
})

how to aggregate in mongoDB

I have a document called user.monthly, in that I have we used store 'day' : no. of clicks .
Here I have given 2 samples for different date
For month January
{
name : "devid",
date : ISODate("2014-01-21T11:32:42.392Z"),
daily: {'1':12,'9':13,'30':13}
}
For month February
{
name : "devid",
date : ISODate("2014-02-21T11:32:42.392Z"),
daily: {'3':12,'12':13,'25':13}
}
How can I aggregate this and get total clicks for January and February ?
Please help me to resolve my problem.
Your current schema is not helping you here as the "daily" field ( which we presume is your clicks per type or something like that ) is represented as a sub-document, which means that you need to explicitly name the path to each field in order to do something with it.
A better approach would be to put this information in an array:
{
"name" : "devid",
"date" : ISODate("2014-02-21T11:32:42.392Z"),
"daily": [
{ "type": "3", "clicks": 12 },
{ "type": "12", "clicks": 13 },
{ "type": "25", "clicks": 13 }
]
}
Then you have an aggregation statement that goes like this:
db.collection.aggregate([
// Just match the dates in January and February
{ "$match": {
"date": {
"$gte": new Date("2014-01-01"), "$lt": new Date("2014-03-01")
}
}},
// Unwind the "daily" array
{ "$unwind": "$daily" },
// Group the values together by "type" on "January" and "February"
{ "$group": {
"_id": {
"year": { "$year": "$date" },
"month": { "$month": "$date" },
"type": "$daily.type"
},
"clicks": { "$sum": "$daily.clicks" }
}},
// Sort the result nicely
{ "$sort": {
"_id.year": 1,
"_id.month": 1,
"_id.type": 1
}}
])
That form is pretty simple. Or even if you do not care about the type as a grouping and just want the month totals:
db.collection.aggregate([
{ "$match": {
"date": {
"$gte": new Date("2014-01-01"), "$lt": new Date("2014-03-01")
}
}},
{ "$unwind": "$daily" },
{ "$group": {
"_id": {
"year": { "$year": "$date" },
"month": { "$month": "$date" },
},
"clicks": { "$sum": "$daily.clicks" }
}},
{ "$sort": { "_id.year": 1, "_id.month": 1 }}
])
But with the current sub-document form you currently have this becomes ugly:
db.collection.aggregate([
{ "$match": {
"date": {
"$gte": new Date("2014-01-01"), "$lt": new Date("2014-03-01")
}
}},
{ "$group": {
"_id": {
"year": { "$year": "$date" },
"month": { "$month": "$date" },
},
"clicks": {
"$sum": {
"$add": [
{ "$ifNull": ["$daily.1", 0] },
{ "$ifNull": ["$daily.3", 0] },
{ "$ifNull": ["$daily.9", 0] },
{ "$ifNull": ["$daily.12", 0] },
{ "$ifNull": ["$daily.25", 0] },
{ "$ifNull": ["$daily.30", 0] },
]
}
}
}}
])
That shows that you have no other option here other than to specify what is essentially every possible field under daily ( so probably much larger ). Then we have to evaluate as that key may possibly not exist for a given document to return a default value.
For example, your first document has no key "daily.3" so without the $ifNull check the returned value would be null and invalidate the whole $sum process so that the total would be "0".
Grouping on those keys as in the first aggregate example gets even worse:
db.collection.aggregate([
// Just match the dates in January and February
{ "$match": {
"date": {
"$gte": new Date("2014-01-01"), "$lt": new Date("2014-03-01")
}
}},
// Project with an array to match all possible values
{ "$project": {
"date": 1,
"daily": 1,
"type": { "$literal": ["1", "3", "9", "12", "25", "30" ] }
}},
// Unwind the "type" array
{ "$unwind": "$type" },
// Project values onto the "type" while grouping
{ "$group" : {
"_id": {
"year": { "$year": "$date" },
"month": { "$month": "$date" },
"type": "$type"
},
"clicks": { "$sum": { "$cond": [
{ "$eq": [ "$type", "1" ] },
"$daily.1",
{ "$cond": [
{ "$eq": [ "$type", "3" ] },
"$daily.3",
{ "$cond": [
{ "$eq": [ "$type", "9" ] },
"$daily.9",
{ "$cond": [
{ "$eq": [ "$type", "12" ] },
"$daily.12",
{ "$cond": [
{ "$eq": [ "$type", "25" ] },
"$daily.25",
"$daily.30"
]}
]}
]}
]}
]}}
}},
{ "$sort": {
"_id.year": 1,
"_id.month": 1,
"_id.type": 1
}}
])
Which is creating one big conditional evaluation using $cond to match out the values to the "type" which we projected all possible values in an array using the $literal operator.
If you do not have MongoDB 2.6 or greater you can always do this in place of the $literal operator statement:
"type": { "$cond": [1, ["1", "3", "9", "12", "25", "30" ], 0] }
Where essentially the true evaluation from $cond returns a "literal" declared value, which is how you specify an array. There is also the hidden $const operator that is not documented, but now exposed as $literal.
As you can see the structure here is doing you no favors, so the best option is to change it. But if you cannot and otherwise find the aggregation concept for this too hard to handle, then mapReduce offers an approach, but the processing will be much slower:
db.collection.mapReduce(
function () {
for ( var k in this.daily ) {
emit(
{
year: this.date.getFullYear(),
month: this.date.getMonth() + 1,
type: k
},
this.daily[k]
);
}
},
function(key,values) {
return Array.sum( values );
},
{
"query": {
"date": {
"$gte": new Date("2014-01-01"), "$lt": new Date("2014-03-01")
}
},
"out": { "inline": 1 }
}
)
The general lesson here is that you will get the cleanest and fastest results by altering the document format and using the aggregation framework. But all the ways to do this are listed here.