MongoDB $match won't work with $group in an aggregate query - mongodb

This is what my collection looks like.
{ "name" : "Lorem", "createdate" : ISODate("2018-09-12T04:02:41.208Z")},
{ "name" : "Impusom", "createdate" : ISODate("2018-09-12T04:02:41.208Z")}
{ "name" : "Lorem", "createdate" : ISODate("2018-11-01T04:02:41.208Z")}
{ "name" : "Damut", "createdate" : ISODate("2018-11-12T04:02:41.208Z")}
{ "name" : "Remetat", "createdate" : ISODate("2019-01-01T04:02:41.208Z")}
I want to extract distinct createdate values so I can mark them on a calendar widget. I wish to ignore time.
Here is the pipeline which is I am currently using.
db.mycollection.aggregate({
'$project': {
'y': {
'$year': '$createdate'
},
'm': {
'$month': '$createdate'
},
'd': {
'$dayOfMonth': '$createdate'
}
}
}, {
'$group': {
'_id': {
'year': '$y',
'month': '$m',
'day': '$d'
}
}
}, {
'$sort': {
'_id.year': 1,
'_id.month': 1,
'_id.day': 1
},
})
The result looks like this
{
"_id" : {
"year" : 2019,
"month" : 1,
"day" : 1
}
},
{
"_id" : {
"year" : 2018,
"month" : 11,
"day" : 12
}
},
{
"_id" : {
"year" : 2018,
"month" : 11,
"day" : 1
}
},
{
"_id" : {
"year" : 2018,
"month" : 9,
"day" : 12
}
}
I wish to improve the query and limit data to a particular year. e.g. 2018
Hence I tried adding a match condition before $project
{ '$match': { "createdate":{ "$gte": ISODate("2017-12-31T18:30:00.000Z"), "$lt": ISODate("2018-12-31T18:30:00.000Z") } }
},
The result is limited to year 2018 but then $group stops working hence I get duplicated records for dates.
I tried moving the $match option after $sort which allows $group to work but won't apply $match condition.
I would really appreciate a nudge in the right direction.
Update ---
So the above query with $match condition is working in mongoplayground.net
But not with my mongoose code.
const pipeline = [{
"$match": {
"createdate": {
"$gte": moment().startOf('year'),
"$lt": moment().endOf('year'),
}
}
},{
'$project': {
'y': {
'$year': '$createdate'
},
'm': {
'$month': '$createdate'
},
'd': {
'$dayOfMonth': '$createdate'
}
}
}, {
'$group': {
'_id': {
'year': '$y',
'month': '$m',
'day': '$d'
}
}
}, {
'$sort': {
'_id.year': 1,
'_id.month': 1,
'_id.day': 1
},
}];
Collection.aggregate(pipeline, (err, dates) => { res.json({ data: dates }); });
The response returns empty array.

Change your $match section to:
"$match": {
"createdate": {
"$gte": moment().startOf('year').toDate(),
"$lt": moment().endOf('year').toDate(),
}
}
And see if this helps. I do not think moment would work as a valid date parameter

Related

Is there a way to group results from multiple documents when performing aggregation

I am new to mongo and trying to perform aggregation query to calculate min/max of timestamps for a given document.
Sample documents are below -
{
"_id" : ObjectId("5c9cd93adddca9ebb2b3fcba"),
"frequency" : 5,
"s_id" : "30081993",
"timestamp" : NumberLong(1546300800000),
"date" : ISODate("2019-01-01T00:00:00.000Z"),
"values" : {
"1547439900000" : {
"number_of_values" : 3,
"min_value" : 32.13,
"max_value" : 81.42
},
"1547440200000" : {
"number_of_values" : 3,
"min_value" : 48.08,
"max_value" : 84.52
},
"1547440500000" : {
"number_of_values" : 2,
"min_value" : 27.39,
"max_value" : 94.64
}
}
}
{
"_id" : ObjectId("5c9cd851dddca9ebb2b3f2ac"),
"frequency" : 5,
"s_id" : "27061995",
"timestamp" : NumberLong(1546300800000),
"date" : ISODate("2019-01-01T00:00:00.000Z"),
"values" : {
"1547539900000" : {
"number_of_values" : 31,
"min_value" : 322.13,
"max_value" : 831.42
},
"1547540200000" : {
"number_of_values" : 3,
"min_value" : 418.08,
"max_value" : 8114.52
},
"1547740500000" : {
"number_of_values" : 2,
"min_value" : 207.39,
"max_value" : 940.64
}
}
}
I have come up with the following query which works for a single document.
db.testdb.aggregate([
{
$match: {
"s_id": "30081993",
"frequency": 5,
}
},
{
$project: {
_id: 1,
valuesarray: {
$objectToArray: "$values"
}
}
},
{
$unwind: "$valuesarray"
},
{
$group: {
"_id": "",
"min_timestamp": {
$min: "$valuesarray.k"
},
"max_timestamp": {
$max: "$valuesarray.k"
}
}
}
]);
The output is below
{
"_id" : "",
"min_timestamp" : "1547439900000",
"max_timestamp" : "1547440500000"
}
I want an aggregation query which can calculate the max/min of timestamps but for multiple documents i.e I want to use a $in operator during the $match stage and get min/max of all s_id. Is this possible?
Expected :
{
"_id" : "30081993",
"min_timestamp" : "1547439900000",
"max_timestamp" : "1547440500000"
}
{
"_id" : "27061995",
"min_timestamp" : "1547539900000",
"max_timestamp" : "1547740500000"
}
Yes, only small changes are required to make this work for multiple documents.
In $match stage, specify your $in query:
$match: {
"s_id": { $in : [ "30081993", "27061995" ] },
"frequency": 5,
}
In $project stage, rename s_id to _id, to ensure we keep the s_id associated with each document:
$project: {
_id: "$s_id",
valuesarray: {
$objectToArray: "$values"
}
}
In $group stage, group by _id (originally s_id), to ensure we correctly group the timestamps together before calculating $min/$max:
$group: {
"_id": "$_id",
"min_timestamp": {
$min: "$valuesarray.k"
},
"max_timestamp": {
$max: "$valuesarray.k"
}
}
Whole pipeline:
db.testdb.aggregate([
{
$match: {
"s_id": { $in : [ "30081993", "27061995" ] },
"frequency": 5,
}
},
{
$project: {
_id: "$s_id",
valuesarray: {
$objectToArray: "$values"
}
}
},
{
$unwind: "$valuesarray"
},
{
$group: {
"_id": "$_id",
"min_timestamp": {
$min: "$valuesarray.k"
},
"max_timestamp": {
$max: "$valuesarray.k"
}
}
}
]);

Mongodb aggregation json format by month result

Hello I am working with the reporting api which will going to use in highcharts and I am new to mongodb.
Below is my aggregation query (suggest me modification) :
db.product_sold.aggregate({
$group: {
_id: { year: { $year: "$solddate" }, month: { $month: "$solddate" }, productid: "$productid" },
totalQty: { $sum: "$qty" }
}
})
Output:
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "11"
},
"totalQty" : 6.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "14"
},
"totalQty" : 7.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(1),
"productid" : "13"
},
"totalQty" : 3.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "10"
},
"totalQty" : 6.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2018),
"month" : NumberInt(2),
"productid" : "12"
},
"totalQty" : 5.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "15"
},
"totalQty" : 8.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(1),
"productid" : "11"
},
"totalQty" : 2.0
}
// ----------------------------------------------
What I want in output is something like :
status: 200,
msg: "SUCCESS"
data: [{
1:[
{
"productid": 11,
"totalQty": 3
},
{
"productid": 12,
"totalQty": 27
}
],
2:[
{
"productid": 11,
"totalQty": 64
},
{
"productid": 12,
"totalQty": 10
}
]
}]
For reference attaching the image of the collection
Is there any way to achieve it using aggregation or anything else or I will have to do it manually by code ?
You can append below aggreagation stages to your current pipeline:
db.product_sold.aggregate([
// your current $group stage
{
$group: {
_id: "$_id.month",
docs: { $push: { productid: "$_id.productid", totalQty: "$totalQty" } }
}
},
{
$project: {
_id: 0,
k: { $toString: "$_id" },
v: "$docs"
}
},
{
$group: {
_id: null,
data: { $push: "$$ROOT" }
}
},
{
$project: {
_id: 0,
data: { $arrayToObject: "$data" }
}
}
])
The idea here is that you can use $group with _id set to null to get all the data into single document and then use $arrayToObject to get month number as key and all the aggregates for that month as value.

How can I group by ID and Month?

How can I group by ID and Month in MongoDB?
My data looks like this:
{
"_id" : ObjectId("597225c62e7cbfc9a0b099f8"),
"LogId" : NumberInt(17351963),
"EntryId" : NumberInt(22),
"Date" : "2013-08-11 00:00:00",
"LogTypeId" : NumberInt(6),
"Count" : NumberInt(1),
"EntryType" : NumberInt(1)
}
{
"_id" : ObjectId("597225c62e7cbfc9a0b099f9"),
"LogId" : NumberInt(17352356),
"EntryId" : NumberInt(23),
"Date" : "2013-08-11 00:00:00",
"LogTypeId" : NumberInt(6),
"Count" : NumberInt(2),
"EntryType" : NumberInt(1)
}
{
"_id" : ObjectId("597225c62e7cbfc9a0b099fa"),
"LogId" : NumberInt(17360483),
"EntryId" : NumberInt(28),
"Date" : "2013-08-11 00:00:00",
"LogTypeId" : NumberInt(6),
"Count" : NumberInt(1),
"EntryType" : NumberInt(1)
}
My simplified aggregation query runs without errors, but it doesn't group:
db.log.aggregate([
{"$group":{"_id":"$EntryId", "Count":{"$sum":"$Count"}}},
{"$sort": {"EntryId": 1}}
])
Ultimately, I want to group by EntryID and the month of the date column.
First of all "convert your strings to date" with a very simple operation:
let ops = [];
db.log.find().forEach(doc => {
ops.push({ "updateOne": {
"filter": { "_id": doc._id },
"update": { "$set": { "Date": new Date(doc.Date.replace(" ","T")) } }
}});
if ( ops.length >= 500 ) {
db.log.bulkWrite(ops);
ops = [];
}
});
if ( ops.length > 0 ) {
db.log.bulkWrite(ops);
ops = [];
};
Then run the new aggregate:
db.log.aggregate([
{ "$group": {
"_id": {
"EntryId": "$EntryId",
"year": { "$year": "$Date" },
"month": { "$month": "$Date" }
},
"Count": { "$sum": 1 }
}},
{ "$sort": { "_id": 1 } }
])
Also noting that even a "compound _id" like this one will sort correctly to it't numeric values.

MongoDB aggregation counts based on value

I have a MongoDB query I am trying to figure out. I'd like to group my data by date and one other field (portfolio) and get the counts for each buildResult in that grouping.
Sample data looks like this:
{
"_id" : ObjectId("52dea764e4b0a491abb54102"),
"type" : "build",
"time" : ISODate("2014-01-21T16:59:16.642Z"),
"data" : {
"buildNumber" : 35,
"buildDuration" : 1034300,
"portfolio" : "Shared",
"buildResult" : "FAILURE"
}
}
{
"_id" : ObjectId("52dea7b7e4b0a491abb54103"),
"type" : "build",
"time" : ISODate("2014-01-21T17:00:39.617Z"),
"data" : {
"buildNumber" : 13,
"buildDuration" : 1186028,
"portfolio" : "Sample",
"buildResult" : "SUCCESS"
}
}
The output I am shooting for would be something like this:
{
"result" : [
{
"_id" : {
"month" : 2,
"day" : 28,
"year" : 2014,
"portfolio" : "Shared"
},
"aborted" : 3,
"failure" : 0,
"unstable" : 0,
"success" : 34
},
{
"_id" : {
"month" : 2,
"day" : 28,
"year" : 2014,
"portfolio" : "Sample"
},
"aborted" : 3,
"failure" : 2,
"unstable" : 0,
"success" : 37
}
],
"ok" : 1
}
My current query is:
db.builds.aggregate([
{ $match: { "data.buildResult" : { $ne : null} }},
{ $group: {
_id: {
month: { $month: "$time" },
day: { $dayOfMonth: "$time" },
year: { $year: "$time" },
portfolio: "$data.portfolio",
},
aborted: { $sum: { "$data.buildResult": "ABORTED" } },
failure: { $sum: { "$data.buildResult": "FAILURE" } },
unstable: { $sum: { "$data.buildResult": "UNSTABLE" } },
success: { $sum: { "$data.buildResult": "SUCCESS" } }
} },
{ $sort: { "_id.day": 1, "_id.month": 1, "_id.year": 1 } }
])
I have tried many variations with the following lines including $match, $in and other operators. Any help would be very appreciated.
aborted: { $sum: { "$data.buildResult": "ABORTED" } },
failure: { $sum: { "$data.buildResult": "FAILURE" } },
unstable: { $sum: { "$data.buildResult": "UNSTABLE" } },
success: { $sum: { "$data.buildResult": "SUCCESS" } }
To achieve that you can use the $cond and $eq operators like this:
aborted: {$sum: {$cond : [{$eq : ["$data.buildResult", "ABORTED"]}, 1, 0]}}
Edit:
As noted by Neil Lunn in the comments, the $cond here is irrelevant because the $eq operator already returns 0 or 1.
aborted: {$sum: {$eq : ["$data.buildResult", "ABORTED"]}}

MongoDB aggregation sort not working

I'm having a problem applying a sort to an aggregation grouping. My raw data looks like the following:
{
"_id" : ObjectId("52deab2fe4b0a491abb54108"),
"type" : "build",
"time" : ISODate("2014-01-21T17:15:27.471Z"),
"data" : {
"buildNumber" : 43,
"buildDuration" : 997308,
"buildProjectName" : "TestABC",
"buildResult" : "SUCCESS"
}
}
I would like to sort this first by buildProjectName and then date. Here is my query:
db.builds.aggregate([
{ $group: {
_id: {
month: { $month: "$time" },
day: { $dayOfYear: "$time" },
year: { $year: "$time" },
buildProjectName: "$data.buildProjectName",
},
buildDuration: { $avg: "$data.buildDuration" }
} },
{ $sort: {buildProjectName: 1, year: 1, month: 1, day: 1} }
])
I've tried switching the order of the sort (i.e.: buildProjectName, day, month, year), but I always get the same result with the dates out of order:
{
"result" : [
{
"_id" : {
"month" : 1,
"day" : 20,
"year" : 2014,
"buildProjectName" : "TestABC"
},
"buildDuration" : 1170723.5
},
{
"_id" : {
"month" : 1,
"day" : 21,
"year" : 2014,
"buildProjectName" : "TestABC"
},
"buildDuration" : 2284863.3333333335
},
{
"_id" : {
"month" : 1,
"day" : 17,
"year" : 2014,
"buildProjectName" : "TestABC"
},
"buildDuration" : 2234662
}
],
"ok" : 1
}
The fields you're sorting on are part of the _id so you need to include that in your $sort field names:
db.builds.aggregate([
{ $group: {
_id: {
month: { $month: "$time" },
day: { $dayOfYear: "$time" },
year: { $year: "$time" },
buildProjectName: "$data.buildProjectName",
},
buildDuration: { $avg: "$data.buildDuration" }
} },
{ $sort: {
'_id.buildProjectName': 1,
'_id.year': 1,
'_id.month': 1,
'_id.day': 1
} }
])