Mongoose aggregate and group by two fields - mongodb

I want to aggregate a table to lookup in other 3 tables, then make a nested group by.
I have 4 models
Order
Order_Batches which have a ref to order
Batch which have a ref to Order_Batch
Event which have a ref to Batch
so i'm selecting all orders then getting all it's orderbatches then find all batch and get all events done on that batch
Code
let order=await Order.aggregate([
{$lookup:{from:'orderbatches',localField:'_id',foreignField:'order',as:"order_batches"}},
{$unwind: {path: "$order_batches", preserveNullAndEmptyArrays: true}},
{$lookup:{from:'batches',localField:'order_batches._id',foreignField:'orderBatches',as:"batches"}},
{$unwind: {path: "$batches", preserveNullAndEmptyArrays: true}},
{$lookup:{from:'events',localField:'batches._id',foreignField:'batch',as:"events"}},
{$group: {
_id: "$_id",
code: {$first: "$code"},
order_batches: {$push: {
batches: "$batches",
events:"$events"
}}
}},
]);
Output
{
"success": true,
"orders": [
{
"_id": "5a5cbdd91ecaff0f8417a10d",
"code": "0",
"order_batches": [
{
"batches": {
"_id": "5a5cbdd91ecaff0f8417a114",
"updatedAt": "2018-01-15T14:42:33.585Z",
"createdAt": "2018-01-15T14:42:33.585Z",
"number": 1,
"quantity": 10,
"orderBatches": "5a5cbdd91ecaff0f8417a10e",
"removed": false,
"__v": 0,
}
"events": []
},
{
"batches": {
"_id": "5a5cbdd91ecaff0f8417a116",
"updatedAt": "2018-01-15T14:42:33.586Z",
"createdAt": "2018-01-15T14:42:33.586Z",
"number": 2,
"quantity": 10,
"orderBatches": "5a5cbdd91ecaff0f8417a10e",
"removed": false,
"__v": 0,
}
"events": [
{
"_id": "5a5cbdd91ecaff0f8417a117",
"updatedAt": "2018-01-15T14:42:33.587Z",
"createdAt": "2018-01-15T14:42:33.587Z",
"batch": "5a5cbdd91ecaff0f8417a116",
"process": [
"5a5cbdd91ecaff0f8417a115"
],
"removed": false,
"__v": 0
}
]
}
]
}
]
}
Expected Output
{
"success": true,
"orders": [
{
"_id": "5a5cbdd91ecaff0f8417a10d",
"code": "0",
"order_batches": [
{
"batches": {
"_id": "5a5cbdd91ecaff0f8417a114",
"updatedAt": "2018-01-15T14:42:33.585Z",
"createdAt": "2018-01-15T14:42:33.585Z",
"number": 1,
"quantity": 10,
"orderBatches": "5a5cbdd91ecaff0f8417a10e",
"removed": false,
"__v": 0,
"events": []
}
},
{
"batches": {
"_id": "5a5cbdd91ecaff0f8417a116",
"updatedAt": "2018-01-15T14:42:33.586Z",
"createdAt": "2018-01-15T14:42:33.586Z",
"number": 2,
"quantity": 10,
"orderBatches": "5a5cbdd91ecaff0f8417a10e",
"removed": false,
"__v": 0,
"events": [
{
"_id": "5a5cbdd91ecaff0f8417a117",
"updatedAt": "2018-01-15T14:42:33.587Z",
"createdAt": "2018-01-15T14:42:33.587Z",
"batch": "5a5cbdd91ecaff0f8417a116",
"process": [
"5a5cbdd91ecaff0f8417a115"
],
"removed": false,
"__v": 0
}
]
}
}
]
}
]
}
I've done this but also not working
let order=await Order.aggregate([
{$lookup:{from:'orderbatches',localField:'_id',foreignField:'order',as:"order_batches"}},
{$unwind: {path: "$order_batches", preserveNullAndEmptyArrays: true}},
{$lookup:{from:'batches',localField:'order_batches._id',foreignField:'orderBatches',as:"order_batches.batches"}},
{$unwind: {path: "$order_batches.batches", preserveNullAndEmptyArrays: true}},
{$lookup:{from:'events',localField:'order_batches.batches._id',foreignField:'batch',as:"order_batches.batches.events"}},
{$group: {
_id: "$order_batches._id",
// code: {$first: "$code"},
"batches": {
"$push": "$order_batches.batches"
}
}},
{$group: {
_id: "$_id",
code: {$first: "$code"},
"order_batches": {
"$push": "$order_batches"
}
}},
]);

I got it to work
let order=await Order.aggregate([
{$lookup:{from:'orderbatches',localField:'_id',foreignField:'order',as:"order_batches"}},
{$lookup:{from:'styles',localField:'style',foreignField:'parent',as:"style"}},
{$lookup:{from:'processes',localField:'style._id',foreignField:'style',as:"processes"}},
{$unwind: {path: "$order_batches", preserveNullAndEmptyArrays: true}},
{$lookup:{from:'batches',localField:'order_batches._id',foreignField:'orderBatches',as:"order_batches.batches"}},
{$unwind: {path: "$order_batches.batches", preserveNullAndEmptyArrays: true}},
{$lookup:{from:'events',localField:'order_batches.batches._id',foreignField:'batch',as:"order_batches.batches.events"}},
{$group: {
_id: "$order_batches._id",
order_id: {$first: "$_id"},
code:{$first:"$code"},
processes:{$first:"$processes"},
style:{$first:"$style"},
"batches": {
"$push": "$order_batches.batches"
}
}},
{$group: {
_id: "$order_id",
"code":{$first:"$code"},
"style":{$first:"$style"},
processes:{$first:"$processes"},
"order_batches": {
"$push": "$batches"
}
}},
]);

Related

Mongodb Aggregations - Group by date including condition

I have a series of documents gathered by aggregation grouping. This is the result for one document:
{
"_id": {
"ip": "79.xxx.xxx.117",
"myDate": "2022-10-19"
},
"date": "2022-10-19",
"allVisitedPages": [
{
"page": "/",
"time": {
"time": "2022-10-19T11:35:44.655Z",
"tz": "-120",
"_id": "634fe1100a011986b7137da0"
}
},
{
"page": "/2",
"time": {
"time": "2022-10-19T12:14:29.536Z",
"tz": "-120",
"_id": "634fea257acb264f23d421f1"
}
},
{
"page": "/",
"time": {
"time": "2022-10-19T15:37:30.002Z",
"tz": "-120",
"_id": "634fea266001ea364eeb38ea"
}
},
],
"visitedPages": 3,
"createdAt": "2022-10-19T11:35:44.920Z"
},
I want to get this (in this case 2 documents as the time difference between array position 2 and 3 is greater than 2 hours):
{
"_id": {
"ip": "79.xxx.xxx.117",
"myDate": "2022-10-19"
},
"date": "2022-10-19",
"allVisitedPages": [
{
"page": "/",
"durationInMinutes": "39",
"time": {
"time": "2022-10-19T11:35:44.655Z",
"tz": "-120",
"_id": "634fe1100a011986b7137da0"
}
},
{
"page": "/2",
"durationInMinutes": "2",
"time": {
"time": "2022-10-19T12:14:29.536Z",
"tz": "-120",
"_id": "634fea257acb264f23d421f1"
}
}
],
"visitedPages": 2,
},
{
"_id": {
"ip": "79.xxx.xxx.117",
"myDate": "2022-10-19"
},
"date": "2022-10-19",
"allVisitedPages": [
{
"page": "/",
"durationInMinutes": "2",
"time": {
"time": "2022-10-19T15:37:30.002Z",
"tz": "-120",
"_id": "634fea266001ea364eeb38ea"
}
},
],
"visitedPages": 1,
},
I want to get a new grouping document if the time between an array position and the following array position is greater than 2 hours. On the last array position it show always show "2".
I tried $divide and $datediff. But this is not possible on the group stage as it's an unary operator. An approach I tried is to calculate the sum of start and end time by dividing. But how to execute this on an array level on the group stage? Maybe someone could point me in the right direction if possible at all?
You can group and then reduce, but another option is to use $setWindowFields to calculate your grouping index before grouping:
db.collection.aggregate([
{$setWindowFields: {
partitionBy: {$concat: ["$ip", "$date"]},
sortBy: {"time.time": 1},
output: {prevtime: {
$push: "$time.time",
window: {documents: [-1, "current"]}
}}
}},
{$addFields: {
minutesDiff: {
$toInt: {
$dateDiff: {
startDate: {$first: "$prevtime"},
endDate: {$last: "$prevtime"},
unit: "minute"
}
}
}
}},
{$addFields: {deltaIndex: {$cond: [{$gt: ["$minutesDiff", 120]}, 1, 0]}}},
{$setWindowFields: {
partitionBy: {$concat: ["$ip", "$date"]},
sortBy: {"time.time": 1},
output: {
groupIndex: {
$sum: "$deltaIndex",
window: {documents: ["unbounded", "current"]}
},
duration: {
$push: "$minutesDiff",
window: {documents: ["current", 1]}
}
}
}
},
{$set: {
duration: {
$cond: [
{$and: [
{$eq: [{$size: "$duration"}, 2]},
{$lte: [{$last: "$duration"}, 120]}
]},
{$last: "$duration"},
2
]
}
}},
{$group: {
_id: {ip: "$ip", myDate: "$date", groupIndex: "$groupIndex"},
date: {$first: "$date"},
allVisitedPages: {$push: {page: "$page", time: "$time", duration: "$duration"}},
visitedPages: {$sum: 1}
}},
{$unset: "_id.groupIndex"}
])
See how it works on the playground example

Mongodb pipeline on parse server document add pointer field with $lookup

To be honest I really know sql but I'm kind of new to mongodb noSql so I'm a bit lost.
I have made a pipeline that's just working fine.
The point was to group by day and mindmapId to count number of user viewed it and sum watching time and save it into a collection in order to make request on it after.
here's sample of data
MindMap
{
"_id": "Yg5uGI3Iy0",
"data": {
"id": "root",
"topic": "Main topic",
"expanded": true
},
"theme": "orange",
"_p_author": "_User$zqPzSKD7EM",
"_created_at": {
"$date": {
"$numberLong": "1658497264836"
}
},
"_updated_at": {
"$date": {
"$numberLong": "1661334292749"
}
}
}
MindmapView
{
"_id": "qWR6HVIcvT",
"startViewDate": {
"$date": {
"$numberLong": "1658669095261"
}
},
"_p_user": "_User$VnrxG9gABO",
"_p_mindmap": "MindMap$Yg5uGI3Iy0",
"_created_at": {
"$date": {
"$numberLong": "1658669095274"
}
},
"_updated_at": {
"$date": {
"$numberLong": "1658669095274"
}
}
}
Pipeline
[{
$group: {
_id: {
day: {
$dateToString: {
format: '%Y-%m-%d',
date: '$startViewDate'
}
},
mindmapId: {
$substr: [
'$_p_mindmap',
8,
-1
]
}
},
watchTime: {
$sum: {
$dateDiff: {
startDate: '$_created_at',
endDate: '$_updated_at',
unit: 'second'
}
}
},
uniqueCount: {
$addToSet: '$_p_user'
}
}
}, {
$project: {
_id: 1,
total: {
$size: '$uniqueCount'
},
watchTime: {
$sum: '$watchTime'
}
}
}]
pipeline results
[{
"_id": {
"day": "2022-08-01",
"mindmapId": "oGCQDQmaNK"
},
"total": 1,
"watchTime": 7
},{
"_id": {
"day": "2022-08-11",
"mindmapId": "7YlZ6FPwiD"
},
"total": 1,
"watchTime": 21
},{
"_id": {
"day": "2022-08-15",
"mindmapId": "7YlZ6FPwiD"
},
"total": 1,
"watchTime": 13
},{
"_id": {
"day": "2022-07-25",
"mindmapId": "7YlZ6FPwiD"
},
"total": 1,
"watchTime": 3
},{
"_id": {
"day": "2022-08-01",
"mindmapId": "YXa8omyChc"
},
"total": 2,
"watchTime": 1306837
},{
"_id": {
"day": "2022-07-25",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 7
},{
"_id": {
"day": "2022-08-17",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 60
},{
"_id": {
"day": "2022-08-06",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 0
},{
"_id": {
"day": "2022-08-11",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 69
},{
"_id": {
"day": "2022-08-10",
"mindmapId": "oGCQDQmaNK"
},
"total": 1,
"watchTime": 4
},{
"_id": {
"day": "2022-08-15",
"mindmapId": "Yg5uGI3Iy0"
},
"total": 1,
"watchTime": 9
},
...
]
However to exploit this data faster I need to include the mindmap author inside the result collection.
The point is to group by day and mindmapId to count number of user viewed it and sum watching time and get the mindmap author and save it into a collection.
To do that I need to use $lookup but the result is kind of messy and the lookup act like a full join in sql. I've tried so much combination before this post.
Here's what I have mainly tried
[{
$group: {
_id: {
day: {
$dateToString: {
format: '%Y-%m-%d',
date: '$startViewDate'
}
},
mindmapId: {
$substr: [
'$_p_mindmap',
8,
-1
]
}
},
watchTime: {
$sum: {
$dateDiff: {
startDate: '$_created_at',
endDate: '$_updated_at',
unit: 'second'
}
}
},
uniqueCount: {
$addToSet: '$_p_user'
}
}
}, {
$lookup: {
from: 'MindMap',
localField: '_objectId',
foreignField: '_id.mindmapId',
as: 'tempMindmapPointer'
}
}, {
$unwind: '$tempMindmapPointer'
}, {
$match: {
'tempMindmapPointer._id': '_id.mindmapId'
}
}, {
$project: {
_id: 1,
total: {
$size: '$uniqueCount'
},
watchTime: {
$sum: '$watchTime'
},
author: {
$substr: [
'$tempMindmapPointer._p_author',
6,
-1
]
}
}
}]
the $match doesn't work here it make me have no results
If I remove $match it act like a full join user list with mindmap id list which I don't want
[{
"_id": {
"day": "2022-08-17",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 60,
"author": "zqPzSKD7EM"
},{
"_id": {
"day": "2022-08-17",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 60,
"author": "zqPzSKD7EM"
},{
"_id": {
"day": "2022-08-17",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 60,
"author": "zqPzSKD7EM"
},{
"_id": {
"day": "2022-08-17",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 60,
"author": "VnrxG9gABO"
},{
"_id": {
"day": "2022-08-17",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 60,
"author": "zqPzSKD7EM"
},{
"_id": {
"day": "2022-08-17",
"mindmapId": "YXa8omyChc"
},
"total": 1,
"watchTime": 60,
"author": "x6kNvG2O0X"
},...
]
I have tried to switch localField: '_objectId' foreignField:'_id.mindmapId' values.
I have also tried to make the lookup first and group by id{day,mindmapId,authorId} but I have never been able to make this compiling.
What could I do to make this request working ? I'm sure there is something to do with $match and $lookup
If I understand you correctly (since you didn't add the requested result), the simple option is:
db.MindmapView.aggregate([
{$group: {
_id: {
day: {$dateToString: {format: "%Y-%m-%d", date: "$startViewDate"}},
mindmapId: {$substr: ["$_p_mindmap", 8, -1]}
},
watchTime: {
$sum: {
$dateDiff: {startDate: "$_created_at", endDate: "$_updated_at", unit: "second"}
}
},
uniqueCount: {$addToSet: "$_p_user"}
}
},
{$project: {_id: 1, total: {$size: "$uniqueCount"}, watchTime: 1}},
{$lookup: {
from: "MindMap",
localField: "_id.mindmapId",
foreignField: "_id",
as: "author"
}
},
{$set: {author: {$first: "$author._p_author"}}}
])
See how it works on the playground example.
There is another option that may be a little more efficient, which is using the '$lookup' with a pipeline, to bring only the author from the MindMap collection instead of bringing the entire document and then filter it.
In this case the $lookup stage will be:
{
$lookup: {
from: "MindMap",
let: {id: "$_id.mindmapId"},
pipeline: [
{$match: {$expr: {$eq: ["$$id", "$_id"]}}},
{$project: {_p_author: 1, _id: 0}}
],
as: "author"
}
}

$addFields is not adding value in document

Query is as follows and result is given below:
What I want is I am adding field called name, in which I want categoryObj[0].categoryName but it is empty.
Tried categoryObj.$.categoryName but giving error.
Once name is obtained as I want i will exclude categoryObj with project opertator.
Thanks for help in advance
let itemsByCategory = await VendorItem.aggregate([
{$match: {vendor: vendorId}},
{$lookup: {
from: "vendorcategories",
localField: "category",
foreignField: "_id",
as: 'categoryDetails'
}},
{$group:{
"_id":"$category",
"count":{"$sum":1},
"items":{"$push":"$$ROOT"},
"categoryObj":{"$addToSet":"$categoryDetails"}
}},
{$project: {"items.categoryDetails":0}},
{$addFields: {"categoryName" : "$categoryObj.categoryName"}},
//{$project: {"categoryObj":0}},
]);
and the result is as follows
{
"itemsByCategory": [
{
"_id": "62296d612a1462a7d5e4b86b",
"count": 1,
"menuItems": [
{
"_id": "622971fa4fda7b4c792a7812",
"category": "62296d612a1462a7d5e4b86b",
"vendor": "62296c6f2a1462a7d5e4b863",
"item": "Dahi Chaat",
"price": 30,
"inStock": true,
"variants": [
{
"variantName": "With Sev",
"variantPrice": 40,
"_id": "622975b9f7bdf6c2a3b7703c"
}
],
"toppings": [
{
"name": "cheese",
"price": 10,
"inStock": true,
"_id": "62297766ff9f01d236c60736"
}
],
"categoryDetails": [
{
"_id": "62296d612a1462a7d5e4b86b",
"categoryName": "Snacks",
"categoryDescription": "Desciption changed!",
"vendor": "621c6c944d6d79e83219e59a",
"__v": 0
}
]
}
],
"categoryObj": [
[
{
"_id": "62296d612a1462a7d5e4b86b",
"categoryName": "Snacks",
"categoryDescription": "Desciption changed!",
"vendor": "621c6c944d6d79e83219e59a",
"__v": 0
}
]
],
"name": []
}
]
}
You can add an $unwind phase in order to "loop" all objects inside "categoryObj", but you will need to group it back afterwards:
{"$addFields": {orig_id: "$_id"}},
{"$unwind": "$categoryObj"},
{"$addFields": {"name": {"$arrayElemAt": ["$categoryObj", 0]}}},
{"$group": {_id: "$orig_id", name: {$push: "$name.categoryName"},
menuItems: {$first: "$menuItems"}, count: {$first: "count"},
}
}
See playground here:
https://mongoplayground.net/p/wsH2Y0UZ_FH

how to transform data using aggregate function mongodb

How to transform data base on parent_id within self join ? Is this possible make the result as expected. Please help on this thanks
db={
post: [
{
"_id": ObjectId("59f9c5629f75813e21a6fe34"),
"parent_id": "0",
"name": "main_category",
"short_desc": "",
"long_desc": "",
"slug": "main_category",
"status": true,
"createdAt": ISODate("2017-11-01T13:00:18.714Z"),
"updatedAt": ISODate("2019-02-19T07:31:20.967Z")
},
{
"_id": ObjectId("59f9c5629f75813e21a6fe73"),
"parent_id": "59f9c5629f75813e21a6fe34",
"name": "sub_category",
"short_desc": "",
"long_desc": "",
"slug": "sub_category",
"status": true,
"createdAt": ISODate("2017-11-01T13:00:18.714Z"),
"updatedAt": ISODate("2019-02-19T07:31:20.967Z")
},
{
"_id": ObjectId("59f9c5629f75813e21a6fe33"),
"parent_id": "59f9c5629f75813e21a6fe73",
"name": "sub_category1",
"short_desc": "",
"long_desc": "",
"slug": "sub_category1",
"status": true,
"createdAt": ISODate("2017-11-01T13:00:18.714Z"),
"updatedAt": ISODate("2019-02-19T07:31:20.967Z")
}
]
}
output should like this. If any more category does not belongs to anything it should stay blank
[
{
mainCategory: 'main_category',
subCategory1: 'sub_category',
subCategory2: 'sub_category1',
subCategory3: '',
subCategory4: '',
subCategory5: ''
}, {
mainCategory: '{if any}',
subCategory1: '{if any}',
subCategory2: '{if any}',
subCategory3: '',
subCategory4: '',
subCategory5: ''
}
];
Any hope to get this stat. ?
$graphLookup reads from the collection specified by its from argument, not from the documents in the pipeline.
In the pipeline you created to change the datatype, use a $merge stage to update the existing documents:
db.post.aggregate([
{$addFields: {
parent_oid: {
$cond: {
if: {$eq: ["$parent_id","0"]},
then: "$parent_id",
else: {$toObjectId: "$parent_id"}
}
}
}
},
{$merge: "post"}
])
Then you can use $graphLookup to form the lists, and transform them to the shape you need:
db.post.aggregate([
{$match: {parent_id: "0" }},
{"$graphLookup": {
"from": "post",
"startWith": "$_id",
"connectFromField": "_id",
"connectToField": "parent_oid",
"as": "response"
}},
{$unwind: "$response"},
{$group: {
_id: "$_id",
main_category: {$first: "$slug"},
subCategories: {$push: {
k: "$response.name",
v: "$response.slug"
}}
}
},
{$replaceRoot: {
newRoot: {
$mergeObjects: [
{mainCategory: "$main_category"},
{$arrayToObject: "$subCategories"}
]
}
}}
])
Output from the sample data:
[
{
"mainCategory": "main_category",
"sub_category": "sub_category",
"sub_category1": "sub_category1"
}
]
Playground

Mongo DB aggrregation on multiple arrays

I want to retrieve from my cart items and bundles that are not deleted.
my cart looks like this:
{
"_id": "589474849d7b3f439797faf1",
"bundles": [{
"id": "57c98e25298cd0f908021c12",
"serial": "xxxx",
"status": ""
}],
"items": [{
"id": "589de9a690d632ccbc10cd64",
"status": "deleted",
"quantity": 1,
"serial": "fffff"
}]
}
What I tried was:
[
{$match: condition},
{$unwind: {"path": "$items", "preserveNullAndEmptyArrays": true}},
{$unwind: {"path": "$bundles", "preserveNullAndEmptyArrays": true}},
{$match: {"items.status": {$ne: "deleted"}}},
{$match: {"bundles.status": {$ne: "deleted"}}},
{
"$group": {
"_id": "$_id",
currency: {$first: "$currency"},
tenant: {$first: "$tenant"},
user: {$first: "$user"},
"items": {"$addToSet": "$items"},
"bundles": {"$addToSet": "$bundles"}
}
}
];
It works fine for all cases except when there is only one deleted item, and multiple bundles. The query return no bundles at all
expected output:
{
"_id": "589474849d7b3f439797faf8",
"items": [{
"id": "589de9a690d632ccbc10cd64",
"quantity": 1,
"serial": "fff"
}, {
"id": "589de9a690d632ccbc10c55",
"quantity": 1,
"serial": "xxx"
}],
"bundles": [{
"id": "57c98e25298cd0f908021c12",
"serial": "pppp"
}]
}
Oh thanks guys I found the solution, filter will do the trick
[{
$match: "condition"
}, {
$project: {
items: {
$filter: {
input: "$items",
as: "item",
cond: {
$ne: ["$$item.status", "deleted"]
}
}
},
bundles: {
$filter: {
input: "$bundles",
as: "bundle",
cond: {
$ne: ["$$bundle.status", "deleted"]
}
}
}
}
}]