Filter by nested arrays/objects values (on different levels) and $push by multiple level - MongoDB Aggregate - mongodb

I have a document with multiple level of embedded subdocument each has some nested array. Using $unwind and sort, do sorting based on day in descending and using push to combine each row records into single array. This Push is working only at one level means it allows only one push. If want to do the same things on the nested level and retains the top level data, got "errmsg" : "Unrecognized expression '$push'".
{
"_id" : ObjectId("5f5638d0ff25e01482432803"),
"name" : "XXXX",
"mobileNo" : 323232323,
"payroll" : [
{
"_id" : ObjectId("5f5638d0ff25e01482432801"),
"month" : "Jan",
"salary" : 18200,
"payrollDetails" : [
{
"day" : "1",
"salary" : 200,
},
{
"day" : "2",
"salary" : 201,
}
]
},
{
"_id" : ObjectId("5f5638d0ff25e01482432802"),
"month" : "Feb",
"salary" : 8300,
"payrollDetails" : [
{
"day" : "1",
"salary" : 300,
},
{
"day" : "2",
"salary" : 400,
}
]
}
],
}
Expected Result:
{
"_id" : ObjectId("5f5638d0ff25e01482432803"),
"name" : "XXXX",
"mobileNo" : 323232323,
"payroll" : [
{
"_id" : ObjectId("5f5638d0ff25e01482432801"),
"month" : "Jan",
"salary" : 18200,
"payrollDetails" : [
{
"day" : "2",
"salary" : 201
},
{
"day" : "1",
"salary" : 200
}
]
},
{
"_id" : ObjectId("5f5638d0ff25e01482432802"),
"month" : "Feb",
"salary" : 8300,
"payrollDetails" : [
{
"day" : "2",
"salary" : 400
},
{
"day" : "1",
"salary" : 300
}
]
}
],
}
Just day will be sorted and remaining things are same
I have tried but it got unrecognized expression '$push'
db.employee.aggregate([
{$unwind: '$payroll'},
{$unwind: '$payroll.payrollDetails'},
{$sort: {'payroll.payrollDetails.day': -1}},
{$group: {_id: '$_id', payroll: {$push: {payrollDetails:{$push:
'$payroll.payrollDetails'} }}}}])

It requires two time $group, you can't use $push operator two times in a field,
$group by main id and payroll id, construct payrollDetails array
$sort by payroll id (you can skip if not required)
$group by main id and construct payroll array
db.employee.aggregate([
{ $unwind: "$payroll" },
{ $unwind: "$payroll.payrollDetails" },
{ $sort: { "payroll.payrollDetails.day": -1 } },
{
$group: {
_id: {
_id: "$_id",
pid: "$payroll._id"
},
name: { $first: "$name" },
mobileNo: { $first: "$mobileNo" },
payrollDetails: { $push: "$payroll.payrollDetails" },
month: { $first: "$payroll.month" },
salary: { $first: "$payroll.salary" }
}
},
{ $sort: { "payroll._id": -1 } },
{
$group: {
_id: "$_id._id",
name: { $first: "$name" },
mobileNo: { $first: "$mobileNo" },
payroll: {
$push: {
_id: "$_id.pid",
month: "$month",
salary: "$salary",
payrollDetails: "$payrollDetails"
}
}
}
}
])
Playground

Related

How to calculate profit using aggregations from two collections in mongodb?

I have two collections, orders and producttypes
ProductTypes:
{
"_id" : 609d79de5909592f2635c64e,
"name" : "T-Shirt",
"subType" : "Round Neck",
"__v" : 0,
"size" : "XXL",
"sellingPrice" : 320,
"createdAt" : ISODate("2021-05-18T05:22:00.695+0000"),
"actualPrice" : 200,
"updatedAt" : ISODate("2021-05-25T12:11:50.986+0000")
},
{
"_id" : 609d79de5909592f2635c64d,
"name" : "T-Shirt",
"subType" : "V Neck",
"__v" : 0,
"size" : "XXL",
"sellingPrice" : 290,
"createdAt" : ISODate("2021-05-18T05:22:00.695+0000"),
"actualPrice" : 200,
"updatedAt" : ISODate("2021-05-25T12:11:50.986+0000")
}
Orders:
{
"_id" : "60a63e369cf3a806c0209bd8",
"items" : [
{
"type" : "609d79de5909592f2635c64e",
"quantity" : 1,
"sellingPrice" : 320
},
{
"type" : "609d79de5909592f2635c64d",
"quantity" : 2,
"sellingPrice" : 290
}
],
"orderId" : "ORD101",
"from" : "Abc",
"to" : "xyz",
"createdAt" : ISODate("2021-05-20T10:47:18.920+0000"),
"__v" : 0,
"tracking" : "12345678"
}
I want to calculate total profit per order like:
{orderId: "ORD101", createdAt: ISODate("2021-05-18T05:22:00.695+0000"), profit: 300}
I don't know how to join these two collections to calculate the profit.
But I tried something like below in node:
Order.aggregate([{
$unwind: '$items'
}, {
$project: {
orderId:1,
quantity: "$items.quantity",
sellingPrice: {
$multiply: [
{"$ifNull": ["$items.quantity", 0]},
{"$ifNull": ["$items.price", 0]}
]
},
type: '$items.type'
}
}])
.exec(function(err, transactions) {
//console.log(transactions);
ProductType.populate(transactions,{path: 'type', select: 'actualPrice' }, function(err, populatedTransactions) {
//res.json(populatedTransactions);
var items = [];
var totalProfit = 0;
if(populatedTransactions){
populatedTransactions.forEach( order => {
if( order.quantity != undefined && order.sellingPrice != undefined && order.sellingPrice > 0){
let profit = order.sellingPrice - (order.quantity * order.type.actualPrice);
totalProfit = totalProfit + profit;
items.push({ orderId: order.orderId, profit: profit });
}
})
res.status(200).json({data: items, totalProfit: totalProfit});
}
});
});
Is this the right way?
Here am using $unwind on the array then populating with producttypes collection to get an actual price, then am doing calculations to get the profit.
$project to show required fields
$unwind deconstruct the items array
$lookup with productTypes collection
calculate the profit
$arrayElemAt to get first element from item actualPrice result
$subtract sellingPrice by actualPrice
$multiply above result with quantity
$group by order _id and get required fields and sum profit
Order.aggregate([
{
$project: {
orderId: 1,
createdAt: 1,
items: 1
}
},
{ $unwind: "$items" },
{
$lookup: {
from: "productTypes", // replace your actual collection name
localField: "items.type",
foreignField: "_id",
as: "item"
}
},
{
$addFields: {
profit: {
$multiply: [
{
$subtract: [
"$items.sellingPrice",
{ $arrayElemAt: ["$item.actualPrice", 0] }
]
},
"$items.quantity"
]
}
}
},
{
$group: {
_id: "$_id",
orderId: { $first: "$orderId" },
createdAt: { $first: "$createdAt" },
profit: { $sum: "$profit" }
}
}
])
Playground

How to get percentage total of data with group by date in MongoDB

How to get percentage total of data with group by date in MongoDB ?
Link example : https://mongoplayground.net/p/aNND4EPQhcb
I have some collection structure like this
{
"_id" : ObjectId("5ccbb96706d1d47a4b2ced4b"),
"date" : "2019-05-03T10:39:53.108Z",
"id" : 166,
"update_at" : "2019-05-03T10:45:36.208Z",
"type" : "image"
}
{
"_id" : ObjectId("5ccbb96706d1d47a4b2ced4c"),
"date" : "2019-05-03T10:39:53.133Z",
"id" : 166,
"update_at" : "2019-05-03T10:45:36.208Z",
"type" : "image"
}
{
"_id" : ObjectId("5ccbb96706d1d47a4b2ced4d"),
"date" : "2019-05-03T10:39:53.180Z",
"id" : 166,
"update_at" : "2019-05-03T10:45:36.208Z",
"type" : "image"
}
{
"_id" : ObjectId("5ccbb96706d1d47a4b2ced4e"),
"date" : "2019-05-03T10:39:53.218Z",
"id" : 166,
"update_at" : "2019-05-03T10:45:36.208Z",
"type" : "image"
}
And I have query in mongodb to get data of collection, how to get percentage of total data. in bellow example query to get data :
db.name_collection.aggregate(
[
{ "$match": {
"update_at": { "$gte": "2019-11-04T00:00:00.0Z", "$lt": "2019-11-06T00:00:00.0Z"},
"id": { "$in": [166] }
} },
{
"$group" : {
"_id": {
$substr: [ '$update_at', 0, 10 ]
},
"count" : {
"$sum" : 1
}
}
},
{
"$project" : {
"_id" : 0,
"date" : "$_id",
"count" : "$count"
}
},
{
"$sort" : {
"date" : 1
}
}
]
)
and this response :
{
"date" : "2019-11-04",
"count" : 39
},
{
"date" : "2019-11-05",
"count" : 135
}
how to get percentage data total from key count ? example response to this :
{
"date" : "2019-11-04",
"count" : 39,
"percentage" : "22%"
},
{
"date" : "2019-11-05",
"count" : 135,
"percentage" : "78%"
}
You have to group by null to get total count and then use $map to calculate the percentage. $round will be a useful operator in such case. Finally you can $unwind and $replaceRoot to get back the same number of documents:
db.collection.aggregate([
// previous aggregation steps
{
$group: {
_id: null,
total: { $sum: "$count" },
docs: { $push: "$$ROOT" }
}
},
{
$project: {
docs: {
$map: {
input: "$docs",
in: {
date: "$$this.date",
count: "$$this.count",
percentage: { $concat: [ { $toString: { $round: { $multiply: [ { $divide: [ "$$this.count", "$total" ] }, 100 ] } } }, '%' ] }
}
}
}
}
},
{
$unwind: "$docs"
},
{
$replaceRoot: { newRoot: "$docs" }
}
])
Mongo Playground

Mongodb aggregate by day and delete duplicate value

I'm trying to clean a huge database.
Sample DB :
{
"_id" : ObjectId("59fc5249d5ab401d99f3de7f"),
"addedAt" : ISODate("2017-11-03T11:26:01.744Z"),
"__v" : 0,
"check" : 17602,
"lastCheck" : ISODate("2018-04-05T11:47:00.609Z"),
"tracking" : [
{
"timeCheck" : ISODate("2017-11-06T13:17:20.861Z"),
"_id" : ObjectId("5a0060e00f3c330012bafe39"),
"rank" : 2395,
},
{
"timeCheck" : ISODate("2017-11-06T13:22:31.254Z"),
"_id" : ObjectId("5a0062170f3c330012bafe77"),
"rank" : 2395,
},
{
"timeCheck" : ISODate("2017-11-06T13:27:40.551Z"),
"_id" : ObjectId("5a00634c0f3c330012bafebe"),
"rank" : 2379,
},
{
"timeCheck" : ISODate("2017-11-06T13:32:41.084Z"),
"_id" : ObjectId("5a0064790f3c330012baff03"),
"rank" : 2395,
},
{
"timeCheck" : ISODate("2017-11-06T13:37:51.012Z"),
"_id" : ObjectId("5a0065af0f3c330012baff32"),
"rank" : 2379,
},
{
"timeCheck" : ISODate("2017-11-07T13:37:51.012Z"),
"_id" : ObjectId("5a0065af0f3c330012baff34"),
"rank" : 2379,
}]
}
I have a lot of duplicate value but I need to clean only by day.
To obtain this for example :
{
"_id" : ObjectId("59fc5249d5ab401d99f3de7f"),
"addedAt" : ISODate("2017-11-03T11:26:01.744Z"),
"__v" : 0,
"check" : 17602,
"lastCheck" : ISODate("2018-04-05T11:47:00.609Z"),
"tracking" : [
{
"timeCheck" : ISODate("2017-11-06T13:17:20.861Z"),
"_id" : ObjectId("5a0060e00f3c330012bafe39"),
"rank" : 2395,
},
{
"timeCheck" : ISODate("2017-11-06T13:27:40.551Z"),
"_id" : ObjectId("5a00634c0f3c330012bafebe"),
"rank" : 2379,
},
{
"timeCheck" : ISODate("2017-11-07T13:37:51.012Z"),
"_id" : ObjectId("5a0065af0f3c330012baff34"),
"rank" : 2379,
}]
}
How can I aggregate by day and after delete last value duplicate?
I need to keep the values per day even if they are identical with another day.
The aggregation framework cannot update data at this stage. However, you can use the following aggregation pipeline in order to get the desired output and then use e.g. a bulk replace to update all your documents:
db.collection.aggregate({
$unwind: "$tracking" // flatten the "tracking" array into separate documents
}, {
$sort: {
"tracking.timeCheck": 1 // sort by timeCheck to allow us to use the $first operator in the next stage reliably
}
}, {
$group: {
_id: { // group by
"_id": "$_id", // "_id" and
"rank": "$tracking.rank", // "rank" and
"date": { // the "date" part of the "timeCheck" field
$dateFromParts : {
year: { $year: "$tracking.timeCheck" },
month: { $month: "$tracking.timeCheck" },
day: { $dayOfWeek: "$tracking.timeCheck" }
}
}
},
"doc": { $first: "$$ROOT" } // only keep the first document per group
}
}, {
$sort: {
"doc.tracking.timeCheck": 1 // restore ascending sort order - may or may not be needed...
}
}, {
$group: {
_id: "$_id._id", // merge everything again per "_id"
"addedAt": { $first: "$doc.addedAt" },
"__v": { $first: "$doc.__v" },
"check": { $first: "$doc.check" },
"lastCheck": { $first: "$doc.lastCheck" },
"tracking": { $push: "$doc.tracking" } // in order to join the tracking values into an array again
}
})

Merge Multiple Document from same collection MongoDB

I have a JSON data like this and i wanted to apply aggregation on this data in such a way that i should group by from data:
{
"series": [
{
"id": "1",
"element": "111",
"data": [
{
"timeFrame": {
"from": "2016-01-01T00:00:00Z",
"to": "2016-01-31T23:59:59Z"
},
"value": 1
},
{
"timeFrame": {
"from": "2016-02-01T00:00:00Z",
"to": "2016-02-29T23:59:59Z"
},
"value": 2
}
]
}
]
}
and i have acheived this by the above aggregation:
db.getCollection('col1').aggregate([
{$unwind: "$data"},
{$group :{
element: {$first:"$relatedElement"},
_id : {
day : {$dayOfMonth: "$values.timeFrame.from"},
month:{$month: "$values.timeFrame.from"},
year:{$year: "$values.timeFrame.from"}
},
fromDate : { $first : "$values.timeFrame.from" },
total : {$sum : "$values.value"},
count : {$sum : 1},
}
},
{
$project: {
_id : 0,
element:1,
fromDate : '$fromDate',
avgValue : { $divide: [ "$total", "$count" ] }
}
}])
OutPut:
{
"id" : "1",
"element" : "3",
"fromDate" : ISODate("2017-05-01T00:00:00.000Z"),
"avgValue" : 0.0378787878787879
}
{
"id" : "1",
"element" : "3",
"fromDate" : ISODate("2017-04-30T22:00:00.000Z"),
"avgValue" : 0.416666666666667
}
But, i am getting two document and this i want to merge as a single document like :
{
"id" : "1",
"element" : "3",
"average" : [
{
"fromDate" : ISODate("2017-05-01T00:00:00.000Z"),
"avgValue" : 0.0378787878787879
},
{
"fromDate" : ISODate("2017-04-30T22:00:00.000Z"),
"avgValue" : 0.416666666666667
}
]
}
Can anyone help me on this.
Add following $group at the end of your aggregate pipeline to merge current output documents into single document -
{$group:{
_id:"$_id",
element: {$first: "$element"},
average:{$push:{
"fromDate": "$fromDate",
"avgValue": "$avgValue"
}}
}}

MongoDB aggregate Timezone for date add

I have a problem of MongoDB's aggregate of timezone is UTC. I have looked for solutions from many other existing issues, but it is still not working. My code as follows:
MongoDB version : 2.2
Data
{ "_id" : ObjectId("52a3c9df46c6a9627eeb0337"), "Counting" : { "id" : "b1a93dfda46c47848f9862031300d24c", "group" : "Salary", "user_id" : "4d4ad2d37a464ad09d9aca2fee4c760c", "subGroup" : "e–ae3?", "bank_id" : "97e0fecc322b49b48c4eb3c8425fea77", "fee" : 646, "isIncome" : "true", "payment" : "", "consumeDate" : ISODate("2013-08-15T16:00:00Z"), "createDate" : ISODate("2013-12-08T01:22:39.008Z"), "bank_name" : "9edb6897-cdb8-4ce4-8f08-f5792cfa83d9" } }
{ "_id" : ObjectId("52a3c9df46c6a9627eeb0338"), "Counting" : { "id" : "33b341fc71314daebe851397c5cbaa40", "group" : "Salary", "user_id" : "cb9e06649cf943e5b368f6b05fc126c6", "subGroup" : "e–ae3?", "bank_id" : "e8da8cdae3ae495ca76f873fb3460b6d", "fee" : 647, "isIncome" : "true", "payment" : "", "consumeDate" : ISODate("2013-02-28T16:00:00Z"), "createDate" : ISODate("2013-12-08T01:22:39.016Z"), "bank_name" : "6913b48a-1a95-48c5-81f5-6920031358d7"} }
{ "_id" : ObjectId("52a3c9df46c6a9627eeb033a"), "Counting" : { "id" : "f0d41ed9f29f47e7b68a05c378cf939d", "group" : "Salary", "user_id" : "847cadbf55f84615af3ee63922446b54", "subGroup" : "e–ae3?", "bank_id" : "f45d62b5e62f4b7fa8172870cd992f19", "fee" : 623, "isIncome" : "true", "payment" : "", "consumeDate" : ISODate("2013-04-18T16:00:00Z"), "createDate" : ISODate("2013-12-08T01:22:39.152Z"), "bank_name" : "30dd169e-723e-4748-93cd-2d7a45b4a3b7"} }
db.Product.aggregate([{
"$group": {
"_id": {
"tyear": {
"$year": [{
"$add": ["$Counting.consumeDate", 28800000]
}]
},
"tMonth": {
"$month": [{
"$add": ["$Counting.consumeDate", 28800000]
}]
},
"tDate": {
"$dayOfMonth": [{
"$add": ["$Counting.consumeDate", 28800000]
}]
},
},
"count": {
"$sum": "$Counting.fee"
}
} }])
Error Message :
"errmsg" : "exception: $add does not support dates"
Reference
How to agregate by year-month-day on a different timezone
I'd recommend doing this in two-steps as a project then a group.
var millisecondsFromUTC = 8 * 60 * 60 * 1000; //PST is -8 hours from UTC
db.Product.aggregate([
{ $project : {
consumeDateLocal: {
$subtract : [ "$Counting.consumeDate", millisecondsFromUTC ]
},
fee: '$Counting.fee" } },
{ $group: {
_id: {
"tyear": { $year: "$consumeDateLocal" },
"tMonth": { "$month": "$consumeDateLocal" },
"tDate": { "$dayOfMonth": "consumeDateLocal" }
},
count: {
$sum: "$fee"
}
} } ], ...);
I do it like this.
millisecondsFromUTC = 8 * 3600 * 1000
Db.collection.aggreagte([
{$match: query},
{
$group: {
_id: {
$dateToString: {
format: "%Y-%m-%d",
date: {$add: ["$date", millisecondsFromUTC]}
},
click: {$sum: '$click'},
money: {$sum: {$divide: ['$money', 10000]}},
pv: {$sum: '$pv'},
req: {$sum: '$req'},
date: {$last: '$date'}
}
}]