MongoDB aggregate $group $sum that matches date inside array of objects - mongodb

I'll explain my problem here and i'll put a tldr at the bottom summarizing the question.
We have a collection called apple_receipt, since we have some apple purchases in our application. That document has some fields that we will be using on this aggregation. Those are: price, currency, startedAt and history. Price, currency and startedAt are self-explanatory. History is a field that is an array of objects containing a price and startedAt. So, what we are trying to accomplish is a query that gets every document between a date of our choice, for example: 06-06-2020 through 10-10-2022 and get the total price combined of all those receipts that have a startedAt between that. We have a document like this:
{
price: 12.9,
currency: 'BRL',
startedAt: 2022-08-10T16:23:42.000+00:00
history: [
{
price: 12.9,
startedAt: 2022-05-10T16:23:42.000+00:00
},
{
price: 12.9,
startedAt: 2022-06-10T16:23:42.000+00:00
},
{
price: 12.9,
startedAt: 2022-07-10T16:23:42.000+00:00
}
]
}
If we query between dates 06-06-2022 to 10-10-2022, we would have a return like this: totalPrice: 38,7.
-total price of the 3 objects that have matched the date inside that value range-
I have tried this so far:
AppleReceipt.aggregate([
{
$project: {
price: 1,
startedAt: 1,
currency: 1,
history: 1,
}
},
{
$unwind: {
path: "$history",
preserveNullAndEmptyArrays: true,
}
},
{
$match: {
$or: [
{ startedAt: {$gte: new Date(filters.begin), $lt: new Date(filters.end)} },
]
}
},
{
$group: {
_id: "$_id",
data: { $push: '$$ROOT' },
totalAmountHelper: { $sum: '$history.price' }
}
},
{
$unwind: "$data"
},
{
$addFields: {
totalAmount: { $add: ['$totalAmountHelper', '$data.price'] }
}
}
])
It does bring me the total value but I couldn't know how to take into consideration the date to make the match stage to only get the sum of the documents that are between that date.
tl;dr: Want to make a query that gets the total sum of the prices of all documents that have startedAt between the dates we choose. Needs to match the ones inside history field - which is an array of objects, and also the startedAt outside of the history field.

https://mongoplayground.net/p/lOvRbX24QI9
db.collection.aggregate([
{
$set: {
"history_total": {
"$reduce": {
"input": "$history",
"initialValue": 0,
"in": {
$sum: [
{
"$cond": {
"if": {
$and: [
{
$gte: [
new Date("2022-06-06"),
{
$dateFromString: {
dateString: "$$this.startedAt"
}
}
]
},
{
$lt: [
{
$dateFromString: {
dateString: "$$this.startedAt"
}
},
new Date("2022-10-10")
]
},
]
},
"then": "$$this.price",
"else": 0
}
},
"$$value",
]
}
}
}
}
},
{
$set: {
"history_total": {
"$sum": [
"$price",
"$history_total"
]
}
}
}
])
Result:
[
{
"_id": ObjectId("5a934e000102030405000000"),
"currency": "BRL",
"history": [
{
"price": 12.9,
"startedAt": "2022-05-10T16:23:42.000+00:00"
},
{
"price": 12.9,
"startedAt": "2022-06-10T16:23:42.000+00:00"
},
{
"price": 12.9,
"startedAt": "2022-07-10T16:23:42.000+00:00"
}
],
"history_total": 325.79999999999995,
"price": 312.9,
"startedAt": "2022-08-10T16:23:42.000+00:00"
}
]
Kudos goes to #user20042973

Related

MongoDB - get datewise/houlty aggregate count of column

I have set of documents in my mongoDB collection. I am looking to get datewise aggregate count of document if date range is more than a day and hourly aggregate count for same column if date query is for single day. The data may have documents with same conversationId, hence it is necessary to group with conversationId as well.Below is sample of data for reference
[
{
"_id":"c438a671-2391-4b85-815c-ecfcb3d2bb54",
"status":"INTERNAL_UPDATE",
"conversationId":"ac44781d-caab-4410-a708-9d6db8480fc3",
"messageIds":[],
"messageId":"4dc02026-ac06-4eb1-aa59-e385fcce4a36",
"responseId":"0c00c83d-61c5-4937-846c-2e6a46aae857",
"conversation":{},
"message":{},
"params":{},
"timestamp":"2021-05-04T11:40:06.552Z",
"source":{}
},
{
"_id":"98370ddf-9ff8-4347-bab7-1f7777ab9e9d",
"status":"NEW",
"conversationId":"b5dc39d2-56a1-4eb6-a728-cdbe33dca580",
"messageIds":[],
"messageId":"ba94b839-f795-44f2-aea0-173d26006f14",
"responseId":"a2b75364-447b-4345-8008-2beccd6cbb34",
"conversation":{},
"message":{},
"params":{},
"timestamp":"2021-05-05T11:40:30.897Z",
"source":{}
},
{
"_id":"db1eae2b-62d9-455c-ab46-dbfc5baf8b67",
"status":"INTERNAL_UPDATE",
"conversationId":"b5dc39d2-56a1-4eb6-a728-cdbe33dcb584",
"messageIds":[],
"messageId":"b83c743b-d36e-4fdd-9c03-21988af47263",
"responseId":"97198c09-0130-48dc-a225-6d0faeff3116",
"conversation":{},
"message":{},
"params":{},
"timestamp":"2021-05-05T11:40:31.418Z",
"source":{}
},
{
"_id":"12a21495-f857-4f18-a06e-f8ba0b951ade",
"status":"NEW",
"conversationId":"8e37c704-add8-4f9f-8e70-d630c24f653b",
"messageIds":[],
"messageId":"51a48362-545c-4f9f-930b-42e4841fc974",
"responseId":"4691468b-a43b-41d1-83df-1349fb554bfa",
"conversation":{},
"message":{},
"params":{},
"timestamp":"2021-05-06T11:43:58.174Z",
"source":{}
},
{
"_id":"4afaa735-4618-40cf-8b4f-00ee83b2c3c5",
"status":"INTERNAL_UPDATE",
"conversationId":"8e37c704-add8-4f9f-8e70-d630c24f653b",
"messageIds":[],
"messageId":"7c860126-bf1e-41b2-a7d3-6bcec3e8d5fb",
"responseId":"09cec9a1-2621-481d-b527-d98b007ef5be",
"conversation":{},
"message":{},
"params":{},
"timestamp":"2021-05-06T11:43:58.736Z",
"source":{}
},
{
"_id":"cf8deeca-2cfd-497e-b92b-03204c84217a",
"status":"NEW",
"conversationId":"3c6870b5-88d6-4e21-8629-28137dea3fee",
"messageIds":[],
"messageId":"da84e414-2269-4812-8ddd-e2cd6c9be4fd",
"responseId":"ae1014b2-0cc1-41f0-9990-cf724ed67ab7",
"conversation":{},
"message":{},
"params":{},
"timestamp":"2021-05-06T13:37:55.060Z",
"source":{}
}
]
Presently I am able to group by conversationId, but unable to get data aggregated datewise or on hourly basis if date range is on single date.
Below is the query for same
db.documentName.aggregate([
{
'$match': {
'$and': [
{
timestamp: {
'$gte': ISODate('2021-05-01T00:00:00.000Z'),
'$lte': ISODate('2021-05-10T23:59:59.999Z')
}
},
{ 'source.author': { '$regex': 'user', '$options': 'i' } },
{},
{}
]
}
},
{ '$group': {
_id: {'conversationId': '$conversationId'} },
{ '$count': 'document_count' }
])
I have tried adding something like, $hour: '$timestamp' with comma separation beside conversationId in $group, but its of no use and is giving error.
The desired result I am trying to get for above data is, something like this
[{"date": "2021-05-04", "doc_count": 1},
{"date": "2021-05-05", "doc_count": 2},
{"date": "2021-05-06", "doc_count": 2}]
As for 2021-05-05 there are 2 docs with different conversationId, and for 2021-05-06 there are 3 docs in total but 2 documents have same conversationId hence aggregate count for 2021-05-06 is also 2. Hope this clarifies my quesiton.
The question is not entirely clear to me, but it sounds like you want something like this:
The groupId is a field to rebuild the date including the hour, or not, according to your condition:
EDIT:
db.collection.aggregate([
{$match: {
timestamp: {
$gte: ISODate("2021-05-01T00:00:00.000Z"),
$lte: ISODate("2021-05-07T23:59:59.999Z")
}
}
},
{$project: {
conversationId: 1,
groupId: {
$dateFromParts: {
year: {$year: "$timestamp"},
month: {$month: "$timestamp"},
day: {$dayOfMonth: "$timestamp"},
hour: {$cond: [
{$gte: [
{$dateDiff: {
startDate: ISODate("2021-05-01T00:00:00.000Z"),
endDate: ISODate("2021-05-07T23:59:59.999Z"),
unit: "day"}}, 1]},
0,
{$hour: "$timestamp"}]}
}
}
}
},
{$group: {_id: {conversationId: "$conversationId", groupId: "$groupId"}}},
{$group: {_id: "$_id.groupId", doc_count: {$sum: 1}}},
{$project: {date: {$toString: "$_id"}, doc_count: 1, _id: 0}}
])
See how it works on the playground example
As suggested by #nimrodserok, for mongo version 4.2.9 the query would be
db.collection.aggregate([
{
$match: {
timestamp: {
$gte: ISODate("2021-05-01T00:00:00.000Z"),
$lte: ISODate("2021-05-07T23:59:59.999Z")
}
}
},
{
$project: {
conversationId: 1,
groupId: {
$dateFromParts: {
year: {
$year: "$timestamp"
},
month: {
$month: "$timestamp"
},
day: {
$dayOfMonth: "$timestamp"
},
hour: {
$cond: [
{
$gte: [
{
$subtract: [
{
$toLong: ISODate("2021-05-07T23:59:59.999Z")
},
{
$toLong: ISODate("2021-05-01T00:00:00.000Z")
}
]
},
86400000
]
},
0,
{
$hour: "$timestamp"
}
]
}
}
}
}
},
{
$group: {
_id: {
conversationId: "$conversationId",
groupId: "$groupId"
}
}
},
{
$group: {
_id: "$_id.groupId",
doc_count: {
$sum: 1
}
}
},
{
$project: {
date: {
$toString: "$_id"
},
doc_count: 1,
_id: 0
}
}
])

MongoDB compare endTime with startTime of next document

I have a similar collection where I have sort them by their startTime:
{"name": 'A', "startTime": '1634626355', "endTime": '1634631405'}
{"name": 'A', "startTime": '1634631406', "endTime": '1634631864'}
{"name": 'A', "startTime": '1634631865', "endTime": '1634656048'}
{"name": 'A', "startTime": '1634712642', "endTime": '1634718856'}
How can I compare the documents such that if the document endTime and the next document startTime duration is less than 5 minutes, merge it.
This is the result I'm trying to achieve (The 1st 3 documents are merged into 1 where it uses the startTime of the 1st document and the endTime of the 3rd document):
{"name": 'A', "startTime": '1634626355', "endTime": '1634656048'}
{"name": 'A', "startTime": '1634712642', "endTime": '1634718856'}
Thanks
First of all, you should never store date/time values as string, it's a design flaw. Store always proper Date object.
This solution works without self-lookup, so it may perform better:
db.collection.aggregate([
{
$set: {
startDateTime: { $toDate: { $multiply: ["$startTime", 1000] } },
endDateTime: { $toDate: { $multiply: ["$endTime", 1000] } }
},
},
{ $sort: { startDateTime: 1 } },
{ $group: { _id: null, data: { $push: "$$ROOT" } } },
{
$set: {
data: {
$reduce: {
input: "$data",
initialValue: [],
in: {
$cond: {
if: {
$or: [
{ $eq: [{ $size: "$$value" }, 0] }, // for the initail element
{
$gt: [
{
$dateDiff: { // calculate difference
endDate: "$$this.startDateTime",
startDate: { $last: "$$value.endDateTime" },
unit: "minute"
}
},
5 // more than 5 Minutes
]
}
]
},
then: { $concatArrays: ["$$value", ["$$this"]] }, // append new element
else: {
$map: {
input: "$$value",
as: "data",
in: {
$cond: {
if: { $eq: ["$$data._id", { $last: "$$value._id" }] }, // find last element
then: { // update last element
$mergeObjects: [
"$$data",
{ endDateTime: "$$this.endDateTime" },
{ endTime: "$$this.endTime" }
]
},
else: "$$data"
}
}
}
}
}
}
}
}
}
},
// some cosmetic
{ $unwind: "$data" },
{ $replaceRoot: { newRoot: "$data" } }
])
Mongo Playground
You can use $lookup in an aggregation pipeline to find out the documents that you need to remove. Then, perform a forEach to remove them.
db.collection.aggregate([
{
$addFields: {
endDateTime: {
"$toDate": {
"$multiply": [
{
$toLong: "$endTime"
},
1000
]
}
}
},
},
{
"$lookup": {
"from": "collection",
let: {
end: "$endDateTime"
},
pipeline: [
{
"$addFields": {
startDateTime: {
"$toDate": {
"$multiply": [
{
$toLong: "$startTime"
},
1000
]
}
}
}
},
{
$match: {
$expr: {
$and: [
{
$lte: [
{
$subtract: [
"$startDateTime",
"$$end"
]
},
300000
]
},
{
$lte: [
"$$end",
"$startDateTime"
]
}
]
}
}
}
],
"as": "lessThan5min"
}
},
{
"$unwind": "$lessThan5min"
},
{
"$replaceRoot": {
"newRoot": "$lessThan5min"
}
}
]).forEach(function(doc){
db.collection.remove({ "_id": doc._id });
});
Here is the Mongo playground to find out the documents that you need to remove for your reference.

Convert the string to integer in mongo aggregate query

My company has inserted numerical values for certain keys in string format. They can't be converted to integer format for some business reason.
Now coming to the query...
I am writing a mongo aggregate query which calculates annual cost for a particular manufacturer like Unilever across shops. It seems I cannot convert a string to integer inside the $cond and $eq blocks using $toInt method.
Please find below the sample collection.
[
{
_id: "ddfdfdfdggfgfgsg",
rate: "3323",
quantity_packs: "343",
shop_name: "Whole Foods",
manufacturer_name: "Unilever"
},
{
_id: "ddfdfdfsdsds",
rate: "434",
quantity_packs: "453",
shop_name: "Carrefour",
manufacturer_name: "Unilever"
},
{
_id: "dfdfdgcvgfgfvvv",
rate: "343",
quantity_packs: "23",
shop_name: "Target",
manufacturer_name: "Beirsdorf"
}
]
The query is
db.collection.aggregate([
{
$match: {
manufacturer_name: {
$in: [ "Unilever" ]
}
}
},
{
$group: {
_id: {
"Shop Name": "$shop_name"
},
"annual_cost": {
$sum: {
$cond: [
{
$eq: ["manufacturer_name", "Unilever"]
},
{ "$toInt": "$rate"},
0
]
}
},
"other_annual_cost": {
$sum: {
$cond: [
{
$ne: [$manufacturer_name, "Unilever"]
}, {"$toInt" : "$rate"},
0
]
}
},
"annual_qty": {
$sum: {
"$toInt": "$quantity_packs"
}
},
}
},
{
$project: {
"Purchase_Cost": {
$multiply: [ "$annual_cost", "$annual_qty" ]
},
"Other Manu Pur Cost": {
$multiply: ["$other_annual_cost", "$annual_qty"]
}
}
}
])
Current Output
[
{
_id: { 'Shop Name': 'Whole Foods' },
Purchase_Cost: 0
}
]
As $rate is of string type, the multiplication has yielded 0 as shown over here. Ideally the result should show some integer value for purchase cost as shown below.
Intended Output
[
{
_id: { 'Shop Name': 'Whole Foods' },
Purchase_Cost: 234
}
]
Any suggestion would be of great help. I want to make this query work somehow.
I have updated the question based on Rajdeep's Answer.
I just corrected this, please take a look
Playground
"annual_cost": {
$sum: {
$cond: [
{
$eq: [
"$manufacturer_name", //added $
"Unilever"
]
},
{
$toInt: "$rate" //added $toInt
},
0
]

How can I select all data based on date range mongoDB?

I have a lists of records like below
[
{
"product": "p1",
"salesdate": "2020-02-01",
"amount": 100
},
{
"product": "p2",
"salesdate": "2020-02-04",
"amount": 200
},
]
On 2nd feb and 3rd feb i don't have data. But I need to add this in my result. My expected result is
[
{
"amount": 100,
"salesdate": "2020-02-01"
},
{
"amount": 0,
"salesdate": "2020-02-02"
},
{
"amount": 0,
"salesdate": "2020-02-03"
}
{
"amount": 200,
"salesdate": "2020-02-04"
}
]
Can I achieve this using mongoDB?
https://mongoplayground.net/p/EiAJdY9jRHn
You can use $reduce for it. Whenever one has to work with data/time values, then I recommend the moment.js library. You don't have to use it, but it makes your life easier.
db.collection.aggregate([
// Put all data into one document
{ $group: { _id: null, data: { $push: "$$ROOT" } } },
// Add missing days
{
$addFields: {
data: {
$reduce: {
// Define the range of date
input: { $range: [0, moment().get('day')] },
initialValue: [],
in: {
$let: {
vars: {
ts: {
$add: [moment().startOf('month').toDate(), { $multiply: ["$$this", 1000 * 60 * 60 * 24] }]
}
},
in: {
$concatArrays: [
"$$value",
[{
$ifNull: [
{ $first: { $filter: { input: "$data", cond: { $eq: ["$$this.salesdate", "$$ts"] } } } },
// Default value for missing days
{ salesdate: "$$ts", amount: 0 }
]
}]
]
}
}
}
}
}
}
},
{ $unwind: "$data" },
{ $replaceRoot: { newRoot: "$data" } }
// If requried add further $group stages
])
Note, this code returns values from first day of current months to current day (not 2020 as in your sample data). You may adapt the ranges - your requirements are not clear from the question.
Mongo Playground

how to convert price from string to int in mongodb and pass it through $project

here I am calculating count and price according to group by I am getting the count but price total is coming as 0. how to send price total .
db.connect(function(err) {
if (err) {
res.sendStatus(510);
} else {
var collection = db.get().collection('Order');
collection.aggregate(
[
{
"$match": {
"$and": [{
"createdDate": {
"$gte": dateFrom
}
},
{
"createdDate": {
"$lte": dateTo
}
}
]
}
},
{
"$project": {
createdDate: {
$dateToString: { format: "%Y-%m-%d", date: "$createdDate" },
},
paymentType: "$paymentType",
price:"$price"
}
},
{
"$group": {
"_id": {
"createdDate": "$createdDate",
"paymentType": "$paymentType"
},
"COUNT(_id)": {"$sum": 1},
"TOTALPRICE": { "$sum" : "$price" }
}
},
{
"$project": {
"COUNT": "$COUNT(_id)",
"TOTALPRICE" : "$TOTALPRICE",
"createdDate": "$_id.createdDate",
"paymentType": "$_id.paymentType",
"_id": 0
}
},
{
"$sort": {
"createdDate": 1
}
}
], {
"allowDiskUse": true
}
)
query response is:
COUNT: 1
TOTALPRICE: 0
createdDate: "2019-09-24"
paymentType: "cod"
TOTALPRICE is coming 0 which is 70.
You need to add price to the $projection step before the $group:
{
"$project": {
createdDate: {
$dateToString: { format: "%Y-%m-%d", date: "$createdDate" },
},
paymentType: "$paymentType",
price: "$price"
}
}
Otherwise you're summing on a field that no longer exists due to the projection phase dropping it, hence the 0.