MongoDB: get sum by year/month with nested values - mongodb

I'm trying to sum (spending by month/year) of a collection with nested amounts - with no luck.
This is the collection (extract):
[
{
"_id" : ObjectId("5faaf88d0657287993e541a5"),
"segment" : {
"l1" : "Segment A",
"l2" : "001"
},
"invoiceNo" : "2020.10283940",
"invoicePos" : 3,
"date" : ISODate("2019-09-06T00:00:00.000Z"),
"amount" : {
"document" : {
"amount" : NumberDecimal("125.000000000000"),
"currCode" : "USD"
},
"local" : {
"amount" : NumberDecimal("123.800000000000"),
"currCode" : "CHF"
},
"global" : {
"amount" : NumberDecimal("123.800000000000"),
"currCode" : "CHF"
}
}
},
...
]
I would like to sum up the aggregated invoice volume per month in "global" currency.
I tried this query on MongoDB:
db.invoices.aggregate(
{$project : {
month : {$month : "$date"},
year : {$year : "$date"},
amount : 1
}},
{$unwind: '$amount'},
{$group : {
_id : {month : "$month" ,year : "$year" },
total : {$sum : "$amount.global.amount"}
}})
I am getting as result this:
/* 1 */
{
"_id" : ObjectId("5faaf88d0657287993e541a5"),
"amount" : {
"document" : {
"amount" : NumberDecimal("125.000000000000"),
"currCode" : "USD"
},
"local" : {
"amount" : NumberDecimal("123.800000000000"),
"currCode" : "CHF"
},
"global" : {
"amount" : NumberDecimal("123.800000000000"),
"currCode" : "CHF"
}
},
"month" : 9,
"year" : 2019
}
/* 2 */
{
"_id" : ObjectId("5faaf88d0657287993e541ac"),
"amount" : {
"document" : {
"amount" : NumberDecimal("105.560000000000"),
"currCode" : "CHF"
},
"local" : {
"amount" : NumberDecimal("105.560000000000"),
"currCode" : "CHF"
},
"global" : {
"amount" : NumberDecimal("105.560000000000"),
"currCode" : "CHF"
}
},
"month" : 11,
"year" : 2020
}
This however does not sum up all invoices per month, but looks like single invoice lines - no aggregation.
I would like to get a result like this:
[
{
"month": 11,
"year": 2020,
"amount" : NumberDecimal("99999.99")
},
{
"month": 10,
"year": 2020,
"amount" : NumberDecimal("99999.99")
},
{
"month": 9,
"year": 2020,
"amount" : NumberDecimal("99999.99")
}
]
What is wrong with my query?

Would this be helpful?
db.invoices.aggregate([
{
$group: {
_id: {
month: {
$month: "$date"
},
year: {
$year: "$date"
}
},
total: {
$sum: "$amount.global.amount"
}
}
},
{$sort:{"_id.year":-1, "_id.month":-1}}
])
Playground
If you need any extra explanation let me know, but the code is pretty short and self-explanatory.

In principle your aggregation pipeline is fine, there a few mistakes:
An aggregation pipeline expects an array
$unwind is useless, because $amount is not an array. One element in -> one document out
You can use date function directly
So, short and simple:
db.invoices.aggregate([
{
$group: {
_id: { month: { $month: "$date" }, year: { $year: "$date" } },
total: { $sum: "$amount.global.amount" }
}
}
])

Related

Creating measures in a mongodb aggregation pipeline

I have a report that has been developed in PowerBI. It runs over a collection of jobs, and for a given month and year counts the number of jobs that were created, due or completed in that month using measures.
I am attempting to reproduce this report using a mongoDB aggregation pipeline. At first, I thought I could just use the $group stage to do this, but quickly realised that grouping by a specific date would exclude jobs.
Some sample documents are below (most fields excluded as they are not relevant):
{
"_id": <UUID>,
"createdOn": ISODate("2022-07-01T00:00"),
"dueOn": ISODate("2022-08-01T00:00"),
"completedOn": ISODate("2022-07-29T00:00")
},
{
"_id": <UUID>,
"createdOn": ISODate("2022-06-01T00:00"),
"dueOn": ISODate("2022-08-01T00:00"),
"completedOn": ISODate("2022-07-24T00:00")
}
For example, if I group by created date, the record for July 2022 would show 1 created job and only 1 completed job, but it should show 2.
How can I go about recreating this report? One idea was that I needed to determine the minimum and maximum of all the possible dates across those 3 date fields in my collection, but I don't know where to go from there
I ended up solving this by using a facet. I followed this process:
Each facet field grouped by a different date field from the source document, and then aggregated the relevant field (e.g. counts, or sums as required). I ensured each of these fields in the facet had a unique name.
I then did a project stage where I took each of the facet stage fields (arrays), and concat them into a single array
I unwound the array, and then replaced the root to make it simpler to work with
I then grouped again by the _id field which was set to the relevant date during the facet field, and then grabbed the relevant fields.
The relevant parts of the pipeline are below:
db.getCollection("jobs").aggregate(
// Pipeline
[
// Stage 3
{
$facet: {
//Facet 1, group by created date, count number of jobs created
//facet 2, group by completed date, count number of jobs completed
//facet 3, group by due date, count number of jobs due
"created" : [
{
$addFields : {
"monthStarting" : {
"$dateFromString" : {
"dateString" : {
"$dateToString" : {
"date" : {
"$dateTrunc" : {
"date" : "$createdAt",
"unit" : "month",
"binSize" : 1.0,
"timezone" : "$timezone",
"startOfWeek" : "mon"
}
},
"timezone" : "$timezone"
}
}
}
},
"yearStarting" : {
"$dateFromString" : {
"dateString" : {
"$dateToString" : {
"date" : {
"$dateTrunc" : {
"date" : "$createdAt",
"unit" : "year",
"binSize" : 1.0,
"timezone" : "$timezone"
}
},
"timezone" : "$timezone"
}
}
}
}
}
},
{
$group : {
"_id" : {
"year" : "$yearStarting",
"month" : "$monthStarting"
},
"monthStarting" : {
"$first" : "$monthStarting"
},
"yearStarting" : {
"$first" : "$yearStarting"
},
"createdCount": {$sum: 1}
}
}
],
"completed" : [
{
$addFields : {
"monthStarting" : {
"$dateFromString" : {
"dateString" : {
"$dateToString" : {
"date" : {
"$dateTrunc" : {
"date" : "$completedDate",
"unit" : "month",
"binSize" : 1.0,
"timezone" : "$timezone",
"startOfWeek" : "mon"
}
},
"timezone" : "$timezone"
}
}
}
},
"yearStarting" : {
"$dateFromString" : {
"dateString" : {
"$dateToString" : {
"date" : {
"$dateTrunc" : {
"date" : "$completedDate",
"unit" : "year",
"binSize" : 1.0,
"timezone" : "$timezone"
}
},
"timezone" : "$timezone"
}
}
}
}
}
},
{
$group : {
"_id" : {
"year" : "$yearStarting",
"month" : "$monthStarting"
},
"monthStarting" : {
"$first" : "$monthStarting"
},
"yearStarting" : {
"$first" : "$yearStarting"
},
"completedCount": {$sum: 1}
}
}
],
"due": [
{
$match: {
"dueDate": {$ne: null}
}
},
{
$addFields : {
"monthStarting" : {
"$dateFromString" : {
"dateString" : {
"$dateToString" : {
"date" : {
"$dateTrunc" : {
"date" : "$dueDate",
"unit" : "month",
"binSize" : 1.0,
"timezone" : "$timezone",
"startOfWeek" : "mon"
}
},
"timezone" : "$timezone"
}
}
}
},
"yearStarting" : {
"$dateFromString" : {
"dateString" : {
"$dateToString" : {
"date" : {
"$dateTrunc" : {
"date" : "$dueDate",
"unit" : "year",
"binSize" : 1.0,
"timezone" : "$timezone"
}
},
"timezone" : "$timezone"
}
}
}
}
}
},
{
$group : {
"_id" : {
"year" : "$yearStarting",
"month" : "$monthStarting"
},
"monthStarting" : {
"$first" : "$monthStarting"
},
"yearStarting" : {
"$first" : "$yearStarting"
},
"dueCount": {$sum: 1},
"salesRevenue": {$sum: "$totalSellPrice"},
"costGenerated": {$sum: "$totalBuyPrice"},
"profit": {$sum: "$profit"},
"avgValue": {$avg: "$totalSellPrice"},
"finalisedRevenue": {$sum: {
$cond: {
"if": {$in: ["$status",["Finalised","Closed"]]},
"then": "$totalSellPrice",
"else": 0
}
}}
}
}
]
}
},
// Stage 4
{
$project: {
"docs": {$concatArrays: ["$created","$completed","$due"]}
}
},
// Stage 5
{
$unwind: {
path: "$docs",
}
},
// Stage 6
{
$replaceRoot: {
// specifications
"newRoot": "$docs"
}
},
// Stage 7
{
$group: {
_id: "$_id",
"monthStarting" : {
"$first" : "$monthStarting"
},
"yearStarting" : {
"$first" : "$yearStarting"
},
"monthStarting" : {
"$first" : "$monthStarting"
},
"createdCountSum" : {
"$sum" : "$createdCount"
},
"completedCountSum" : {
"$sum" : "$completedCount"
},
"dueCountSum" : {
"$sum" : "$dueCount"
},
"salesRevenue" : {
"$sum" : "$salesRevenue"
},
"costGenerated" : {
"$sum" : "$costGenerated"
},
"profit" : {
"$sum" : "$profit"
},
"finalisedRevenue" : {
"$sum" : "$finalisedRevenue"
},
"avgJobValue": {
$sum: "$avgValue"
}
}
},
],
);

Counting distinct number of users from beginning

I have a MongoDB aggregation pipeline that has been frustrating me for a while now, because it never seems to be accurate or correct to my needs. The aim is to count the number of new unique users each day per chatbot, starting from the very beginning.
Here's what my pipeline looks like right now.
[
{
"$project" : {
"_id" : 0,
"bot_id" : 1,
"customer_id" : 1,
"timestamp" : {
"$ifNull" : [
'$incoming_log.created_at', '$outcome_log.created_at'
]
}
}
},
{
"$project" : {
"customer_id" : 1,
"bot_id" : 1,
"timestamp" : {
"$dateFromString" : {
"dateString" : {
"$substr" : [
"$timestamp", 0, 10
]
}
}
}
}
},
{
"$group" : {
"_id" : "$customer_id",
"timestamp" : {
"$first" : "$timestamp"
},
"bot_id" : {
"$addToSet" : "$bot_id"
}
}
},
{
"$unwind" : "$bot_id"
},
{
"$group" : {
"_id" : {
"bot_id" : "$bot_id",
"customer_id" : "$_id"
},
"timestamp" : {
"$first" : "$timestamp"
}
}
},
{
"$project" : {
"_id" : 0,
"timestamp" : 1,
"customer_id" : "$_id.customer_id",
"bot_id" : "$_id.bot_id"
}
},
{
"$group" : {
"_id": {
"timestamp" : "$timestamp",
"bot_id" : "$bot_id"
},
"new_users" : {
"$sum" : 1
}
}
},
{
"$project" : {
"_id" : 0,
"timestamp" : "$_id.timestamp",
"bot_id" : "$_id.bot_id",
"new_users" : 1
}
}
]
Some sample data for an idea of what the data looks like...
{
"mid" : "...",
"bot_id" : "...",
"bot_name" : "JOBBY",
"customer_id" : "U122...",
"incoming_log" : {
"created_at" : ISODate("2020-12-08T09:14:16.237Z"),
"event_payload" : "",
"event_type" : "text"
},
"outcome_log" : {
"created_at" : ISODate("2020-12-08T09:14:18.145Z"),
"distance" : 0.25,
"incoming_msg" : "🥺"
}
}
My expected outcome is something along the lines of:
{
"new_users" : 1187.0,
"timestamp" : ISODate("2021-01-27T00:00:00.000Z"),
"bot_id" : "5ffd......."
},
{
"new_users" : 1359.0,
"timestamp" : ISODate("2021-01-27T00:00:00.000Z"),
"bot_id" : "6def......."
}
Have I overcomplicated my pipeline somewhere? I seem to get a reasonable number of new users per bot each day, but for some reason my colleague tells me that the number is too high. I need some tips, please!
I have really no idea what you are looking for.
"The aim is to count the number of new unique users each day per chatbot, starting from the very beginning."
What is "new unique users"? What do you mean by "starting from the very beginning"? You ask for count per day but you use {"$group": {"_id": "$customer_id", "timestamp": { "$first": "$timestamp" } } }
For me your grouping does not make any sense. With only one single sample document, it is almost impossible to guess what you like to count.
Regarding group per day: I prefer to work always with Date values, rather than strings. It is less error prone. Maybe you have to consider time zones, because UTC midnight is not your local midnight. When you work with Dates then you have better control over it.
The $project stages are useless when you do $group afterwards. Typically you have only one $project stage at the end.
So, put something to start.
db.collection.aggregate([
{
$set: {
day: {
$dateToParts: {
date: { $ifNull: ["$incoming_log.created_at", "$outcome_log.created_at"] }
}
}
}
},
{
$group: {
_id: "$customer_id",
timestamp: {$min: { $dateFromParts: { year: "$day.year", month: "$day.month", day: "$day.day" } }}
}
}
]);

I want to return sum of values by month from JAN-DEC in a year in mongodb

I want to write a Mongoose query to get the sum total for every month (Jan-Dec) in a year.
Currently the query I wrote only returns a value of the sum total for a month and skips all other months. I want a situation where all other months with no sum total should just return 0
Here is my query
db.getCollection('sales_records').aggregate([
{ $match: { createdDate: { $gt: ISODate('2020-01-01 00:00:00.000Z'), $lt: ISODate('2020-12-25 00:00:00.000Z')} } },
{ $match: { merchantId: 5547 } },
{
$group : {
_id: {
year: { $year : "$createdDate" },
month: { $month : "$createdDate" },
//week: { $week : "$createdDate" },
// day: { $day : { $week : "$createdDate" } },
//hour: { $hour : "$createdDate" },
},
merchantId: { $first: "$merchantId" },
date: { $first: "$createdDate" },
total: { $sum : "$totalAmount"}
}
},
{
$sort: { date: -1 }
}
])
{
"_id" : {
"year" : 2020,
"month" : 4
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-04-01T00:00:00.000Z"),
"total" : 2100.0
}
/* 2 */
{
"_id" : {
"year" : 2020,
"month" : 3
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-03-02T09:34:23.870Z"),
"total" : 396511341.0
}
But I wanted to return something like this
{
"_id" : {
"year" : 2020,
"month" : 6
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-04-01T00:00:00.000Z"),
"total" : 0
}
{
"_id" : {
"year" : 2020,
"month" : 5
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-04-01T00:00:00.000Z"),
"total" : 0
}
{
"_id" : {
"year" : 2020,
"month" : 4
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-04-01T00:00:00.000Z"),
"total" : 2100.0
}
{
"_id" : {
"year" : 2020,
"month" : 3
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-03-02T09:34:23.870Z"),
"total" : 396511341.0
}
{
"_id" : {
"year" : 2020,
"month" : 2
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-03-02T09:34:23.870Z"),
"total" : 0
}
{
"_id" : {
"year" : 2020,
"month" : 1
},
"merchantId" : NumberLong(5547),
"date" : ISODate("2020-03-02T09:34:23.870Z"),
"total" : 0
}

Mongodb aggregation json format by month result

Hello I am working with the reporting api which will going to use in highcharts and I am new to mongodb.
Below is my aggregation query (suggest me modification) :
db.product_sold.aggregate({
$group: {
_id: { year: { $year: "$solddate" }, month: { $month: "$solddate" }, productid: "$productid" },
totalQty: { $sum: "$qty" }
}
})
Output:
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "11"
},
"totalQty" : 6.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "14"
},
"totalQty" : 7.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(1),
"productid" : "13"
},
"totalQty" : 3.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "10"
},
"totalQty" : 6.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2018),
"month" : NumberInt(2),
"productid" : "12"
},
"totalQty" : 5.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(2),
"productid" : "15"
},
"totalQty" : 8.0
}
// ----------------------------------------------
{
"_id" : {
"year" : NumberInt(2019),
"month" : NumberInt(1),
"productid" : "11"
},
"totalQty" : 2.0
}
// ----------------------------------------------
What I want in output is something like :
status: 200,
msg: "SUCCESS"
data: [{
1:[
{
"productid": 11,
"totalQty": 3
},
{
"productid": 12,
"totalQty": 27
}
],
2:[
{
"productid": 11,
"totalQty": 64
},
{
"productid": 12,
"totalQty": 10
}
]
}]
For reference attaching the image of the collection
Is there any way to achieve it using aggregation or anything else or I will have to do it manually by code ?
You can append below aggreagation stages to your current pipeline:
db.product_sold.aggregate([
// your current $group stage
{
$group: {
_id: "$_id.month",
docs: { $push: { productid: "$_id.productid", totalQty: "$totalQty" } }
}
},
{
$project: {
_id: 0,
k: { $toString: "$_id" },
v: "$docs"
}
},
{
$group: {
_id: null,
data: { $push: "$$ROOT" }
}
},
{
$project: {
_id: 0,
data: { $arrayToObject: "$data" }
}
}
])
The idea here is that you can use $group with _id set to null to get all the data into single document and then use $arrayToObject to get month number as key and all the aggregates for that month as value.

How to preserve a UNIX datestamp with mongodb aggregation

I would like to aggregate data in meteor/MongoDB. I have a few thousand entries formatted like the following
{_id: sadsadjhsjdys7ad67as8d, t: 1464162907, prod: 123, sys: xyz}
I would like to sort them into their relative dates and aggregate the prod field.
I currently have the following
var project = {
"$project" : {
"_id" : 0,
"y" : {
"$year" : {
"$add" : [
new Date(0), {
"$multiply" : [1000, "$t"]
}
]
}
},
"m" : {
"$month" : {
"$add" : [
new Date(0), {
"$multiply" : [1000, "$t"]
}
]
}
},
"d" : {
"$dayOfMonth" : {
"$add" : [
new Date(0), {
"$multiply" : [1000, "$t"]
}
]
}
},
"prod" : "$prod",
"sys" : "$sys"
}
};
var group = {
"$group" : {
"_id" : {
"year" : "$y",
"month" : "$m",
"day" : "$d",
"test" : "$test"
},
sys : "sys",
prod : {
$sum : "$prod"
}
}
};
var result = power_stats.aggregate([match, project, group]);
However this makes my _id into [y: 2016, m: 5, d: 25] which is the form of aggregation that I want, however it will be completely useless if I want to do a conditional find with the date later on with this data.
How can I preserve a UNIX datestamp with MongoDBe aggregation?
to achieve that you can do:
Add t field in $project phase and keep populating it thru pipeline - so there will be always reference to oryginal timestamp
Create dateTime field 'time: new Date("$_id.year", "$_id.month", "_id.day");'
var group = {
"$group" : {
"_id" : {
"year" : "$y",
"month" : "$m",
"day" : "$d",
"test" : "$test"
},
prod : {
$sum : "$prod"
},
dateTime : {
$addToSet : new Date("$_id.year", "$_id.month", "$_id.day")
}
}
};