Related
I have two collections "datasets" and "users".
I tried to lookup datasets.assignedTo = users.id that's working fine. Also, I want to match the field of datasets.firstBillable >= users.prices.beginDate date field are matched to get the current index price value. And also check users.prices.endDate is less than or equal to users.prices.beginDate.
For example:
cgPrices: 45
https://mongoplayground.net/p/YQps9EozlAL
Collections:
db={
users: [
{
id: 1,
name: "Aravinth",
prices: [
{
beginDate: "2022-08-24T07:29:01.639Z",
endDate: "2022-08-31T07:29:01.639Z",
price: 45
}
]
},
{
id: 2,
name: "Raja",
prices: [
{
beginDate: "2022-07-25T07:29:01.639Z",
endDate: "2022-07-30T07:29:01.639Z",
price: 55
}
]
}
],
datasets: [
{
color: "braun, rose gold",
firstBillable: "2022-08-24T07:29:01.639Z",
assignedTo: 1
},
{
color: "beige, silber",
firstBillable: "2022-07-25T07:29:01.639Z",
assignedTo: 2
}
]
}
My current implementation:
db.datasets.aggregate([
{
"$lookup": {
"from": "users",
"as": "details",
let: {
assigned_to: "$assignedTo",
first_billable: "$firstBillable"
},
pipeline: [
{
"$match": {
$expr: {
"$and": [
{
"$eq": [
"$id",
"$$assigned_to"
]
},
{
"$gte": [
"$first_billable",
"$details.prices.beginDate"
]
},
{
"$lte": [
"$first_billable",
"$details.prices.endDate"
]
}
]
}
}
}
]
}
},
{
"$addFields": {
"details": 0,
"cg": {
$first: {
"$first": "$details.prices.price"
}
}
}
}
])
Output i needed:
[
{
"_id": ObjectId("5a934e000102030405000000"),
"assignedTo": 1,
"cg": 45,
"color": "braun, rose gold",
"details": 0,
"firstBillable": "2022-08-24T07:29:01.639Z"
},
{
"_id": ObjectId("5a934e000102030405000001"),
"assignedTo": 2,
"cg": 55,
"color": "beige, silber",
"details": 0,
"firstBillable": "2022-07-25T07:29:01.639Z"
}
]
https://mongoplayground.net/p/YQps9EozlAL
Concerns:
You should compare the date as Date instead of string, hence you are required to convert the date strings to Date before comparing.
In users collection, prices is an array. You need to deconstruct the array to multiple documents first before compare the date fields in price.
The query should be:
db.datasets.aggregate([
{
"$lookup": {
"from": "users",
"as": "details",
let: {
assigned_to: "$assignedTo",
first_billable: {
$toDate: "$firstBillable"
}
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$id",
"$$assigned_to"
]
}
}
},
{
$unwind: "$prices"
},
{
"$match": {
$expr: {
"$and": [
{
"$gte": [
"$$first_billable",
{
$toDate: "$prices.beginDate"
}
]
},
{
"$lte": [
"$$first_billable",
{
$toDate: "$prices.endDate"
}
]
}
]
}
}
}
]
}
},
{
"$addFields": {
"details": 0,
"cg": {
$first: "$details.prices.price"
}
}
}
])
Demo # Mongo Playground
I'm trying to get a list of current holders at specific times from a collection. My collection looks like this:
[
{
"time": 1,
"holdings": [
{ "owner": "A", "tokens": 2 },
{ "owner": "B", "tokens": 1 }
]
},
{
"time": 2,
"holdings": [
{ "owner": "B", "tokens": 2 }
]
},
{
"time": 3,
"holdings": [
{ "owner": "A", "tokens": 3 },
{ "owner": "B", "tokens": 1 },
{ "owner": "C", "tokens": 1 }
]
},
{
"time": 4,
"holdings": [
{ "owner": "C", "tokens": 0 }
]
}
]
tokens show the current holdings of an owner if the holdings have changed to the last document. I would like to change the collection so that holdings always includes the full current holdings for any point in time.
At time: 1, the holdings are: A: 2, B: 1.
At time: 2, the holdings are: A: 2, B: 2. The collections does not include A's holdings however, because they haven't changed. So what I'd like to get is:
[
{
"time": 1,
"holdings": [
{ "owner": "A", "tokens": 2 },
{ "owner": "B", "tokens": 1 }
]
},
{
"time": 2,
"holdings": [
{ "owner": "A", "tokens": 2 }, // merged from prev doc.
{ "owner": "B", "tokens": 2 }
]
},
{
"time": 3,
"holdings": [
{ "owner": "A", "tokens": 3 },
{ "owner": "B", "tokens": 1 },
{ "owner": "C", "tokens": 1 }
]
},
{
"time": 4,
"holdings": [
{ "owner": "A", "tokens": 3 }, // merged from prev
{ "owner": "B", "tokens": 1 }, // merged from prev
{ "owner": "C", "tokens": 0 }
]
}
]
From what I understand $mergeObjects does that, but I don't understand how I can merge all previous docs in order up to the current doc for each doc. So I'm looking for a way to combine setWindowFields with mergeObjects I think.
This is a nice challenge.
So far, I got this complicated solution:
Get all of our timestamps in all of our documents. This is the purpose of the first 4 steps. $setWindowFields is used to accumulate this data.
$group by owner and calculate the empty timestamps as wantedTimes- next 5 steps.
$set empty timestamps with tokens: null to be filled with actual data and $unwind to separate - next 3 steps
Use $setWindowFields to find the last known token for each owner at each timestamp.
Fill this last known state for documents with unknown token - 2 steps
$group and format answer:
db.collection.aggregate([
{
$setWindowFields: {
sortBy: {time: 1},
output: {
allTimes: {$addToSet: "$time", window: {documents: ["unbounded", "current"]}
}
}
}
},
{
$setWindowFields: {
sortBy: {time: -1},
output: {
allTimes: {$addToSet: "$allTimes", window: {documents: ["unbounded", "current"]}
}
}
}
},
{
$set: {
allTimes: {
$reduce: {
input: "$allTimes",
initialValue: [],
in: {"$concatArrays": ["$$value", "$$this"]}
}
}
}
},
{$set: {allTimes: {$setIntersection: "$allTimes"}}},
{$unwind: "$holdings"},
{$sort: {time: 1}},
{$group: { _id: "$holdings.owner",
tokens: {$push: {tokens: "$holdings.tokens", time: "$time"}},
times: {$push: "$time"}, firstTime: {$first: "$time"},
allTimes: {$first: "$allTimes"}}
},
{
$addFields: {
wantedTimes: {
$filter: {
input: "$allTimes",
as: "item",
cond: {$gte: ["$$item", "$firstTime"]}
}
}
}
},
{
$project: {
tokens: 1,
wantedTimes: {$setDifference: ["$wantedTimes", "$times"]}
}
},
{
$set: {
data: {
$map: {
input: "$wantedTimes",
as: "item",
in: {time: "$$item", tokens: null}
}
}
}
},
{$project: {tokens: {"$concatArrays": ["$tokens", "$data"]}}},
{$unwind: "$tokens"},
{
$setWindowFields: {
partitionBy: "$_id",
sortBy: {"tokens.time": 1},
output: {
lastTokens: {
$push: "$tokens.tokens",
window: {documents: ["unbounded", "current"]}
}
}
}
},
{
$set: {
lastTokens: {
$filter: {
input: "$lastTokens",
as: "item",
cond: {$ne: ["$$item", null]}
}
}
}
},
{
$set: {
"tokens.tokens": {$ifNull: ["$tokens.tokens", {$last: "$lastTokens"}]}
}
},
{
$group: {
_id: "$tokens.time",
holdings: {$push: {owner: "$_id", tokens: "$tokens.tokens" }}
}
},
{$project: {time: "$_id", holdings: 1, _id: 0}},
{$sort: {time: 1}}
])
Playground example
From a performance perspective I recommend you split it into 2 calls, the first will be a quick findOne just to get the maximum time value in the collection.
Once you have that value the pipeline can be much leaner:
const maxItem = await db.collection.findOne({}).sort({ time: -1 });
db.collection.aggregate([
{
$unwind: "$holdings"
},
{
$group: {
_id: "$holdings.owner",
times: {
$push: {
time: "$time",
tokens: "$holdings.tokens"
}
},
minTime: {
$min: "$time"
}
}
},
{
$addFields: {
times: {
$reduce: {
input: {
$range: [
"$minTime",
maxItem.time + 1 // this is max time
]
},
initialValue: {
values: [],
lastIndex: 0
},
in: {
values: {
"$concatArrays": [
"$$value.values",
[
{
$cond: [
{
$in: [
"$$this",
"$times.time"
]
},
{
"$arrayElemAt": [
"$times",
"$$value.lastIndex"
]
},
{
"$mergeObjects": [
{
tokens: 0
},
{
"$arrayElemAt": [
"$times",
{
$subtract: [
"$$value.lastIndex",
1
]
}
]
},
{
time: "$$this"
}
]
}
]
}
]
]
},
lastIndex: {
$cond: [
{
$in: [
"$$this",
"$times.time"
]
},
{
$sum: [
"$$value.lastIndex",
1
]
},
"$$value.lastIndex"
]
}
}
}
}
}
},
{
$unwind: "$times.values"
},
{
$group: {
_id: "$times.values.time",
holdings: {
$push: {
owner: "$_id",
tokens: "$times.values.tokens"
}
}
}
},
{
$project: {
_id: 0,
time: "$_id",
holdings: 1
}
},
{
$sort: {
time: 1
}
}
])
This is still quite a heavy query as it requires to $unwind and $group the entire collection, however there is no workaround this due to the requirements. if the collection is too big for this approach I recommend iteration owner by owner, or time by time and doing separate updates accordingly.
Mongo Playground
If you don't care about performance at all and want it in a single query you can still use the same pipeline, you will have to first extract the max time in the collection, this will require you to add an initial $group stage, like so:
db.collection.aggregate([
{
$group: {
_id: null,
maxTime: {
$max: "$time"
},
roots: {
$push: "$$ROOT"
}
}
},
{
$unwind: "$roots"
},
{
$replaceRoot: {
newRoot: {
"$mergeObjects": [
"$roots",
{
maxTime: "$maxTime"
}
]
}
}
},
... same pipeline ...
])
I want to filter the dataset to extract documents which were created 7 days ago OR a Month ago OR Documents created at any date.
filter documents based on createdAt field in document.
Dataset:-
[
{
"_id": ObjectId("6257047cffd61ab62864c1ae"),
"type": "A",
"source": "B",
"user": ObjectId("622b55ff0b0af6b049c387d3"),
"createdAt": ISODate("2022-04-17T07:55:00.368Z"),
"updatedAt": ISODate("2022-04-17T07:55:00.368Z"),
},
{
"_id": ObjectId("6257047cffd61ab62864c1ad"),
"type": "B",
"source": "A",
"user": ObjectId("622b55ff0b0af6b049c387d3"),
"createdAt": ISODate("2022-04-23T07:55:00.368Z"),
"updatedAt": ISODate("2022-04-23T07:55:00.368Z"),
},
{
"_id": ObjectId("6257047cffd61ab62864c1ce"),
"type": "A",
"source": "C",
"user": ObjectId("622b55ff0b0af6b049c387d3"),
"createdAt": ISODate("2022-04-17T07:55:00.368Z"),
"updatedAt": ISODate("2022-04-17T07:55:00.368Z"),
},
{
"_id": ObjectId("6257047cffd61ab62864c1cb"),
"type": "A",
"source": "B",
"user": ObjectId("622b56250b0af6b049c387d6"),
"createdAt": ISODate("2022-04-24T07:55:00.368Z"),
"updatedAt": ISODate("2022-04-24T07:55:00.368Z"),
},
{
"_id": ObjectId("6257047cffd61ab62864c1cb"),
"type": "A",
"source": "B",
"user": ObjectId("622b56250b0af6b049c387d6"),
"createdAt": ISODate("2022-03-24T07:55:00.368Z"),
"updatedAt": ISODate("2022-03-24T07:55:00.368Z"),
},
{
"_id": ObjectId("6257047cffd61ab62864c1ce"),
"type": "A",
"source": "C",
"user": ObjectId("622b55ff0b0af6b049c387d3"),
"createdAt": ISODate("2022-03-17T07:55:00.368Z"),
"updatedAt": ISODate("2022-03-17T07:55:00.368Z"),
},
]
MongoDB aggregate query:-
db.collection.aggregate([
{
$addFields: {
paramType: "All",
paramSource: "All",
paramCreatedAt:"All",
}
},
{
$match: {
$and: [
{
user: ObjectId("622b55ff0b0af6b049c387d3")
},
{
$or: [
{
paramType: {
$eq: "All"
}
},
{
$expr: {
$eq: [
"$paramType",
"$type"
],
}
}
]
},
{
$or: [
{
paramSource: {
$eq: "All"
}
},
{
$expr: {
$eq: [
"$paramSource",
"$source"
]
}
}
]
}
]
}
},
{
$setWindowFields: {
output: {
totalCount: {
$count: {}
}
}
}
},
{
$sort: {
createdAt: -1
}
},
{
$skip: 0
},
{
$limit: 6
},
{
"$project": {
"paramSource": false,
"paramType": false,
}
}
])
how to filter to get documents created in the last 7 days or 30 days or any date.
paramCreatedAt will take one of the following values [All dates, 7 days ago, a month ago]
Example:-
If the All dates filter is applied then display all records.
If 7 days filter is applied display records created from the current date (which can be any day not necessary that it should be sunday) to 7 days back.
If 30 days filter applied then display records created in last 30 days
Your skeleton is pretty neat and you are actually quite close. For the date filtering, just use $dateDiff to return the date difference in days and compare it with the days interval your selected(i.e. 7 days or 30 days) by using $switch
db.collection.aggregate([
{
$addFields: {
paramType: "All",
paramSource: "All",
paramCreatedAt: "All dates"// [All dates, 7 days ago, a month ago]
}
},
{
$match: {
$and: [
{
user: ObjectId("622b55ff0b0af6b049c387d3")
},
{
$or: [
{
paramType: {
$eq: "All"
}
},
{
$expr: {
$eq: [
"$paramType",
"$type"
],
}
}
]
},
{
$or: [
{
paramSource: {
$eq: "All"
}
},
{
$expr: {
$eq: [
"$paramSource",
"$source"
]
}
}
]
},
{
$or: [
{
paramCreatedAt: {
$eq: "All dates"
}
},
{
$expr: {
$and: [
{
"$in": [
"$paramCreatedAt",
[
"7 days ago",
"a month ago"
]
]
},
{
$lte: [
{
"$dateDiff": {
"startDate": "$createdAt",
"endDate": "$$NOW",
"unit": "day"
}
},
{
"$switch": {
"branches": [
{
"case": {
$eq: [
"$paramCreatedAt",
"7 days ago"
]
},
"then": 7
},
{
"case": {
$eq: [
"$paramCreatedAt",
"a month ago"
]
},
"then": 30
}
]
}
}
]
}
]
}
}
]
}
]
}
},
{
$setWindowFields: {
output: {
totalCount: {
$count: {}
}
}
}
},
{
$sort: {
createdAt: -1
}
},
{
$skip: 0
},
{
$limit: 6
},
{
"$project": {
"paramSource": false,
"paramType": false,
}
}
])
Here is the Mongo playground for your reference.
Here's an alternate approach using $facet. $facet is very handy because it allows you to "match and group in parallel" and create overlapping buckets of documents. A single pipeline with $group and $cond on the aggregation field works well for "if/then/elif/elif/else" constructions where overlaps are not desired and an order of precedence is desired.
db.foo.aggregate([
// Initial filter(s):
{$match: {user: ObjectId("622b55ff0b0af6b049c387d3")}},
// Create a single version of "now" from the perspective of the
// CLIENT to use in queries to follow.
// To create such a target date from the perspective of the SERVER,
// use {$addFields: {DD: '$$NOW'}}
// Probably overkill but OK.
{$addFields: {DD: new ISODate()}},
{$facet: {
"all": [ ], // not exciting! :-)
"exactly_7_days_ago": [
{$match: {$expr:
{$eq: [7, {$floor: {$divide:[{$subtract:['$DD', '$createdAt'] }, 1000 * 60 * 60 * 24]}} ]}
}}
],
"everything_from_last_month": [
{$match: {$expr:
{$eq: [1, {$subtract:[{$month: '$DD'}, {$month: '$createdAt'} ]} ]}
}}
],
"only_one_day_from_last_month": [
{$match: {$expr:
{$and: [
{$eq: [1, {$subtract:[{$month: '$DD'}, {$month: '$createdAt'}]} ]},
{$eq: [0, {$subtract:[{$dayOfMonth: '$DD'}, {$dayOfMonth: '$createdAt'} ]} ]}
]}
}}
],
}}
]);
How to bring age group base data from a collection in MongoDB i.e how many people are 0-18, 19-24, 25-34, 35+
[
{
"_id": ObjectId("608be7c608c7de2367c89638"),
"status": true,
"gender": "Male",
"first_name": "Vinter",
"last_name": "R",
"dob": "1-2-1999"
},
{
"_id": ObjectId("608be7c608c7de2267c89639"),
"status": true,
"gender": "Male",
"first_name": "Ray",
"last_name": "Morgan",
"dob": "1-2-2015"
}
....
]
See the Mongo Playground:
https://mongoplayground.net/p/4ydNg9Plh6P
Interesting question!
Would like to credit to #Takis and #YuTing.
Good hint from #Takis's comment on $bucket.
#YuTing's answer is good.
Think this answer is shorter by utilizing the feature provided by MongoDB.
$toDate - Convert date string to Date (supported for version 4.0 above).
$dateDiff - Date subtraction and get the unit (Supported in version 5).
$$CURRENT - Variable to get the current iterated document. For adding into persons array field (via $push).
$switch - To display group value based on conditions (Optional).
db.collection.aggregate([
{
"$addFields": {
"age": {
$dateDiff: {
startDate: {
$toDate: "$dob"
},
endDate: "$$NOW",
unit: "year"
}
}
}
},
{
$bucket: {
groupBy: "$age",
// Field to group by
boundaries: [
0,
19,
25,
35
],
// Boundaries for the buckets
default: "Other",
// Bucket id for documents which do not fall into a bucket
output: {
// Output for each bucket
"count": {
$sum: 1
},
"persons": {
$push: "$$CURRENT"
}
}
}
},
{
$project: {
_id: 0,
group: {
$switch: {
branches: [
{
case: {
$lt: [
"$_id",
19
]
},
then: "0-18"
},
{
case: {
$lt: [
"$_id",
25
]
},
then: "19-24"
},
{
case: {
$lt: [
"$_id",
35
]
},
then: "25-34"
}
],
default: "35+"
}
},
count: 1,
persons: 1
}
}
])
Sample Mongo Playground
use $bucket
db.collection.aggregate([
{
$bucket: {
groupBy: {
"$subtract": [
{
$year: new Date()
},
{
$toInt: {
$substr: [
"$dob",
{
$subtract: [
{
$strLenCP: "$dob"
},
4
]
},
4
]
}
}
]
},
// Field to group by
boundaries: [
0,
19,
25,
35,
100
],
// Boundaries for the buckets
default: "Other",
// Bucket id for documents which do not fall into a bucket
output: {
// Output for each bucket
"count": {
$sum: 1
},
"artists": {
$push: {
"name": {
$concat: [
"$first_name",
" ",
"$last_name"
]
},
"age": {
"$subtract": [
{
$year: new Date()
},
{
$toInt: {
$substr: [
"$dob",
{
$subtract: [
{
$strLenCP: "$dob"
},
4
]
},
4
]
}
}
]
}
}
}
}
}
}
])
mongoplayground
I have a similar collection where I have sort them by their startTime:
{"name": 'A', "startTime": '1634626355', "endTime": '1634631405'}
{"name": 'A', "startTime": '1634631406', "endTime": '1634631864'}
{"name": 'A', "startTime": '1634631865', "endTime": '1634656048'}
{"name": 'A', "startTime": '1634712642', "endTime": '1634718856'}
How can I compare the documents such that if the document endTime and the next document startTime duration is less than 5 minutes, merge it.
This is the result I'm trying to achieve (The 1st 3 documents are merged into 1 where it uses the startTime of the 1st document and the endTime of the 3rd document):
{"name": 'A', "startTime": '1634626355', "endTime": '1634656048'}
{"name": 'A', "startTime": '1634712642', "endTime": '1634718856'}
Thanks
First of all, you should never store date/time values as string, it's a design flaw. Store always proper Date object.
This solution works without self-lookup, so it may perform better:
db.collection.aggregate([
{
$set: {
startDateTime: { $toDate: { $multiply: ["$startTime", 1000] } },
endDateTime: { $toDate: { $multiply: ["$endTime", 1000] } }
},
},
{ $sort: { startDateTime: 1 } },
{ $group: { _id: null, data: { $push: "$$ROOT" } } },
{
$set: {
data: {
$reduce: {
input: "$data",
initialValue: [],
in: {
$cond: {
if: {
$or: [
{ $eq: [{ $size: "$$value" }, 0] }, // for the initail element
{
$gt: [
{
$dateDiff: { // calculate difference
endDate: "$$this.startDateTime",
startDate: { $last: "$$value.endDateTime" },
unit: "minute"
}
},
5 // more than 5 Minutes
]
}
]
},
then: { $concatArrays: ["$$value", ["$$this"]] }, // append new element
else: {
$map: {
input: "$$value",
as: "data",
in: {
$cond: {
if: { $eq: ["$$data._id", { $last: "$$value._id" }] }, // find last element
then: { // update last element
$mergeObjects: [
"$$data",
{ endDateTime: "$$this.endDateTime" },
{ endTime: "$$this.endTime" }
]
},
else: "$$data"
}
}
}
}
}
}
}
}
}
},
// some cosmetic
{ $unwind: "$data" },
{ $replaceRoot: { newRoot: "$data" } }
])
Mongo Playground
You can use $lookup in an aggregation pipeline to find out the documents that you need to remove. Then, perform a forEach to remove them.
db.collection.aggregate([
{
$addFields: {
endDateTime: {
"$toDate": {
"$multiply": [
{
$toLong: "$endTime"
},
1000
]
}
}
},
},
{
"$lookup": {
"from": "collection",
let: {
end: "$endDateTime"
},
pipeline: [
{
"$addFields": {
startDateTime: {
"$toDate": {
"$multiply": [
{
$toLong: "$startTime"
},
1000
]
}
}
}
},
{
$match: {
$expr: {
$and: [
{
$lte: [
{
$subtract: [
"$startDateTime",
"$$end"
]
},
300000
]
},
{
$lte: [
"$$end",
"$startDateTime"
]
}
]
}
}
}
],
"as": "lessThan5min"
}
},
{
"$unwind": "$lessThan5min"
},
{
"$replaceRoot": {
"newRoot": "$lessThan5min"
}
}
]).forEach(function(doc){
db.collection.remove({ "_id": doc._id });
});
Here is the Mongo playground to find out the documents that you need to remove for your reference.