Nested $addFields in MongoDB - mongodb

I have the following document:
[
{
"callId": "17dac51e-125e-499e-9064-f20bd3b1a9d8",
"caller": {
"firstName": "Test",
"lastName": "Testing",
"phoneNumber": "1231231234"
},
"inquiries": [
{
"inquiryId": "b0d14381-ce75-49aa-a66a-c36ae20b72a8",
"routeHistory": [
{
"assignedUserId": "cfa0ffe9-c77d-4eec-87d7-4430f7772e81",
"routeDate": "2020-01-01T06:00:00.000Z",
"status": "routed"
},
{
"assignedUserId": "cfa0ffe9-c77d-4eec-87d7-4430f7772e81",
"routeDate": "2020-01-03T06:00:00.000Z",
"status": "routed"
}
]
},
{
"inquiryId": "9d743be9-7613-46d7-8f9b-a04b4b899b56",
"routeHistory": [
{
"assignedUserId": "cfa0ffe9-c77d-4eec-87d7-4430f7772e81",
"routeDate": "2020-01-01T06:00:00.000Z",
"status": "ended"
},
{
"assignedUserId": "cfa0ffe9-c77d-4eec-87d7-4430f7772e81",
"routeDate": "2020-01-03T06:00:00.000Z",
"status": "ended"
}
]
}
]
}
]
And I'm using the following aggregate:
{
$unwind: '$inquiries',
},
{
$addFields: {
'inquiries.routeHistory': {
$filter: {
input: '$inquiries.routeHistory',
cond: {
$eq: [{ $max: '$inquiries.routeHistory.routeDate' }, '$$this.routeDate'],
},
},
},
},
},
{
$group: {
_id: '$_id',
callId: { $first: '$callId' },
caller: { $first: '$caller' },
inquiries: { $push: '$inquiries' },
},
}
I would like to expand this query to be able to further filter at the inquiry grain, so that I am returning only the inquiry that contains my specified criteria. E.g. if I wanted to find where inquiry.routeHistory.status = ended, I would expect the following results:
[
{
"callId": "17dac51e-125e-499e-9064-f20bd3b1a9d8",
"caller": {
"firstName": "Test",
"lastName": "Testing",
"phoneNumber": "1231231234"
},
"inquiries": [
{
"inquiryId": "9d743be9-7613-46d7-8f9b-a04b4b899b56",
"routeHistory": [
{
"assignedUserId": "cfa0ffe9-c77d-4eec-87d7-4430f7772e81",
"routeDate": "2020-01-03T06:00:00.000Z",
"status": "ended"
}
]
}
]
}
]
Is there a way to do nested $addField or is there another route I could take?

Since you're using $unwind you can do that easily by adding $match since expression: "inquiries.routeHistory.status": "ended" will return true if there's any document in routeHistory having such status:
db.collection.aggregate([
{
$unwind: "$inquiries"
},
{
$match: {
"inquiries.routeHistory.status": "ended"
}
},
{
$addFields: {
"inquiries.routeHistory": {
$filter: {
input: "$inquiries.routeHistory",
cond: {
$eq: [ { $max: "$inquiries.routeHistory.routeDate" }, "$$this.routeDate" ]
}
}
}
}
},
{
$group: {
_id: "$_id",
callId: { $first: "$callId" },
caller: { $first: "$caller" },
inquiries: { $push: "$inquiries" }
}
}
])
Mongo Playground

Related

MongoDB aggregate using $match with $expr with array

MongoDB 5.0.9
I am trying to get
value of application within course and their specification
value of paid application ( status : paid) based on course and their specification
courses collection having multiple courses with specification which might be there maybe not
[
{
"_id": {
"$oid": "62aab6669b3740313d881a30"
},
"course_name": "Master",
"fees": "Rs.1000.0/-",
"course_specialization": [
{
"spec_name": "Social Work",
"is_activated": true
}
],
"college_id": {
"$oid": "628dfd41ef796e8f757a5c13"
},
"is_pg": true
},
{
"_id": {
"$oid": "62aab6669b3740313d881a38"
},
"college_id": {
"$oid": "628dfd41ef796e8f757a5c13"
},
"course_name": "BBA",
"fees": "Rs.1000.0/-",
"is_pg": false,
"course_specialization": null
},
{
"_id": {
"$oid": "628f3967cb69fc0789e69181"
},
"course_name": "BTech",
"fees": "Rs.1000.0/-",
"course_specialization": [
{
"spec_name": "Computer Science and Engineering",
"is_activated": true
},
{
"spec_name": "Mutiple Specs",
"is_activated": true
}
],
"college_id": {
"$oid": "628dfd41ef796e8f757a5c13"
},
"is_pg": false
},
{
"_id": {
"$oid": "628f35a1cb69fc0789e6917e"
},
"course_name": "Bachelor",
"fees": "Rs.1000.0/-",
"course_specialization": [
{
"spec_name": "Social Work",
"is_activated": true
}
],
"college_id": {
"$oid": "628dfd41ef796e8f757a5c13"
},
"is_pg": false
}
],
Student Application forms collection where we are storing student application forms details
[
{
"_id": {
"$oid": "62cd476adbc878a0490e20ee"
},
"spec_name1": "Social Work",
"spec_name2": "",
"spec_name3": "",
"student_id": {
"$oid": "62cd1374dbc878a0490e20a5"
},
"course_id": {
"$oid": "62aab6669b3740313d881a30"
},
"current_stage": 2.5,
"declaration": true,
"payment_info": {
"payment_id": "123458",
"status": "paid"
},
"enquiry_date": {
"$date": {
"$numberLong": "1657620330432"
}
},
"last_updated_time": {
"$date": {
"$numberLong": "1657621796062"
}
}
},
{
"_id": {
"$oid": "62cd476adbc878a0490e20ef"
},
"spec_name1": "",
"spec_name2": "",
"spec_name3": "",
"student_id": {
"$oid": "62cd1374dbc878a0490e20a5"
},
"course_id": {
"$oid": "62aab6669b3740313d881a38"
},
"current_stage": 2.5,
"declaration": true,
"payment_info": {
"payment_id": "123458",
"status": "paid"
},
"enquiry_date": {
"$date": {
"$numberLong": "1657620330432"
}
},
"last_updated_time": {
"$date": {
"$numberLong": "1657621796062"
}
}
},
{
"_id": {
"$oid": "62cdc12000b820f5ea58cc60"
},
"spec_name1": "Social Work",
"spec_name2": "",
"spec_name3": "",
"student_id": {
"$oid": "62cdad90a9b64d58b15e6976"
},
"course_id": {
"$oid": "628f35a1cb69fc0789e6917e"
},
"current_stage": 6.25,
"declaration": false,
"payment_info": {
"payment_id": "",
"status": ""
},
"enquiry_date": {
"$date": {
"$numberLong": "1657651488511"
}
},
"last_updated_time": {
"$date": {
"$numberLong": "1657651987155"
}
}
}
]
Desired output with every specification within the course
[
"_id": {
"coursename": "Master",
"spec": "Social Work",
"Application_Count": 1,
"Paid_Application_Count:0
},
{
"_id": {
"coursename": "Bachelor"
"spec":"" ,
"Application_Count": 1,
"Paid_Application_Count:0
},
{
"_id": {
"coursename": "BBA"
"spec":"" ,
"Application_Count": 1,
"Paid_Application_Count:1
},
]
Aggregation Query
[{
$match: {
college_id: ObjectId('628dfd41ef796e8f757a5c13')
}
}, {
$project: {
_id: 1,
course_name: 1,
course_specialization: 1
}
}, {
$unwind: {
path: '$course_name',
includeArrayIndex: 'course_index',
preserveNullAndEmptyArrays: true
}
}, {
$unwind: {
path: '$course_specialization',
includeArrayIndex: 'course_specs_index',
preserveNullAndEmptyArrays: true
}
}, {
$lookup: {
from: 'studentApplicationForms',
'let': {
id: '$_id',
spec: '$course_specialization.spec_name'
},
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: [
'$course_id',
'$$id'
]
},
{
$eq: [
'$spec_name1',
'$$spec'
]
}
]
}
}
},
{
$project: {
student_id: 1,
payment_info: 1,
spec_name1: 1,
spec_name2: 1,
spec_name3: 1
}
}
],
as: 'student_application'
}
}, {
$unwind: {
path: '$student_application',
includeArrayIndex: 'application',
preserveNullAndEmptyArrays: true
}
}, {
$facet: {
course: [
{
$group: {
_id: {
course_name: '$course_name',
spec: '$course_specialization'
},
count: {
$count: {}
}
}
}
],
declatration: [
{
$group: {
_id: {
course_name: '$course_name',
spec: '$course_specialization'
},
count_dec: {
$sum: {
$cond: [
'$student_application.declaration',
1,
0
]
}
}
}
}
],
payment: [
{
$group: {
_id: {
course_name: '$course_name',
spec: '$course_specialization'
},
payment: {
$sum: {
$eq: [
'$student_application.payment_info.status',
'paid'
]
}
}
}
}
]
}
}]
Problem :
I am able to get application count but it is not getting unique value if 2 specs are same then duplicate value is coming as you can see on sample application collection Social Work is in two different course . So my aggregations is not grouping them based in course name.specs
Not able to find correct Paid_Application_Count and Application_Count
Update :
Updated JSON Data Matching use cases with different type of data
MongoDB Playground
You can do it in several different ways, I took the liberty to simplify the pipeline a little bit.
I will just mention that the structure does not fully make sense to me, and there are some additional contradictions between the sample input you provided and the "text" description/pipeline description.
Just a tiny example is payment_info_status being paid in the sample and capture in the pipeline.
These things will not change the pipeline structure, will just need to be fixed by you based on the actual needs.
db.courses.aggregate([
{
$project: {
_id: 1,
course_name: 1,
course_specialization: 1
}
},
{
$unwind: {
path: "$course_specialization",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: "studentApplicationForms",
"let": {
courseId: "$_id",
spec: {
$ifNull: [
"$course_specialization.spec_name",
""
]
}
},
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: [
"$spec_name1",
"$$spec"
]
},
{
$eq: [
"$$courseId",
"$course_id"
]
}
]
}
}
},
{
$project: {
student_id: 1,
payment_info: 1,
spec_name1: 1,
spec_name2: 1,
spec_name3: 1,
declaration: 1,
}
},
{
$group: {
_id: null,
count: {
$sum: 1
},
declatration: {
$sum: {
$cond: [
"$declaration",
1,
0
]
}
},
paid: {
$sum: {
$cond: [
{
$eq: [
"$payment_info.status",
"paid"
]
},
1,
0
]
}
},
}
}
],
as: "student_application"
}
},
{
$project: {
_id: {
coursename: "$course_name",
spec: "$course_specialization.spec_name",
Application_count: {
$ifNull: [
{
$first: "$student_application.count"
},
0
]
},
Declaration_count: {
$ifNull: [
{
$first: "$student_application.declatration"
},
0
]
},
Paid_Application_Count: {
$ifNull: [
{
$first: "$student_application.paid"
},
0
]
},
}
}
}
])
Mongo Playground

MongoDB query inside an array

I want to use this mongoDB collection:
[
{
"_id": {
"$oid": "627c4eb87e7c2b8ba510ac4c"
},
"Contact": [
{
"name": "ABC",
"phone": 5501234,
"mail": "abc#mail.com"
},
{
"name": "DEF",
"phone": 6001234,
"mail": "def#mail.com"
}
],
"nomatter": "trash"
}
]
search for {"name":"ABC"} and return only {"mail":"abc#mail.com"}.
It's possible to use find or it's necessary to use aggregate?
Try this one:
db.collection.aggregate([
{ $match: { "Contact.name": "ABC" } },
{
$project: {
Contact: {
$filter: {
input: "$Contact",
cond: { $eq: [ "$$this.name", "ABC" ] }
}
}
}
},
{ "$replaceWith": { mail: { $first: "$Contact.mail" } } }
])
Mongo Playground

Mongo Query to fetch distinct nested documents

I need to fetch distinct nested documents.
Please find the sample document:
{
"propertyId": 1001820437,
"date": ISODate("2020-07-17T00:00:00.000Z"),
"HList":[
{
"productId": 123,
"name": "Dubai",
"tsh": true
}
],
"PList":[
{
"productId": 123,
"name": "Dubai",
"tsh": false
},
{
"productId": 234,
"name": "India",
"tsh": true
}
],
"CList":[
{
"productId": 234,
"name": "India",
"tsh": false
}
]
}
Expected result is:
{
"produts":[
{
"productId": 123,
"name": "Dubai"
},
{
"productId": 234,
"name": "India"
}
]
}
I tried with this query:
db.property.aggregate([
{
$match: {
"propertyId": 1001820437,
"date": ISODate("2020-07-17T00:00:00.000Z")
}
},
{
"$project": {
"_id": 0,
"unique": {
"$filter": {
"input": {
"$setDifference": [
{
"$concatArrays": [
"$HList.productId",
"$PList.productId",
"$CList.productId"
]
},
[]
]
},
"cond": {
"$ne": [ "$$this", "" ]
}
}
}
}
}
]);
Is $setDifference aggregation is correct choice here?
My query returns only unique product ids but i need a productId with name.
Could someone help me to solve this?
Thanks in advance
You can use $projectfirst to get rid of tsh field and then run $setUnion which ignores duplicated entries:
db.collection.aggregate([
{
$project: {
"HList.tsh": 0,
"PList.tsh": 0,
"CList.tsh": 0,
}
},
{
$project: {
products: {
$setUnion: [ "$HList", "$PList", "$CList" ]
}
}
}
])
Mongo Playground
The following two aggregations return the expected and same result (you can use any of the two):
db.collection.aggregate( [
{
$project: {
_id: 0,
products: {
$reduce: {
input: { $setUnion: [ "$HList", "$PList", "$CList" ] },
initialValue: [],
in: {
$setUnion: [ "$$value", [ { productId: "$$this.productId", name: "$$this.name" } ] ]
}
}
}
}
}
] )
This one is little verbose:
db.collection.aggregate( [
{
$project: { list: { $setUnion: [ "$HList", "$PList", "$CList" ] } }
},
{
$unwind: "$list"
},
{
$group: {
_id: null,
products: { $addToSet: { "productId": "$list.productId", "name": "$list.name" } }
}
},
{
$project: { _id: 0 }
}
] )
db.collection.aggregate([
{
$match: {
"propertyId": 1001820437,
"date": ISODate("2020-07-17T00:00:00.000Z")
}
},
{
$project: {
products: {
$filter: {
input: { "$setUnion" : ["$CList", "$HList", "$PList"] },
as: 'product',
cond: {}
}
}
}
},
{
$project: {
"_id":0,
"products.tsh": 1,
"products.name": 1,
}
},
])

How to use $mergeObjects to merge arrays corresponding to existing groups?

I'm trying to merge two arrays in my aggregation pipeline. After performing $facet, my MongoDB document has this format:
{
"final": [
{
"key": "TP-1",
"status_map": [
{ "status": "Closed", "final": [ "a", "b"]},
{ "status": "Done", "final": ["c","d" ] }
]
},
{
"key": "TP-2",
"status_map": [
{ "status": "Closed", "final": [ "x","y"] }
]
}
],
"start": [
{
"key": "TP-1",
"status_map": [
{ "status": "Closed", "start": [ "h"]},
{ "status": "Done", "start": ["a"]}
]
},
{
"key": "TP-2",
"status_map": [{ "status": "Done", "start": ["l","m"]}
]
}
]
}
Expected Output:
I need to merge final and start array corresponding to two groups:
Based on key and then
Based on status
{
"data": [
{
"key": "TP-1",
"status_map": [
{ "status": "Closed","final": ["a","b"],"start":["h"]},
{ "status": "Done","final": ["c","d"],"start":["a"]}
]
},
{
"key": "TP-2",
"status_map": [
{ "status": "Closed", "final":[ "x","y"],"start": []},
{ "status": "Done", "final": [ ],"start": [ "l","m"]}
]
}
]
}
How to achieve this use case?
There are several ways to approach this, not necessarily with $mergeObjects. But since you mentioned $mergeObjects this is one that uses it:
Note that, with this approach, we are merging objects of the same key and status, the values in the arrays will not get concatenated if the same key exists for multiple documents, The arrays will get replaced instead.
db.collection.aggregate([
{
$project: {
all: { $concatArrays: ["$final","$start"] }
}
},
{
$unwind: "$all"
},
{
$unwind: "$all.status_map"
},
{
$group: {
_id: {
_id: "$_id", // keep _id in $group to apply the group for each document, otherwise if you want to apply group on all documents, omit this
key: "$all.key",
status: "$all.status_map.status"
},
status_map: { $mergeObjects: "$$ROOT.all.status_map" }
}
},
{ // some data don't have start or end at all, we have to set a default empty array
$addFields: { // you can skip this stage if you allow data without start and final keys
"status_map.start": { $ifNull: ["$status_map.start", []] },
"status_map.final": { $ifNull: ["$status_map.final", []] }
}
},
{
$group: {
_id: { _id: "$_id._id", key: "$_id.key" },
key: { $first: "$_id.key" },
status_map: { $push: "$status_map" }
}
}
])
Mongo Playground
With no assumptions (for example for both keys to always appear) my strategy was to concat both arrays, unwind and finally group by the key.
db.collection.aggregate([
{
$project: {
concat: {
$concatArrays: [
"$final",
"$start"
]
}
}
},
{
$unwind: "$concat"
},
{
$unwind: "$concat.status_map"
},
{
$group: {
_id: {
k: "$concat.key",
status: "$concat.status_map.status"
},
final: {
$push: "$concat.status_map.final"
},
start: {
$push: "$concat.status_map.start"
}
}
},
{
$group: {
_id: "$_id.k",
status_map: {
$push: {
status: "$_id.status",
final: "$final",
start: "$start"
}
}
}
},
{
$project: {
key: "$_id",
status_map: 1,
_id: 0
}
}
])
Mongo Playground
Adding to #Tom Slabbaert's answer,
Mongo Playground
Here, final and start array is of format array of array. But It has to be simply an array.
It can be achieved by using $unwind on status_map and $reduce on status_map.final and status_map.start arrays.
Final query:
db.collection.aggregate([
{
$project: {
concat: {
$concatArrays: [
"$final",
"$start"
]
}
}
},
{
$unwind: "$concat"
},
{
$unwind: "$concat.status_map"
},
{
$group: {
_id: {
k: "$concat.key",
status: "$concat.status_map.status"
},
final: {
$push: "$concat.status_map.final"
},
start: {
$push: "$concat.status_map.start"
}
}
},
{
$group: {
_id: "$_id.k",
status_map: {
$push: {
status: "$_id.status",
final: "$final",
start: "$start"
}
}
}
},
{
$project: {
key: "$_id",
status_map: 1,
_id: 0
}
},
{
$unwind: "$status_map"
},
{
$project: {
key: 1,
"status_map.status": 1,
final: {
$reduce: {
input: "$status_map.final",
initialValue: [],
in: {
$concatArrays: [
"$$value",
"$$this"
]
}
}
},
start: {
$reduce: {
input: "$status_map.start",
initialValue: [],
in: {
$concatArrays: [
"$$value",
"$$this"
]
}
}
}
}
},
{
$group: {
_id: "$key",
status_map: {
$push: {
status: "$status_map.status",
final: "$final",
start: "$start"
}
}
}
}
])
Mongo Playground

Group on field while getting the last document for each field with MongoDB

Problem
I'm trying to group a stock inventory by products. At first, my stock entries was fully filled each time so I made this aggregate:
[
{ $sort: { date: 1 } },
{
$group: {
_id: '$userId',
stocks: { $last: '$stocks' },
},
},
{ $unwind: '$stocks' },
{
$group: {
_id: '$stocks.productId',
totalQuantity: { $sum: '$stocks.quantity' },
stocks: { $push: { userId: '$_id', quantity: '$stocks.quantity' } },
},
},
]
Now, it can be possible that a stock entry doesn't contain all the products filled. So I'm stuck while writing the new aggregate.
Basically I need to group every products by productId and have an array of the last entry for each user.
Output
This is my expected output:
[
{
"_id": ObjectId("5e75eae1359fc8159d5b6073"),
"totalQuantity": 33,
"stocks": [
{
"userId": ObjectId("5e75f498359fc8159d5b6075"),
"lastDate": "2020-03-21T11:45:53.077Z",
"quantity": 33
}
]
},
{
"_id": ObjectId("5e75eaea359fc8159d5b6074"),
"totalQuantity": 2,
"stocks": [
{
"userId": ObjectId("5e75f498359fc8159d5b6075"),
"lastDate": "2020-03-21T11:45:53.077Z",
"quantity": 2
}
]
}
]
Documents
Documents (when fully filled):
{
"_id": ObjectId("5e75fe71e4a3e0323ba47e0a"),
"date": "2020-03-21T11:45:53.077Z",
"userId": ObjectId("5e75f498359fc8159d5b6075"),
"stocks": [
{
"productId": ObjectId("5e75eae1359fc8159d5b6073"),
"quantity": 33
},
{
"productId": ObjectId("5e75eaea359fc8159d5b6074"),
"quantity": 2
}
]
}
Sometimes it won't be filled for the whole inventory (that's why I need the lastDate):
{
"_id": ObjectId("5e75fe71e4a3e0323ba47e0a"),
"date": "2020-03-21T11:45:53.077Z",
"userId": ObjectId("5e75f498359fc8159d5b6075"),
"stocks": [
{
"productId": ObjectId("5e75eae1359fc8159d5b6073"),
"quantity": 33
}
]
}
Try this one:
db.collection.aggregate([
{
$group: {
_id: "$userId",
root: {
$push: "$$ROOT"
}
}
},
{
$addFields: {
root: {
$map: {
input: "$root",
as: "data",
in: {
"stocks": {
$map: {
input: "$$data.stocks",
as: "stock",
in: {
"productId": "$$stock.productId",
"userId": "$$data.userId",
"quantity": "$$stock.quantity",
"lastDate": "$$data.date"
}
}
}
}
}
}
}
},
{
$unwind: "$root"
},
{
$replaceRoot: {
newRoot: "$root"
}
},
{
$unwind: "$stocks"
},
{
$sort: {
"stocks.lastDate": 1
}
},
{
$group: {
_id: "$stocks.productId",
totalQuantity: {
$last: "$stocks.quantity"
},
stocks: {
$last: "$stocks"
}
}
},
{
$addFields: {
stocks: [
{
"lastDate": "$stocks.lastDate",
"quantity": "$stocks.quantity",
"userId": "$stocks.userId"
}
]
}
}
])
MongoPlayground