Mongodb group and push with empty arrays - mongodb

I'm having a problem with a group when there is an array that could be empty.
The collection could be like this:
{
"_id" : "Contract_1",
"ContactId" : "Contact_1",
"Specifications" : [ ]
}
{
"_id" : "Contract_2",
"ContactId" : "Contact_2",
"Specifications" : [
{
"Description" : "Descrizione1",
"VehicleId" : "Vehicle_1",
"Customizations" : [
{
"Description" : "Random furniture",
"ContactId" : "Contact_5"
},
{
"Description" : "Random furniture 2",
"ContactId" : "Contact_3"
}
]
},
{
"Description" : "Descrizione2",
"VehicleId" : "Vehicle_2",
"Customizations" : [
{
"Description" : "Random furniture",
"ContactId" : "Contact_5"
},
{
"Description" : "Random furniture 2",
"ContactId" : "Contact_3"
}
]
}
]
}
{
"_id" : "Contract_3",
"ContactId" : "Contact_25",
"Specifications" : [
{
"Description" : "Descrizione1",
"VehicleId" : "Vehicle_1",
"Customizations" : []
},
{
"Description" : "Descrizione2",
"VehicleId" : "Vehicle_2",
"Customizations" : []
}
]
}
As you can see, sometimes Specifications can be null, and also Customizations.
And this is the query that I execute:
db.getCollection("Contract").aggregate([
{ "$lookup": {
"from": "Contact",
"localField": "ContactId",
"foreignField": "_id",
"as": "Contact"
}},
{ "$unwind": {"path":"$Contact", "preserveNullAndEmptyArrays":true }},
{ "$unwind": { "path": "$Specifications", "preserveNullAndEmptyArrays":true }},
{ "$lookup": {
"from": "Vehicle",
"localField": "Specifications.VehicleId",
"foreignField": "_id",
"as": "Specifications.Vehicle"
}},
{ "$unwind": {"path": {"$Specifications.Vehicle","preserveNullAndEmptyArrays":true} },
{ "$unwind": {"path": {"$Specifications.Customizations","preserveNullAndEmptyArrays":true} },
{ "$lookup": {
"from": "Contact",
"localField": "Specifications.Customizations.ContactId",
"foreignField": "_id",
"as": "Specifications.Customizations.Contact"
}},
{ "$unwind": {"path": {"$Specifications.Customizations.Contact","preserveNullAndEmptyArrays":true} },
{ "$group": {
"_id": {
"_id": "$_id",
"Description": "$Specifications.Description"
},
"ContactId": { "$first": "$ContactId" },
"Contact": { "$first": "$Contact" },
"Specifications": {
"$push": "$Specifications.Customizations"
}
}},
{ "$group": {
"_id": "$_id._id",
"ContactId": { "$first": "$ContactId" },
"Contact": { "$first": "$Contact" },
"Specifications": {
"$push": {
"Description": "$_id.Description",
"Customizations": "$Specifications"
}
}
}}
])
}},
{ "$group": {
"_id": "$_id._id",
"ContactId": { "$first": "$ContactId" },
"Contact": { "$first": "$Contact" },
"Specifications": {
"$push": {
"Description": "$_id.Description",
"Customizations": "$Specifications"
}
}
}}
])
Once the query execute, when it's doing the 2 $group it creates a problem, since for the first one when pushing $Specifications.Customizations will create an array with an empty element inside. What I want is that If Specifications is an empty array, will stay so without adding an empty element inside.

This is I can see one of the drawback of the $unwind and $group for the nested arrays. To get rid from this you need to add one more stage $addFields to filter out the empty nested arrays.
Add this at the end of the pipeline
{ "$addFields": {
"Specifications": {
"$filter": {
"input": "$Specifications",
"cond": { "$ne": ["$$this.Description", undefined] }
}
}
}}

For anyone still having the same problem like me and for whom the answer from #Ashh isnt working (I cant figure out the reason why it doesnt work for me). $ifNull instead of $ne is what worked for me, like this:
{ "$addFields": {
"Specifications": {
"$filter": {
"input": "$Specifications",
"cond": { "$ifNull": ["$$this.Description", false] }
}
}
}}

Related

How to push all values in single array in mongodb

Colleges
/* 1 createdAt:5/9/2019, 7:00:04 PM*/
{
"_id" : ObjectId("5cd42b5c65b41027845938ae"),
"clgID" : "100",
"name" : "Anna University"
},
/* 2 createdAt:5/9/2019, 7:00:04 PM*/
{
"_id" : ObjectId("5cd42b5c65b41027845938ad"),
"clgID" : "200",
"name" : "National"
}
Subjects:
/* 1 createdAt:5/9/2019, 7:03:24 PM*/
{
"_id" : ObjectId("5cd42c2465b41027845938b0"),
"name" : "Hindi",
"members" : {
"student" : [
"123"
]
},
"college" : {
"collegeID" : "100"
}
},
/* 2 createdAt:5/9/2019, 7:03:24 PM*/
{
"_id" : ObjectId("5cd42c2465b41027845938af"),
"name" : "English",
"members" : {
"student" : [
"456",
"789"
]
},
"college" : {
"collegeID" : "100"
}
}
Here i am having two collection and i want to join Colleges table is clgID and Subjects table iscollege.collegeID , then i want to take members.student values and push into single array based on college.collegeID.
My Expected Output
{
"GroupDetails" : [ ],
"clgName" : "National"
},
{
"GroupDetails" : [
"123",
"456",
"789"
],
"clgName" : "Anna University"
}
My Code
db.Colleges.aggregate([
{ $match : { "clgID" : { $in : ["100", "200"] } } },
{ $lookup: { from: "Subjects", localField: "clgID", foreignField: "college.collegeID", as: "GroupDetails" } },
//{ $unwind: "$GroupDetails" },
{ $project: { '_id' : false, 'clgName' : '$name', 'GroupDetails.members.student' : true } }
])
I am getting like this
/* 1 */
{
"GroupDetails" : [ ],
"clgName" : "National"
},
/* 2 */
{
"GroupDetails" : [
{
"members" : {
"student" : [
"456"
]
}
},
{
"members" : {
"student" : [
"123"
]
}
}
],
"clgName" : "Anna University"
}
You can use below aggregation with mongodb 3.6 and above
db.Colleges.aggregate([
{ "$match": { "clgID": { "$in": ["100", "200"] } } },
{ "$lookup": {
"from": "Subjects",
"let": { "clgId": "$clgID" },
"pipeline": [
{ "$match": { "$expr": { "$eq": ["$$clgId", "$college.collegeID"] } } },
{ "$group": {
"_id": "$college.collegeID",
"groupDetails": { "$push": "$members.student" }
}},
{ "$project": {
"groupDetails": {
"$reduce": {
"input": "$groupDetails",
"initialValue": [],
"in": { "$concatArrays": ["$$this", "$$value"] }
}
}
}}
],
"as": "clg"
}},
{ "$unwind": { "path": "$clg", "preserveNullAndEmptyArrays": true } },
{ "$project": {
"clgName": "$name",
"groupDetails": { "$ifNull": ["$clg.groupDetails", []] }
}}
])
MongoPlayground
Or with the mongodb 3.4 and below
db.Colleges.aggregate([
{ "$match": { "clgID": { "$in": ["100", "200"] }}},
{ "$lookup": {
"from": "Subjects",
"localField": "clgID",
"foreignField": "college.collegeID",
"as": "clg"
}},
{ "$unwind": { "path": "$clg", "preserveNullAndEmptyArrays": true }},
{ "$group": {
"_id": { "clgId": "$clg.college.collegeID", "_id": "$_id" },
"groupDetails": { "$push": "$clg.members.student" },
"clgName": { "$first": "$name" }
}},
{ "$project": {
"_id": "$_id._id",
"clgName": 1,
"groupDetails": {
"$reduce": {
"input": "$groupDetails",
"initialValue": [],
"in": { "$concatArrays": ["$$this", "$$value"] }
}
}
}}
])
MongoPlayground

exclude fields in $lookup aggregation

I am querying between 3 collections I want to exclude _id everywhere in output
My output is:
{
"_id" : ObjectId("5b6aed5f9bcdb5d4ae64aef5"),
"userID" : "1",
"skills" : [
{
"_id" : ObjectId("5b766b5f1365a4940bb6050f"),
"skillID" : "javaid",
"skillname" : "जावा",
"languageID" : "hindiid"
},
{
"_id" : ObjectId("5b766b8c1365a4940bb60535"),
"skillID" : "pythonid",
"skillname" : "पायथन",
"languageID" : "hindiid"
}
],
"gender" : {
"_id" : ObjectId("5b7687cd2a2329043e2383d5"),
"genderID" : "femaleid",
"gendername" : "महिला",
"languageID" : "hindiid"
}
}
Query:
db.User.aggregate([
{ "$match": { "userID":"1" }},
{ "$lookup":{
"from": "Skill",
"pipeline": [
{ "$match": { "languageID": "hindiid", "skillID": { "$in": [ "javaid","pythonid" ] }}},
],
"as": "skills"
}},
{ "$lookup": {
"from": "Gender",
"pipeline": [
{ "$match": { "languageID": "hindiid", "genderID" : "femaleid" }},
],
"as": "gender"
}},
{ "$unwind": { "path": "$gender", "preserveNullAndEmptyArrays": true }},
{ "$project": { "userID": 1, "skills": 1, "gender": 1 }}
])
In output for every object has _id.Example for skill list every object has _id i want exclude _id field every where. How I can exclude?
In mongodb 3.6 you can use projection ($project) inside $lookup pipeline... Something like this
db.User.aggregate([
{ "$match": { "userID":"1" }},
{ "$lookup":{
"from": "Skill",
"pipeline": [
{ "$match": { "languageID": "hindiid", "skillID": { "$in": [ "javaid","pythonid" ] }}},
{ "$project": { "_id": 0 }}
],
"as": "skills"
}}
])

$lookup with nested data in mongodb

How do I combine 2 array objects using mongoDB NoSQL? Because I have tried to find some of the same problems here that I got, but I have not found the answers and problems that match what I got.
If someone here wants to help me, here are the problems I want to solve.
Example: I tried using noSQL in mongoDB like this:
db.tables.aggregate([
{ $lookup: { from: 'reservations', localField: '_id', foreignField: 'tableId', as: 'reservation' }},
{ $unwind: { path: '$reservation', 'preserveNullAndEmptyArrays': true }},
{ $lookup: { from: 'orders', localField: 'reservation._id', foreignField: 'reservationId', as: 'orders' }},
{ $lookup: { from: 'products', localField: 'orders.productId', foreignField: '_id', as: 'products' }},
{
$project: {
'_id': 1,
'initial': 1,
'description': 1,
'reservation._id': 1,
'reservation.guest': 1,
'orders._id': 1,
'orders.status': 1,
'orders.quantity': 1,
'orders.productId': 1,
'products._id': 1,
'products.name': 1
}
},
]);
After running noSQL mongoDB above, I got the results below:
{
"_id" : ObjectId("5b63e519514cf01c2864749a"),
"description" : "Kursi VIP 01",
"reservation" : {
"_id" : ObjectId("5b63f104514cf01c286474b6"),
"guest" : "Jhon Doe"
},
"orders" : [
{
"_id" : ObjectId("5b63f239514cf01c286474bb"),
"productId" : ObjectId("5b63e72d514cf01c286474a3"),
"status" : "3",
"quantity" : "2"
},
{
"_id" : ObjectId("5b63f252514cf01c286474bc"),
"productId" : ObjectId("5b63e7de514cf01c286474a6"),
"status" : "2",
"quantity" : "3"
},
{
"_id" : ObjectId("5b63f267514cf01c286474bd"),
"productId" : ObjectId("5b63e937514cf01c286474ac"),
"status" : "0",
"quantity" : "2"
}
],
"products" : [
{
"_id" : ObjectId("5b63e72d514cf01c286474a3"),
"name" : "AQUA 600ML"
},
{
"_id" : ObjectId("5b63e7de514cf01c286474a6"),
"name" : "Nasi Goreng Kecap Asin"
},
{
"_id" : ObjectId("5b63e937514cf01c286474ac"),
"name" : "Daging Ayam Goreng"
}
]
}
Now, my Question is. How to merge/combine 2 Object Array ("orders and products"), So I can get results like this:
{
"_id" : ObjectId("5b63e519514cf01c2864749a"),
"description" : "Kursi VIP 01",
"reservation" : {
"_id" : ObjectId("5b63f104514cf01c286474b6"),
"guest" : "Jhon Doe"
},
"orders" : [
{
"_id" : ObjectId("5b63f239514cf01c286474bb"),
"productId" : ObjectId("5b63e72d514cf01c286474a3"),
"name" : "AQUA 600ML",
"status" : "3",
"quantity" : "2"
},
{
"_id" : ObjectId("5b63f252514cf01c286474bc"),
"productId" : ObjectId("5b63e7de514cf01c286474a6"),
"name" : "Nasi Goreng Kecap Asin",
"status" : "2",
"quantity" : "3"
},
{
"_id" : ObjectId("5b63f267514cf01c286474bd"),
"productId" : ObjectId("5b63e937514cf01c286474ac"),
"name" : "Daging Ayam Goreng"
"status" : "0",
"quantity" : "2"
}
]
}
I hope, someone can help me.
Thanks in advance.
You can try below aggregation with mongodb 3.4
You need to $unwind the orders array to add the field($addFields) name inside orders and then $group to rollback orders again to the make an array field
db.tables.aggregate([
{ "$lookup": {
"from": "reservations",
"localField": "_id",
"foreignField": "tableId",
"as": "reservation"
}},
{ "$unwind": { "path": '$reservation', 'preserveNullAndEmptyArrays': true }},
{ "$lookup": {
"from": "orders",
"localField": "reservation._id",
"foreignField": "reservationId",
"as": "orders",
}},
{ "$unwind": { "path": '$orders', 'preserveNullAndEmptyArrays': true }},
{ "$lookup": {
"from": "products",
"localField": "orders.productId",
"foreignField": "_id",
"as": "orders.products"
}},
{ "$unwind": { "path": '$orders.products', 'preserveNullAndEmptyArrays': true }},
{ "$addFields": {
"orders.name": "$orders.products.name"
}},
{ "$group": {
"_id": "$_id",
"description": { "$first": "$description" },
"reservation": { "$first": "$reservation" },
"orders": { "$push": "$orders" }
}},
{ "$project": { "orders.products": 0 }}
])
Which is far simple with mongodb 3.6 nested $lookup version
db.tables.aggregate([
{ "$lookup": {
"from": "reservations",
"let": { "reservationId": "$_id" },
"pipeline": [
{ "$match": { "$expr": { "$eq": [ "$tableId", "$$reservationId" ] } } }
],
"as": "reservations"
}},
{ "$lookup": {
"from": "orders",
"let": { "reservationId": "$reservation._id" },
"pipeline": [
{ "$match": { "$expr": { "$eq": [ "$reservationId", "$$reservationId" ] } } },
{ "$lookup": {
"from": "products",
"let": { "productId": "$productId" },
"pipeline": [
{ "$match": { "$expr": { "$eq": [ "$_id", "$$productId" ] } } },
{ "$project": { "_id": false }}
],
"as": "products"
}},
{ "$unwind": "$products" },
{ "$addFields": { "name": "$products.name" } },
{ "$project": { "products": 0 }}
],
"as": "orders"
}}
])

$lookup when localField is nested

MongoDB version 3.4.10 (Application is using Meteor framework)
Objective: Aggregate documents that are referenced by _id into the containing document as required at runtime.
I have Materials, Models, and Catalog collections with the following documents:
Materials
{ "_id" : "cf4KgXw7ZK6ukdzR7", "name" : "parquet_wood_mahogany" }
Models
{
"_id" : "Mwp5eYYZ4GZzvZuoK",
"name" : "top_square_chamfered",
"type" : "top"
}
{
"_id" : "CqhS2m2RcLZ2Bm4eb",
"name" : "skirt_square",
"type" : "skirt"
}
{
"_id" : "dYP22ajALnWBwpBj2",
"name" : "leg_square",
"type" : "leg"
}
Catalog
{
"_id" : "EcRGzPAq79giYKrbY",
...,
"specs" : {
...,
"models" : [
{
"mesh" : "Mwp5eYYZ4GZzvZuoK",
"material" : "cf4KgXw7ZK6ukdzR7"
},
{
"mesh" : "CqhS2m2RcLZ2Bm4eb",
"material" : "cf4KgXw7ZK6ukdzR7"
},
{
"mesh" : "dYP22ajALnWBwpBj2",
"material" : "cf4KgXw7ZK6ukdzR7"
}
]
}
}
Desired returned document format after aggregation:
{
"_id" : "EcRGzPAq79giYKrbY",
...,
"specs" : {
"dimensions" : {
...,
},
"models" : [
{
"mesh" : {
"_id" : "Mwp5eYYZ4GZzvZuoK",
"name" : "top_square_chamfered",
"type" : "top"
},
"material" : {
"_id" : "cf4KgXw7ZK6ukdzR7",
"name" : "parquet_wood_mahogany"
}
},
{
"mesh" : {
"_id" : "CqhS2m2RcLZ2Bm4eb",
"name" : "skirt_square",
"type" : "skirt"
},
"material" : {
"_id" : "cf4KgXw7ZK6ukdzR7",
"name" : "parquet_wood_mahogany"
}
},
{
"mesh" : {
"_id" : "dYP22ajALnWBwpBj2",
"name" : "leg_square",
"type" : "leg"
},
"material" : {
"_id" : "cf4KgXw7ZK6ukdzR7",
"name" : "parquet_wood_mahogany"
}
}
]
}
}
I haven't included any of my query code because it is so far off the mark as to just be noise. I've been trying to use aggregate, with $lookup combinations, but I'm not getting anywhere close to what I'm after. The MongoDB v3.6 pipeline syntax would make this much easier... but I'm at a complete loss in v3.4.
I would like to avoid using multiple database requests to combine this information if at all possible. Any assistance of advice would be greatly appreciated!
EDIT: Working solution -
db.catalog.aggregate([
{ "$lookup": {
"from": 'models',
"localField": "specs.models.mesh",
"foreignField": "_id",
"as": "models.mesh"
}},
{ "$lookup": {
"from": 'materials',
"localField": "specs.models.material",
"foreignField": "_id",
"as": "models.material"
}},
{ "$unwind": "$models.mesh" },
{ "$unwind": "$models.material" },
{ "$group":{
"_id": "$_id",
"title": { "$first": "$title" },
"desc": { "$first": "$desc" },
"thumbnail": { "$first": "$thumbnail" },
"createdBy": { "$first": "$createdBy" },
"createdAt": { "$first": "$createdAt" },
"specs": { "$first": "$specs" },
"models": { "$push": "$models" }
}},
{ "$project": {
"_id": "$_id",
"title": "$title",
"desc": "$desc",
"thumbnail": "$thumbnail",
"createdBy": "$createdBy",
"createdAt": "$createdAt",
"specs.dimensions": "$specs.dimensions",
"specs.models": "$models",
}}
])
You can try below aggregation
db.catalog.aggregate([
{ "$lookup": {
"from": 'models',
"localField": "specs.models.mesh",
"foreignField": "_id",
"as": "models.mesh"
}},
{ "$lookup": {
"from": 'materials',
"localField": "specs.models.material",
"foreignField": "_id",
"as": "models.material"
}},
{ "$unwind": "$models.mesh" },
{ "$unwind": "$models.material" },
{ "$group":{
"_id": "$_id",
"title": { "$first": "$title" },
"desc": { "$first": "$desc" },
"thumbnail": { "$first": "$thumbnail" },
"createdBy": { "$first": "$createdBy" },
"createdAt": { "$first": "$createdAt" },
"specs": { "$first": "$specs" },
"models": { "$push": "$models" }
}},
{ "$project": {
"title": "$title",
"desc": "$desc",
"thumbnail": "$thumbnail",
"createdBy": "$createdBy",
"createdAt": "$createdAt",
"specs.dimensions": "$specs.dimensions",
"specs.models": "$models",
}}
])

$lookup nested array in mongodb

I am struggling with the newish (lovely) lookup operator in MongoDB. I have 3 collections:
artists
{
"_id" : ObjectId("5b0d2b2c7ac4792df69a9942"),
"name" : "Dream Theater",
"started_in" : NumberInt(1985),
"active" : true,
"country" : "US",
"current_members" : [
ObjectId("5b0d2a7c7ac4792df69a9941")
],
"previous_members" : [
ObjectId("5b0d2bf57ac4792df69a9954")
],
"albums" : [
ObjectId("5b0d16ee7ac4792df69a9924"),
ObjectId("5b0d47667ac4792df69a9994")
],
"genres" : [
"prog metal",
"prog rock"
]
}
Albums
{
"_id" : ObjectId("5b0d16ee7ac4792df69a9924"),
"title" : "Images and words",
"released" : ISODate("1992-07-07T00:00:00.000+0000"),
"songs" : [
ObjectId("5b0d15ab7ac4792df69a9916"),
ObjectId("5b0d15ee7ac4792df69a991e"),
ObjectId("5b0d2db37ac4792df69a995d"),
ObjectId("5b0d2dbe7ac4792df69a995e"),
ObjectId("5b0d2dcb7ac4792df69a995f"),
ObjectId("5b0d2dd87ac4792df69a9960"),
ObjectId("5b0d2de27ac4792df69a9961"),
ObjectId("5b0d2dec7ac4792df69a9962")
],
"type" : "LP"
}
{
"title" : "Awake",
"released" : ISODate("1994-10-04T00:00:00.000+0000"),
"songs" : [
ObjectId("5b0d470d7ac4792df69a9991")
],
"type" : "LP",
"_id" : ObjectId("5b0d47667ac4792df69a9994")
}
Songs
{
"_id" : ObjectId("5b0d15ab7ac4792df69a9916"),
"title" : "Pull me under"
}
{
"_id" : ObjectId("5b0d15ee7ac4792df69a991e"),
"title" : "Another day"
}
{
"title" : "Take the time",
"_id" : ObjectId("5b0d2db37ac4792df69a995d")
}
{
"title" : "Surrounded",
"_id" : ObjectId("5b0d2dbe7ac4792df69a995e")
}
{
"title" : "Metropolis - part I",
"_id" : ObjectId("5b0d2dcb7ac4792df69a995f")
}
{
"title" : "Under a glass moon",
"_id" : ObjectId("5b0d2dd87ac4792df69a9960")
}
{
"title" : "Wait for sleep",
"_id" : ObjectId("5b0d2de27ac4792df69a9961")
}
{
"title" : "Learning to live",
"_id" : ObjectId("5b0d2dec7ac4792df69a9962")
}
{
"title" : "6:00",
"_id" : ObjectId("5b0d470d7ac4792df69a9991")
}
I can easily do an aggregation with $lookup to get the detailed albums array, but how do I get also the detailed songs in the corresponding albums?
I would like to extend the following query:
db.artists.aggregate([ {
$lookup: {
from: "albums",
localField: "albums",
foreignField: "_id",
as: "albums"
}
}]).pretty()
If you have mongodb version 3.6 then you can try with nested $lookup aggregation...
db.collection.aggregate([
{ "$lookup": {
"from": Albums.collection.name,
"let": { "albums": "$albums" },
"pipeline": [
{ "$match": { "$expr": { "$in": [ "$_id", "$$albums" ] } } },
{ "$lookup": {
"from": Songs.collection.name,
"let": { "songs": "$songs" },
"pipeline": [
{ "$match": { "$expr": { "$in": [ "$_id", "$$songs" ] } } }
],
"as": "songs"
}}
],
"as": "albums"
}}
])
And for long-winded explanation you can go through $lookup multiple levels without $unwind?
Or If you have mongodb version prior to 3.6
db.collection.aggregate([
{ "$lookup": {
"from": Albums.collection.name,
"localField": "albums",
"foreignField": "_id",
"as": "albums"
}},
{ "$unwind": "$albums" },
{ "$lookup": {
"from": Songs.collection.name,
"localField": "albums.songs",
"foreignField": "_id",
"as": "albums.songs",
}},
{ "$group": {
"_id": "$_id",
"name": { "$first": "$name" },
"started_in": { "$first": "$started_in" },
"active": { "$first": "$active" },
"country": { "$first": "$country" },
"albums": {
"$push": {
"_id": "$albums._id",
"title": "$albums.title",
"released": "$albums.released",
"type": "$albums.type",
"songs": "$albums.songs"
}
}
}}
])