mongodb aggregation lookup with multiple conditions and ids - mongodb

Having the following collections and data on them
db.a.insert([
{ "_id" : ObjectId("5b56989172ebcb00105e8f41"), "items" : [{id:ObjectId("5b56989172ebcb00105e8f41"), "instock" : 120}]},
{ "_id" : ObjectId("5b56989172ebcb00105e8f42"), "items" : [{id:ObjectId("5b56989172ebcb00105e8f42"), "instock" : 120}] },
{ "_id" : ObjectId("5b56989172ebcb00105e8f43"), "items" : [{ObjectId("5b56989172ebcb00105e8f43"), "instock" : 80}] }
])
db.b.insert([
{ "_id" : ObjectId("5b56989172ebcb00105e8f41")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f42")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f43")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f44")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f45")}
])
executing an lookup aggregation like
db.b.aggregate([
{
$lookup:
{
from: "b",
let: { bId: "$_id", qty: 100 },
pipeline: [
{ $match:
{ $expr:
{ $and:
[
{ $eq: [ "$items.id", "$$bId" ] },
{ $gte: [ "$instock", "$$qty" ] }
]
}
}
}
],
as: "a"
}
}
])
does not bring any results in the expected lookup operation. Is there any restriction to use ObjectId as a comparison? In the official documentations does not say any about it and it works like a charm with any other kind of data type, like strings

I am not sure if this is a bug in mongodb or not but the query only works after adding an $unwind stage first.
db.b.aggregate([
{
$lookup:
{
from: "a",
let: { bId: "$_id", qty: 100 },
pipeline: [
{
$unwind: {
path: "$items"
}
},
{ $match:
{ $expr:
{ $and:
[
{ $eq: [ "$items.id", "$$bId" ] },
{ $gte: [ "$items.instock", "$$qty" ] },
]
}
}
}
],
as: "a"
}
}
]);
Note: Join Conditions and Uncorrelated Sub-queries were added in mongo 3.6

Related

I have this two collections namely Inward and Outward, Both collections have similar embeded sub documents contains product, batch and quantity fields

Inward collections
{"ord" : 1,
"products" : [
{
"name" : "apple",
"qty" : "10",
"batch" : "jun-2021"
},
{
"name" : "banana",
"qty" : 20,
"batch" : "jan-2021"
}
]
}
outward collections
{
"_id" : ObjectId("5edde5487957d9efea972a74"),
"inv" : 1,
"products" : [
{
"name" : "apple",
"qty" : 13,
"batch" : "jun-2021"
}
]
}
Now, I would like to perform actual stock quantity check for particular product and batch (grouping together) both the collections
You may try this way:
We join them with inward.ord = outward.inv condition.
Flatten products field.
Group by product's name and batch to sum qty value.
db.inward.aggregate([
{
$lookup: {
from: "outward",
let: {
ord: "$ord",
products: "$products"
},
pipeline: [
{
$match: {
$expr: {
$eq: [ "$$ord", "$inv" ]
}
}
},
{
$project: {
products: {
$concatArrays: [
"$$products",
"$products"
]
}
}
},
{
$unwind: "$products"
},
{
$replaceWith: "$products"
}
],
as: "products"
}
},
{
$unwind: "$products"
},
{
$group: {
_id: {
batch: "$products.batch",
name: "$products.name"
},
qty: {
$sum: "$products.qty"
}
}
}
])
MongoPlayground
Note: You need to have MongoDB v4.2

mongodb aggregate to find,count and project unique documnets

Below are the sample collection.
col1:
"_id" : ObjectId("5ec293782bc00b43b463b67c")
"status" : ["running"],
"name" : "name1 ",
"dcode" : "dc001",
"address" : "address1",
"city" : "city1"
col2:
"_id" : ObjectId("5ec296182bc00b43b463b68f"),
"scode" : ObjectId("5ec2933df6079743c0a2a1f8"),
"ycode" : ObjectId("5ec293782bc00b43b463b67c"),
"city" : "city1",
"lockedDate" : ISODate("2020-05-20T00:00:00Z"),
"_id" : ObjectId("5ec296182bc00b43b463688b"),
"scode" : ObjectId("5ec2933df6079743c0a2a1ff"),
"ycode" : ObjectId("5ec293782bc00b43b463b67c"),
"city" : "city1",
"lockedDate" : ISODate("2020-05-20T00:00:00Z"),
"_id" : ObjectId("5ec296182bc00b43b44fc6cb"),
"scode" :null,
"ycode" : ObjectId("5ec293782bc00b43b463b67c"),
"city" : "city1",
"lockedDate" : ISODate("2020-05-20T00:00:00Z"),
problemStatement:
I want to display name from col1 & count of documents from col2 according to ycode where scode is != null
Tried attempt:
db.col1.aggregate([
{'$match':{
city:'city1'
}
},
{
$lookup:
{
from: "col2",
let: {
ycode: "$_id",city:'$city'
},
pipeline: [
{
$match: {
scode:{'$ne':null},
lockedDate:ISODate("2020-05-20T00:00:00Z"),
$expr: {
$and: [
{
$eq: [
"$ycode",
"$$ycode"
]
},
{
$eq: [
"$city",
"$$city"
]
}
]
},
},
},
], as: "col2"
}
},
{'$unwind':'$col2'},
{'$count':'ycode'},
{
$project: {
name: 1,
status: 1,
}
},
])
now problem with this query is it either displays the count or project the name & status i.e if i run this query in the current format it gives {} if I remove {'$count':'ycode'} then it project the values but doesn't give the count and if I remove $project then i do get the count {ycode:2} but then project doesn't work but I want to achieve both in the result. Any suggestions
ORM: mongoose v>5, mongodb v 4.0
You can try below query :
db.col1.aggregate([
{ "$match": { city: "city1" } },
{
$lookup: {
from: "col2",
let: { id: "$_id", city: "$city" }, /** Create local variables from fields of `col1` but not from `col2` */
pipeline: [
{
$match: { scode: { "$ne": null }, lockedDate: ISODate("2020-05-20T00:00:00Z"),
$expr: { $and: [ { $eq: [ "$ycode", "$$id" ] }, { $eq: [ "$city", "$$city" ] } ] }
}
},
{ $project: { _id: 1 } } // Optional, But as we just need count but not the entire doc, holding just `_id` helps in reduce size of doc
],
as: "col2" // will be an array either empty (If no match found) or array of objects
}
},
{
$project: { _id: 0, name: 1, countOfCol2: { $size: "$col2" } }
}
])
Test : mongoplayground

How to $lookup by avoiding null values in mongodb aggregate

In here i'm using $lookup to to a left join from other collections, the query works fine but when some records missing values it returns
errmsg : $in requires an array as a second argument, found: null
Heres the querying document structure :
{
"no" : "2020921008981",
"sale" : {
"soldItems" : [
{
"itemId" : "5b55ac7f0550de00210a3b24",
},
{
"itemId" : "5b55ac7f0550de00215584re",
}
],
"bills" : [
{
"billNo" : "2020921053467",
"insurancePlanId" : "160",
},
{
"billNo" : "2020921053467",
"insurancePlanId" : "170",
}
],
"visitIds" : [
5b55ac7f0550de00210a3b24, 5b55ac7f0550de00210a3b24
]
}
}
the query :
db.case.aggregate([
{
$lookup: {
from: "insurance",
let: { ipids: "$sale.bill.insurancePlanId" },
pipeline: [
{
$unwind: "$coveragePlans"
},
{
$match: { $expr: { $in: ["$coveragePlans._id", "$$ipids"] } }
},
{
$project: { _id: 0, name: 1 }
}
],
as: "insurances"
}
},
{
$lookup: {
from: "item",
let: { iid: "$salesOrder.purchaseItems.itemRefId" },
pipeline: [
{
$match: {
$expr: {
$in: ["$_id", {
$map: {
input: "$$iid",
in: { $toObjectId: "$$this" }
}
}
]
}
}
}
],
as: "items"
}
}
])
insurance collection :
{
"_id" : ObjectId("5b55aca20550de00210a6d25"),
"name" : "HIJKL"
"coveragePlans" : [
{
"_id" : "160",
"name" : "UVWZ",
},
{
"_id" : "161",
"name" : "LMNO",
}
]
},
{
"_id" : ObjectId("5b55aca20550de00210a6d25"),
"name" : "WXYZ"
"coveragePlans" : [
{
"_id" : "169",
"name" : "5ABC",
},
{
"_id" : "170",
"name" : "4XYZ",
}
]
}
item collection :
{
"_id" : ObjectId("5b55ac7f0550de00210a3b24"),
"code" : "ABCDE"
},
{
"_id" : ObjectId("5b55ac7f0550de00215584re"),
"code" : "PQRST"
}
How to avoid this and do null checks effectively before pipe-lining into the next stages? Tried with { $match: { "fieldName": { $exists: true, $ne: null } } } but it returns mongo error regarding the format. If its the way to go please mention the stage i should put that.. Thanks in advance
You can use $ifNull operator
let: { ipids: {$ifNull:["$sale.bill.insurancePlanId", [] ]} },
EDIT: To skip empty "$salesOrder.purchaseItems.itemRefId" values
let: { iid: {$filter: {input:"$salesOrder.purchaseItems.itemRefId", cond:{$ne:["$$this", ""]}}} },
You can get around that by not using $in.
It looks like this $map is executed separately for every document in the items collection. If you were to run the map in an $addFields stage, you could used the simple form of lookup to match the added field to _id, which would automagically handle missing, null, and array.
Remove the added field with a $project stage if necessary.
db.case.aggregate([
{$lookup: {
from: "insurance",
let: { ipids: "$sale.bill.insurancePlanId" },
pipeline: [
{$unwind: "$coveragePlans"},
{$match: { $expr: { $in: ["$coveragePlans._id", "$$ipids"] } }},
{$project: { _id: 0, name: 1 }}
],
as: "insurances"
}}
{$addFields:{
matchArray:{$map: {
input: "$$iid",
in: { $toObjectId: "$$this" }
}}
}},
{$lookup: {
from: "item",
localField: "matchArray",
foreignField:"_id",
as: "items"
}},
{$project:{
arrayField: 0
}}
])

Lookup and aggregate multiple levels of subdocument in Mongodb

I've tried many answers to similar problems using $lookup, $unwind, and $match, but I can't get this to work for my sub-sub-subdocument situation.
I have this collection, Things:
{
"_id" : ObjectId("5a7241f7912cfc256468cb27"),
"name" : "Fortress of Solitude",
"alias" : "fortress_of_solitude",
},
{
"_id" : ObjectId("5a7247ec548c9ad042f579e2"),
"name" : "Batcave",
"alias" : "batcave",
},
{
"_id" : ObjectId("6a7247bc548c9ad042f579e8"),
"name" : "Oz",
"alias" : "oz",
},
and this one-document collection, Venues:
{
"_id" : ObjectId("5b9acabbbf71f39223f8de6e"),
"name" : "The Office",
"floors" : [
{
"name" : "1st Floor",
"places" : [
{
"name" : "Front Entrance",
"alias" : "front_entrance"
}
]
},
{
"name" : "2nd Floor",
"places" : [
{
"name" : "Batcave",
"alias" : "batcave"
},
{
"name" : "Oz",
"alias" : "oz"
}
]
}
]
}
I want to return all the Things, but with the Venue's floors.places.name aggregated with each Thing if it exists if the aliases match between Things and Venues. So, I want to return:
{
"_id" : ObjectId("5a7241f7912cfc256468cb27"),
"name" : "Fortress of Solitude",
"alias" : "fortress_of_solitude",
<-- nothing added here because
<-- it's not found in Venues
},
{
"_id" : ObjectId("5a7247ec548c9ad042f579e2"),
"name" : "Batcave",
"alias" : "batcave",
"floors" : [ <-- this should be
{ <-- returned
"places" : [ <-- because
{ <-- the alias
name" : "Batcave" <-- matches
} <-- in Venues
] <--
} <--
] <--
},
{
"_id" : ObjectId("6a7247bc548c9ad042f579e8"),
"name" : "Oz",
"alias" : "oz",
"floors" : [ <-- this should be
{ <-- returned
"places" : [ <-- because
{ <-- the alias
name" : "Oz" <-- matches
} <-- in Venues
] <--
} <--
] <--
}
I've gotten as far as the following query, but it only returns the entire Venues.floors array as an aggregate onto each Thing, which is way too much extraneous data aggregated. I just want to merge each relevant floor.place sub-subsubdocument from Venues into its corresponding Thing if it exists in Venues.
db.getCollection('things').aggregate([
{$lookup: {from: "venues",localField: "alias",foreignField: "floors.places.alias",as: "matches"}},
{
$replaceRoot: { newRoot: { $mergeObjects: [ { $arrayElemAt: [ "$matches", 0 ] }, "$$ROOT" ] } }
},
{ $project: { matches: 0 } }
])
I'm struggling with existing answers, which seem to change at MongoDB version 3.2, 3.4, 3.6, or 4.2 to include or not include $unwind, $pipeline, and other terms. Can someone explain how to get a sub-sub-subdocument aggregated like this? Thanks!
You can try this :
db.things.aggregate([
{
$lookup:
{
from: "venues",
let: { alias: "$alias" },
pipeline: [
{ $unwind: { path: "$floors", preserveNullAndEmptyArrays: true } },
{ $match: { $expr: { $in: ['$$alias', '$floors.places.alias'] } } },
/** Below stages are only if you've docs like doc 2 in Venues */
{ $addFields: { 'floors.places': { $filter: { input: '$floors.places', cond: { $eq: ['$$this.alias', '$$alias'] } } } } },
{ $group: { _id: '$_id', name: { $first: '$name' }, floors: { $push: '$floors' } } },
{$project : {'floors.places.alias': 1, _id :0}} // Optional
],
as: "matches"
}
}
])
Test : MongoDB-Playground
Since MongoDB v3.6, we may perform uncorrelated sub-queries which gives us more flexibility to join two collections.
Try this:
db.things.aggregate([
{
$lookup: {
from: "venues",
let: {
"alias": "$alias"
},
pipeline: [
{
$unwind: "$floors"
},
{
$project: {
_id: 0,
places: {
$filter: {
input: "$floors.places",
cond: {
$eq: [
"$$alias",
"$$this.alias"
]
}
}
}
}
},
{
$match: {
"places.0": {
$exists: true
}
}
},
{
$unset: "places.name"
}
],
as: "floors"
}
}
])
MongoPlayground

MongoDB Query, get document only if there is results in lookup

I have collection of events and collection of devices:
Events:
{
"_id" : ObjectId("5a3e9f2613e8867c1300002a"),
"AS_CloudsID" : 397,
"TerminalsID" : 1,
"TABLE_NAME" : "Products",
"ItemsID" : 43,
"UpdateNew" : 0,
"RepType" : 1
}
{
"_id" : ObjectId("5a3eafa813e886e407000029"),
"AS_CloudsID" : 377,
"TerminalsID" : 1,
"TABLE_NAME" : "Products",
"ItemsID" : 14812,
"UpdateNew" : 0,
"RepType" : 1
}
Devices:
{
"_id" : ObjectId("5a3ea0999c1d80a468094d34"),
"DateCreated" : ISODate("2017-12-23T18:29:45.569Z"),
"cloudID" : 397,
"terminalID" : 1
}
When I am executing this query:
db.getCollection('events').aggregate([
{
$lookup:
{
from: "devices",
let: {
cloudID: "$AS_CloudsID",
terminalID: "$TerminalsID",
},
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: [
"$terminalID",
"$$terminalID"
]
},
{
$eq: [
"$cloudID",
"$$cloudID"
]
}
]
}
}
}
],
as: "Devices"
}
}
])
I get good results but I only need the events that have Devices, so I need to add some where: Devices.length > 1 How I can do it?
Right now I get documents (from event collection) with empty array of Devices.
Use $match pipeline after $lookup to get events only with devices.
db.getCollection('events').aggregate(
[{
$lookup: {
from: "devices",
let: {
cloudID: "$AS_CloudsID",
terminalID: "$TerminalsID",
},
pipeline: [{
$match: {
$expr: {
$and: [{
$eq: [
"$terminalID",
"$$terminalID"
]
},
{
$eq: [
"$cloudID",
"$$cloudID"
]
}
]
}
}
}],
as: "Devices"
}
},
{
$match: {
Devices: { $size: { $gt: 0 } }
}
}
])