MongoDB Query, get document only if there is results in lookup - mongodb

I have collection of events and collection of devices:
Events:
{
"_id" : ObjectId("5a3e9f2613e8867c1300002a"),
"AS_CloudsID" : 397,
"TerminalsID" : 1,
"TABLE_NAME" : "Products",
"ItemsID" : 43,
"UpdateNew" : 0,
"RepType" : 1
}
{
"_id" : ObjectId("5a3eafa813e886e407000029"),
"AS_CloudsID" : 377,
"TerminalsID" : 1,
"TABLE_NAME" : "Products",
"ItemsID" : 14812,
"UpdateNew" : 0,
"RepType" : 1
}
Devices:
{
"_id" : ObjectId("5a3ea0999c1d80a468094d34"),
"DateCreated" : ISODate("2017-12-23T18:29:45.569Z"),
"cloudID" : 397,
"terminalID" : 1
}
When I am executing this query:
db.getCollection('events').aggregate([
{
$lookup:
{
from: "devices",
let: {
cloudID: "$AS_CloudsID",
terminalID: "$TerminalsID",
},
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: [
"$terminalID",
"$$terminalID"
]
},
{
$eq: [
"$cloudID",
"$$cloudID"
]
}
]
}
}
}
],
as: "Devices"
}
}
])
I get good results but I only need the events that have Devices, so I need to add some where: Devices.length > 1 How I can do it?
Right now I get documents (from event collection) with empty array of Devices.

Use $match pipeline after $lookup to get events only with devices.
db.getCollection('events').aggregate(
[{
$lookup: {
from: "devices",
let: {
cloudID: "$AS_CloudsID",
terminalID: "$TerminalsID",
},
pipeline: [{
$match: {
$expr: {
$and: [{
$eq: [
"$terminalID",
"$$terminalID"
]
},
{
$eq: [
"$cloudID",
"$$cloudID"
]
}
]
}
}
}],
as: "Devices"
}
},
{
$match: {
Devices: { $size: { $gt: 0 } }
}
}
])

Related

How to $lookup by avoiding null values in mongodb aggregate

In here i'm using $lookup to to a left join from other collections, the query works fine but when some records missing values it returns
errmsg : $in requires an array as a second argument, found: null
Heres the querying document structure :
{
"no" : "2020921008981",
"sale" : {
"soldItems" : [
{
"itemId" : "5b55ac7f0550de00210a3b24",
},
{
"itemId" : "5b55ac7f0550de00215584re",
}
],
"bills" : [
{
"billNo" : "2020921053467",
"insurancePlanId" : "160",
},
{
"billNo" : "2020921053467",
"insurancePlanId" : "170",
}
],
"visitIds" : [
5b55ac7f0550de00210a3b24, 5b55ac7f0550de00210a3b24
]
}
}
the query :
db.case.aggregate([
{
$lookup: {
from: "insurance",
let: { ipids: "$sale.bill.insurancePlanId" },
pipeline: [
{
$unwind: "$coveragePlans"
},
{
$match: { $expr: { $in: ["$coveragePlans._id", "$$ipids"] } }
},
{
$project: { _id: 0, name: 1 }
}
],
as: "insurances"
}
},
{
$lookup: {
from: "item",
let: { iid: "$salesOrder.purchaseItems.itemRefId" },
pipeline: [
{
$match: {
$expr: {
$in: ["$_id", {
$map: {
input: "$$iid",
in: { $toObjectId: "$$this" }
}
}
]
}
}
}
],
as: "items"
}
}
])
insurance collection :
{
"_id" : ObjectId("5b55aca20550de00210a6d25"),
"name" : "HIJKL"
"coveragePlans" : [
{
"_id" : "160",
"name" : "UVWZ",
},
{
"_id" : "161",
"name" : "LMNO",
}
]
},
{
"_id" : ObjectId("5b55aca20550de00210a6d25"),
"name" : "WXYZ"
"coveragePlans" : [
{
"_id" : "169",
"name" : "5ABC",
},
{
"_id" : "170",
"name" : "4XYZ",
}
]
}
item collection :
{
"_id" : ObjectId("5b55ac7f0550de00210a3b24"),
"code" : "ABCDE"
},
{
"_id" : ObjectId("5b55ac7f0550de00215584re"),
"code" : "PQRST"
}
How to avoid this and do null checks effectively before pipe-lining into the next stages? Tried with { $match: { "fieldName": { $exists: true, $ne: null } } } but it returns mongo error regarding the format. If its the way to go please mention the stage i should put that.. Thanks in advance
You can use $ifNull operator
let: { ipids: {$ifNull:["$sale.bill.insurancePlanId", [] ]} },
EDIT: To skip empty "$salesOrder.purchaseItems.itemRefId" values
let: { iid: {$filter: {input:"$salesOrder.purchaseItems.itemRefId", cond:{$ne:["$$this", ""]}}} },
You can get around that by not using $in.
It looks like this $map is executed separately for every document in the items collection. If you were to run the map in an $addFields stage, you could used the simple form of lookup to match the added field to _id, which would automagically handle missing, null, and array.
Remove the added field with a $project stage if necessary.
db.case.aggregate([
{$lookup: {
from: "insurance",
let: { ipids: "$sale.bill.insurancePlanId" },
pipeline: [
{$unwind: "$coveragePlans"},
{$match: { $expr: { $in: ["$coveragePlans._id", "$$ipids"] } }},
{$project: { _id: 0, name: 1 }}
],
as: "insurances"
}}
{$addFields:{
matchArray:{$map: {
input: "$$iid",
in: { $toObjectId: "$$this" }
}}
}},
{$lookup: {
from: "item",
localField: "matchArray",
foreignField:"_id",
as: "items"
}},
{$project:{
arrayField: 0
}}
])

How to avoid possible null error scenarios in mongodb Aggregate

I've set up a fairly long mongo aggregate query to join several mongo collections together and shape up them into output of set of string fields. The query works fine as long as all the required values (ie : ids) exists but it breaks when it encounters null or empty values when doing the $lookup.
Following is the patientFile collection thats being queried :
{
"no" : "2020921008981",
"startDateTime" : ISODate("2020-04-01T05:19:02.263+0000")
"saleId" : "5e8424464475140d19c6941b",
"patientId" : "5e8424464475140d1955941b"
}
sale collection :
{
"_id" : ObjectId("5e8424464475140d19c6941b"),
"invoices" : [
{
"billNumber" : "2020921053467",
"type" : "CREDIT",
"insurancePlanId" : "160"
},
{
"billNumber" : "2020921053469",
"type" : "DEBIT",
"insurancePlanId" : "161"
}
],
"status" : "COMPLETE"
}
insurance collection :
{
"_id" : ObjectId("5b55aca20550de00210a6d25"),
"name" : "HIJKL"
"plans" : [
{
"_id" : "160",
"name" : "UVWZ",
},
{
"_id" : "161",
"name" : "LMNO",
}
]
}
patient collection :
{
"_id" : ObjectId("5b55cc5c0550de00217ae0f3"),
"name" : "TAN NAI",
"userId" : {
"number" : "787333128H"
}
}
Heres the aggregate query :
db.getCollection("patientFile").aggregate([
{ $match: { "startDateTime": { $gte: ISODate("2020-01-01T00:00:00.000Z"),
$lt: ISODate("2020-05-01T00:00:00.000Z") } } },
{
$lookup:
{
from: "patient",
let: { pid: "$patientId" },
pipeline: [
{
$match: {
$expr: {
$eq: ["$_id", { $toObjectId: "$$pid" }]
}
}
},
{ "$project": { "name": 1, "userId.number": 1, "_id": 0 } }
],
as: "patient"
}
},
{
$lookup:
{
from: "sale",
let: { sid: "$saleId" },
pipeline: [
{
$match: {
$expr: {
$eq: ["$_id", { $toObjectId: "$$sid" }]
}
}
}
],
as: "sale"
}
},
{ $unwind: "$sale" },
{ $unwind: "$patient" },
{
$lookup: {
from: "insurance",
let: { pid: {$ifNull:["$sale.bill.insurancePlanId", [] ]} },
pipeline: [
{
$unwind: "$plans"
},
{
$match: { $expr: { $in: ["$plans._id", "$$pid"] } }
},
{
$project: { _id: 0, name: 1 }
}
],
as: "insurances"
}
},
{ $match: { "insurances.name": { $exists: true, $ne: null } } },
{
$addFields: {
invoice: {
$reduce: {
input: {$ifNull:["$sale.bill.billNumber", [] ]},
initialValue: "",
in: {
$cond: [{ "$eq": ["$$value", ""] }, "$$this", { $concat: ["$$value", "\n", "$$this"] }]
}
}
},
insurances: {
$reduce: {
input: {$ifNull:["$insurances.name", [] ]},
initialValue: "",
in: {
$cond: [{ "$eq": ["$$value", ""] }, "$$this", { $concat: ["$$value", "\n", "$$this"] }]
}
}
}
}
},
{
"$project": {
"startDateTime": 1,
"patientName": "$patient.name",
"invoice": 1,
"insurances": 1
}
}
],
{ allowDiskUse: true }
)
Error :
Unable to execute the selected commands
Mongo Server error (MongoCommandException): Command failed with error 241 (ConversionFailure): 'Failed to parse objectId '' in $convert with no onError value: Invalid string length for parsing to OID, expected 24 but found 0' on server localhost:27017.
The full response is:
{
"ok" : 0.0,
"errmsg" : "Failed to parse objectId '' in $convert with no onError value: Invalid string length for parsing to OID, expected 24 but found 0",
"code" : NumberInt(241),
"codeName" : "ConversionFailure"
}
As a solution i have found, used $ifNull but this error keeps coming. What would be the best step to take for this scenario?
I see a couple of ways:
Instead of converting the string value to an ObjectId to test, convert the ObjectId to a string
$match: {
$expr: {
$eq: [{$toString: "$_id"}, "$$pid" ]
}
}
Instead of the $toObjectId helper, use $convert and provide onError and/or onNull values:
$match: {
$expr: {
$eq: ["$_id", { $convert: {
input: "$$pid",
to: "objectId",
onError: {error:true},
onNull: {isnull:true}
}}]
}
}

$unwind, $aggregation manipulation in mongodb nodejs

please check this query
db.billsummaryofthedays.aggregate([
{
'$match': {
'userId': ObjectId('5e43de778b57693cd46859eb'),
'adminId': ObjectId('5e43e5cdc11f750864f46820'),
'date': ISODate("2020-02-11T16:30:00Z"),
}
},
{
$lookup:
{
from: "paymentreceivables",
let: { userId: '$userId', adminId: '$adminId' },
pipeline: [
{
$match:
{
paymentReceivedOnDate:ISODate("2020-02-11T16:30:00Z"),
$expr:
{
$and:
[
{ $eq: ["$userId", "$$userId"] },
{ $eq: ["$adminId", "$$adminId"] }
]
}
}
},
{ $project: { amount: 1, _id: 0 } }
],
as: "totalPayment"
}
}, {'$unwind':'$totalPayment'},
{ $group:
{ _id:
{ date: '$date',
userId: '$userId',
adminId: '$adminId' },
totalBill:
{
$sum: '$billOfTheDay'
},
totalPayment:
{
$sum: '$totalPayment.amount'
}
}
},
}
}])
this is the result i am getting in the shell
{
"_id" : {
"date" : ISODate("2020-02-11T18:30:00Z"),
"userId" : ObjectId("5e43de778b57693cd46859eb"),
"adminId" : ObjectId("5e43e5cdc11f750864f46820")
},
"totalBill" : 1595.6799999999998,
"totalPayments" : 100
}
now this is not what i expected,i assume due to {'$unwind':'$totalPayment'} it takes out all the values from the array and because of which every document is getting counted 2 times. When i remove {'$unwind':'$totalPayment'} then totalBill sum turns out to be correct but totalPayment is 0.
I have tried several other ways but not able to achieve the desired result
Below are my collections:
// collection:billsummaryofthedays//
{
"_id" : ObjectId("5e54f784f4032c1694535c0e"),
"userId" : ObjectId("5e43de778b57693cd46859eb"),
"adminId" : ObjectId("5e43e5cdc11f750864f46820"),
"date" : ISODate("2020-02-11T16:30:00Z"),
"UID":"acex01"
"billOfTheDay" : 468,
}
{
"_id" : ObjectId("5e54f784f4032c1694535c0f"),
"UID":"bdex02"
"userId" : ObjectId("5e43de778b57693cd46859eb"),
"adminId" : ObjectId("5e43e5cdc11f750864f46820"),
"date" : ISODate("2020-02-11T16:30:00Z"),
"billOfTheDay" : 329.84,
}
// collection:paymentreceivables//
{
"_id" : ObjectId("5e43e73169fe1e3fc07eb7c5"),
"paymentReceivedOnDate" : ISODate("2020-02-11T16:30:00Z"),
"adminId" : ObjectId("5e43e5cdc11f750864f46820"),
"userId" : ObjectId("5e43de778b57693cd46859eb"),
"amount" : 20,
}
{
"_id" : ObjectId("5e43e73b69fe1e3fc07eb7c6"),
"paymentReceivedOnDate" : ISODate("2020-02-11T16:30:00Z"),
"adminId" : ObjectId("5e43e5cdc11f750864f46820"),
"userId" : ObjectId("5e43de778b57693cd46859eb"),
"amount" : 30,
}
desired result should be totalBill:797.83 i.e[468+329.84,] and totalPayment:50 i.e[30+20,] but i am getting double the expected result and even if i am able to calculate one of the value correctly the other one result 0.How to tackle this??
Since you've multiple documents with same data in billsummaryofthedays collection then you can group first & then do $lookup - that way JOIN between two collections would be 1-Vs-many rather than many-Vs-many as like it's currently written, So you can try below query for desired o/p & performance gains :
db.billsummaryofthedays.aggregate([
{
"$match": {
"userId": ObjectId("5e43de778b57693cd46859eb"),
"adminId": ObjectId("5e43e5cdc11f750864f46820"),
"date": ISODate("2020-02-11T16:30:00Z"),
}
},
{
$group: {
_id: {
date: "$date",
userId: "$userId",
adminId: "$adminId"
},
totalBill: {
$sum: "$billOfTheDay"
}
}
},
{
$lookup: {
from: "paymentreceivables",
let: {
userId: "$_id.userId",
adminId: "$_id.adminId"
},
pipeline: [
{
$match: {
paymentReceivedOnDate: ISODate("2020-02-11T16:30:00Z"),
$expr: {
$and: [
{
$eq: [
"$userId",
"$$userId"
]
},
{
$eq: [
"$adminId",
"$$adminId"
]
}
]
}
}
},
{
$project: {
amount: 1,
_id: 0
}
}
],
as: "totalPayment"
}
},
{
$addFields: {
totalPayment: {
$reduce: {
input: "$totalPayment",
initialValue: 0,
in: {
$add: [
"$$value",
"$$this.amount"
]
}
}
}
}
}
])
Test : MongoDB-Playground

Conditional lookup for nested localfield

I want to get user's friends and their information. But only wanna join for users where "status" equals to 2.
Here is my object
{
"user_id" : ObjectId("5d2f574b1c27807fd7eb133d"),
"relationship" : [
{
"user_id" : ObjectId("5d2f57661c27807fd7eb133e"),
"status" : 2
},
{
"user_id" : ObjectId("5d2f57c01c27807fd7eb133f"),
"status" : 1
}
]
}
My query
db.getCollection('relationships').aggregate([
{$match: {
user_id: ObjectId("5d2f574b1c27807fd7eb133d")}
},
{ $lookup: {
from: "users",
as: "users",
let: { id: "$_id" },
pipeline: [
{ $match: {
$expr: { $and: [
{ $eq: [ "$relationship.user_id", "$$id" ] },
{ $eq: [ "$relationship.status", 2 ] }
] }
} }
],
} }
])
expected output
{
"user_id" : ObjectId("5d2f574b1c27807fd7eb133d"),
"friends" : [
{
"_id" : ObjectId("5d2f57661c27807fd7eb133e"),
"name" : "Mike"
}
]
}
What's the proper way to do this ?

mongodb aggregation lookup with multiple conditions and ids

Having the following collections and data on them
db.a.insert([
{ "_id" : ObjectId("5b56989172ebcb00105e8f41"), "items" : [{id:ObjectId("5b56989172ebcb00105e8f41"), "instock" : 120}]},
{ "_id" : ObjectId("5b56989172ebcb00105e8f42"), "items" : [{id:ObjectId("5b56989172ebcb00105e8f42"), "instock" : 120}] },
{ "_id" : ObjectId("5b56989172ebcb00105e8f43"), "items" : [{ObjectId("5b56989172ebcb00105e8f43"), "instock" : 80}] }
])
db.b.insert([
{ "_id" : ObjectId("5b56989172ebcb00105e8f41")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f42")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f43")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f44")},
{ "_id" : ObjectId("5b56989172ebcb00105e8f45")}
])
executing an lookup aggregation like
db.b.aggregate([
{
$lookup:
{
from: "b",
let: { bId: "$_id", qty: 100 },
pipeline: [
{ $match:
{ $expr:
{ $and:
[
{ $eq: [ "$items.id", "$$bId" ] },
{ $gte: [ "$instock", "$$qty" ] }
]
}
}
}
],
as: "a"
}
}
])
does not bring any results in the expected lookup operation. Is there any restriction to use ObjectId as a comparison? In the official documentations does not say any about it and it works like a charm with any other kind of data type, like strings
I am not sure if this is a bug in mongodb or not but the query only works after adding an $unwind stage first.
db.b.aggregate([
{
$lookup:
{
from: "a",
let: { bId: "$_id", qty: 100 },
pipeline: [
{
$unwind: {
path: "$items"
}
},
{ $match:
{ $expr:
{ $and:
[
{ $eq: [ "$items.id", "$$bId" ] },
{ $gte: [ "$items.instock", "$$qty" ] },
]
}
}
}
],
as: "a"
}
}
]);
Note: Join Conditions and Uncorrelated Sub-queries were added in mongo 3.6