mongodb aggregation conditional $lookup pipeline performance - mongodb
i made aggregation query like below. this query works great and generate data that i want. what i want to ask is aggregation like this is good in performace especially using pipeline in conditional $lookup like this?
ReactiveAggregate(this, Product, [
{$match: {slug}},
{$lookup: {
from: 'suppliers',
let: {supplierId: '$supplier_id'},
pipeline: [
{
$match: {
$expr: {
$eq: ['$_id', '$$supplierId']
}
}
},
{
$lookup: {
from: 'addresses',
let: {supplierAddress: '$address'},
pipeline: [
{
$match: {
$expr: {
$eq: ['$_id', '$$supplierAddress']
}
}
}
],
as: 'address'
}
},
{$unwind: {path: '$address', preserveNullAndEmptyArrays: false}},
{
$lookup: {
from: 'supplier_assets',
let: {supplierId: '$_id'},
pipeline: [
{
$match: {
$expr: {
$and: [
{$eq: ['$meta.supplier_id', '$$supplierId']},
{$eq: ['$meta.type', 'logo']}
]
}
}
}
],
as: 'logo'
}
},
{$unwind: {path: '$logo', preserveNullAndEmptyArrays: false}},
{
$lookup: {
from: 'products',
let: {supplierId: '$_id'},
pipeline: [
{
$match: {
$expr: {
$and: [
{$eq: ['$supplier_id', '$$supplierId']},
{$eq: ['$isApproved', true]}
]
}
},
},
{
$limit: 5
},
{$project: {
name: 1,
slug: 1,
supplier_id: 1,
variantMaster: {
$filter: {
input: '$variants',
as: 'variantMaster',
cond: {$eq: ['$$variantMaster.is_master', true]}
}
}
}
},
{$unwind: {path: '$variantMaster', preserveNullAndEmptyArrays: true}},
{
$lookup: {
from: 'product_assets',
let: {variantMasterId: '$variantMaster._id'},
pipeline: [
{
$match: {
$expr: {
$or: [
{
$and: [
{$eq: ['$meta.variant_id', '$$variantMasterId']},
{$eq: ['$meta.is_primary', true]}
]
},
{
$eq: ['$meta.variant_id', '$$variantMasterId'],
}
]
}
}
},
],
as: 'images'
}
},
{$lookup: {
from: 'product_sale',
let: {productId: '$_id'},
pipeline: [
{
$match: {
$expr: {
$and: [
{$eq: ['$product_id', '$$productId']},
{$gt: ['$end_date', new Date()]},
{$lte: ['$start_date', new Date()]},
{$eq: ['$active', true]},
{$eq: ['$approved', true]},
]
}
}
},
],
as: 'sale'
}},
{
$project:{
name: 1,
slug: 1,
supplierName: '$supplier.name',
price: '$variantMaster.price',
sale: {$arrayElemAt: ['$sale', 0]},
image: {$arrayElemAt: ['$images', 0]}
}
}
],
as: 'relatedProducts'
}
},
{
$project: {
name: 1,
url: 1,
relatedProducts: 1,
logo: 1,
city: '$address.city',
state: '$address.state',
}
}
],
as: 'supplier'
}},
{
$unwind: '$supplier'
},
{$project: {
name: 1,
slug: 1,
supplier: 1,
variants: 1,
description: 1,
variantMaster: {
$filter: {
input: '$variants',
as: 'variantMaster',
cond: {$eq: ['$$variantMaster.is_master', true]}
}
}
}
},
{
$unwind: '$variantMaster'
},
{
$lookup: {
from: 'product_assets',
localField: 'variantMaster._id',
foreignField: 'meta.variant_id',
as: 'images'
}
},
{
$lookup: {
from: 'product_sale',
let: {productId: '$_id'},
pipeline: [
{
$match: {
$expr: {
$and: [
{$eq: ['$product_id', '$$productId']},
{$gt: ['$end_date', new Date()]},
{$lte: ['$start_date', new Date()]},
{$eq: ['$active', true]},
{$eq: ['$approved', true]},
]
}
}
},
],
as: 'sale'
},
},
{
$unwind: {path: '$sale', preserveNullAndEmptyArrays: true}
},
{
$lookup: {
from: 'stock',
localField: 'variantMaster.stock',
foreignField: '_id',
as: 'stock'
}
},
{
$unwind: {path: '$stock', preserveNullAndEmptyArrays: true}
},
{
$lookup: {
from: 'product_review',
let: {productId: '$_id'},
pipeline: [
{
$match: {
$expr: {$eq: ['$product_id', '$$productId']}
},
},
{
$lookup: {
from: 'user_data',
let: {reviewProfileId: '$user_profile_id'},
pipeline: [
{
$match: {
$expr: {$eq: ['$_id', '$$reviewProfileId']}
}
},
{
$lookup: {
from: 'avatar',
let: {profileId: '$_id'},
pipeline: [
{
$match: {
$expr: {$eq: ['$meta.profile_id', '$$profileId']}
}
}
],
as: 'avatar'
}
},
{
$project: {
user_id: 1,
avatar: {$arrayElemAt: ['$avatar', 0]},
}
}
],
as: 'user'
}
},
{$unwind: {path: '$user', preserveNullAndEmptyArrays: true}},
{
$project: {
user: 1,
rating: 1,
review: 1,
user_fullname: 1,
postedAt: 1,
owner_reply: 1,
repliedAt: 1,
}
}
],
as: 'reviews'
}
},
{
$lookup: {
from: 'favorited_products',
let: {productId: '$_id'},
pipeline: [
{
$match: {
$expr: {
$and: [
{$eq: ['$product_id', '$$productId']},
{$eq: ['$user_id', this.userId]}
]
}
}
}
],
as: 'favorite'
}
},
{
$unwind: {path: '$favorite', preserveNullAndEmptyArrays: true}
},
{
$project: {
name: 1,
images: 1,
description: 1,
price: '$variantMaster.price',
variantMaster: 1,
supplier: 1,
is_ready_stock: 1,
is_custom: 1,
is_culinary: 1,
slug: 1,
reviews: 1,
sale: 1,
variants: 1,
stock: 1,
favorite: 1,
}
}
])
here is the log i got from mongo shell
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 5,
"totalKeysExamined" : 1,
"totalDocsExamined" : 1,
// rest..
sure it's fast because only 1 data was returned, but what if 100 or 1000 data. I'm still learning mongodb. please share pros and cons of doing aggregation query like this in mongodb. what is good approach when making aggregation query? i know i can refer to this page for optimize aggregation pipeline
Thanks.
Related
MongoDB : Add match to nested pipeline lookup
I have this BD : db={ "dashboard": [ { "_id": "dashboard1", "name": "test", "user": 1 } ], "templatefolders": [ { "dashboardId": "dashboard1", "folderId": "folder123", "name": "folder", "region": "XXX" }, { "dashboardId": "dashboard1", "folderId": "folder123", "name": "folder1", "region": "YYY" } ], "folders": [ { "_id": "folder123", "name": "test1" } ] } I have this query : db.dashboard.aggregate([ { $lookup: { from: "templatefolders", let: { "boardId": "$_id" }, pipeline: [ { $match: { $expr: { $eq: [ "$dashboardId", "$$boardId" ] } } }, { $lookup: { from: "folders", let: { "folderId": "$folderId" }, pipeline: [ { $match: { $expr: { $eq: [ "$_id", "$$folderId" ] } } }, ], as: "joinFolder" }, }, ], as: "joinDashboard" } } ]) I want to fetch all dashbords where dashbord.templateFolder.name=="folder" OR dashbord.templateFolder.folders.name=="test1" Please, how i can achieve this ? How i can add many conditions in the nested pipeline lookup, where one condition is related to another condition.
One option is use $lookup and then $match. In your case: db.templatefolders.aggregate([ {$lookup: { from: "folders", let: {"folderId": "$folderId"}, pipeline: [ {$match: { $expr: {$and: [ {$eq: ["$_id", "$$folderId"]}, {$eq: ["$name", "test1"]} ]} }} ], as: "folder" } }, {$match: {$or: [{"folder.0": {$exists: true}}, {name: "folder"}]}}, {$lookup: { from: "dashboard", as: "dashboard", localField: "dashboardId", foreignField: "_id" }}, {$project: {_id: 0, dashboard: {$first: "$dashboard"}}}, {$replaceRoot: {newRoot: "$dashboard"}} ]) See how it works on the playground example You did not define your requested result, but you can use group if you have duplicates and you want to remove them.
Mongo db - How to get two counts from same collection with a single lookup?
I have a working query below: { $lookup: { from: "watchList", as: "watchList", let: { postId: "$_id", userId: userCurrent }, pipeline: [ { $match: { $and: [ { $expr: {$eq: ["$postId","$$postId"]} }, { $expr: {$eq: ["$$userId", "$user"]} }, ] } }, { $count: "watchCount" }, ] } }, The above query gives me a count of two conditions combined in AND: { $expr: {$eq: ["$postId","$$postId"]} }, { $expr: {$eq: ["$$userId", "$user"]} }, In addition, is it possible to get the count of ONLY { $expr: {$eq: ["$postId","$$postId"]} } without doing another lookup ? So there will be two counts - Count 1: AND condition for { $expr: {$eq: ["$postId","$$postId"]} }, { $expr: {$eq: ["$$userId", "$user"]} }, Count 2: Single condition { $expr: {$eq: ["$postId","$$postId"]} }
You can do something like this, $group first match count to firstCount and second match count to secondCount { $lookup: { from: "watchList", as: "watchList", let: { postId: "$_id", userId: userCurrent }, pipeline: [ { $group: { _id: null, firstCount: { $sum: { $cond: [ { $and: [ { $eq: ["$postId", "$$postId"] }, { $eq: ["$$userId", "$user"] } ] }, 1, 0 ] } }, secondCount: { $sum: { $cond: [{ $eq: ["$postId", "$$postId"] }, 1, 0] } } } } ] } }
how to reduce unnecessary unwind stages from aggregation pipeline
Like if i'm applying many lookup stages in aggregation pipeline and each lookup is followed by an unwind(just to covert into object) first question does it affect query performance? and if yes how to do that in optimised manner Note: all lookup's will return only one object For Ex: xyz.aggregate([ { $lookup:{ ----}} //first lookup {$unwind :{----}} //first unwind { $lookup:{ ----}} //second lookup {$unwind :{----}} //second unwind { $lookup:{ ----}} //third lookup {$unwind :{----}} //third unwind { $lookup:{ ----}} //fourth lookup {$unwind :{----}} //fourth unwind ])
In reference to comments, here is advanced $lookup: $lookup: { from: 'accounts', let: { "localAccountField": "$account" }, pipeline: [ { $match: { $expr: { $eq: ["$_id", "$$localAccountField"] } } }, { $project: { _id: 1, user: 1 } }, { $lookup: { from: 'users', let: { 'localUserField': "$user" }, pipeline: [ { $match: { $expr: { $eq: ["$_id", "$$localUserField"] } } }, { $project: { _id: 1, username: "$uid", phone:"$phoneNumber", email: "$email.add", name: { $concat: [ "$profile.name.first", ' ', "$profile.name.last" ] }, } } ], as: "users" } }, { $lookup: { from: 'documents', let: { 'localDocumentField': "$user" }, pipeline: [ { $match: { $expr: { $eq: ["$user", "$$localDocumentField"] }, status:"verified", "properties.expirydate": { $exists: true, $ne: "" }, name: "idcard" } }, { $project: { _id: 0, cnic: "$properties.number" } } ], as: "documents" } } ], as: 'account' }
Mongodb if then condition under filter how to make
db.main.aggregate([ { $lookup: { from: "history", localField: "history_id", foreignField: "history_id", as: "History" } }, { $project: { "History": { $filter: { input: "$History", as: "his", if: { $eq: [ "5e4a8d2d3952132a08ae5764", "$$his.user_id" ] }, then: { cond: { $and: [ { $lt: [ "$$his.date", "$date" ] }, { $eq: [ "5e4a8d2d3952132a08ae5764", "$$his.user_id" ] } ] } }, else: {} } }, data: 1, history_id: 1, sender_id: 1, text: 1, date: 1 } }, { $unwind: "$History" } ]) MongoPlayground Play with if condition under filter getting error. My purpose is specified user_id match with history's user_id then condition work other wise not. How to fix it please guide or alternative approaches are welcome.
After discussion in chat, it seems the overall problem was how to select documents from the main collection based on 2 criteria of the related history documents: db.main.aggregate([ {$lookup: { from: "history", localField: "history_id", foreignField: "history_id", as: "History" }}, {$match: { $expr: { $eq: [ false, {$reduce: { input: "$History", initialValue: false, in: { $or: [ "$$value", {$and: [ {$eq: [ "$$this.user_id", ObjectId("5e4a8d2d3952132a08ae5764") ]}, {$gte: [ "$$this.date", "$date" ]} ]} ] } }} ] } }} ]) Playground
Full Outer join in MongoDB
I want to do a Full Outer Join in MongoDB by lookup mongoDB query. Is this possible? Is a Full Outer Join supported by MongoDB by any other alternative? [Update:] I want to achieve result from Collection1 & Collection2 as following attachment: Example: Result Required In above result column there may be different arithmetic operations and will be further used in calculations.
You can use $unionWith (starting 4.4) Something like this: db.c1.aggregate([ {$set: { mark1: "$marks" }}, {$unionWith: { coll: 'c2', pipeline: [{$set: {mark2: "$marks"}}] }}, {$group: { _id: "$name", result: { $sum: "$marks" }, mark1: {$first: {$ifNull: ["$mark1", 0]}}, mark2: {$first: {$ifNull: ["$mark2", 0]}} }}])
I have named the collections as coll1 and coll2 then just use this query it will give you the required output. db.getCollection('coll1').aggregate([ { $facet: { commonRecords: [{ $lookup: { from: "coll2", localField: 'name', foreignField: 'name', as: "coll2" } }, { $unwind: { path: '$coll2', preserveNullAndEmptyArrays: true } } ] } }, { $lookup: { from: "coll2", let: { names: { $map: { input: '$commonRecords', as: 'commonRecord', in: '$$commonRecord.name' } } }, pipeline: [{ $match: { $expr: { $eq: [{ $indexOfArray: ['$$names', '$name'] }, -1] } } }, ], as: "coll2" } }, { $addFields: { coll2: { $map: { input: '$coll2', as: 'doc', in: { coll2: '$$doc' } } } } }, { $project: { records: { $concatArrays: ['$commonRecords', '$coll2'] } } }, { $unwind: '$records' }, { $replaceRoot: { newRoot: '$records' } }, { $project: { _id: 0, name: { $ifNull: ['$name', '$coll2.name'] }, marks1: { $ifNull: ['$marks', 0] }, marks2: { $ifNull: ['$coll2.marks', 0] } } }, { $addFields: { result: { $add: ['$marks1', '$marks2'] } } } ])
This is a sample: { $lookup: { from: [collection to join], local_Field: [field from the input documents], foreign_Field: [field from the documents of the "from" collection], as: [output field] } } show this link