Related
I am writing this aggregation function to get distinct data according to post_id and doing lookup for the same post_id. But I am not getting null value if the post_id does not match in campusposts collection. How do I get the match data and also null if not matched?
return await complaint.aggregate([
{
$group: {
_id: "$post_id",
count: { $sum: 1 },
post_id: { $first: "$post_id" },
complaint_against: { $first: "$complaint_against" },
}
},
{
$lookup: {
from: "campusposts",
localField: "_id",
foreignField: "_id",
as: "post"
}
},
{
$unwind: "$post"
},
{
// also show null values
$match: {
post: { $ne: null }
}
},
{
$lookup: {
from: "users",
localField: "complaint_against",
foreignField: "_id",
as: "complaint_against"
}
},
{
$unwind: "$complaint_against"
},
{
$project: {
_id: 0,
post: {
_id: 1,
title: 1,
createdAt: 1,
updatedAt: 1,
},
count: 1,
complaint_against: {
_id: 1,
name: 1,
email: 1,
roleType: 1,
number: 1,
}
}
}
])
I have 3 collections to aggregate.
1st is colors collection
{
{
_id: 1, <- mongoose objectId
name: red
},
{
_id: 2, <- mongoose objectId
name: green
}
}
2nd is products
{
{
_id: Id777, <- mongoose objectId
productName: test prod 777
},
{
_id: Id888, <- mongoose objectId
productName: test prod 888
}
}
and 3rd it move collection
{
....other fields here
items: [
{
_id: an mongoose id,
itemId: Id777 <- in products collection,
itemColor: 1 <- id in colors collection,
coutn: 7,
....other fields
},
{
_id: an mongoose id,
itemId: Id888 <- in products collection,
itemColor: 2 <- id in colors collection
cout: 10
....other fields
}
]
}
I need to have an output like this:
{
////information from collection
items: [
{
itemId: test prod 777, itemColor: red, count: 7
},
{
itemId: test prod 888, itemColor: green, count: 10
}
]
}
My code is:
const moves = await ProductMoves.aggregate([
{ $match: query }, // this is my query
{
$lookup: {
from: 'products',
localField: 'items.itemId',
foreignField: '_id',
as: 'productName'
}
},
{
$unwind: { path: "$productName" , preserveNullAndEmptyArrays: true }
},
{
$lookup: {
from: 'colors',
localField: 'items.itemColor',
foreignField: '_id',
as: 'cName'
}
},
{
$unwind: { path: "$cName" , preserveNullAndEmptyArrays: true }
},
{
$addFields: {
mItems: {
prName: "$productName.productName",
prColor: "$cName.colorName"
},
productName: 0,
cName: 0
}
}
])
.sort({addedDate: -1})
.skip(+req.query.offset)
.limit(+req.query.limit)
but it returns only 1 element from the object array. probably I need something like a for loop, but i couldn't do it.
thank you for your responses, and have a good day!
$unwind deconstruct items array
$lookup with products collection
$lookup with colors collection
$addFields, $arrayElemAt to get first element from lookup result
$group by _id and reconstruct items array and pass other fields as well
there is no external methods in an aggregate function, you have to use stages for sort, skip and limit like below
$sort by addedDate in descending order
$skip and $limit result
const moves = await ProductMoves.aggregate([
{ $match: query }, // this is my query
{ $unwind: "$items" },
{
$lookup: {
from: "products",
localField: "items.itemId",
foreignField: "_id",
as: "itemId"
}
},
{
$lookup: {
from: "colors",
localField: "items.itemColor",
foreignField: "_id",
as: "itemColor"
}
},
{
$addFields: {
"items.itemId": { $arrayElemAt: ["$itemId.productName", 0] },
"items.itemColor": { $arrayElemAt: ["$itemColor.name", 0] }
}
},
{
$group: {
_id: "$_id",
items: { $push: "$items" },
addedDate: { $first: "$addedDate" }
// add other fields that you want in result like "addedDate"
}
},
{ $sort: { addedDate: -1 } },
{ $skip: +req.query.offset },
{ $limit: +req.query.limit }
])
Playground
I have a MongodDB database on MongoDB Atlas.
It has an "orders", "products", "itemTypes" and "brands".
"orders" only keep track of product id ordered.
"products" only keep track of brand id and itemType id
"itemTypes" keep track of item type name
"brands" keep track of brand name.
If I aggregate orders + products + itemTypes it is ok:
[{
$unwind: {
path: '$orders'
}
}, {
$lookup: {
from: 'products',
localField: 'orders.productId',
foreignField: 'productId',
as: 'products'
}
}, {
$lookup: {
from: 'itemTypes',
localField: 'products.typeId',
foreignField: 'typeId',
as: 'itemTypes'
}
}, {
$set: {
'orders.price': {
$arrayElemAt: ['$products.price', 0]
},
'orders.brandId': {
$arrayElemAt: ['$products.brandId', 0]
},
'orders.typeId': {
$arrayElemAt: ['$products.typeId', 0]
},
'orders.typeName': {
$arrayElemAt: ['$itemTypes.name', 0]
}
}
}, {
$group: {
_id: '$_id',
createdAt: {
$first: '$createdAt'
},
status: {
$first: '$status'
},
retailerId: {
$first: '$retailerId'
},
retailerName: {
$first: '$retailerName'
},
orderId: {
$first: '$orderId'
},
orders: {
$push: '$orders'
}
}
}]
If I aggregate orders + products + itemTypes + brands, either Mongo Compass or the web UI of Mongo Atlas aggregation builder will give operation timeout error.
[{
$unwind: {
path: '$orders'
}
}, {
$lookup: {
from: 'products',
localField: 'orders.productId',
foreignField: 'productId',
as: 'products'
}
}, {
$lookup: {
from: 'itemTypes',
localField: 'products.typeId',
foreignField: 'typeId',
as: 'itemTypes'
}
}, {
$lookup: {
from: 'brands',
localField: 'products.brandId',
foreignField: 'brandId',
as: 'brands'
}
}, {
$set: {
'orders.price': {
$arrayElemAt: ['$products.price', 0]
},
'orders.brandId': {
$arrayElemAt: ['$products.brandId', 0]
},
'orders.typeId': {
$arrayElemAt: ['$products.typeId', 0]
},
'orders.typeName': {
$arrayElemAt: ['$itemTypes.name', 0]
},
'orders.brandName': {
$arrayElemAt: ['$brands.name', 0]
}
}
}, {
$group: {
_id: '$_id',
createdAt: {
$first: '$createdAt'
},
status: {
$first: '$status'
},
retailerId: {
$first: '$retailerId'
},
retailerName: {
$first: '$retailerName'
},
orderId: {
$first: '$orderId'
},
orders: {
$push: '$orders'
}
}
}]
This is a demo of the aggregation that timed out:
https://mongoplayground.net/p/Jj6EhSl58MS
We have approximately 50k orders, 14k products, 200 brands, 89 item types.
Is there anyway to optimise this aggregation so that it won't timeout?
P/s: My ultimate goal is to visualise popular brands and item types ordered using beautiful chart in the Mongodb Charts function.
If you are on Mongo Atlas, you can use Triggers to run the aggregation query in the background - either when the database is updated or as a scheduled trigger (https://docs.mongodb.com/realm/triggers/).
When the trigger runs, you can save the result of the aggregation pipeline in a new collection using the "$merge" operation.
exports = function() {
const mongodb = context.services.get(CLUSTER_NAME);
const orders = mongodb.db(DATABASE_NAME).collection("orders");
const ordersSummary = mongodb.db(DATABASE_NAME).collection("orders.summary");
const pipeline = [
{
YOUR_PIPELINE
},
{ $merge: { into: "orders.summary", on: "_id", whenMatched: "replace", whenNotMatched: "insert" } }
];
orders.aggregate(pipeline);
};
This way, your charts will be very fast, since they only have to do a simple query from the new collection.
Do you have index on the collections you $lookup from:
products (productId) + itemTypes (typeId) + brands (brandId).
Otherwise, the lookups can take a long time to complete.
I have the following code:
const profiles = await Profile.aggregate([
{
$lookup: {
from: "users",
localField: "user",
foreignField: "_id",
as: "user",
},
},
{
$unwind: "$user",
},
{
$match: {
"user.name": {
$regex: q.trim(),
$options: "i",
},
},
},
{
$skip: req.params.page ? (req.params.page - 1) * 10 : 0,
},
{
$limit: 11,
},
{
$group: {
_id: "$_id",
skills:{skills}
user: { name: "$name" },
user: { avatar: "$avatar" },
},
},
]);
I want to return only specific fields like skills _id and user.name and user.avatar, but this doesn't work. I'm pretty sure that the problem is in $group. I want to receive only these fields
[
{
_id: 5ef78d005d23020ca847aa76,
skills: [ 'asd' ],
user: {
_id: 5ef78c7c5d23020ca847aa75,
name: 'Simeon Lazarov',
avatar: 'uploads\\1593286096227 - background.jpg',
}
}
]
You can make use of $project to get specific fields.
After grouping add the below:
{
$project: {_id:1, skills:1, user:1}
}
Projection value of 0 means that the field needs to be excluded, Value 1 represents inclusion of the field.
Document reference: https://docs.mongodb.com/manual/reference/operator/aggregation/project/
For some reason, I can't retrieve the author name from another collection on my aggregate query.
db.getCollection('books').aggregate([
{
$match: {
authorId: { $nin: [ObjectId('5b9a008575c50f1e6b02b27b'), ObjectId('5ba0fb3275c50f1e6b02b2f5'), ObjectId('5bc058b6ae9a2a4d6df330b1')]},
isBorrowed: { $in: [null, false] },
status: 'ACTIVE',
},
},
{
$lookup: {
from: "authors",
localField: "authorId", // key of author id in "books" collection
foreignField: "_id", // key of author id in "authors" collection
as: "bookAuthor",
}
},
{
$group: {
_id: {
author: '$authorId',
},
totalSalePrice: {
$sum: '$sale.amount',
},
},
},
{
$project: {
author: '$_id.author',
totalSalePrice: '$totalSalePrice',
authorName: '$bookAuthor.name', // I can't make this appear
_id: 0,
},
},
{ $sort: { totalSalePrice: -1 } },
])
Any advice on where I had it wrong? Thanks for the help.
Two things that are missing here: you need $unwind to convert bookAuthor from an array into single object and then you need to add that object to your $group stage (so that it will be available in next stages), try:
db.getCollection('books').aggregate([
{
$match: {
authorId: { $nin: [ObjectId('5b9a008575c50f1e6b02b27b'), ObjectId('5ba0fb3275c50f1e6b02b2f5'), ObjectId('5bc058b6ae9a2a4d6df330b1')]},
isBorrowed: { $in: [null, false] },
status: 'ACTIVE',
},
},
{
$lookup: {
from: "authors",
localField: "authorId",
foreignField: "_id",
as: "bookAuthor", // this will be an array
}
},
{
$unwind: "$bookAuthor"
},
{
$group: {
_id: {
author: '$authorId',
},
bookAuthor: { $first: "$bookAuthor" },
totalSalePrice: {
$sum: '$sale.amount',
},
},
},
{
$project: {
author: '$_id.author',
totalSalePrice: '$totalSalePrice',
authorName: '$bookAuthor.name',
_id: 0,
},
},
{ $sort: { totalSalePrice: -1 } },
])
Actually you have lost the bookAuthor field in the $group stage. You have to use $first accumulator to get it in the next $project stage.
{ "$group": {
"_id": { "author": "$authorId" },
"totalSalePrice": { "$sum": "$sale.amount" },
"authorName": { "$first": "$bookAuthor" }
}},
{ "$project": {
"author": "$_id.author",
"totalSalePrice": "$totalSalePrice",
"authorName": { "$arrayElemAt": ["$bookAuthor.name", 0] }
"_id": 0,
}}