I have two collections one is bids and another one is auctions. I am able to get bid inside customer wise count bids collection inside almost one million records. and auctions collection have 500k records. I need this same result as quick fetching in mongodb.
this is getting almost 29 seconds for response time. but i need quick time to get response
{ $match: { customer: '00000000823026' } },
{ $group: {
_id: '$auctioncode',
data: {
$last: '$$ROOT'
}
}
},
{
$lookup: {
from: 'auctions',
let: { auctioncode: '$_id' },
pipeline: [
{
$match: {
$expr: {
$and: [{ $eq: ['$_id', '$$auctioncode'] }],
},
},
},
],
as: 'auction',
},
},
{ $match: { auction: { $exists: true, $not: { $size: 0 } } } },
{
$addFields: {
_id: '$data._id',
auctioncode: '$data.auctioncode',
amount: '$data.amount',
customer: '$data.customer',
customerName: '$data.customerName',
maxBid: '$data.maxBid',
stockcode: '$data.stockcode',
watchlistHidden: '$data.watchlistHidden',
winner: '$data.winner'
}
},
{
$match: {
$and: [
{
// to get only RUNNING auctions in bid history
'auction.status': { $ne: 'RUNNING'},
// to filter auctions based on status
// tslint:disable-next-line:max-line-length
},
],
},
},
{ $sort: { 'auction.enddate': -1 } },
{ $count: 'totalCount'}
current result is totalCount 2640
how to optimize and need to find a way to performance changes in mongodb
If all that you require is the count of results, the below code is more optimized.
Index customer key for even better execution time.
Note: You can make use of pipeline method of $lookup if you are using MongoDB version >= 5.0 as it makes use if indexes unlike the lower version.
db.collection.aggregate([
{
$match: {
customer: '00000000823026'
}
},
{
$group: {
_id: '$auctioncode',
data: {
$last: '$$ROOT'
}
}
},
{
$lookup: {
from: 'auctions',
localField: '_id',
foreignField: '_id',
as: 'auction',
},
},
{
$match: {
// auction: {$ne: []},
// to get only RUNNING auctions in bid history
'auction.status': { $ne: 'RUNNING'},
// to filter auctions based on status
// tslint:disable-next-line:max-line-length
// {
// $addFields: { <-- Not Required
// _id: '$data._id',
// auctioncode: '$data.auctioncode',
// amount: '$data.amount',
// customer: '$data.customer',
// customerName: '$data.customerName',
// maxBid: '$data.maxBid',
// stockcode: '$data.stockcode',
// watchlistHidden: '$data.watchlistHidden',
// winner: '$data.winner'
// }
// },
// { $sort: { 'auction.enddate': -1 } }, <-- Not Required
{ $count: 'totalCount'}
], {
allowDiskUse: true
})
Related
I am quite new to MongoDB. And I have a Mongo aggregate query that takes 5 minutes on 10k+ documents. That doesn't seem right. Basically Products and SKUs have a 1-to-many relationship and I want to find all products with SKUs greater than 0.00001
products = await this.productModel.aggregate([
{
$lookup: {
from: 'skus',
let: { myid: '$_id' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$product', '$$myid'] },
{ $gt: ['$quantity', 0.00001] },
],
},
},
},
],
as: 'skus',
},
},
{
$addFields: {
id: '$_id',
skusize: { $size: '$skus' },
},
},
{
$match: { skusize: { $gt: 0 } },
},
]);
I have College model which has ref :"University". each college has 1 university when the college is created.
so I'm trying to get all the colleges where the uuid of the university matches the request query.
So i tried the following :
const myAggregate = College.aggregate([
{
$match: {
language: req.query.language,
display_language: req.query.display_language
}
},
{
$lookup: {
from: 'unis', // UNIVERSITY COLLECTION
pipeline: [
{
$match: {
$expr: {
$eq: ["$uuid", req.query.id]
}
}
}
],
as: 'univ',
},
},
{
$sort: { createdAt: -1 },
},
])
so the above code when the query is provided and it does not equal the university UUID , it fetches all the colleges but the univ array is empty of each college.
is it possible to only show the colleges when the uuid equals the query id and not show an empty array of univ ?
You could do unwind to remove empty ones.
Doc reference:https://docs.mongodb.com/manual/reference/operator/aggregation/unwind/
const myAggregate = College.aggregate([
{
$match: {
language: req.query.language,
display_language: req.query.display_language
}
},
{
$lookup: {
from: 'unis', // UNIVERSITY COLLECTION
pipeline: [
{
$match: {
$expr: {
$eq: ["$uuid", req.query.id]
}
}
}
],
as: 'univ',
},
},
{"$unwind":{
path: '$univ',
preserveNullAndEmptyArrays: false}},
{
$sort: { createdAt: -1 },
},
])
Right now I have a lookup stage that matches a collection of answers with their respective questions. The problem I'm having is that on the $match if the question does not having a matching answer document in the lookup collection it is removed from the results of the aggregation. How can I avoid this?
{
$lookup: {
from: 'groupansphotos',
let: { question_id: '$questions._id' },
pipeline: [
{
$match: {
$expr: { $eq: ['$_id', '$$question_id'] },
},
},
{ $unwind: '$answers' },
{ $match: { 'answers.reported': { $lt: 1 } } },
{ $sort: { 'answers.helpful': -1 } },
{ $limit: 2 },
{
$project: {
_id: 0,
k: { $toString: '$answers._id' }, // k
v: '$$ROOT.answers', // v
},
},
],
as: 'answers',
},
},
This is my first experience using aggregate pipeline. I'm not able to get a "$match" expression to work inside the pipeline. If I remove the "_id" match, I get every document in the collection past the start date, but once I add the $eq expression, it returns empty.
I read a lot of other examples and tried many different ways, and this seems like it is correct. But the result is empty.
Any suggestions?
let now = new Date()
let doc = await Team.aggregate([
{ $match: { created_by: mongoose.Types.ObjectId(req.params.user_oid)} },
{ $sort: { create_date: 1 } },
{ $lookup: {
from: 'events',
let: { "team_oid": "$team_oid" },
pipeline: [
{ $addFields: { "team_oid" : { "$toObjectId": "$team_oid" }}},
{ $match: {
$expr: {
$and: [
{ $gt: [ "$start", now ] },
{ $eq: [ "$_id", "$$team_oid" ] }
]
},
}
},
{
$sort: { start: 1 }
},
{
$limit: 1
}
],
as: 'events',
}},
{
$group: {
_id: "$_id",
team_name: { $first: "$team_name" },
status: { $first: "$status" },
invited: { $first: "$invited" },
uninvited: { $first: "$uninvited" },
events: { $first: "$events.action" },
dates: { $first: "$events.start" } ,
team_oid: { $first: "$events.team_oid" }
}
}])
Example Docs (added by request)
Events:
_id:ObjectId("60350837c57b3a15a414d265")
invitees:null
accepted:null
sequence:7
team_oid:ObjectId("60350837c57b3a15a414d263")
type:"Calendar Invite"
action:"Huddle"
status:"Questions Issued"
title:"Huddle"
body:"This is a Huddle; you should receive new questions 5 days befor..."
creator_oid:ObjectId("5ff9e50a206b1924dccd691e")
start:2021-02-26T07:00:59.999+00:00
end:2021-02-26T07:30:59.999+00:00
__v:0
Team:
_id:ObjectId("60350837c57b3a15a414d263")
weekly_schedule:1
status:"Live"
huddle_number:2
reminders:2
active:true
created_by:ObjectId("5ff9e50a206b1924dccd691e")
team_name:"tESTI"
create_date:2021-02-23T13:50:47.172+00:00
__v:0
This is just a guess since you don't have schema in your question. But it looks like your have some of your _ids mixed up. Where you are currently trying to $match events whose _id is equal to a team_oid. Rather than the event's team_oid field being equal to the current 'team' _id.
I'm pretty confident this will produce the correct output. If you post any schema or sample docs I will edit it.
https://mongoplayground.net/p/5i1w2Ii7KCR
let now = new Date()
let doc = await Team.aggregate([
{ $match: { created_by: mongoose.Types.ObjectId(req.params.user_oid)} },
{ $sort: { create_date: 1 } },
{ $lookup: {
from: 'events',
// Set tea_oid as the current team _id
let: { "team_oid": "$_id" },
pipeline: [
{ $match: {
$expr: {
$and: [
{ $gt: [ "$start", now ] },
// Match events whose 'team_oid' field matches the 'team' _id set above
{ $eq: [ "$team_oid", "$$team_oid" ] }
]
},
}
},
{
$sort: { start: 1 }
},
{
$limit: 1
}
],
as: 'events',
}},
{
$group: {
_id: "$_id",
team_name: { $first: "$team_name" },
status: { $first: "$status" },
invited: { $first: "$invited" },
uninvited: { $first: "$uninvited" },
events: { $first: "$events.action" },
dates: { $first: "$events.start" } ,
team_oid: { $first: "$events.team_oid" }
}
}])
I am trying to get the total count of documents which match my pipeline operators and then after I get that count of all of them I would like to use $skip and $limit to return a subset for pagination. Right now I am basically doing the same aggregation twice - once to get the count of all matches and once to do the skip/limit. Can this be done using one aggregation pipeline?
To get the count I do
let [count] = await MyModel.aggregate([
{
$match: {
store: mongoose.Types.ObjectId(storeId),
},
},
{
$lookup: {
from: "resaleitems",
let: {
store: "$store",
},
as: "itemsForSale",
pipeline: [
{
$match: {
$expr: {
$eq: ["$store", "$$store"],
},
},
},
],
},
},
{
$match: {
"itemsForSale.0": { $exists: true },
},
},
{ $count: "totalCount" },
]).exec();
and then right after to get the skipped and limited results I do basically the same thing, but have added the $skip and $limit steps
let items = await MyModel.aggregate([
{
$match: {
store: mongoose.Types.ObjectId(storeId),
},
},
{ $skip: skip.toNumber() },
{ $limit: bnLimit.toNumber() },
{
$lookup: {
from: "resaleitems",
let: {
store: "$store",
},
as: "itemsForSale",
pipeline: [
{
$match: {
$expr: {
$eq: ["$store", "$$store"],
},
},
},
],
},
},
{
$match: {
"itemsForSale.0": { $exists: true },
},
},
]).exec();