MongoDB very slow $count after $lookup - mongodb

Help, I am using MongoDB 4.2.6, and writing an aggregate to obtain the number of filtered data from collections with 40000+ data. Before applying the $count method, I need to $lookup an extra collection as well.
Here is my aggregate
db.exams.aggregate([{
$match: {
schoolId: ObjectId("5d91c9ec098506001b426cb5")
}
}, {
$lookup: {
from: 'students',
localField: 'studentId',
foreignField: '_id',
as: 'student'
}
}, {
$unwind: "$student"
}, {
$match: {
"student.gender": 1
}
},{
$count: 'count'
}])
But it looks more than 10 seconds. I have already add indexes on every ID: exams._id, students._id, exams.studentId, exams.schoolId, student.gender, etc...
Can someone gives me some suggestions in order to make the query faster?
Explains:
{
stages: [
{
$cursor: {
query: {
schoolId: ObjectId('5d91c9ec098506001b426cb5')
},
fields: {
_id: 1
},
queryPlanner: {
plannerVersion: 1,
namespace: 'happya.exams',
indexFilterSet: false,
parsedQuery: {
schoolId: {
$eq: ObjectId('5d91c9ec098506001b426cb5')
}
},
queryHash: '9533F340',
planCacheKey: 'CE7F9610',
winningPlan: {
stage: 'FETCH',
inputStage: {
stage: 'IXSCAN',
keyPattern: {
schoolId: 1
},
indexName: 'schoolId_1',
isMultiKey: false,
multiKeyPaths: {
schoolId: []
},
isUnique: false,
isSparse: false,
isPartial: false,
indexVersion: 2,
direction: 'forward',
indexBounds: {
schoolId: [
"[ObjectId('5d91c9ec098506001b426cb5'), ObjectId('5d91c9ec098506001b426cb5')]"
]
}
}
},
rejectedPlans: [
{
stage: 'FETCH',
inputStage: {
stage: 'IXSCAN',
keyPattern: {
schoolId: 1,
referenceNo: 1
},
indexName: 'schoolId_1_referenceNo_1',
isMultiKey: false,
multiKeyPaths: {
schoolId: [],
referenceNo: []
},
isUnique: true,
isSparse: false,
isPartial: false,
indexVersion: 2,
direction: 'forward',
indexBounds: {
schoolId: [
"[ObjectId('5d91c9ec098506001b426cb5'), ObjectId('5d91c9ec098506001b426cb5')]"
],
referenceNo: ['[MinKey, MaxKey]']
}
}
}
]
}
}
},
{
$lookup: {
from: 'students',
as: 'student',
localField: 'studentId',
foreignField: '_id',
unwinding: {
preserveNullAndEmptyArrays: false
},
matching: {
gender: {
$eq: 1
}
}
}
},
{
$group: {
_id: {
$const: null
},
count: {
$sum: {
$const: 1
}
}
}
},
{
$project: {
_id: false,
count: true
}
}
],
serverInfo: {
host: 'a98010d6dcf4',
port: 27017,
version: '4.2.6',
gitVersion: '20364840b8f1af16917e4c23c1b5f5efd8b352f8'
},
ok: 1,
$clusterTime: {
clusterTime: Timestamp(1597720010, 1),
signature: {
hash: BinData(0, '1PiNaAzDzNRrnZl/mpVJP4oneyU='),
keyId: NumberLong('6819213090182135813')
}
},
operationTime: Timestamp(1597720010, 1)
};

Related

Simplifying a MongoDB aggregate query

I made that aggregate query below to handle a social network main feed that will be able to show the posts and then answer then. Is there a way to simplify that query or the right choice for that case is to lookup, unwind and project?
`[
{
$match: {
_id: mongoose.Types.ObjectId(myId),
},
},
{
$addFields: {
following: {
$concatArrays: ["$following", [mongoose.Types.ObjectId(myId)]],
},
},
},
{
$lookup: {
from: "users",
localField: "following",
foreignField: "_id",
as: "following",
},
},
{
$unwind: {
path: "$following",
preserveNullAndEmptyArrays: true,
},
},
{
$project: {
_id: "$following._id",
name: "$following.name",
surname: "$following.surname",
profilePicPath: "$following.profilePicPath",
},
},
{
$lookup: {
from: "posts",
localField: "_id",
foreignField: "parentId",
as: "posts",
},
},
{
$unwind: {
path: "$posts",
preserveNullAndEmptyArrays: true,
},
},
{
$project: {
name: true,
surname: true,
profilePicPath: true,
_id: "$posts._id",
parentId: "$posts.parentId",
content: "$posts.content",
date: "$posts.date",
},
},
{
$lookup: {
from: "answerposts",
localField: "_id",
foreignField: "parentId",
as: "answerPosts",
},
},
{
$unwind: {
path: "$answerPosts",
preserveNullAndEmptyArrays: true,
},
},
{
$project: {
name: true,
surname: true,
profilePicPath: true,
_id: true,
parentId: true,
content: true,
date: true,
answerPosts_id: "$answerPosts._id",
answerPostsOwnerId: "$answerPosts.ownerId",
answerPostsContent: "$answerPosts.content",
answerPostsDate: "$answerPosts.date",
},
},
{
$lookup: {
from: "users",
localField: "answerPostsOwnerId",
foreignField: "_id",
as: "answerPostsOwner",
},
},
{
$unwind: {
path: "$answerPostsOwner",
preserveNullAndEmptyArrays: true,
},
},
{
$project: {
name: true,
surname: true,
profilePicPath: true,
_id: true,
parentId: true,
content: true,
date: true,
answerPosts_id: true,
answerPostsContent: true,
answerPostsDate: true,
answerPostsOwner_name: "$answerPostsOwner.name",
answerPostsOwner_surname: "$answerPostsOwner.surname",
answerPostsOwner_profilePicPath: "$answerPostsOwner.profilePicPath",
},
},
{
$group: {
_id: "$_id",
parentId: {
$first: "$parentId",
},
name: {
$first: "$name",
},
surname: {
$first: "$surname",
},
profilePicPath: {
$first: "$profilePicPath",
},
content: {
$first: "$content",
},
date: {
$first: "$date",
},
answerPosts: {
$push: {
_id: "$answerPosts_id",
name: "$answerPostsOwner_name",
surname: "$answerPostsOwner_surname",
content: "$answerPostsContent",
profilePicPath: "$answerPostsOwner_profilePicPath",
date: "$answerPostsDate",
},
},
},
},
{
$addFields: {
answerPosts: {
$cond: [
{
$in: [{}, "$answerPosts"],
},
[],
"$answerPosts",
],
},
},
},
{
$match: {
_id: {
$ne: null,
},
},
},
{
$sort: {
date: -1,
"answerPosts.date": 1,
},
},
]`
It works that way but I want to know if there is a simpler way to make this query.

MongoDB aggregate. Create new groups for non-existing items

My collection of documents contains information about users, their sessions and CRUD operations they performed during these sessions:
{
user_id: '1',
sessions: [
{
actions: [
{
type: 'create',
created_at: ISODate('2020-01-01T00:00:00'),
},
{
type: 'read',
created_at: ISODate('2022-01-01T00:00:00'),
},
{
type: 'read',
created_at: ISODate('2021-01-01T00:00:00'),
}
],
}
]
}
I need to get a summary for each user, which includes the amount of CRUD operations and the date of the last one:
{
user_id: '1',
actions: [
{
type: 'create',
last: ISODate('2020-01-01T00:00:00'),
count: 1,
},
{
type: 'read',
last: ISODate('2022-01-01T00:00:00'),
count: 2,
},
// Problematic part:
{
type: 'update',
last: null,
count: 0,
},
{
type: 'delete',
last: null,
count: 0,
},
]
}
I came up with this solution:
db.users.aggregate([
{$unwind:'$sessions'},
{$unwind:'$sessions.actions'},
{
$group:{
_id:{user_id:'$user_id', type:'$sessions.actions.type'},
last:{$max:'$sessions.actions.created_at'},
count:{$sum:1},
}
},
{
$group:{
_id:{user_id:'$_id.user_id'},
actions:{$push:{type:'$_id.type', last:'$last', count:'$count'}}
}
},
{
$project:{
_id:0,
user_id: '$_id.user_id',
actions: '$actions'
}
}
])
The problem here is that I cannot figure out, how can I add missing actions, like in 'update' and 'delete' in the example above
Try this,
db.collection.aggregate([
{
$unwind: "$sessions"
},
{
$unwind: "$sessions.actions"
},
{
$group: {
_id: {
user_id: "$user_id",
type: "$sessions.actions.type"
},
last: {
$max: "$sessions.actions.created_at"
},
count: {
$sum: 1
},
}
},
{
$group: {
_id: {
user_id: "$_id.user_id"
},
actions: {
$push: {
type: "$_id.type",
last: "$last",
count: "$count"
}
}
}
},
{
$project: {
_id: 0,
user_id: "$_id.user_id",
actions: {
"$function": {
"body": "function(doc) { const ops = {read:0, delete:0, update: 0, create: 0}; const actions = doc.actions; actions.forEach(action => { ops[action.type] = 1 }); Object.keys(ops).filter(key => ops[key] === 0).forEach(key => actions.push({count: 0, last: null, type: key})); return actions }",
"args": [
"$$ROOT"
],
"lang": "js"
}
},
}
},
])
Here, we use $function and provide a small JS function to populate the missing entries.
Playground link.

MongoDB Aggregation - Select only same value from array inside lookup

I have aggregation like this:
Produk.aggregate([
{
$lookup: {
from: "kis_m_kategoriproduks",
localField: "idSubKategori",
foreignField: "subKategori._id",
as: "kategori",
},
},
{ $unwind: "$kategori" },
{ $sort: { produk: 1 } },
{
$project: {
_id: 0,
id: "$id",
dataKategori: {
idKategori: "$kategori._id",
kategori: "$kategori.kategori",
idSubKategori: "$idSubKategori",
subKategori: "$kategori.subKategori",
},
},
},
])
current result is :
{
"status": "success",
"data": [
{
"dataKategori": {
"idKategori": "6195bbec8ee419e6a9b8329d",
"kategori": "Kuliner",
"idSubKategori": "6195bc0f8ee419e6a9b832a2",
"subKategori": [
{
"nama": "Food",
"_id": "6195bc0f8ee419e6a9b832a2"
},
{
"nama": "Drink",
"_id": "6195bc258ee419e6a9b832a8"
}
]
}
}
]
}
I only want to display data in subKategori that the _id match with idSubKategori. this what I expected:
{
"status": "success",
"data": [
{
"dataKategori": {
"idKategori": "6195bbec8ee419e6a9b8329d",
"kategori": "Kuliner",
"idSubKategori": "6195bc0f8ee419e6a9b832a2",
"subKategori": [
{
"nama": "Food",
"_id": "6195bc0f8ee419e6a9b832a2"
}
]
}
}
]
}
here is my $kategori schema:
const schema = mongoose.Schema(
{
kategori: {
type: String,
required: true,
unique: true,
},
subKategori: [
{
id: mongoose.Types.ObjectId,
nama: String,
},
],
},
{
timestamps: false,
}
);
any suggestion?
I fix the problem by add $filter inside $project like this:
dataKategori: {
idKategori: "$kategori._id",
kategori: "$kategori.kategori",
subKategori: {
$arrayElemAt: [
{
$filter: {
input: "$kategori.subKategori",
as: "sub",
cond: { $eq: ["$$sub._id", "$idSubKategori"] },
},
},
0,
],
},
},
reference: https://stackoverflow.com/a/42490320/6412375

MongoDB query to find top store from list of orders

I'm pretty new to Mongo. I have two collections that look as follows.
Order collection
[
{
id: 1,
price: 249,
store: 1,
status: true
},
{
id: 2,
price: 230,
store: 1,
status: true
},
{
id: 3,
price: 240,
store: 1,
status: true
},
{
id: 4,
price: 100,
store: 2,
status: true
},
{
id: 5,
price: 150,
store: 2,
status: true
},
{
id: 6,
price: 500,
store: 3,
status: true
},
{
id: 7,
price: 70,
store: 4,
status: true
},
]
Store Collection
[
{
id: 1,
name: "Store A",
status: true
},
{
id: 2,
name: "Store B",
status: true
},
{
id: 3,
name: "Store C",
status: true
},
{
id: 4,
name: "Store D",
status: false
}
]
How to find the top store from the list of orders, which should be based on the total sales in each store.
I have tried the following
db.order.aggregate([
{
"$match": {
status: true
}
},
{
"$group": {
"_id": "$store",
"totalSale": {
"$sum": "$price"
}
}
},
{
$sort: {
totoalSale: -1
}
}
])
I got the sorted list of stores from the above snippets. But I want to add store details along with total sales.
For more: https://mongoplayground.net/p/V3UH1r6YRnS
Expected Output
[
{
id: 1,
name: "Store A",
status: true,
totalSale: 719
},
{
id: 1,
name: "Store c",
status: true,
totalSale: 500
},
{
_id: 2,
id: 1,
name: "Store B",
status: true,
totalSale: 250
},
{
_id: 4,
name: "Store D",
status: true,
totalSale: 70
}
]
$lookup - store collection joins order collection and generate new field store_orders.
$set - Filter order with status: true from store_orders.
$set - totalSale field sum for store_orders.price.
$sort - Sort totalSale by descending.
$unset - Remove store_orders field.
db.store.aggregate([
{
$lookup: {
from: "order",
localField: "id",
foreignField: "store",
as: "store_orders"
}
},
{
$set: {
"store_orders": {
$filter: {
input: "$store_orders",
as: "order",
cond: {
$eq: [
"$$order.status",
true
]
}
}
}
}
},
{
$set: {
"totalSale": {
"$sum": "$store_orders.price"
}
}
},
{
$sort: {
totalSale: -1
}
},
{
$unset: "store_orders"
}
])
Sample Mongo Playground
You can start from store collection, $lookup the order collection, $sum the totalSales, then wrangle to your expected form
db.store.aggregate([
{
"$lookup": {
"from": "order",
let: {
id: "$id"
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$$id",
"$store"
]
}
}
},
{
$group: {
_id: null,
totalSale: {
$sum: "$price"
}
}
}
],
"as": "totalSale"
}
},
{
$unwind: "$totalSale"
},
{
$addFields: {
totalSale: "$totalSale.totalSale"
}
},
{
$sort: {
totalSale: -1
}
}
])
Here is the Mongo playground for youre reference.

MongoDB add $lookup result to array

I have two collections
users:
{
{ _id: 1, name: 'John' },
{ _id: 2, name: 'Sarah' },
{ _id: 3, name: 'Mike' }
}
services:
{
{ _id: 1,
payment: [
{ uid: 1, paid: true },
{ uid: 2, paid: false }
]
},
{ _id: 2,
payment: [
{ uid: 3, paid: true }
]
}
}
I need result like this (from services):
{
{ _id: 1,
payment: [
{ uid: 1, paid: true, user: { _id: 1, name: 'John' } },
{ uid: 2, paid: false, user: { _id: 2, name: 'Sarah' } }
]
},
{ _id: 2,
payment: [
{ uid: 3, paid: true, user: { _id: 3, name: 'Mike' } }
]
}
}
I can $lookup by uid field, but how to add "paid" field to each item in lookup result? I know that it's must be really simple... but not for me now ;)
Thanks in advance!
db.getCollection('services').aggregate([
{ "$match": {} },
{ "$unwind": '$payment' },
{ "$lookup": {
"from": "users",
"localField": "payment.cid",
"foreignField": "_id",
"as": "user_data"
}},
{ "$unwind": '$user_data' },
{ "$addFields": {
"payment.user.name": "$user_data.name",
}},
{ "$group": {
"_id": "$_id",
"payment": { "$push": "$payment" }
}}
])