Understanding explain() to optimize complex queries - mongodb

Question

I have a aggregate pipeline that takes over 5 seconds to return 200 rows.
I'm trying to optimize my current pipeline using explain("executionStats").
This is my pipeline:
db.getCollection("content_topics").explain("executionStats").aggregate([{
"$lookup": {
from: "users",
localField: "creator",
foreignField: "_id",
as: "user"
}
},
{
$unwind: "$user"
},
{
"$match": {
created_at: {
"$gte": 1528914600,
"$lte": 1534271400
},
dash_status: 3,
language: "hi",
parent_topic_id: {
"$eq": null
},
status: 1,
"user.device_os": {
"$ne": "BOT"
}
}
},
{
"$sort": {
created_at: -1
}
},
{
"$addFields": {
user_handle: "$user.handle",
user_phone: "$user.phone",
user_status: "$user.status"
}
},
{
"$project": {
topic_id: 1,
n_vokes: 1,
message: 1,
title: 1,
language: 1,
description: 1,
voice_desc: 1,
image: 1,
image_share: 1,
hashtag: 1,
location: 1,
default_text: 1,
creator: 1,
created_at: 1,
status: 1,
ref_id: 1,
weightage: 1,
username: 1,
slug_generated: 1,
user_handle: 1,
user_phone: 1,
user_status: 1
}
},
{
"$skip": 0
},
{
"$limit": 200
}
]);
And this is what mongo explain()ed:
{
"stages" : [
{
"$cursor" : {
"query" : {
"$and" : [
{
"created_at" : {
"$gte" : 1528914600.0
}
},
{
"created_at" : {
"$lte" : 1534271400.0
}
},
{
"dash_status" : {
"$eq" : 3.0
}
},
{
"language" : {
"$eq" : "hi"
}
},
{
"parent_topic_id" : {
"$eq" : null
}
},
{
"status" : {
"$eq" : 1.0
}
}
]
},
"fields" : {
"created_at" : 1.0,
"creator" : 1.0,
"default_text" : 1.0,
"description" : 1.0,
"hashtag" : 1.0,
"image" : 1.0,
"image_share" : 1.0,
"language" : 1.0,
"location" : 1.0,
"message" : 1.0,
"n_vokes" : 1.0,
"ref_id" : 1.0,
"slug_generated" : 1.0,
"status" : 1.0,
"title" : 1.0,
"topic_id" : 1.0,
"user.handle" : 1.0,
"user.phone" : 1.0,
"user.status" : 1.0,
"user_handle" : 1.0,
"user_phone" : 1.0,
"user_status" : 1.0,
"username" : 1.0,
"voice_desc" : 1.0,
"weightage" : 1.0,
"_id" : 1.0
},
"queryPlanner" : {
"plannerVersion" : 1.0,
"namespace" : "vokalapp.content_topics",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"dash_status" : {
"$eq" : 3.0
}
},
{
"language" : {
"$eq" : "hi"
}
},
{
"parent_topic_id" : {
"$eq" : null
}
},
{
"status" : {
"$eq" : 1.0
}
},
{
"created_at" : {
"$lte" : 1534271400.0
}
},
{
"created_at" : {
"$gte" : 1528914600.0
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"parent_topic_id" : {
"$eq" : null
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dash_status" : 1.0,
"status" : 1.0,
"language" : 1.0,
"parent_topic_id" : 1.0,
"created_at" : 1.0
},
"indexName" : "index_for_dashboard",
"isMultiKey" : false,
"multiKeyPaths" : {
"dash_status" : [
],
"status" : [
],
"language" : [
],
"parent_topic_id" : [
],
"created_at" : [
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2.0,
"direction" : "forward",
"indexBounds" : {
"dash_status" : [
"[3.0, 3.0]"
],
"status" : [
"[1.0, 1.0]"
],
"language" : [
"[\"hi\", \"hi\"]"
],
"parent_topic_id" : [
"[null, null]"
],
"created_at" : [
"[1528914600.0, 1534271400.0]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"dash_status" : {
"$eq" : 3.0
}
},
{
"language" : {
"$eq" : "hi"
}
},
{
"parent_topic_id" : {
"$eq" : null
}
},
{
"status" : {
"$eq" : 1.0
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created_at" : 1.0
},
"indexName" : "created_at_index",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2.0,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[1528914600.0, 1534271400.0]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"dash_status" : {
"$eq" : 3.0
}
},
{
"parent_topic_id" : {
"$eq" : null
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"language" : 1.0,
"status" : 1.0,
"created_at" : 1.0,
"answers.type" : 1.0
},
"indexName" : "language_status_created_at_answer_type_index",
"isMultiKey" : true,
"multiKeyPaths" : {
"language" : [
],
"status" : [
],
"created_at" : [
],
"answers.type" : [
"answers"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2.0,
"direction" : "forward",
"indexBounds" : {
"language" : [
"[\"hi\", \"hi\"]"
],
"status" : [
"[1.0, 1.0]"
],
"created_at" : [
"[1528914600.0, 1534271400.0]"
],
"answers.type" : [
"[MinKey, MaxKey]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"dash_status" : {
"$eq" : 3.0
}
},
{
"parent_topic_id" : {
"$eq" : null
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"language" : -1.0,
"status" : -1.0,
"weightage" : -1.0,
"created_at" : -1.0
},
"indexName" : "language_status_weightage_created_at_index",
"isMultiKey" : false,
"multiKeyPaths" : {
"language" : [
],
"status" : [
],
"weightage" : [
],
"created_at" : [
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2.0,
"direction" : "forward",
"indexBounds" : {
"language" : [
"[\"hi\", \"hi\"]"
],
"status" : [
"[1.0, 1.0]"
],
"weightage" : [
"[MaxKey, MinKey]"
],
"created_at" : [
"[1534271400.0, 1528914600.0]"
]
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 7072.0,
"executionTimeMillis" : 984.0,
"totalKeysExamined" : 7072.0,
"totalDocsExamined" : 7072.0,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"parent_topic_id" : {
"$eq" : null
}
},
"nReturned" : 7072.0,
"executionTimeMillisEstimate" : 40.0,
"works" : 7073.0,
"advanced" : 7072.0,
"needTime" : 0.0,
"needYield" : 0.0,
"saveState" : 63.0,
"restoreState" : 63.0,
"isEOF" : 1.0,
"invalidates" : 0.0,
"docsExamined" : 7072.0,
"alreadyHasObj" : 0.0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 7072.0,
"executionTimeMillisEstimate" : 0.0,
"works" : 7073.0,
"advanced" : 7072.0,
"needTime" : 0.0,
"needYield" : 0.0,
"saveState" : 63.0,
"restoreState" : 63.0,
"isEOF" : 1.0,
"invalidates" : 0.0,
"keyPattern" : {
"dash_status" : 1.0,
"status" : 1.0,
"language" : 1.0,
"parent_topic_id" : 1.0,
"created_at" : 1.0
},
"indexName" : "index_for_dashboard",
"isMultiKey" : false,
"multiKeyPaths" : {
"dash_status" : [
],
"status" : [
],
"language" : [
],
"parent_topic_id" : [
],
"created_at" : [
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2.0,
"direction" : "forward",
"indexBounds" : {
"dash_status" : [
"[3.0, 3.0]"
],
"status" : [
"[1.0, 1.0]"
],
"language" : [
"[\"hi\", \"hi\"]"
],
"parent_topic_id" : [
"[null, null]"
],
"created_at" : [
"[1528914600.0, 1534271400.0]"
]
},
"keysExamined" : 7072.0,
"seeks" : 1.0,
"dupsTested" : 0.0,
"dupsDropped" : 0.0,
"seenInvalidated" : 0.0
}
}
}
}
},
{
"$lookup" : {
"from" : "users",
"as" : "user",
"localField" : "creator",
"foreignField" : "_id",
"unwinding" : {
"preserveNullAndEmptyArrays" : false
},
"matching" : {
"$nor" : [
{
"device_os" : {
"$eq" : "BOT"
}
}
]
}
}
},
{
"$sort" : {
"sortKey" : {
"created_at" : -1.0
},
"limit" : NumberLong(200)
}
},
{
"$addFields" : {
"user_handle" : "$user.handle",
"user_phone" : "$user.phone",
"user_status" : "$user.status"
}
},
{
"$project" : {
"_id" : true,
"hashtag" : true,
"voice_desc" : true,
"description" : true,
"location" : true,
"language" : true,
"topic_id" : true,
"image" : true,
"slug_generated" : true,
"n_vokes" : true,
"ref_id" : true,
"image_share" : true,
"message" : true,
"created_at" : true,
"default_text" : true,
"weightage" : true,
"user_handle" : true,
"title" : true,
"status" : true,
"creator" : true,
"user_status" : true,
"username" : true,
"user_phone" : true
}
}
],
"ok" : 1.0
}
This is after I created the index "index_for_dashboard" with the keys that I use in "$match". I also created an index for "device_os" from user collection. But to no dice, no improvements in the response time.
Possible culprits:
Is $lookup expensive. If it is can I fetch only the fields I need.
Can the indexing be done better. Should I use a different set of
fields?
Is $addField expensive. If it is I can offload it to
application level.
How should I troubleshoot(and understand) with the explain result to help myself optimize the query?
I can't make an educated call here, need some directions.

Answer 1

First, you want to make sure that you are running at least v3.6.3 of MongoDB because in v3.6 there's a new feature that allows $lookup to specify sub-pipelines. Those pipelines can actually use indexes, however, there was a bug which only got fixed in the version mentioned above.
The following should be as fast as things can get here:
db.getCollection("content_topics").createIndex({ created_at: -1, dash_status: 1, language: 1, parent_topic_id: 1, status: 1 }); // this index will get used by the main $match and the $sort stage
db.getCollection("users").createIndex({ device_os: 1 }); // this index will get used by the sub-pipeline in $lookup
db.getCollection("content_topics").aggregate([{
"$match": { // filter at the start in order to be able to use indexes
created_at: {
"$gte": 1528914600,
"$lte": 1534271400
},
dash_status: 3,
language: "hi",
parent_topic_id: {
"$eq": null
},
status: 1
// see the below $lookup stage in case you're wondering where the user filter went
}
},
{
"$sort": {
created_at: -1 // sort straight away so the index can be used
}
},
{
"$lookup": {
from: "users",
let: { "creator": "$creator" },
pipeline: [{ // use new v3.6 pipeline syntax to be able to leverage indexes
$match: {
$expr: {
$and: [
{ $eq: [ "$_id", "$$creator" ] },
{ $ne: [ "$device_os", "BOT" ] } // here is the "device_os" filter inside the pipeline so index can be used
]
}
}
}, {
$project: {
_id: 0, // "_id" field is not needed
user_handle: 1, // only those fields are of interest
user_phone: 1,
user_status: 1
}
}],
as: "user"
}
},
{
$unwind: "$user"
},
{
"$project": {
topic_id: 1,
n_vokes: 1,
message: 1,
title: 1,
language: 1,
description: 1,
voice_desc: 1,
image: 1,
image_share: 1,
hashtag: 1,
location: 1,
default_text: 1,
creator: 1,
created_at: 1,
status: 1,
ref_id: 1,
weightage: 1,
username: 1,
slug_generated: 1,
user_handle: "$user.handle", // no extra $addFields stage needed
user_phone: "$user.phone", // same here
user_status: "$user.status" // and here
}
},
{
"$skip": 0
},
{
"$limit": 200
}
]);