Analyze performance of a query - mongoDB - mongodb

I have to analyze the performance of a MongoDB query. I read I must use:
cursor.explain("executionStats")
So in my case, I used to analyze my query:
> db.team.find({common_name:"Milan"},{_id:0, "stadium.name":1}).explain("executionStats")
And this is the result:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "Progettino.team",
"indexFilterSet" : false,
"parsedQuery" : {
"common_name" : {
"$eq" : "Milan"
}
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"_id" : 0,
"stadium.name" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"common_name" : {
"$eq" : "Milan"
}
},
"direction" : "forward"
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 87,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 89,
"advanced" : 1,
"needTime" : 87,
"needFetch" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"_id" : 0,
"stadium.name" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"common_name" : {
"$eq" : "Milan"
}
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 89,
"advanced" : 1,
"needTime" : 87,
"needFetch" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 87
}
}
}
I want to know how long the query takes to be executed. I don't understand which is the the field containing this information.

The time your looking for is the executionTimeInMillis however your query is only going over it looks like 87 documents so the execution time is extremely small.

Related

MongoDB sort and limit performance issue

I'm facing with a strange issue. The explain of the query shows usage of index with a fast execution time in all stages but the last stage of LIMIT breaks it all. I have execution time of 60s and more!
The DB is used for marketing tool and we collect data on the campaign activity. We have 100k+ records in DB per each hour and I want to select all the 100k in batches of 5k (I tried to reduce it to 1k also) in order to make statistics aggregation.
Just for tests, I will add an example of the query explanation.
If I reduce the created_at date range to 10 minutes and I set the limit to 1000, it works fast.
If I set the created_at date range to 30 minutes and I set the limit to 1000, it stock again.
I have 8 CPU and 64 memory
Storage is with 6000 IOPS
The table include 900 million records in total
I have the following indexes:
created_at: 1
_id: 1
{created_at: 1, _id: 1} - not used in query
Slow query for 30 minutes
> db.logs.explain('allPlansExecution').aggregate([{"$match":{"created_at":{"$gte":ISODate('2021-06-02T20:00:00.000+00:00'),"$lte":ISODate('2021-06-02T20:30:00.000+00:00')}}},{"$sort":{"_id":1}},{"$limit":1000}], { allowDiskUse: true });
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "webpush.campaign_action_logs",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"optimizedPipeline" : true,
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 100,
"executionTimeMillis" : 183411,
"totalKeysExamined" : 257959,
"totalDocsExamined" : 257959,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"works" : 258061,
"advanced" : 100,
"needTime" : 257960,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"totalDataSizeSorted" : 187871472,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 533,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"docsExamined" : 257959,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 265,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
},
"keysExamined" : 257959,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"allPlansExecution" : [
{
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"totalKeysExamined" : 257959,
"totalDocsExamined" : 257959,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"works" : 258060,
"advanced" : 100,
"needTime" : 257960,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"totalDataSizeSorted" : 187871472,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 533,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"docsExamined" : 257959,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 265,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
},
"keysExamined" : 257959,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
{
"nReturned" : 70,
"executionTimeMillisEstimate" : 178092,
"totalKeysExamined" : 258060,
"totalDocsExamined" : 258060,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 70,
"executionTimeMillisEstimate" : 178092,
"works" : 258060,
"advanced" : 70,
"needTime" : 257990,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"nReturned" : 70,
"executionTimeMillisEstimate" : 178057,
"works" : 258060,
"advanced" : 70,
"needTime" : 257990,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"docsExamined" : 258060,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 258060,
"executionTimeMillisEstimate" : 645,
"works" : 258060,
"advanced" : 258060,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 258060,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
]
},
"serverInfo" : {
"host" : "ip-10-0-3-171",
"port" : 27017,
"version" : "4.4.6",
"gitVersion" : "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
"ok" : 1
}
Faster query for 10 minutes:
> db.logs.explain('allPlansExecution').aggregate([{"$match":{"created_at":{"$gte":ISODate('2021-06-02T20:00:00.000+00:00'),"$lte":ISODate('2021-06-02T20:10:00.000+00:00')}}},{"$sort":{"_id":1}},{"$limit":1000}], { allowDiskUse: true });
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "webpush.campaign_action_logs",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"optimizedPipeline" : true,
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 1000,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1000,
"executionTimeMillis" : 1502,
"totalKeysExamined" : 58027,
"totalDocsExamined" : 58027,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1000,
"executionTimeMillisEstimate" : 122,
"works" : 59029,
"advanced" : 1000,
"needTime" : 58028,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"totalDataSizeSorted" : 42213931,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 96,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"docsExamined" : 58027,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 40,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
},
"keysExamined" : 58027,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"allPlansExecution" : [
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 122,
"totalKeysExamined" : 58027,
"totalDocsExamined" : 58027,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 101,
"executionTimeMillisEstimate" : 122,
"works" : 58129,
"advanced" : 101,
"needTime" : 58028,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"totalDataSizeSorted" : 42213931,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 96,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 1,
"docsExamined" : 58027,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 40,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
},
"keysExamined" : 58027,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
{
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"totalKeysExamined" : 58129,
"totalDocsExamined" : 58129,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"works" : 58129,
"advanced" : 3,
"needTime" : 58126,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"limitAmount" : 1000,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"works" : 58129,
"advanced" : 3,
"needTime" : 58126,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"docsExamined" : 58129,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58129,
"executionTimeMillisEstimate" : 17,
"works" : 58129,
"advanced" : 58129,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 58129,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
]
},
"serverInfo" : {
"host" : "ip-10-0-3-171",
"port" : 27017,
"version" : "4.4.6",
"gitVersion" : "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
"ok" : 1
}
Questions:
If the issue is the amount of doc scanned and we used index, how can I solve it? I do not think that looping over 5000k results in a sort order in a 300k docs should cuz an issue but maybe I'm wrong.
Why the LIMIT stage (with another stage of FETCH) causing the issue?
Thank you very much for all experts in advance!
The issue is the sort. Because you are using skip/limit on a sorted set, it must load the entire set from disk, then sort it in memory (possibly spilling to disk) in order to apply the skip/limit.
This means that if you have 100k documents matching the time range, you load all 100k to get the first batch, and then load 100k again to get the second batch.
An index in mongodb can support a sort if the index includes the field that is sorted on, and any fields that precede that key in the index creation spec are matched with an equality predicate.
Fields matched with inequality or ranges can be listed after the sort key.
The existing index on { _id:1 } is inefficient because it is non-selective and therefore requires reading the entire collection from diks.
The index on { created_at:1 } requires loading only those document that match the query, but they must then be sorted in memory.
To support this query, create an index on { _id:1, created_at:1 }. This index should significantly improve the performance because it can both eliminate the in-memory sort, and only require the query executor to load from disk those documents that match the query. This also has the benefit that the query executor can terminate as soon as the limit is satisfied.

How does Mongodb $pull work under the hood

Imagine the following two documents in a myCollection:
{_id: '1', fruits: ['Apple', 'Banana', 'Lemon']}
{_id: '1', fruits: ['Apple', 'Lemon']}
Is there a difference of performance between:
db.myCollection.update({}, {$pull: 'Banana'}, {multi: true)}
and:
db.myCollection.update({fruits: {$elemMatch: 'Banana'}}, {$pull: 'Banana'}, {multi: true)}
//output from mongo client
//ran below queries and compared the output from explain. I did not find any difference in //execution stats for performance
> db.myCollection.explain("executionStats").update(
... {},
... {$pull:{fruits:{$in:["Banana"]}}},
... {multi: true}
... );
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.myCollection",
"indexFilterSet" : false,
"parsedQuery" : {
},
"winningPlan" : {
"stage" : "UPDATE",
"inputStage" : {
"stage" : "COLLSCAN",
"direction" : "forward"
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 2,
"executionStages" : {
"stage" : "UPDATE",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"nMatched" : 2,
"nWouldModify" : 2,
"wouldInsert" : false,
"inputStage" : {
"stage" : "COLLSCAN",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 2,
"needTime" : 1,
"needYield" : 0,
"saveState" : 2,
"restoreState" : 2,
"isEOF" : 1,
"direction" : "forward",
"docsExamined" : 2
}
}
},
"serverInfo" : {
"host" : "XXXX",
"port" : XXXX,
"version" : "4.2.6",
"gitVersion" : "XXXX"
},
"ok" : 1
}
>
--
> db.myCollection.explain("executionStats").update(
... {},
... {$pull:
... {fruits:{$elemMatch:{$in:["Banana"]}}}
... },
... {multi: true}
... );
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.myCollection",
"indexFilterSet" : false,
"parsedQuery" : {
},
"winningPlan" : {
"stage" : "UPDATE",
"inputStage" : {
"stage" : "COLLSCAN",
"direction" : "forward"
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 2,
"executionStages" : {
"stage" : "UPDATE",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"nMatched" : 2,
"nWouldModify" : 2,
"wouldInsert" : false,
"inputStage" : {
"stage" : "COLLSCAN",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 2,
"needTime" : 1,
"needYield" : 0,
"saveState" : 2,
"restoreState" : 2,
"isEOF" : 1,
"direction" : "forward",
"docsExamined" : 2
}
}
},
"serverInfo" : {
"host" : "XXXX",
"port" : XXX,
"version" : "4.2.6",
"gitVersion" : "XXXX"
},
"ok" : 1
}
>

Mongodb query execution time

I used Query below its take to much time
Query
db.saleOrder.find({"currentStatus._id":"147"},{"_id":1}).limit(10).explain("executionStats")
ExecutionStats result
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "db_erp_tube.saleOrder",
"indexFilterSet" : false,
"parsedQuery" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "PROJECTION",
"transformBy" : {
"_id" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"direction" : "forward"
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 8673,
"totalKeysExamined" : 0,
"totalDocsExamined" : 3458482,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 8460,
"works" : 3458484,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 10,
"inputStage" : {
"stage" : "PROJECTION",
"nReturned" : 10,
"executionTimeMillisEstimate" : 8450,
"works" : 3458483,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 0,
"invalidates" : 0,
"transformBy" : {
"_id" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"nReturned" : 10,
"executionTimeMillisEstimate" : 8400,
"works" : 3458483,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 0,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 3458482
}
}
}
},
"serverInfo" : {
"host" : "172.16.109",
"port" : 27017,
"version" : "4.0.0",
"gitVersion" : "3b07af3d4f471ae89e8186d33bbb1d5259597d51"
},
"ok" : 1,
"operationTime" : Timestamp(1556365275, 114),
"$clusterTime" : {
"clusterTime" : Timestamp(1556365275, 114),
"signature" : {
"hash" : BinData(0,"ppu91nKmeiC//+UvdsEbjrBTDLU="),
"keyId" : NumberLong("6633468944474701825")
}
}
}
The query took more than 8 seconds(8673ms) to execute because it had to scan all 3458482 documents in the saleOrder collection before returning 10 documents.
This is stated in the filter stage of the explain output.
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
COLLSCAN indicate full collection scan,
"totalDocsExamined" : 3458482,
This is the number of document scanned before an result is returned.
"executionTimeMillis" : 8673,
This is the total time taken for the query to run.
As #the_mahasagar suggested, build a index on currentStatus._id can speed up the query a lot, use the command below.
db.saleOrder.createIndex({ "currentStatus._id": 1})

MongoDB find query very slow even with index

I don't understand why the following query is so slow, the explain shows it will take 74+ seconds to finish, in spite of the presence of index.
both dev_id and _id are indexed I can assure you, it seems just not helpful at all.
db.DeviceLoginLog.find({"dev_id": "xxx"}).skip(0).limit(10).sort({"_id": -1}).explain("executionStats");
Can anyone help me to interpret explain output and advise me how to speed up?
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "example.DeviceLoginLog",
"indexFilterSet" : false,
"parsedQuery" : {
"dev_id" : {
"$eq" : "xxx"
}
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 10,
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dev_id" : 1
},
"indexName" : "dev_id_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"dev_id" : [
"[\"xxx\", \"xxx\"]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 4,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"dev_id" : {
"$eq" : "xxx"
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 74867,
"totalKeysExamined" : 9203,
"totalDocsExamined" : 9203,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 49860,
"works" : 9537,
"advanced" : 10,
"needTime" : 9204,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"sortPattern" : {
"_id" : -1
},
"memUsage" : 1890,
"memLimit" : 33554432,
"limitAmount" : 10,
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 49820,
"works" : 9525,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 49820,
"works" : 9525,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"docsExamined" : 9203,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 10,
"works" : 9204,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 0,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"keyPattern" : {
"dev_id" : 1
},
"indexName" : "dev_id_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"dev_id" : [
"[\"xxx\", \"xxx\"]"
]
},
"keysExamined" : 9203,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "iZ231ear7c9Z",
"port" : 27017,
"version" : "3.0.3",
"gitVersion" : "b40106b36eecd1b4407eb1ad1af6bc60593c6105"
},
"ok" : 1
}
Thanks in advanceļ¼
The query used index {dev_id: 1} to find and then had to fetch matched docs into memory to sort without using an index. Create a compound index as {dev_id: 1, _id: -1} will improve performance.
See sort-and-non-prefix-subset-of-an-index.
"totalKeysExamined" : 9203,
"totalDocsExamined" : 9203,
That shows that the query had to examine each individual document. You aren't using an index on the _id field. Also depending on which fields you are returning, consider creating an index that allows for a covered query.
A covered query is a query that can be satisfied entirely using an
index and does not have to examine any documents. An index covers a
query when both of the following apply:
all the fields in the query are part of an index, and
all the fields returned in the results are in the same index.
https://docs.mongodb.com/manual/core/query-optimization/

MongoDB possibly scanning documents for an operation that could be covered by an index

I have a collection with a locked field in each document.
I have the following index:
{
locked : 1
}
when I perform this explain over a count operation
db.scheduled.find({locked: false}).explain({executionStats:1})
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "connectivity_recruiter.scheduled",
"indexFilterSet" : false,
"parsedQuery" : {
"locked" : {
"$eq" : false
}
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"locked" : 1
},
"indexName" : "locked_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"locked" : [
"[false, false]"
]
}
}
},
.....
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 53045,
"executionTimeMillis" : 299,
"totalKeysExamined" : 53045,
"totalDocsExamined" : 53045,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 53045,
"executionTimeMillisEstimate" : 180,
"works" : 53046,
"advanced" : 53045,
"needTime" : 0,
"needFetch" : 0,
"saveState" : 417,
"restoreState" : 417,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 53045,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 53045,
"executionTimeMillisEstimate" : 70,
"works" : 53046,
"advanced" : 53045,
"needTime" : 0,
"needFetch" : 0,
"saveState" : 417,
"restoreState" : 417,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"locked" : 1
},
"indexName" : "locked_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"locked" : [
"[false, false]"
]
},
"keysExamined" : 53045,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0
}
},
...........
}
totalDocsExamined seems indicate that all documents are being scanned in order to count them, while this operation could be performed by using the index alone.
What is happening? Is this normal? Is a full scan of the collection going on?
Thanks
All the returned docs were examined, the index was used only to filter not to retrieve the documents.
If you look at your explain, you'll notice that the number of docs is equals the number of documents examined.
Why that? Your index only contains one field while you're fetching the entire document, what mongodb does is to query the index for the keys and then go for the collection to fetch the document.
The only situation were no document will need to be examined is for covered queries, when the index contains all the projected fields.
See more at this link: https://docs.mongodb.com/manual/core/query-optimization/#covered-query