I'm facing with a strange issue. The explain of the query shows usage of index with a fast execution time in all stages but the last stage of LIMIT breaks it all. I have execution time of 60s and more!
The DB is used for marketing tool and we collect data on the campaign activity. We have 100k+ records in DB per each hour and I want to select all the 100k in batches of 5k (I tried to reduce it to 1k also) in order to make statistics aggregation.
Just for tests, I will add an example of the query explanation.
If I reduce the created_at date range to 10 minutes and I set the limit to 1000, it works fast.
If I set the created_at date range to 30 minutes and I set the limit to 1000, it stock again.
I have 8 CPU and 64 memory
Storage is with 6000 IOPS
The table include 900 million records in total
I have the following indexes:
created_at: 1
_id: 1
{created_at: 1, _id: 1} - not used in query
Slow query for 30 minutes
> db.logs.explain('allPlansExecution').aggregate([{"$match":{"created_at":{"$gte":ISODate('2021-06-02T20:00:00.000+00:00'),"$lte":ISODate('2021-06-02T20:30:00.000+00:00')}}},{"$sort":{"_id":1}},{"$limit":1000}], { allowDiskUse: true });
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "webpush.campaign_action_logs",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"optimizedPipeline" : true,
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 100,
"executionTimeMillis" : 183411,
"totalKeysExamined" : 257959,
"totalDocsExamined" : 257959,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"works" : 258061,
"advanced" : 100,
"needTime" : 257960,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"totalDataSizeSorted" : 187871472,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 533,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"docsExamined" : 257959,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 265,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
},
"keysExamined" : 257959,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"allPlansExecution" : [
{
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"totalKeysExamined" : 257959,
"totalDocsExamined" : 257959,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"works" : 258060,
"advanced" : 100,
"needTime" : 257960,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"totalDataSizeSorted" : 187871472,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 533,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"docsExamined" : 257959,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 265,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
},
"keysExamined" : 257959,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
{
"nReturned" : 70,
"executionTimeMillisEstimate" : 178092,
"totalKeysExamined" : 258060,
"totalDocsExamined" : 258060,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 70,
"executionTimeMillisEstimate" : 178092,
"works" : 258060,
"advanced" : 70,
"needTime" : 257990,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"nReturned" : 70,
"executionTimeMillisEstimate" : 178057,
"works" : 258060,
"advanced" : 70,
"needTime" : 257990,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"docsExamined" : 258060,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 258060,
"executionTimeMillisEstimate" : 645,
"works" : 258060,
"advanced" : 258060,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 258060,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
]
},
"serverInfo" : {
"host" : "ip-10-0-3-171",
"port" : 27017,
"version" : "4.4.6",
"gitVersion" : "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
"ok" : 1
}
Faster query for 10 minutes:
> db.logs.explain('allPlansExecution').aggregate([{"$match":{"created_at":{"$gte":ISODate('2021-06-02T20:00:00.000+00:00'),"$lte":ISODate('2021-06-02T20:10:00.000+00:00')}}},{"$sort":{"_id":1}},{"$limit":1000}], { allowDiskUse: true });
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "webpush.campaign_action_logs",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"optimizedPipeline" : true,
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 1000,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1000,
"executionTimeMillis" : 1502,
"totalKeysExamined" : 58027,
"totalDocsExamined" : 58027,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1000,
"executionTimeMillisEstimate" : 122,
"works" : 59029,
"advanced" : 1000,
"needTime" : 58028,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"totalDataSizeSorted" : 42213931,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 96,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"docsExamined" : 58027,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 40,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
},
"keysExamined" : 58027,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"allPlansExecution" : [
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 122,
"totalKeysExamined" : 58027,
"totalDocsExamined" : 58027,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 101,
"executionTimeMillisEstimate" : 122,
"works" : 58129,
"advanced" : 101,
"needTime" : 58028,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"totalDataSizeSorted" : 42213931,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 96,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 1,
"docsExamined" : 58027,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 40,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
},
"keysExamined" : 58027,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
{
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"totalKeysExamined" : 58129,
"totalDocsExamined" : 58129,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"works" : 58129,
"advanced" : 3,
"needTime" : 58126,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"limitAmount" : 1000,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"works" : 58129,
"advanced" : 3,
"needTime" : 58126,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"docsExamined" : 58129,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58129,
"executionTimeMillisEstimate" : 17,
"works" : 58129,
"advanced" : 58129,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 58129,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
]
},
"serverInfo" : {
"host" : "ip-10-0-3-171",
"port" : 27017,
"version" : "4.4.6",
"gitVersion" : "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
"ok" : 1
}
Questions:
If the issue is the amount of doc scanned and we used index, how can I solve it? I do not think that looping over 5000k results in a sort order in a 300k docs should cuz an issue but maybe I'm wrong.
Why the LIMIT stage (with another stage of FETCH) causing the issue?
Thank you very much for all experts in advance!
The issue is the sort. Because you are using skip/limit on a sorted set, it must load the entire set from disk, then sort it in memory (possibly spilling to disk) in order to apply the skip/limit.
This means that if you have 100k documents matching the time range, you load all 100k to get the first batch, and then load 100k again to get the second batch.
An index in mongodb can support a sort if the index includes the field that is sorted on, and any fields that precede that key in the index creation spec are matched with an equality predicate.
Fields matched with inequality or ranges can be listed after the sort key.
The existing index on { _id:1 } is inefficient because it is non-selective and therefore requires reading the entire collection from diks.
The index on { created_at:1 } requires loading only those document that match the query, but they must then be sorted in memory.
To support this query, create an index on { _id:1, created_at:1 }. This index should significantly improve the performance because it can both eliminate the in-memory sort, and only require the query executor to load from disk those documents that match the query. This also has the benefit that the query executor can terminate as soon as the limit is satisfied.
Imagine the following two documents in a myCollection:
{_id: '1', fruits: ['Apple', 'Banana', 'Lemon']}
{_id: '1', fruits: ['Apple', 'Lemon']}
Is there a difference of performance between:
db.myCollection.update({}, {$pull: 'Banana'}, {multi: true)}
and:
db.myCollection.update({fruits: {$elemMatch: 'Banana'}}, {$pull: 'Banana'}, {multi: true)}
//output from mongo client
//ran below queries and compared the output from explain. I did not find any difference in //execution stats for performance
> db.myCollection.explain("executionStats").update(
... {},
... {$pull:{fruits:{$in:["Banana"]}}},
... {multi: true}
... );
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.myCollection",
"indexFilterSet" : false,
"parsedQuery" : {
},
"winningPlan" : {
"stage" : "UPDATE",
"inputStage" : {
"stage" : "COLLSCAN",
"direction" : "forward"
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 2,
"executionStages" : {
"stage" : "UPDATE",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"nMatched" : 2,
"nWouldModify" : 2,
"wouldInsert" : false,
"inputStage" : {
"stage" : "COLLSCAN",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 2,
"needTime" : 1,
"needYield" : 0,
"saveState" : 2,
"restoreState" : 2,
"isEOF" : 1,
"direction" : "forward",
"docsExamined" : 2
}
}
},
"serverInfo" : {
"host" : "XXXX",
"port" : XXXX,
"version" : "4.2.6",
"gitVersion" : "XXXX"
},
"ok" : 1
}
>
--
> db.myCollection.explain("executionStats").update(
... {},
... {$pull:
... {fruits:{$elemMatch:{$in:["Banana"]}}}
... },
... {multi: true}
... );
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.myCollection",
"indexFilterSet" : false,
"parsedQuery" : {
},
"winningPlan" : {
"stage" : "UPDATE",
"inputStage" : {
"stage" : "COLLSCAN",
"direction" : "forward"
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 2,
"executionStages" : {
"stage" : "UPDATE",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"nMatched" : 2,
"nWouldModify" : 2,
"wouldInsert" : false,
"inputStage" : {
"stage" : "COLLSCAN",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 2,
"needTime" : 1,
"needYield" : 0,
"saveState" : 2,
"restoreState" : 2,
"isEOF" : 1,
"direction" : "forward",
"docsExamined" : 2
}
}
},
"serverInfo" : {
"host" : "XXXX",
"port" : XXX,
"version" : "4.2.6",
"gitVersion" : "XXXX"
},
"ok" : 1
}
>
I used Query below its take to much time
Query
db.saleOrder.find({"currentStatus._id":"147"},{"_id":1}).limit(10).explain("executionStats")
ExecutionStats result
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "db_erp_tube.saleOrder",
"indexFilterSet" : false,
"parsedQuery" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "PROJECTION",
"transformBy" : {
"_id" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"direction" : "forward"
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 8673,
"totalKeysExamined" : 0,
"totalDocsExamined" : 3458482,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 8460,
"works" : 3458484,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 10,
"inputStage" : {
"stage" : "PROJECTION",
"nReturned" : 10,
"executionTimeMillisEstimate" : 8450,
"works" : 3458483,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 0,
"invalidates" : 0,
"transformBy" : {
"_id" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"nReturned" : 10,
"executionTimeMillisEstimate" : 8400,
"works" : 3458483,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 0,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 3458482
}
}
}
},
"serverInfo" : {
"host" : "172.16.109",
"port" : 27017,
"version" : "4.0.0",
"gitVersion" : "3b07af3d4f471ae89e8186d33bbb1d5259597d51"
},
"ok" : 1,
"operationTime" : Timestamp(1556365275, 114),
"$clusterTime" : {
"clusterTime" : Timestamp(1556365275, 114),
"signature" : {
"hash" : BinData(0,"ppu91nKmeiC//+UvdsEbjrBTDLU="),
"keyId" : NumberLong("6633468944474701825")
}
}
}
The query took more than 8 seconds(8673ms) to execute because it had to scan all 3458482 documents in the saleOrder collection before returning 10 documents.
This is stated in the filter stage of the explain output.
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
COLLSCAN indicate full collection scan,
"totalDocsExamined" : 3458482,
This is the number of document scanned before an result is returned.
"executionTimeMillis" : 8673,
This is the total time taken for the query to run.
As #the_mahasagar suggested, build a index on currentStatus._id can speed up the query a lot, use the command below.
db.saleOrder.createIndex({ "currentStatus._id": 1})
I don't understand why the following query is so slow, the explain shows it will take 74+ seconds to finish, in spite of the presence of index.
both dev_id and _id are indexed I can assure you, it seems just not helpful at all.
db.DeviceLoginLog.find({"dev_id": "xxx"}).skip(0).limit(10).sort({"_id": -1}).explain("executionStats");
Can anyone help me to interpret explain output and advise me how to speed up?
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "example.DeviceLoginLog",
"indexFilterSet" : false,
"parsedQuery" : {
"dev_id" : {
"$eq" : "xxx"
}
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 10,
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dev_id" : 1
},
"indexName" : "dev_id_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"dev_id" : [
"[\"xxx\", \"xxx\"]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 4,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"dev_id" : {
"$eq" : "xxx"
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 74867,
"totalKeysExamined" : 9203,
"totalDocsExamined" : 9203,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 49860,
"works" : 9537,
"advanced" : 10,
"needTime" : 9204,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"sortPattern" : {
"_id" : -1
},
"memUsage" : 1890,
"memLimit" : 33554432,
"limitAmount" : 10,
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 49820,
"works" : 9525,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 49820,
"works" : 9525,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"docsExamined" : 9203,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 10,
"works" : 9204,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 0,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"keyPattern" : {
"dev_id" : 1
},
"indexName" : "dev_id_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"dev_id" : [
"[\"xxx\", \"xxx\"]"
]
},
"keysExamined" : 9203,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "iZ231ear7c9Z",
"port" : 27017,
"version" : "3.0.3",
"gitVersion" : "b40106b36eecd1b4407eb1ad1af6bc60593c6105"
},
"ok" : 1
}
Thanks in advanceļ¼
The query used index {dev_id: 1} to find and then had to fetch matched docs into memory to sort without using an index. Create a compound index as {dev_id: 1, _id: -1} will improve performance.
See sort-and-non-prefix-subset-of-an-index.
"totalKeysExamined" : 9203,
"totalDocsExamined" : 9203,
That shows that the query had to examine each individual document. You aren't using an index on the _id field. Also depending on which fields you are returning, consider creating an index that allows for a covered query.
A covered query is a query that can be satisfied entirely using an
index and does not have to examine any documents. An index covers a
query when both of the following apply:
all the fields in the query are part of an index, and
all the fields returned in the results are in the same index.
https://docs.mongodb.com/manual/core/query-optimization/