I have the following command:
db.mycol.find({sourceId:ObjectId("596bac5a6f473e1a042bFFFF"),myFlag:false}).count()
It was taking a lot of time.
So I decided to add an index { "sourceId" : -1, "myFlag" : -1 }
MongoDB still kept taking the same time.
When I ran explain() on the find command, I noticed that mongoDB still kept using my old indexes.
db.mycol.find({sourceId:ObjectId("596bac5a6f473e1a042bFFFF"),myFlag:false}).explain(true)
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mydb.mycol",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"sourceId" : {
"$eq" : ObjectId("596bac5a6f473e1a042bFFFF")
}
},
{
"myFlag" : {
"$eq" : false
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"myFlag" : {
"$eq" : false
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"sourceId" : 1
},
"indexName" : "sourceId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"sourceId" : {
"$eq" : ObjectId("596bac5a6f473e1a042bFFFF")
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"myFlag" : -1
},
"indexName" : "myFlag_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"myFlag" : [
"[false, false]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"myFlag" : {
"$eq" : false
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"sourceId" : -1,
"timestamp" : -1
},
"indexName" : "sourceId_-1_timestamp_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
],
"timestamp" : [
"[MaxKey, MinKey]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"myFlag" : {
"$eq" : false
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"sourceId" : -1,
"timestamp" : 1
},
"indexName" : "sourceId_-1_timestamp_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
],
"timestamp" : [
"[MinKey, MaxKey]"
]
}
}
},
{
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"sourceId" : -1,
"myFlag" : -1
},
"indexName" : "sourceId_-1_myFlag_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
],
"myFlag" : [
"[false, false]"
]
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 2074,
"executionTimeMillis" : 33936,
"totalKeysExamined" : 2074,
"totalDocsExamined" : 2074,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"myFlag" : {
"$eq" : false
}
},
"nReturned" : 2074,
"executionTimeMillisEstimate" : 31710,
"works" : 2075,
"advanced" : 2074,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 2074,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 2074,
"executionTimeMillisEstimate" : 0,
"works" : 2075,
"advanced" : 2074,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"sourceId" : 1
},
"indexName" : "sourceId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
]
},
"keysExamined" : 2074,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [
{
"nReturned" : 0,
"executionTimeMillisEstimate" : 1630,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"sourceId" : {
"$eq" : ObjectId("596bac5a6f473e1a042bFFFF")
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 1630,
"works" : 101,
"advanced" : 0,
"needTime" : 101,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"myFlag" : -1
},
"indexName" : "myFlag_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"myFlag" : [
"[false, false]"
]
},
"keysExamined" : 101,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"myFlag" : {
"$eq" : false
}
},
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"sourceId" : -1,
"timestamp" : -1
},
"indexName" : "sourceId_-1_timestamp_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
],
"timestamp" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 101,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 10,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"myFlag" : {
"$eq" : false
}
},
"nReturned" : 101,
"executionTimeMillisEstimate" : 10,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"sourceId" : -1,
"timestamp" : 1
},
"indexName" : "sourceId_-1_timestamp_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
],
"timestamp" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 101,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1554,
"restoreState" : 1554,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"sourceId" : -1,
"myFlag" : -1
},
"indexName" : "sourceId_-1_myFlag_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
],
"myFlag" : [
"[false, false]"
]
},
"keysExamined" : 101,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"myFlag" : {
"$eq" : false
}
},
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"sourceId" : 1
},
"indexName" : "sourceId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"sourceId" : [
"[ObjectId('596bac5a6f473e1a042bFFFF'), ObjectId('596bac5a6f473e1a042bFFFF')]"
]
},
"keysExamined" : 101,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
]
},
"serverInfo" : {
"host" : "gdm3d",
"port" : 27017,
"version" : "3.2.15",
"gitVersion" : "e11e3c1b9c9ce3f7b4a79493e16f5e4504e01140"
},
"ok" : 1
}
For some reason, it rejects the plan to use sourceId_-1_myFlag_-1 and goes with sourceId_1 and then filters myFlag document by document.
When I forced it to use the index, it returned the result immediately!
db.mycol.find({sourceId:ObjectId("596bac5a6f473e1a042bFFFF"),myFlag:false}).hint("sourceId_-1_myFlag_-1").count()
I read that periodically mongoDB runs all the plans and checks which one is faster, then uses that one in future.
I even tried restarting the mongodb, clearing query plan cache, restarting the computer. For some sourceIds with very less data, it did use the correct index automatically (I checked the index usage) but then it again started using the old ones.
I don't want to force it to use the index. How do I make sure it automatically does that?
Update:
It started using the correct one after some time. Atleast thats what the indexStats says.
However, the command still takes time, and when i give it hint, it gives output immediately.
Related
I'm facing with a strange issue. The explain of the query shows usage of index with a fast execution time in all stages but the last stage of LIMIT breaks it all. I have execution time of 60s and more!
The DB is used for marketing tool and we collect data on the campaign activity. We have 100k+ records in DB per each hour and I want to select all the 100k in batches of 5k (I tried to reduce it to 1k also) in order to make statistics aggregation.
Just for tests, I will add an example of the query explanation.
If I reduce the created_at date range to 10 minutes and I set the limit to 1000, it works fast.
If I set the created_at date range to 30 minutes and I set the limit to 1000, it stock again.
I have 8 CPU and 64 memory
Storage is with 6000 IOPS
The table include 900 million records in total
I have the following indexes:
created_at: 1
_id: 1
{created_at: 1, _id: 1} - not used in query
Slow query for 30 minutes
> db.logs.explain('allPlansExecution').aggregate([{"$match":{"created_at":{"$gte":ISODate('2021-06-02T20:00:00.000+00:00'),"$lte":ISODate('2021-06-02T20:30:00.000+00:00')}}},{"$sort":{"_id":1}},{"$limit":1000}], { allowDiskUse: true });
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "webpush.campaign_action_logs",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"optimizedPipeline" : true,
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 100,
"executionTimeMillis" : 183411,
"totalKeysExamined" : 257959,
"totalDocsExamined" : 257959,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"works" : 258061,
"advanced" : 100,
"needTime" : 257960,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"totalDataSizeSorted" : 187871472,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 533,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"docsExamined" : 257959,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 265,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
},
"keysExamined" : 257959,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"allPlansExecution" : [
{
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"totalKeysExamined" : 257959,
"totalDocsExamined" : 257959,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 100,
"executionTimeMillisEstimate" : 625,
"works" : 258060,
"advanced" : 100,
"needTime" : 257960,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 100,
"type" : "simple",
"totalDataSizeSorted" : 187871472,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 533,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"docsExamined" : 257959,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 257959,
"executionTimeMillisEstimate" : 265,
"works" : 257960,
"advanced" : 257959,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622665800000)]"
]
},
"keysExamined" : 257959,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
{
"nReturned" : 70,
"executionTimeMillisEstimate" : 178092,
"totalKeysExamined" : 258060,
"totalDocsExamined" : 258060,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 70,
"executionTimeMillisEstimate" : 178092,
"works" : 258060,
"advanced" : 70,
"needTime" : 257990,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:30:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"nReturned" : 70,
"executionTimeMillisEstimate" : 178057,
"works" : 258060,
"advanced" : 70,
"needTime" : 257990,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"docsExamined" : 258060,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 258060,
"executionTimeMillisEstimate" : 645,
"works" : 258060,
"advanced" : 258060,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9571,
"restoreState" : 9571,
"isEOF" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 258060,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
]
},
"serverInfo" : {
"host" : "ip-10-0-3-171",
"port" : 27017,
"version" : "4.4.6",
"gitVersion" : "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
"ok" : 1
}
Faster query for 10 minutes:
> db.logs.explain('allPlansExecution').aggregate([{"$match":{"created_at":{"$gte":ISODate('2021-06-02T20:00:00.000+00:00'),"$lte":ISODate('2021-06-02T20:10:00.000+00:00')}}},{"$sort":{"_id":1}},{"$limit":1000}], { allowDiskUse: true });
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "webpush.campaign_action_logs",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"optimizedPipeline" : true,
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 1000,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1000,
"executionTimeMillis" : 1502,
"totalKeysExamined" : 58027,
"totalDocsExamined" : 58027,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1000,
"executionTimeMillisEstimate" : 122,
"works" : 59029,
"advanced" : 1000,
"needTime" : 58028,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"totalDataSizeSorted" : 42213931,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 96,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"docsExamined" : 58027,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 40,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
},
"keysExamined" : 58027,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"allPlansExecution" : [
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 122,
"totalKeysExamined" : 58027,
"totalDocsExamined" : 58027,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 101,
"executionTimeMillisEstimate" : 122,
"works" : 58129,
"advanced" : 101,
"needTime" : 58028,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memLimit" : 104857600,
"limitAmount" : 1000,
"type" : "simple",
"totalDataSizeSorted" : 42213931,
"usedDisk" : false,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 96,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 1,
"docsExamined" : 58027,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58027,
"executionTimeMillisEstimate" : 40,
"works" : 58028,
"advanced" : 58027,
"needTime" : 0,
"needYield" : 0,
"saveState" : 121,
"restoreState" : 121,
"isEOF" : 1,
"keyPattern" : {
"created_at" : 1
},
"indexName" : "created_at_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"created_at" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"created_at" : [
"[new Date(1622664000000), new Date(1622664600000)]"
]
},
"keysExamined" : 58027,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
{
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"totalKeysExamined" : 58129,
"totalDocsExamined" : 58129,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"works" : 58129,
"advanced" : 3,
"needTime" : 58126,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"limitAmount" : 1000,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"created_at" : {
"$lte" : ISODate("2021-06-02T20:10:00Z")
}
},
{
"created_at" : {
"$gte" : ISODate("2021-06-02T20:00:00Z")
}
}
]
},
"nReturned" : 3,
"executionTimeMillisEstimate" : 935,
"works" : 58129,
"advanced" : 3,
"needTime" : 58126,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"docsExamined" : 58129,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 58129,
"executionTimeMillisEstimate" : 17,
"works" : 58129,
"advanced" : 58129,
"needTime" : 0,
"needYield" : 0,
"saveState" : 122,
"restoreState" : 122,
"isEOF" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 58129,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
]
},
"serverInfo" : {
"host" : "ip-10-0-3-171",
"port" : 27017,
"version" : "4.4.6",
"gitVersion" : "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
"ok" : 1
}
Questions:
If the issue is the amount of doc scanned and we used index, how can I solve it? I do not think that looping over 5000k results in a sort order in a 300k docs should cuz an issue but maybe I'm wrong.
Why the LIMIT stage (with another stage of FETCH) causing the issue?
Thank you very much for all experts in advance!
The issue is the sort. Because you are using skip/limit on a sorted set, it must load the entire set from disk, then sort it in memory (possibly spilling to disk) in order to apply the skip/limit.
This means that if you have 100k documents matching the time range, you load all 100k to get the first batch, and then load 100k again to get the second batch.
An index in mongodb can support a sort if the index includes the field that is sorted on, and any fields that precede that key in the index creation spec are matched with an equality predicate.
Fields matched with inequality or ranges can be listed after the sort key.
The existing index on { _id:1 } is inefficient because it is non-selective and therefore requires reading the entire collection from diks.
The index on { created_at:1 } requires loading only those document that match the query, but they must then be sorted in memory.
To support this query, create an index on { _id:1, created_at:1 }. This index should significantly improve the performance because it can both eliminate the in-memory sort, and only require the query executor to load from disk those documents that match the query. This also has the benefit that the query executor can terminate as soon as the limit is satisfied.
I have several questions about the way mongoDb choose the index which will be more efficient to retrieve datas.
I have the following query :
db.getCollection('myCollection').find({
"TenantId":"1a1a1a1a",
"ContractId": 1111,
"Lists":1111,
"Email":{"$exists":true,"$nin":["",null]},
"Optouts.Media":{"$nin":["EMAIL",null]},
"Deleted":false
})
This query is used in a mapReduce which failed. When I run it with count(), I'll get the result of 290000 in 47s.
So, to understand what happen, I execute the query again with explain mode. And indeed, the index used seems to don't be the best (TenantId_1_ContractId_1_GUID_1_Deleted_1).
I re-run the same query with hint(TenantId_1_ContractId_1_Lists_1_Deleted_1) and this time, I'll get result in 2 sec.
How mongo select the index to retrieve datas?
Good to know:
For the current contractId I've 1,6 million entry.
When filter on the lists, I've got 700 000 entries
In documents, lists is an array of values
I have retrieve index planExecution to try to get more informations, but I can't figure out. See below result of explain('allPlansExecution').
{
"queryPlanner" : {
"mongosPlannerVersion" : 1,
"winningPlan" : {
"stage" : "SINGLE_SHARD",
"shards" : [
{
"shardName" : "rs1",
"connectionString" : "...",
"serverInfo" : {...},
"plannerVersion" : 1,
"namespace" : "",
"indexFilterSet" : false,
"parsedQuery" : {...},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {...},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"TenantId" : 1.0,
"ContractId" : 1.0,
"GUID" : 1.0,
"Deleted" : 1.0
},
"indexName" : "TenantId_1_ContractId_1_GUID_1_Deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"TenantId" : [],
"ContractId" : [],
"GUID" : [],
"Deleted" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {...}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {...},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"TenantId" : 1.0,
"ContractId" : 1.0,
"Lists" : 1.0,
"Deleted" : 1.0
},
"indexName" : "TenantId_1_ContractId_1_Lists_1_Deleted_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"TenantId" : [],
"ContractId" : [],
"Lists" : [
"Lists"
],
"Deleted" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {...}
}
},
{...}
]
}
]
}
},
"executionStats" : {...},
"allPlansExecution" : [
undefined,
{
"shardName" : "rs1",
"allPlans" : [
{
"nReturned" : 0,
"executionTimeMillisEstimate" : 51,
"totalKeysExamined" : 10618,
"totalDocsExamined" : 10618,
"executionStages" : {
"stage" : "FETCH",
"filter" : {...},
"nReturned" : 0,
"executionTimeMillisEstimate" : 51,
"works" : 10618,
"advanced" : 0,
"needTime" : 10618,
"needYield" : 0,
"saveState" : 664,
"restoreState" : 664,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 10618,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 10618,
"executionTimeMillisEstimate" : 20,
"works" : 10618,
"advanced" : 10618,
"needTime" : 0,
"needYield" : 0,
"saveState" : 664,
"restoreState" : 664,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"TenantId" : 1.0,
"ContractId" : 1.0,
"Lists" : 1.0,
"Deleted" : 1.0
},
"indexName" : "TenantId_1_ContractId_1_Lists_1_Deleted_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"TenantId" : [],
"ContractId" : [],
"Lists" : [
"Lists"
],
"Deleted" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {...},
"keysExamined" : 10618,
"seeks" : 1,
"dupsTested" : 10618,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"totalKeysExamined" : 10618,
"totalDocsExamined" : 10616,
"executionStages" : {
"stage" : "FETCH",
"filter" : {...},
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 10618,
"advanced" : 101,
"needTime" : 10517,
"needYield" : 0,
"saveState" : 664,
"restoreState" : 664,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 10616,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 10616,
"executionTimeMillisEstimate" : 0,
"works" : 10618,
"advanced" : 10616,
"needTime" : 2,
"needYield" : 0,
"saveState" : 664,
"restoreState" : 664,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"TenantId" : 1.0,
"ContractId" : 1.0,
"GUID" : 1.0,
"Deleted" : 1.0
},
"indexName" : "TenantId_1_ContractId_1_GUID_1_Deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"TenantId" : [],
"ContractId" : [],
"GUID" : [],
"Deleted" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {...},
"keysExamined" : 10618,
"seeks" : 3,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
]
}
]
}
}
I have remove some lines {...}, if you want more details, don't hesitate to ask me.
Thanks a lot for your help!
I was checking that my query is optimum or not. For that I used Cursor.explain("executionStats"). At there few keys present which I am curious to know about.
So I found needTime. I researched on MongoDB Official Documentation but unfortunately, I was not able to understand. Is there anyone who help me out regarding this.
This is my result:
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"views" : 0,
},
"inputStage" : {
"stage" : "SORT",
"sortPattern" : {
"createdate" : -1
},
"limitAmount" : 20,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"post_status" : {
"$eq" : 1
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"user_id" : 1,
"lastmodified" : -1
},
"indexName" : "user_id_1_lastmodified_-1",
"isMultiKey" : true,
"multiKeyPaths" : {
"user_id" : [ ],
"lastmodified" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"user_id" : [
"[MinKey, MaxKey]"
],
"lastmodified" : [
"[MaxKey, MinKey]"
]
}
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 20,
"executionTimeMillis" : 1,
"totalKeysExamined" : 386,
"totalDocsExamined" : 375,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 20,
"executionTimeMillisEstimate" : 0,
"works" : 409,
"advanced" : 20,
"needTime" : 388,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"views" : 0,
},
"inputStage" : {
"stage" : "SORT",
"nReturned" : 20,
"executionTimeMillisEstimate" : 0,
"works" : 409,
"advanced" : 20,
"needTime" : 388,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"createdate" : -1
},
"memUsage" : 19348,
"memLimit" : 33554432,
"limitAmount" : 20,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 225,
"executionTimeMillisEstimate" : 0,
"works" : 388,
"advanced" : 225,
"needTime" : 162,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"post_status" : {
"$eq" : 1
}
},
"nReturned" : 225,
"executionTimeMillisEstimate" : 0,
"works" : 387,
"advanced" : 225,
"needTime" : 161,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 375,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 375,
"executionTimeMillisEstimate" : 0,
"works" : 387,
"advanced" : 375,
"needTime" : 11,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"user_id" : 1,
"lastmodified" : -1
},
"indexName" : "user_id_1_lastmodified_-1",
"isMultiKey" : true,
"multiKeyPaths" : {
"user_id" : [ ],
"lastmodified" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"user_id" : [
"[MinKey, MaxKey]"
],
"lastmodified" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 386,
"seeks" : 1,
"dupsTested" : 386,
"dupsDropped" : 11,
"seenInvalidated" : 0
}
}
}
}
}
},
I have a mongo document as below.
{
"_id":"1234",
"empId" : "1234"
"manager":"456",
"name" :"Mike",
"mgrHierarchy":"456>678>789>901>999",
"mgrs": [
"456",
"678",
"789",
"901",
"999"
]
}
Goal is to get the hierarchy for the given empId. Using mgrHierarchy for sorting, so that the hierarchy can be retrieved easily.
mgrs - will have all the manager from 1st level to the top level.
Mgrs & mgrHierarchy will hold the samething in a different format.
Have index on empId, mgrHierarchy, mgrs, mgrs.0, mgrs.1, mgrs.2, mgrs.3, mgrs.4, .... This collection has around 20 indexes in total.
Query :-
{ "$or" : [ { "mgrs.2" : "901"} , { "_id" : { "$in" : [ "901"]}} , { "mgrs.0" : "901"} , { "mgrs.1" : "901"} , { "mgrs.4" : "901"} , { "mgrs.3" : "901"}]}).sort({ mgrHierarchy: 1 }
This query count returns 100k.
This query is returning memory error.
Exec error: OperationFailed: Sort operation used more than the maximum
33554432 bytes of RAM. Add an index, or specify a smaller limit.,
state: FAILURE.
explain("executionStats") output,
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.test",
"indexFilterSet" : false,
"parsedQuery" : {
"$or" : [
{
"_id" : {
"$eq" : "901"
}
},
{
"mgrs.0" : {
"$eq" : "901"
}
},
{
"mgrs.1" : {
"$eq" : "901"
}
},
{
"mgrs.2" : {
"$eq" : "901"
}
},
{
"mgrs.3" : {
"$eq" : "901"
}
},
{
"mgrs.4" : {
"$eq" : "901"
}
}
]
},
"winningPlan" : {
"stage" : "SUBPLAN",
"inputStage" : {
"stage" : "PROJECTION",
"transformBy" : {
},
"inputStage" : {
"stage" : "SORT",
"sortPattern" : {
"mgrHierarchy" : 1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[\"901\", \"901\"]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"mgrs.0" : 1
},
"indexName" : "mgrs.0_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"mgrs.0" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.0" : [
"[\"901\", \"901\"]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"mgrs.1" : 1
},
"indexName" : "mgrs.1_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"mgrs.1" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.1" : [
"[\"901\", \"901\"]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"mgrs.2" : 1
},
"indexName" : "mgrs.2_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"mgrs.2" : [
"mgrs"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.2" : [
"[\"901\", \"901\"]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"mgrs.3" : 1
},
"indexName" : "mgrs.3_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"mgrs.3" : [
"mgrs"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.3" : [
"[\"901\", \"901\"]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"mgrs.4" : 1
},
"indexName" : "mgrs.4_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"mgrs.4" : [
"mgrs"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.4" : [
"[\"901\", \"901\"]"
]
}
}
]
}
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : false,
"errorMessage" : "Exec error: OperationFailed: Sort operation used more than the maximum 33554432 bytes of RAM. Add an index, or specify a smaller limit., state: FAILURE",
"errorCode" : 96,
"nReturned" : 0,
"executionTimeMillis" : 1120,
"totalKeysExamined" : 47184,
"totalDocsExamined" : 47184,
"executionStages" : {
"stage" : "SUBPLAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 1109,
"works" : 47191,
"advanced" : 0,
"needTime" : 47190,
"needYield" : 0,
"saveState" : 2067,
"restoreState" : 2067,
"isEOF" : 0,
"invalidates" : 0,
"inputStage" : {
"stage" : "PROJECTION",
"nReturned" : 0,
"executionTimeMillisEstimate" : 232,
"works" : 47191,
"advanced" : 0,
"needTime" : 47190,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 0,
"invalidates" : 0,
"transformBy" : {
},
"inputStage" : {
"stage" : "SORT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 221,
"works" : 47191,
"advanced" : 0,
"needTime" : 47190,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 0,
"invalidates" : 0,
"sortPattern" : {
"mgrHierarchy" : 1
},
"memUsage" : 33555113,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 47184,
"executionTimeMillisEstimate" : 201,
"works" : 47190,
"advanced" : 47184,
"needTime" : 6,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 0,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 47184,
"executionTimeMillisEstimate" : 141,
"works" : 47189,
"advanced" : 47184,
"needTime" : 5,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 47184,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "OR",
"nReturned" : 47184,
"executionTimeMillisEstimate" : 91,
"works" : 47189,
"advanced" : 47184,
"needTime" : 5,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 0,
"invalidates" : 0,
"dupsTested" : 47184,
"dupsDropped" : 0,
"recordIdsForgotten" : 0,
"inputStages" : [
{
"stage" : "IXSCAN",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 2,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[\"901\", \"901\"]"
]
},
"keysExamined" : 1,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 22,
"executionTimeMillisEstimate" : 0,
"works" : 23,
"advanced" : 22,
"needTime" : 0,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"mgrs.0" : 1
},
"indexName" : "mgrs.0_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"mgrs.0" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.0" : [
"[\"901\", \"901\"]"
]
},
"keysExamined" : 22,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 349,
"executionTimeMillisEstimate" : 0,
"works" : 350,
"advanced" : 349,
"needTime" : 0,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"mgrs.1" : 1
},
"indexName" : "mgrs.1_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"mgrs.1" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.1" : [
"[\"901\", \"901\"]"
]
},
"keysExamined" : 349,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 2859,
"executionTimeMillisEstimate" : 0,
"works" : 2860,
"advanced" : 2859,
"needTime" : 0,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"mgrs.2" : 1
},
"indexName" : "mgrs.2_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"mgrs.2" : [
"mgrs"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.2" : [
"[\"901\", \"901\"]"
]
},
"keysExamined" : 2859,
"seeks" : 1,
"dupsTested" : 2859,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 16475,
"executionTimeMillisEstimate" : 40,
"works" : 16476,
"advanced" : 16475,
"needTime" : 0,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"mgrs.3" : 1
},
"indexName" : "mgrs.3_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"mgrs.3" : [
"mgrs"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.3" : [
"[\"901\", \"901\"]"
]
},
"keysExamined" : 16475,
"seeks" : 1,
"dupsTested" : 16475,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 27478,
"executionTimeMillisEstimate" : 20,
"works" : 27478,
"advanced" : 27478,
"needTime" : 0,
"needYield" : 0,
"saveState" : 371,
"restoreState" : 371,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"mgrs.4" : 1
},
"indexName" : "mgrs.4_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"mgrs.4" : [
"mgrs"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"mgrs.4" : [
"[\"901\", \"822717775\"]"
]
},
"keysExamined" : 27478,
"seeks" : 1,
"dupsTested" : 27478,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
]
}
}
}
}
}
}
},
"serverInfo" : {
"host" : "test",
"port" : 10904,
"version" : "3.4.14",
"gitVersion" : "fd954412dfc10e4d1e3e2dd4fac040f8b476b268"
},
"ok" : 1
}
I have indexes for all the fields, not sure why still getting this error. Any pointer for this.
I'm going through the existing indexes on a fairly old MongoDB database which has recently been upgraded from 2.something to 3.4.
The code accessing it is convoluted, automagic, and spaghetti-heavy, plus it allows web-facing frontend code to do arbitrary queries, so it's pretty difficult to get an overview of exactly what queries are made. Doing a bit of profiling, I quickly discovered that a lot of queries were missing indexes, either because logical indexes were missing or misspelled (yeah!), or because someone had assumed that a compound index would support queries on non-prefix parts of them.
Such an example was with a users collection where a compound index, { boxes._id: 1, _deleted: 1 } was missed by a ton of queries filtering by { _deleted: { $exists: false } }.
I wiped it, and created single-field indexes instead, having read up on the index intersection feature which is now part of the version of the MongoDB server backing this app:
db.users.dropIndex({ 'boxes._id': 1, _deleted: 1 });
db.users.ensureIndex({ 'boxes._id': 1 });
db.users.ensureIndex({ '_deleted': 1 });
So far, so good. The dataset here is not very large, so I saw no immediate issue, performance-wise, by dropping the compound index and support index intersection instead. AFAIU from the docs and the interwebs, the performance gains from using compound indexes are mostly felt with really large collections, and for this one, we're looking at about 11k documents.
Simple count queries on either of the two indexed fields yield satisfying responses while hitting the keys:
db.runCommand({ explain: { count: 'users', query: { 'boxes._id': ObjectId('597745846ca2582d8b364c38') }, verbosity: 'executionStats' }})
// "executionTimeMillis" : 8
db.runCommand({ explain: { count: 'users', query: { _deleted: { $exists: false } }, verbosity: 'executionStats' }})
// "executionTimeMillis" : 35
(Note that this is on my old MacBook Air, and compared to query times I'm seeing in the performance log when not using explain, then "normal" response times seem to be less than 25% of those reported with explain in the mongo shell)
However, if I filter on both those fields thus triggering the use of an intersection index, I see a massive performance penalty:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : { "_deleted" : [ "[null, null]" ] }
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"inputStage" : {
"stage" : "AND_SORTED",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
}
}
]
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 11098,
"executionTimeMillis" : 731,
"totalKeysExamined" : 11098,
"totalDocsExamined" : 11098,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
}
]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 702,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 127,
"restoreState" : 127,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 11098,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 11098,
"executionTimeMillisEstimate" : 49,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 127,
"restoreState" : 127,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 11098,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
},
"keysExamined" : 101,
"seeks" : 1,
"dupsTested" : 101,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
{
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"totalKeysExamined" : 101,
"totalDocsExamined" : 50,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"works" : 101,
"advanced" : 50,
"needTime" : 51,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 50,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "AND_SORTED",
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"works" : 101,
"advanced" : 50,
"needTime" : 51,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"flagged" : 0,
"failedAnd_0" : 0,
"failedAnd_1" : 0,
"inputStages" : [
{
"stage" : "IXSCAN",
"nReturned" : 51,
"executionTimeMillisEstimate" : 0,
"works" : 51,
"advanced" : 51,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
},
"keysExamined" : 51,
"seeks" : 1,
"dupsTested" : 51,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"works" : 50,
"advanced" : 50,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 50,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
]
}
}
},
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 11,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
}
]
},
"nReturned" : 101,
"executionTimeMillisEstimate" : 11,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 101,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
]
}
}
The same query hitting the compound index:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1,
"_deleted" : 1
},
"indexName" : "boxes._id_1__deleted_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
],
"_deleted" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
],
"_deleted" : [ "[null, null]" ]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 37,
"totalKeysExamined" : 11098,
"totalDocsExamined" : 11098,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 35,
"works" : 11099,
"advanced" : 0,
"needTime" : 11098,
"needYield" : 0,
"saveState" : 87,
"restoreState" : 87,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 11098,
"nSkipped" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 35,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 87,
"restoreState" : 87,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 11098,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 11098,
"executionTimeMillisEstimate" : 23,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 87,
"restoreState" : 87,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1,
"_deleted" : 1
},
"indexName" : "boxes._id_1__deleted_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
],
"_deleted" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
],
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 11098,
"seeks" : 1,
"dupsTested" : 11098,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"allPlansExecution" : [ ]
}
}
Replacing the compound index with just a single index on boxes._id yields pretty much identical performance to having the compound key on both filtered fields:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 31,
"totalKeysExamined" : 11098,
"totalDocsExamined" : 11098,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 34,
"works" : 11099,
"advanced" : 0,
"needTime" : 11098,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 11098,
"nSkipped" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 34,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 11098,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 11098,
"executionTimeMillisEstimate" : 11,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
},
"keysExamined" : 11098,
"seeks" : 1,
"dupsTested" : 11098,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"allPlansExecution" : [ ]
}
}
Finally, dropping all indexes on these fields, triggering a COLLSCAN yields performance pretty much identical to the index intersection one:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"ideaboxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"ideaboxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"direction" : "forward"
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 848,
"totalKeysExamined" : 0,
"totalDocsExamined" : 11109,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 847,
"works" : 11111,
"advanced" : 0,
"needTime" : 11110,
"needYield" : 0,
"saveState" : 123,
"restoreState" : 123,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 11098,
"nSkipped" : 0,
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"ideaboxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 847,
"works" : 11111,
"advanced" : 11098,
"needTime" : 12,
"needYield" : 0,
"saveState" : 123,
"restoreState" : 123,
"isEOF" : 1,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 11109
}
},
"allPlansExecution" : [ ]
}
}
Surely getting COLLSCAN performance with index intersection queries can't be right? I admit I'm very new with Mongo, so I'm also quite ignorant when trying to decipher this report — the answer may be staring me in the face.
I've tested the same queries on Mongo 3.2 on an ObjectRocket instance, and while their servers are much faster than my ageing laptop, they exhibit the same numbers relatively speaking.
Any ideas why this is happening?
Thank you :)