MongoDB Not Using Index for $in or $or Queries - mongodb

Question

I have a MongoDB collection with about 350k documents in it, and I am doing simple count queries based on one of the integer fields, usually using $in. The field is indexed with both db.myColl.createIndex({indexedField: 1}) and db.myColl.createIndex({indexedField: -1}).
When I run a query matching one value, the response comes quickly, as expected:
> db.myColl.explain("executionStats").count({indexedField: 1})
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "myDb.myColl",
"indexFilterSet" : false,
"parsedQuery" : {
"indexedField" : {
"$eq" : 1
}
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "COUNT_SCAN",
"keyPattern" : {
"indexedField" : 1
},
"indexName" : "indexedField_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"indexedField" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"indexBounds" : {
"startKey" : {
"indexedField" : 1
},
"startKeyInclusive" : true,
"endKey" : {
"indexedField" : 1
},
"endKeyInclusive" : true
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 171,
"totalKeysExamined" : 354783,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 101,
"works" : 354783,
"advanced" : 0,
"needTime" : 354782,
"needYield" : 0,
"saveState" : 2772,
"restoreState" : 2772,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 354782,
"nSkipped" : 0,
"inputStage" : {
"stage" : "COUNT_SCAN",
"nReturned" : 354782,
"executionTimeMillisEstimate" : 91,
"works" : 354783,
"advanced" : 354782,
"needTime" : 0,
"needYield" : 0,
"saveState" : 2772,
"restoreState" : 2772,
"isEOF" : 1,
"invalidates" : 0,
"keysExamined" : 354783,
"keyPattern" : {
"indexedField" : 1
},
"indexName" : "indexedField_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"indexedField" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"indexBounds" : {
"startKey" : {
"indexedField" : 1
},
"startKeyInclusive" : true,
"endKey" : {
"indexedField" : 1
},
"endKeyInclusive" : true
}
}
}
},
"serverInfo" : {
"host" : "...",
"port" : 27017,
"version" : "3.4.4",
"gitVersion" : "888390515874a9debd1b6c5d36559ca86b44babd"
},
"ok" : 1
}
However, when I attempt to query for more than one value for indexedField using $in, it slows to a crawl:
> db.myColl.explain("executionStats").count({indexedField: {$in: [1, 2]}})
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "myDb.myColl",
"indexFilterSet" : false,
"parsedQuery" : {
"indexedField" : {
"$in" : [
1,
2
]
}
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"indexedField" : 1
},
"indexName" : "indexedField_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"indexedField" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"indexedField" : [
"[1.0, 1.0]",
"[2.0, 2.0]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"indexedField" : -1
},
"indexName" : "indexedField_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"indexedField" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"indexedField" : [
"[2.0, 2.0]",
"[1.0, 1.0]"
]
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 354782,
"executionTimeMillis" : 215153,
"totalKeysExamined" : 354782,
"totalDocsExamined" : 354782,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 354782,
"executionTimeMillisEstimate" : 214871,
"works" : 354783,
"advanced" : 354782,
"needTime" : 0,
"needYield" : 0,
"saveState" : 11371,
"restoreState" : 11371,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 354782,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 354782,
"executionTimeMillisEstimate" : 748,
"works" : 354783,
"advanced" : 354782,
"needTime" : 0,
"needYield" : 0,
"saveState" : 11371,
"restoreState" : 11371,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"indexedField" : 1
},
"indexName" : "indexedField_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"indexedField" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"indexedField" : [
"[1.0, 1.0]",
"[2.0, 2.0]"
]
},
"keysExamined" : 354782,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "...",
"port" : 27017,
"version" : "3.4.4",
"gitVersion" : "888390515874a9debd1b6c5d36559ca86b44babd"
},
"ok" : 1
}
Using $or instead of $in yields similar bad results:
> db.myColl.explain("executionStats").count({$or: [{indexedField: 1}, {indexedField: 2}] })
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "myDb.myColl",
"indexFilterSet" : false,
"parsedQuery" : {
"$or" : [
{
"indexedField" : {
"$eq" : 1
}
},
{
"indexedField" : {
"$eq" : 2
}
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "SUBPLAN",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"indexedField" : 1
},
"indexName" : "indexedField_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"indexedField" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"indexedField" : [
"[1.0, 1.0]",
"[2.0, 2.0]"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 219269,
"totalKeysExamined" : 354782,
"totalDocsExamined" : 354782,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 219170,
"works" : 354783,
"advanced" : 0,
"needTime" : 354782,
"needYield" : 0,
"saveState" : 11384,
"restoreState" : 11384,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 354782,
"nSkipped" : 0,
"inputStage" : {
"stage" : "SUBPLAN",
"nReturned" : 354782,
"executionTimeMillisEstimate" : 219090,
"works" : 354783,
"advanced" : 354782,
"needTime" : 0,
"needYield" : 0,
"saveState" : 11384,
"restoreState" : 11384,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 354782,
"executionTimeMillisEstimate" : 219040,
"works" : 354783,
"advanced" : 354782,
"needTime" : 0,
"needYield" : 0,
"saveState" : 11383,
"restoreState" : 11383,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 354782,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 354782,
"executionTimeMillisEstimate" : 686,
"works" : 354783,
"advanced" : 354782,
"needTime" : 0,
"needYield" : 0,
"saveState" : 11383,
"restoreState" : 11383,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"indexedField" : 1
},
"indexName" : "indexedField_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"indexedField" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"indexedField" : [
"[1.0, 1.0]",
"[2.0, 2.0]"
]
},
"keysExamined" : 354782,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "...",
"port" : 27017,
"version" : "3.4.4",
"gitVersion" : "888390515874a9debd1b6c5d36559ca86b44babd"
},
"ok" : 1
}
Any idea what is going wrong here?
You should be able to reproduce this by applying the following script to a Mongo installation.
Create a file create_test_database.js:
db.indexTestColl.createIndex({indexedField: 1});
var RECORDS_TO_CREATE = 5000000;
for (var i = 0; i < RECORDS_TO_CREATE; i++)
{
// Populate indexedField with random numbers [1 - 3].
db.indexTestColl.insertOne({"indexedField": NumberInt((Math.random() * 10) % 3 + 1)});
if ((i + 1) % 10000 == 0) print("Inserted " + (i + 1) + " documents.");
}
Create and populate the collection:
mongo localhost:27017/indexTestDb create_test_database.js
Then test it with these queries:
use indexTestDb
db.indexTestColl.explain("executionStats").count({indexedField: 1})
db.indexTestColl.explain("executionStats").count({indexedField: {$in: [1, 2]}})
db.indexTestColl.explain("executionStats").count({$or: [{indexedField: 1}, {indexedField: 2}] })
Am I correct in assuming that the $in and $or queries should benefit from the index being there?