Mongo Distinct Index for Dot Notated - mongodb

I have a mongodb v4.2.2 hosted on Atlas. Have a collection
objects and two fields in it: metadata as array of objects, each
object has one field key AND second field named as model. Added compound index
model->metadata.key as regular one. While running db command
db.objects.explain('executionStats').distinct('metadata.key',
{model: ObjectId('5e18aff58a5aaffdc3d6f26d')}) from console it
doesn't use my model->metadata.key index and doesn't use DISTINCT_SCAN.
But if I move data to another collection lets say objectKeys that
has two fields like key, model and again add regular compound
index model->key and run command
db.objectKeys.explain('executionStats').distinct('key', {model:
ObjectId('5e18aff58a5aaffdc3d6f26d')}) it is successfully applying DISTINCT_SCAN
and use the index.
Question. How to force MongoDB to use indexes for dot notated
field during distinct operation?
UPDATE DETAILS:
1.
db.objects.getIndexes();
{
"v" : 2,
"key" : {
"model" : 1,
"metadata.key" : 1,
"metadata.value" : 1
},
"name" : "model_1_metadata.key_1_metadata.value_1",
"ns" : "my_db.objects",
"background" : true
}
db.objects.explain('executionStats').distinct('metadata.key', {model: ObjectId('5e18aff58a5aaffdc3d6f26d')})
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"keyPattern" : {
"model" : 1,
"metadata.key" : 1,
"metadata.value" : 1
},
"indexName" : "model_1_metadata.key_1_metadata.value_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"model" : [ ],
"metadata.key" : [
"metadata"
],
"metadata.value" : [
"metadata"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"model" : [
"[ObjectId('5e18aff58a5aaffdc3d6f26d'), ObjectId('5e18aff58a5aaffdc3d6f26d')]"
],
"metadata.key" : [
"[MinKey, MaxKey]"
],
"metadata.value" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 0,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
2.
db.object_keys.getIndexes();
{
"v" : 2,
"key" : {
"models" : 1,
"key" : 1
},
"name" : "models_1_key_1",
"ns" : "my_db.object_keys",
"background" : true
},
db.object_keys.explain('executionStats').distinct('key', {models: ObjectId('5e18aff58a5aaffdc3d6f26d')})
"executionStages" : {
"stage" : "PROJECTION_COVERED",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"transformBy" : {
"_id" : 0,
"key" : 1
},
"inputStage" : {
"stage" : "DISTINCT_SCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"keyPattern" : {
"models" : 1,
"key" : 1
},
"indexName" : "models_1_key_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"models" : [
"models"
],
"key" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"models" : [
"[ObjectId('5e18aff58a5aaffdc3d6f26d'), ObjectId('5e18aff58a5aaffdc3d6f26d')]"
],
"key" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 0
}
}
So as you can see in the first case it uses IXSCAN means index scan and in the second case it uses DISTINCT_SCAN that is considered more faster. The differences is that in first case the metadata field is array of objects and in the second case the field key is string

Related

mongodb query slow in different version and env

I have two env mongodbs,
the difference between them is:
test mongodb version: 3.2.20 , prod mongodb version : 4.0.18
and test env query plan first stage is Limit , however the other is Sort.
in my test env, it's very quick and totalDocsExamined == limit
they both hit the index:
{
"v" : 1,
"key" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"name" : "appIds_1_ctime_-1_background_1",
"ns" : "newsmine.newstoapp"
}
query: db.newstoapp.find({"appIds":{"$in":[999]}}).sort({"ctime":-1}).limit(10).explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "newsmine.newstoapp",
"indexFilterSet" : false,
"parsedQuery" : {
"appIds" : {
"$in" : [
999
]
}
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[999.0, 999.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 0,
"totalKeysExamined" : 10,
"totalDocsExamined" : 10,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 0,
"works" : 11,
"advanced" : 10,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 10,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 10,
"executionTimeMillisEstimate" : 0,
"works" : 10,
"advanced" : 10,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 10,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 10,
"executionTimeMillisEstimate" : 0,
"works" : 10,
"advanced" : 10,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[999.0, 999.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 10,
"dupsTested" : 10,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
},
"serverInfo" : {
"host" : "",
"port" : ,
"version" : "3.2.20",
"gitVersion" : "a7a144f40b70bfe290906eb33ff2714933544af8"
},
"ok" : 1
}
in my prod env, it's getting slow query
query: datamongo:PRIMARY> db.newstoapp.find({"appIds":{"$in":[1460]}}).sort({"ctime":-1}).limit(10).explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "newsmine.newstoapp",
"indexFilterSet" : false,
"parsedQuery" : {
"appIds" : {
"$eq" : 1460
}
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"ctime" : -1
},
"limitAmount" : 10,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[1460.0, 1460.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"ctime" : -1
},
"limitAmount" : 10,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"appIds" : 1
},
"indexName" : "appIds_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[1460.0, 1460.0]"
]
}
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"appIds" : {
"$eq" : 1460
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"ctime" : 1
},
"indexName" : "ctime_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"ctime" : [
"[MaxKey, MinKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 40,
"totalKeysExamined" : 405,
"totalDocsExamined" : 405,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 3,
"works" : 418,
"advanced" : 10,
"needTime" : 407,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"ctime" : -1
},
"memUsage" : 8471,
"memLimit" : 33554432,
"limitAmount" : 10,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 405,
"executionTimeMillisEstimate" : 3,
"works" : 407,
"advanced" : 405,
"needTime" : 1,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 405,
"executionTimeMillisEstimate" : 3,
"works" : 406,
"advanced" : 405,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 405,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 405,
"executionTimeMillisEstimate" : 1,
"works" : 406,
"advanced" : 405,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[1460.0, 1460.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 405,
"seeks" : 1,
"dupsTested" : 405,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "",
"port" : ,
"version" : "4.0.18",
"gitVersion" : "6883bdfb8b8cff32176b1fd176df04da9165fd67"
},
"ok" : 1,
"operationTime" : Timestamp(1629988625, 146),
"$clusterTime" : {
"clusterTime" : Timestamp(1629988625, 146),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
i followed Joe, changed my index, ctime before appIds.
but it does not work very well. there still be slow log for my new sql. it's hard to find out why it is slow
2021-10-19T22:38:16.918+0800 I COMMAND [conn2434281164] command newsmine.newstoapp command: find { find: "newstoapp", filter: { appIds: { $elemMatch: { $in: [ 2433 ] } }, ctime: { $gte: 0 } }, sort: { ctime: -1 }, hint: { ctime: -1, appIds: 1 }, skip: 0, limit: 50, batchSize: 50, $readPreference: { mode: "secondaryPreferred" }, $db: "newsmine" } planSummary: IXSCAN { ctime: -1, appIds: 1 } keysExamined:1471582 docsExamined:50 cursorExhausted:1 numYields:11496 nreturned:50 reslen:34043 locks:{ Global: { acquireCount: { r: 11497 } }, Database: { acquireCount: { r: 11497 } }, Collection: { acquireCount: { r: 11497 } } } storage:{ data: { bytesRead: 44958, timeReadingMicros: 618 } } protocol:op_query 7038ms
The cause of the slowness is that MongoDB 4.0.18 hash a blocking sort stage, so all matching documents must be found, retrieved, and sorted in memory before returning the requested batch.
In prior versions of MongoDB it was found that under certain conditions using a multi-key index to support a sort would provide incorrect result.
I never fully understood these conditions or why the results were incorrect, so if you are able to find those details, please edit or comment.
Prior to MongoDB 3.4 the index metadata contained a boolean value to indicate whether or not the index was multi-key (indexed a field that contained an array for at least one document).
MongoDB 3.4 introduced a new index version that also keeps track of which fields in the index are multi-key.
MongoDB 3.6 introduced a change to sorting to avoid the situations where results would be incorrect. This is why your query has a sort stage and is taking longer.
There are a couple things you could try to get back to the previous behavior without a blocking sort:
Drop and rebuild the index.
The existing index is version 1, which does not track multi-key paths. When rebuilding, the index should be created at version 2, which does track these, and may permit the query executor to use the index for sorting.
Create a new index with ctime before appIds.
A multi-key index has an entry in the index for each value in the indexed array. This may cause the query planner to assume it will disrupt sorting on a following key.
An index on {ctime:-1, appIds:1, background:1} would place the sort key ahead of the multi-key field, and while this may require reading more of the index, it may also permit the query executor to use the index for sorting.

How to query Mongodb on Single Field with multiple ranges efficiently?

I am trying to build custom geospatial indexes using MongoDB's B-tree indexes, as I find Mongo's native implementation limiting for my own case. In order to fulfill my geospatial queries which will effectively search Mongo using a compound index, I need to filter by the location.locIndexKey field with multiple ranges, among other fields.
So far, the only solution I could come up with to support this kind of queryies was using Mongo's $or operator. However, this performed badly since it's an or query and Mongo has to examine the same keys on the index again and again. In order to overcome this inefficiency, I need a way to make Mongo use multiple index-bounds on that field instead of replicating the query with or phrases for each defined bound in the query.
This is my query:
db.users.find({
"gender":2,
"preferences.feed.gender":1,
"age":{"$gte":18,"$lte":55},
"feedPrefChangeDay":{"$gte":1553461200,"$lte":1554066000},
"$or":[{"location.locIndexKey":{"$gte":NumberLong(1493233547543052300),"$lte":NumberLong(1493242343636074500)}},{"location.locIndexKey":{"$gte":NumberLong(1493242343636074500),"$lte":NumberLong(1493251139729096700)}},{"location.locIndexKey":{"$gte":NumberLong(1493287011295953000),"$lte":NumberLong(1493287148734906400)}}]
}).limit(20);
As you can see, in order to express multiple ranges on field location.locIndexKey, I had to use the $or operator. This is the shortened version of query planner's execution stats:
{
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 17762,
"totalKeysExamined" : 196192,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 351,
"works" : 196193,
"advanced" : 0,
"needTime" : 196191,
"needYield" : 0,
"saveState" : 19944,
"restoreState" : 19944,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 20,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 0,
"executionTimeMillisEstimate" : 351,
"works" : 196193,
"advanced" : 0,
"needTime" : 196191,
"needYield" : 0,
"saveState" : 19944,
"restoreState" : 19944,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "OR",
"nReturned" : 0,
"executionTimeMillisEstimate" : 351,
"works" : 196192,
"advanced" : 0,
"needTime" : 196191,
"needYield" : 0,
"saveState" : 19944,
"restoreState" : 19944,
"isEOF" : 1,
"invalidates" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"recordIdsForgotten" : 0,
"inputStages" : [
{
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 10,
"works" : 4534,
"advanced" : 0,
"needTime" : 4533,
"needYield" : 0,
"saveState" : 19944,
"restoreState" : 19944,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"gender" : 1.0,
"preferences.feed.gender" : 1.0,
"age" : 1.0,
"feedPrefChangeDay" : 1.0,
"location.locIndexKey" : 1.0
},
"indexName" : "gender_1_preferences.feed.gender_1_age_1_feedPrefChangeDay_1_location.locIndexKey_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [],
"preferences.feed.gender" : [],
"age" : [],
"feedPrefChangeDay" : [],
"location.locIndexKey" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[2.0, 2.0]"
],
"preferences.feed.gender" : [
"[1.0, 1.0]"
],
"age" : [
"[18.0, 55.0]"
],
"feedPrefChangeDay" : [
"[1553461200.0, 1554066000.0]"
],
"location.locIndexKey" : [
"[1493569998101151700, 1493572197124407300]"
]
},
"keysExamined" : 4534,
"seeks" : 4534,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4534,
"advanced" : 0,
"needTime" : 4533,
"needYield" : 0,
"saveState" : 19944,
"restoreState" : 19944,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"gender" : 1.0,
"preferences.feed.gender" : 1.0,
"age" : 1.0,
"feedPrefChangeDay" : 1.0,
"location.locIndexKey" : 1.0
},
"indexName" : "gender_1_preferences.feed.gender_1_age_1_feedPrefChangeDay_1_location.locIndexKey_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [],
"preferences.feed.gender" : [],
"age" : [],
"feedPrefChangeDay" : [],
"location.locIndexKey" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[2.0, 2.0]"
],
"preferences.feed.gender" : [
"[1.0, 1.0]"
],
"age" : [
"[18.0, 55.0]"
],
"feedPrefChangeDay" : [
"[1553461200.0, 1554066000.0]"
],
"location.locIndexKey" : [
"[1493587581697261600, 1493587590287196200]"
]
},
"keysExamined" : 4534,
"seeks" : 4534,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4534,
"advanced" : 0,
"needTime" : 4533,
"needYield" : 0,
"saveState" : 19944,
"restoreState" : 19944,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"gender" : 1.0,
"preferences.feed.gender" : 1.0,
"age" : 1.0,
"feedPrefChangeDay" : 1.0,
"location.locIndexKey" : 1.0
},
"indexName" : "gender_1_preferences.feed.gender_1_age_1_feedPrefChangeDay_1_location.locIndexKey_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [],
"preferences.feed.gender" : [],
"age" : [],
"feedPrefChangeDay" : [],
"location.locIndexKey" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[2.0, 2.0]"
],
"preferences.feed.gender" : [
"[1.0, 1.0]"
],
"age" : [
"[18.0, 55.0]"
],
"feedPrefChangeDay" : [
"[1553461200.0, 1554066000.0]"
],
"location.locIndexKey" : [
"[1493981215449940000, 1493990011542962200]"
]
},
"keysExamined" : 4534,
"seeks" : 4534,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
If you check indexBounds above, you will see that each range for location.locIndexKey is applied to a single query and combined with or. However, if I choose to run the same query using Mongo's native geospatial operator $geoWithin:
db.users.find({
"gender" : 2.0,
"preferences.feed.gender" : 1.0,
"age" : {
"$gte" : 18.0,
"$lte" : 55.0
},
"feedPrefChangeDay" : {
"$gte" : 1553461200.0,
"$lte" : 1554066000.0
},
"location.loc" : {
"$geoWithin" : {
"$centerSphere" : [
[
0.0,
0.0
],
0.00784806152880239
]
}
}
}).limit(20);
I get the following response from the query planner:
{
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 7,
"totalKeysExamined" : 4506,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4506,
"advanced" : 0,
"needTime" : 4505,
"needYield" : 0,
"saveState" : 35,
"restoreState" : 35,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 20,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"location.loc" : {
"$geoWithin" : {
"$centerSphere" : [
[
0.0,
0.0
],
0.00784806152880239
]
}
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4506,
"advanced" : 0,
"needTime" : 4505,
"needYield" : 0,
"saveState" : 35,
"restoreState" : 35,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4506,
"advanced" : 0,
"needTime" : 4505,
"needYield" : 0,
"saveState" : 35,
"restoreState" : 35,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"gender" : 1.0,
"preferences.feed.gender" : 1.0,
"age" : 1.0,
"feedPrefChangeDay" : 1.0,
"location.loc" : "2dsphere"
},
"indexName" : "gender_1_preferences.feed.gender_1_age_1_feedPrefChangeDay_1_location.loc_2dsphere",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [],
"preferences.feed.gender" : [],
"age" : [],
"feedPrefChangeDay" : [],
"location.loc" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[2.0, 2.0]"
],
"preferences.feed.gender" : [
"[1.0, 1.0]"
],
"age" : [
"[18.0, 55.0]"
],
"feedPrefChangeDay" : [
"[1553461200.0, 1554066000.0]"
],
"location.loc" : [
"[360287970189639680, 360287970189639680]",
"[378302368699121664, 378302368699121664]",
"[382805968326492160, 382805968326492160]",
"[383931868233334784, 383931868233334784]",
"[384213343210045440, 384213343210045440]",
"[384230935396089856, 384230935396089856]",
"[384235333442600960, 384235333442600960]",
"[384236432954228736, 384236432954228736]",
"[384236432954228737, 384236982710042623]",
"[384266119768178688, 384266119768178688]",
"[384266119768178689, 384274915861200895]",
"[384274915861200897, 384283711954223103]",
"[384283711954223104, 384283711954223104]",
"[384283711954223105, 384318896326311935]",
"[384318896326311937, 384354080698400767]",
"[1080863910568919040, 1080863910568919040]",
"[1134907106097364992, 1134907106097364992]",
"[1148417904979476480, 1148417904979476480]",
"[1151795604700004352, 1151795604700004352]",
"[1152640029630136320, 1152640029630136320]",
"[1152789563211513857, 1152798359304536063]",
"[1152798359304536064, 1152798359304536064]",
"[1152798359304536065, 1152807155397558271]",
"[1152833543676624896, 1152833543676624896]",
"[1152846737816158208, 1152846737816158208]",
"[1152850036351041536, 1152850036351041536]",
"[1152850586106855425, 1152851135862669311]",
"[1152851135862669312, 1152851135862669312]",
"[1152851135862669313, 1152859931955691519]",
"[1152868728048713728, 1152868728048713728]",
"[1152877524141735937, 1152886320234758143]",
"[1152886320234758145, 1152921504606846975]",
"[1152921504606846977, 1152956688978935807]",
"[1152956688978935809, 1152991873351024639]",
"[1152991873351024640, 1152991873351024640]",
"[1152991873351024641, 1152992423106838527]",
"[1152992972862652416, 1152992972862652416]",
"[1152996271397535744, 1152996271397535744]",
"[1153009465537069056, 1153009465537069056]",
"[1153035853816135681, 1153044649909157887]",
"[1153044649909157888, 1153044649909157888]",
"[1153044649909157889, 1153053446002180095]",
"[1153202979583557632, 1153202979583557632]",
"[1154047404513689600, 1154047404513689600]",
"[1157425104234217472, 1157425104234217472]",
"[1170935903116328960, 1170935903116328960]",
"[1224979098644774912, 1224979098644774912]",
"[1921488928515293185, 1921524112887382015]",
"[1921524112887382017, 1921559297259470847]",
"[1921559297259470848, 1921559297259470848]",
"[1921559297259470849, 1921594481631559679]",
"[1921606026503651329, 1921606576259465215]",
"[1921606576259465216, 1921606576259465216]",
"[1921607675771092992, 1921607675771092992]",
"[1921612073817604096, 1921612073817604096]",
"[1921629666003648512, 1921629666003648512]",
"[1921911140980359168, 1921911140980359168]",
"[1923037040887201792, 1923037040887201792]",
"[1927540640514572288, 1927540640514572288]",
"[1945555039024054272, 1945555039024054272]"
]
},
"keysExamined" : 4506,
"seeks" : 4506,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
As you can see, Mongo takes advantage of multiple index bounds to fulfill this query and its much more effective.
I believe the inefficiency in the original query happens because Mongo's query planner doesn't check what's inside the $or expression. I think that it should be more clever to understand that there is just one field with multiple ranges inside the expression and build the query using multiple index bounds. Sadly, this is not the case.
My question: is there any way that I can force Mongo to use multiple index bounds for my query, so that it's as efficient as the native geospatial query?
Any help would be appreciated.
Thanks!

Sort of MongoDB using index

Below is the status of the index status of the collection that I want to let you see.
> db.histories.getIndexes();
[
{
"v" : 1,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "development.histories"
},
{
"v" : 1,
"key" : {
"hoge_id" : 1,
"created_at" : 1
},
"name" : "hoge_id_1_created_at_1",
"ns" : "development.histories",
"background" : true
},
{
"v" : 1,
"key" : {
"created_at" : 1
},
"name" : "created_at_1",
"ns" : "development.histories",
"background" : true
}
]
And, I executed the following query.
> db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { created_at: -1 } ).limit(1).explain("executionStats");
And, the result was below.
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "development.histories",
"indexFilterSet" : false,
"parsedQuery" : {
"hoge_id" : {
"$eq" : ObjectId("5a5c171010ebfb1a2c901008")
}
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 1,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MaxKey, MinKey]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 0,
"totalKeysExamined" : 1,
"totalDocsExamined" : 1,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 2,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 1,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 1,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
},
"serverInfo" : {
"host" : "b9cb1b8d1fc1",
"port" : 27017,
"version" : "3.2.18",
"gitVersion" : "4c1bae566c0c00f996a2feb16febf84936ecaf6f"
},
"ok" : 1
}
The result is fast, I guess it's due to creating index on created_at.
ref. "totalDocsExamined" : 1, "executionTimeMillis" : 0
Then, I did exection the following query. The difference of previous is the field used for sort.
> db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { id: -1 } ).limit(1).explain("executionStats");
And, the result was below.
> db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { id: -1 } ).limit(1).explain("executionStats");
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "development.histories",
"indexFilterSet" : false,
"parsedQuery" : {
"hoge_id" : {
"$eq" : ObjectId("5a5c171010ebfb1a2c901008")
}
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"id" : -1
},
"limitAmount" : 1,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 1215,
"totalKeysExamined" : 1034353,
"totalDocsExamined" : 1034353,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1,
"executionTimeMillisEstimate" : 1120,
"works" : 1034357,
"advanced" : 1,
"needTime" : 1034355,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"id" : -1
},
"memUsage" : 297,
"memLimit" : 33554432,
"limitAmount" : 1,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 0,
"executionTimeMillisEstimate" : 950,
"works" : 1034355,
"advanced" : 0,
"needTime" : 1,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 1034353,
"executionTimeMillisEstimate" : 650,
"works" : 1034354,
"advanced" : 1034353,
"needTime" : 0,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 1034353,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1034353,
"executionTimeMillisEstimate" : 330,
"works" : 1034354,
"advanced" : 1034353,
"needTime" : 0,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 1034353,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "b9cb1b8d1fc1",
"port" : 27017,
"version" : "3.2.18",
"gitVersion" : "4c1bae566c0c00f996a2feb16febf84936ecaf6f"
},
"ok" : 1
}
>
The result is late this time.
ref. "totalDocsExamined" : 1034353, "executionTimeMillis" : 1215
About totalDocsExamined, That's all in all documents.
Regardress that id is enable for index as created_at, but, when it is sorted using id, the result is late?
For your 1st query:
db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { created_at: -1 } ).limit(1).explain("executionStats");
MongoDB is optimizing the performance by using the compound index on hoge_id and created_at. It firstly looks at the hoge_id and then it uses the index of created_at to sort the query results. In this way, the sort operation can be very fast because of efficient usage of compound index.
However, for your 2nd query:
db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { id: -1 } ).limit(1).explain("executionStats");
Since there is no compound index on hoge_id and id(you only have a single index on id), MongoDB is actually manually sorting results by id.
More info on sorting with compound index can be found here.

Difficulty optimizing Mongo distinct query to use indexes

I am having difficulty persuading Mongo to run a distinct query that looks like it should be covered by the indexes without fetching a large number of documents in the collection.
My documents have the general form:
{
_tenantId: 'someString',
_productCategory: 'some string from a smallish set'
...
}
I have an index on (_tenantId, _productCategory).
I want to find out what the set of distinct product categories is for a given tenant, so the query is:
db.products.distinct( '_productCategory', { _tenantId: '463171c3-d15f-4699-893d-3046327f8e1f'})
This runs rather slowly (several seconds for a collection of around half a million products against a local DB, which is Mongo 3.2.9). Against our pre-production SaaS-based Mongo (which is probably more memory constrained than my local instance which has free run of my machine) it take several 10s of seconds for the same data.
Explaining the query yields:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "engage-prod.products",
"indexFilterSet" : false,
"parsedQuery" : {
"_tenantId" : {
"$eq" : "463171c3-d15f-4699-893d-3046327f8e1f"
}
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_tenantId" : 1,
"_productCategory" : 1
},
"indexName" : "_tenantId_1__productCategory_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_tenantId" : [
"[\"463171c3-d15f-4699-893d-3046327f8e1f\", \"463171c3-d15f-4699-893d-3046327f8e1f\"]"
],
"_productCategory" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 406871,
"executionTimeMillis" : 358,
"totalKeysExamined" : 406871,
"totalDocsExamined" : 406871,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 406871,
"executionTimeMillisEstimate" : 80,
"works" : 406872,
"advanced" : 406871,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3178,
"restoreState" : 3178,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 406871,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 406871,
"executionTimeMillisEstimate" : 40,
"works" : 406872,
"advanced" : 406871,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3178,
"restoreState" : 3178,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_tenantId" : 1,
"_productCategory" : 1
},
"indexName" : "_tenantId_1__productCategory_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_tenantId" : [
"[\"463171c3-d15f-4699-893d-3046327f8e1f\", \"463171c3-d15f-4699-893d-3046327f8e1f\"]"
],
"_productCategory" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 406871,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "Stevens-MacBook-Pro.local",
"port" : 27017,
"version" : "3.2.9",
"gitVersion" : "22ec9e93b40c85fc7cae7d56e7d6a02fd811088c"
},
"ok" : 1
}
Note that even though it runs an IXSCAN it still returns over 400K documents (nReturned).
If I create a compound field _tenantAndProductCategory containing a lexical concatenation (with a : separator) and index that so it's a single field index, then the query:
db.products.explain('executionStats').distinct( '_productTenantAndCategory', { _productTenantAndCategory: {$gte: '463171c3-d15f-4699-893d-3046327f8e1f',$lt: '463171c3-d15f-4699-893d-3046327f8e1g'}})
works entirely within the index and yields:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "engage-prod.products",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"_productTenantAndCategory" : {
"$lt" : "463171c3-d15f-4699-893d-3046327f8e1g"
}
},
{
"_productTenantAndCategory" : {
"$gte" : "463171c3-d15f-4699-893d-3046327f8e1f"
}
}
]
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"_id" : 0,
"_productTenantAndCategory" : 1
},
"inputStage" : {
"stage" : "DISTINCT_SCAN",
"keyPattern" : {
"_productTenantAndCategory" : 1
},
"indexName" : "_productTenantAndCategory_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_productTenantAndCategory" : [
"[\"463171c3-d15f-4699-893d-3046327f8e1f\", \"463171c3-d15f-4699-893d-3046327f8e1g\")"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 62,
"executionTimeMillis" : 0,
"totalKeysExamined" : 63,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 62,
"executionTimeMillisEstimate" : 0,
"works" : 63,
"advanced" : 62,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"_id" : 0,
"_productTenantAndCategory" : 1
},
"inputStage" : {
"stage" : "DISTINCT_SCAN",
"nReturned" : 62,
"executionTimeMillisEstimate" : 0,
"works" : 63,
"advanced" : 62,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_productTenantAndCategory" : 1
},
"indexName" : "_productTenantAndCategory_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_productTenantAndCategory" : [
"[\"463171c3-d15f-4699-893d-3046327f8e1f\", \"463171c3-d15f-4699-893d-3046327f8e1g\")"
]
},
"keysExamined" : 63
}
}
},
"serverInfo" : {
"host" : "Stevens-MacBook-Pro.local",
"port" : 27017,
"version" : "3.2.9",
"gitVersion" : "22ec9e93b40c85fc7cae7d56e7d6a02fd811088c"
},
"ok" : 1
}
Having to build single field indexes with manually compounded keys for all the aggregation queries I need is not a very desirable path to follow. Since all the information is present in the compound index I started with, why can't Mongo execute the original distinct query with cover by that index? Is there anything I can do to overcome this in the way of query optimization?
Note This is actually a sub-problem of a slightly more complex one involving an aggregation pipeline to actually count the number of occurrences of each category, but I am restricting my question for now to the simpler distinct query since it seems to capture the essence of failure to use an index that should cover things (which I was also seeing in the aggregation pipeline case), while being a simpler overall query.

MongoDB efficient way to search for objectId field with null using Index?

I am trying to find an efficient way to search for items in which specific field is null.
In the MongoDB, I have folder schema which has parent field of its parent folder's ObjectId and parent is indexed. For root folders, parent fields are null.
When I try to find all the root folders with parent:null, explain displays indexOnly: false
db.folders.find({parent: null}, {parent: 1, _id: 0}).explain()
{
"cursor" : "BtreeCursor parent_1",
"isMultiKey" : false,
"n" : 126,
"nscannedObjects" : 126,
"nscanned" : 126,
"nscannedObjectsAllPlans" : 126,
"nscannedAllPlans" : 126,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 1,
"nChunkSkips" : 0,
"millis" : 0,
"indexBounds" : {
"parent" : [
[
null,
null
]
]
},
"server" : "c268.candidate.36:10268",
"filterSet" : false,
"stats" : {
"type" : "PROJECTION",
"works" : 128,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "KEEP_MUTATIONS",
"works" : 128,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 128,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 126,
"children" : [
{
"type" : "IXSCAN",
"works" : 127,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ parent: 1.0 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['parent']: [null, null]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 126,
"children" : []
}
]
}
]
}
]
}
}
I tried to use $type to find any ones which parent folder is not ObjectId. But still indexOnly:false.
db.folders.find({parent: {$ne: {$type: 7}}}, {parent: 1, _id: 0}).explain()
"indexOnly": false
Is there a way to search null value only using index? If not, is there a better value to store instead of null to be able to search with index?
Additional
example of root folder
{
"_id" : ObjectId("55a04a2d754971030059b7ad"),
"active" : true,
"modified" : ISODate("2016-02-05T22:30:08.053Z"),
"created" : ISODate("2015-07-10T22:41:49.009Z"),
"user" : ObjectId("54d3ae187a738c0300f59e61"),
"name" : "2nd Grade",
"parent" : null,
"clientModified" : ISODate("2016-02-05T22:30:07.872Z"),
"userCreated" : ISODate("2015-07-10T22:41:48.328Z"),
"ancestors" : [],
"__v" : 2
}
example of child folder
{
"_id" : ObjectId("56d0b4edb6f05e03009bcabc"),
"active" : true,
"modified" : ISODate("2016-02-26T20:26:21.328Z"),
"created" : ISODate("2016-02-26T20:26:21.328Z"),
"user" : ObjectId("54d3ae187a738c0300f59e61"),
"name" : "music",
"parent" : ObjectId("55a04a2d754971030059b7ad"),
"clientModified" : ISODate("2016-02-26T20:26:20.398Z"),
"userCreated" : ISODate("2016-02-26T20:26:20.398Z"),
"ancestors" : [
ObjectId("55a04a2d754971030059b7ad")
],
"__v" : 0
}
db.folders.getIndexes()
{
"0" : {
"v" : 1,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "app29099188.folders"
},
"1" : {
"v" : 1,
"key" : {
"user" : 1,
"_fts" : "text",
"_ftsx" : 1
},
"name" : "user_1_name_text",
"ns" : "app29099188.folders",
"background" : true,
"safe" : null,
"weights" : {
"name" : 1
},
"default_language" : "english",
"language_override" : "language",
"textIndexVersion" : 2
},
"2" : {
"v" : 1,
"key" : {
"user" : 1,
"parent" : 1
},
"name" : "user_1_parent_1",
"ns" : "app29099188.folders",
"background" : true,
"safe" : null
},
"3" : {
"v" : 1,
"key" : {
"parent" : 1.0000000000000000
},
"name" : "parent_1",
"ns" : "app29099188.folders"
}
}
After comment - update:
The way to eliminate docScan is to have a value in parent field. It can be zeroed objectId or just "/" as a root.
db.satoko.insert({"test":"sdsf", parent: "/"})
db.satoko.insert({"test":"sds33f", parent: "/"})
db.satoko.insert({"parent":ObjectId("56d8b2879bd059e7247a6096"), "test":"sdsf"})
explain results:
db.satoko.find({parent:{$eq:"/"} }, {parent: 1, _id: 0}).explain("allPlansExec
ution")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.satoko",
"indexFilterSet" : false,
"parsedQuery" : {
"parent" : {
"$eq" : "/"
}
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"parent" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"parent" : 1
},
"indexName" : "parent_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"parent" : [
"[\"/\", \"/\"]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 2,
"executionTimeMillis" : 0,
"totalKeysExamined" : 2,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 3,
"advanced" : 2,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"parent" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 3,
"advanced" : 2,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"parent" : 1
},
"indexName" : "parent_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"parent" : [
"[\"/\", \"/\"]"
]
},
"keysExamined" : 2,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [ ]
},
"serverInfo" : {
"host" : "greg",
"port" : 27017,
"version" : "3.2.3",
"gitVersion" : "b326ba837cf6f49d65c2f85e1b70f6f31ece7937"
},
"ok" : 1
}