Why so many scanned Objects? - mongodb

I have a table with the following objects:
> db.shapes.find()
{ "_id" : "P1", "amenity" : "restaurant", "shape" : { "type" : "Point",
"coordinates" : [ 2, 2 ] } }
{ "_id" : "P2", "amenity" : "restaurant", "shape" : { "type" : "Point",
"coordinates" : [ 2, 4 ] } }
{ "_id" : "P3", "amenity" : "police", "shape" : { "type" : "Point",
"coordinates" : [ 4, 2 ] } }
{ "_id" : "P4", "amenity" : "police", "shape" : { "type" : "Point",
"coordinates" : [ 4, 4 ] } }
The explain() on the following query gives a strange (in my opinion) result:
> db.shapes.find({shape:{$nearSphere:{$geometry:{type: "Point", coordinates:
[0,0]}}}}, {id:1, amenity:1}).limit(2).explain()
{
"cursor" : "S2NearCursor",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 22,
"nscanned" : 22,
"nscannedObjectsAllPlans" : 22,
"nscannedAllPlans" : 22,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 1,
"indexBounds" : {
},
"server" : "DBTest:27017",
"filterSet" : false
}
Why there are this much objects scanned ? I mean, the table only has 4
objects and mongodb scans 22 ?
I am thankful for any explanation.
Bye, Andre
> db.shapes.find({shape:{$nearSphere:{$geometry:{type: "Point", coordinates:
... [0,0]}}}}, {id:1, amenity:1}).limit(2).explain(1)
{
"cursor" : "S2NearCursor",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 22,
"nscanned" : 22,
"nscannedObjectsAllPlans" : 22,
"nscannedAllPlans" : 22,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 1,
"indexBounds" : {
},
"allPlans" : [
{
"cursor" : "S2NearCursor",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 22,
"nscanned" : 22,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
}
}
],
"server" : "DBTest:27017",
"filterSet" : false,
"stats" : {
"type" : "LIMIT",
"works" : 22,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 2,
"needTime" : 20,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "PROJECTION",
"works" : 22,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 2,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 0,
"children" : [
{
"type" : "GEO_NEAR_2DSPHERE",
"works" : 22,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 2,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 0,
"children" : [ ]
}
]
}
]
}
}

It looks like you are using version 2.4 and are running into https://jira.mongodb.org/browse/SERVER-12231 which was fixed for version 2.6. If you upgrade, it's likely you will no longer see the bogus nscanned numbers in explain() output.

Related

count slow in mongodb 2.6.8

I've a find request which takes 0.031sec, but when I try to do the same request with a count, it takes over 1sec.
I tried different indexes but it's always the same problem.
Count is still slow.
any idea?
volume
1600000 documents
My request
db.books.find(
{
"categories" : { $eq : null},
"theme" : "comics"
}
)
My Index
{
"categories" : 1,
"theme" : 1
}
Explain
{
"cursor" : "BtreeCursor categories_1_theme_1",
"isMultiKey" : false,
"n" : 353912,
"nscannedObjects" : 353912,
"nscanned" : 353912,
"nscannedObjectsAllPlans" : 354821,
"nscannedAllPlans" : 354821,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 2771,
"nChunkSkips" : 0,
"millis" : 1111,
"indexBounds" : {
"theme" : [
[
"comics",
"comics"
]
],
"categories" : [
[
null,
null
]
]
},
"server" : "xxxmongoxxx:27017",
"filterSet" : false,
"stats" : {
"type" : "KEEP_MUTATIONS",
"works" : 353913,
"yields" : 2771,
"unyields" : 2771,
"invalidates" : 0,
"advanced" : 353912,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 353913,
"yields" : 2771,
"unyields" : 2771,
"invalidates" : 0,
"advanced" : 353912,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 353912,
"children" : [
{
"type" : "IXSCAN",
"works" : 353913,
"yields" : 2771,
"unyields" : 2771,
"invalidates" : 0,
"advanced" : 353912,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ categories: 1, theme: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['categories']: [\"comics\", \"comics\"], field #1['theme']: [null, null]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 353912,
"children" : []
}
]
}
]
}
}

Mongo selecting the wrong index

I'm using mongo db 2.6.9.
I created 2 indexes on a collection and i don`t understand the reson that the query planer allways selects the wrong index.
I would like to understand what I'm missing.
First Index:
{
"TimeStamp":1,
"A":1,
"B":1,
}
Second Index:
{
"TimeStamp":1,
"A":1,
"C":1,
}
When I'm useing the following query the planner selects the first index:
db.collection.find({"TimeStamp":{ "$gte" : ISODate("2015-04-14T00:00:00Z"), "$lt" : ISODate("2015-04-15T00:00:00Z") },"C":2137,"A":1}).explain()
explain({verbose:1}) results:
{
"cursor" : "BtreeCursor IX_First",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"nscannedObjectsAllPlans" : 0,
"nscannedAllPlans" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 0,
"indexBounds" : {
"TimeStamp" : [
[
ISODate("2015-04-14T00:00:00Z"),
ISODate("2015-04-15T00:00:00Z")
]
],
"A" : [
[
1,
1
]
],
"B" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
},
"allPlans" : [
{
"cursor" : "BtreeCursor IX_First",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"TimeStamp" : [
[
ISODate("2015-04-14T00:00:00Z"),
ISODate("2015-04-15T00:00:00Z")
]
],
"A" : [
[
1,
1
]
],
"B" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
},
{
"cursor" : "BtreeCursor IIX_Second",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"TimeStamp" : [
[
ISODate("2015-04-14T00:00:00Z"),
ISODate("2015-04-15T00:00:00Z")
]
],
"A" : [
[
1,
1
]
],
"C" : [
[
2137,
2137
]
]
}
}
],
"server" : "mongo2:27017",
"filterSet" : false,
"stats" : {
"type" : "KEEP_MUTATIONS",
"works" : 2,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 1,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 1,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ TimeStamp: 1, A: 1, B: 1}",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['TimeStamp']: [new Date(1428969600000), new Date(1429056000000)), field #1['A']: [1.0, 1.0], field #2['B']: [MinKey, MaxKey]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 0,
"children" : [ ]
}
]
}
]
}
}
That's because the order of the index fields matters in this case. Try swapping A and C:
db.collection.find({"TimeStamp":{ "$gte" : ISODate("2015-04-14T00:00:00Z"), "$lt" : ISODate("2015-04-15T00:00:00Z") },"A":1,"C":2137}).explain()

MongoDB find() query scans documents twice (duplicate cursor used) when using limit() + sort()?

I'm fairly new to MongoDB, though I haven't been able to find an explanation for what I'm seeing.
I have a small dataset of about 200 documents, when I run the following query:
db.tweets.find({user:22438186})
I get n / nscannedObjects / nscanned / nscannedObjectsAllPlans / nscannedAllPlans all at 9. The cursor is BtreeCursor user_1. All good.
Introducting Sort()
If I append a sort to the query:
db.tweets.find({user:22438186}).sort({created_at:1})
nscannedObjectsAllPlans / nscannedAllPlans have increased to 30. I can see under the allPlans field:
[
{
"cursor" : "BtreeCursor user_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
},
{
"cursor" : "BtreeCursor created_at_1",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 21,
"nscanned" : 21,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"created_at" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
}
]
BtreeCursor created_at_1 scanned 21 documents and matched 2? I'm not sure what is going on here as I thought sort() was applied to the documents returned by find(), which appears to be 9 from the user_1 index. In writing this up I'm gathering from the allPlans field that it's also using my created_at_1 index for some reason.
Limit(>n) combined with Sort() == duplicate cursor & document scans?
When I append limit(10) or higher, n remains at 9, nscannedObjects / nscanned are both at 18 and nscannedObjectsAllPlans / nscannedAllPlans now return 60. Why have all but n doubled? The cursor is now QueryOptimizerCursor, There is a clauses field in my explain(true) results, both child objects are exactly the same, the same cursor was used twice causing the duplication? Is this behaviour normal?
{
"cursor" : "BtreeCursor user_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
}
I've tried a few different limit values and noticed that using a limit of 9, nscannedObjects / nscanned both return back to values of 9 and nscannedObjectsAllPlans / nscannedAllPlans drop down to 29, decrementing by 1 as I decrement the limit.
Under clauses however the 2nd child object is not the same as limit queries of 10 and higher. The cursor field now displays BtreeCursor omitting user_1 for some reason, all the n fields have a value of 0 instead of 9, besides that the rest of the object is the same. For all of these limit queries allPlans field lists the clauses field and another for BtreeCursor created_at_1 (which is used as the cursor for a query with limit of 1).
Actual Question
So what exactly is causing my documents to be scanned twice when limit() and sort() are both used in a find()? The issue only seems to happen if the limit exceeds either nscannedObjects or nscanned. When querying with only limit() or sort() documents are not scanned twice.
Update
Sorry for the confusion, the first code block shows cursor data under the allPlans field. The actual cursor used was *BtreeCursor user_1*.
The 2nd code block is from a query with limit() and sort(). I am providing cursor data listed under clauses, the clauses field lists the same cursor information twice (duplicate). The actual cursor field for that query was *QueryOptimizerCursor*. The duplicate cursors under clauses are *BtreeCursor user_1*.
I've since added a compound index {user:1, created_at:1}, The results for n fields is 9, and nAllPlans 18. Regardless of limit() value or usage with sort(). For some reason under allPlans my original user_id_1 index is still being run alongside the new compound index. If a limit is applied to the query instead of the index user_id_1/BtreeCursor user_1 being used, QueryOptimizerCursor with the two cursors in clauses is being used.
I've been looking into this further and it seems to be the Query Planner uses other indexes in parallel and selecting the optimal index result? I'm not sure if each time I perform this query this 'competition' occurs again or if it is cached.
db.tweets.find({user:22438186}).sort({created_at:1}).limit(10)
Running the query without the compound index produces the following:
{
"clauses" : [
{
"cursor" : "BtreeCursor user_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
},
{
"cursor" : "BtreeCursor user_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 9,
"nscannedObjects" : 18,
"nscanned" : 18,
"nscannedObjectsAllPlans" : 60,
"nscannedAllPlans" : 60,
"scanAndOrder" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 0,
"allPlans" : [
{
"clauses" : [
{
"cursor" : "BtreeCursor user_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
},
{
"cursor" : "BtreeCursor user_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 9,
"nscannedObjects" : 18,
"nscanned" : 18,
"scanAndOrder" : false,
"nChunkSkips" : 0
},
{
"cursor" : "BtreeCursor created_at_1",
"isMultiKey" : false,
"n" : 3,
"nscannedObjects" : 42,
"nscanned" : 42,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"created_at" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
}
],
"server" : "HOME-PC:27017",
"filterSet" : false,
"stats" : {
"type" : "KEEP_MUTATIONS",
"works" : 43,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 32,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "OR",
"works" : 42,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 32,
"needFetch" : 0,
"isEOF" : 1,
"dupsTested" : 18,
"dupsDropped" : 9,
"locsForgotten" : 0,
"matchTested_0" : 0,
"matchTested_1" : 0,
"children" : [
{
"type" : "SORT",
"works" : 21,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 10,
"needFetch" : 0,
"isEOF" : 1,
"forcedFetches" : 0,
"memUsage" : 6273,
"memLimit" : 33554432,
"children" : [
{
"type" : "FETCH",
"works" : 10,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 10,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ user: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['user']: [22438186.0, 22438186.0]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 9,
"children" : []
}
]
}
]
},
{
"type" : "SORT",
"works" : 21,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 10,
"needFetch" : 0,
"isEOF" : 1,
"forcedFetches" : 0,
"memUsage" : 6273,
"memLimit" : 33554432,
"children" : [
{
"type" : "FETCH",
"works" : 10,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 10,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ user: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['user']: [22438186.0, 22438186.0]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 9,
"children" : []
}
]
}
]
}
]
}
]
}
}
With the compound index:
{
"cursor" : "BtreeCursor user_1_created_at_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"nscannedObjectsAllPlans" : 18,
"nscannedAllPlans" : 18,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
],
"created_at" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor user_1_created_at_1",
"isMultiKey" : false,
"n" : 9,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
],
"created_at" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
},
{
"clauses" : [
{
"cursor" : "BtreeCursor user_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
},
{
"cursor" : "BtreeCursor ",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : true,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"user" : [
[
22438186,
22438186
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 9,
"nscanned" : 9,
"scanAndOrder" : false,
"nChunkSkips" : 0
}
],
"server" : "HOME-PC:27017",
"filterSet" : false,
"stats" : {
"type" : "LIMIT",
"works" : 11,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 11,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 10,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 9,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ user: 1, created_at: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['user']: [22438186.0, 22438186.0], field #1['created_at']: [MinKey, MaxKey]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 9,
"children" : []
}
]
}
]
}
}
Hope that clears up the confusion.
If you see the explain() plan, you can see that:
db.tweets.find({user:22438186})
uses the user_1 index.
db.tweets.find({user:22438186}).sort({created_at:1}) uses the created_at_1 index.
This indicates that mongodb has chosen created_at_1 over user_1 for the fact that sort operations perform better when they use an index, and the sort operation is based on the created_at field. That makes mongodb ignore the user_1 index and perform a full collection scan.
So we need to define our indexes carefully in these cases. If we have a compound index on both user_1 and created_at_1, a full table scan will not occur and mongodb will choose the index that supports both the find and the sort operations, which in case would be the compound index.
JIRA has a beautiful explanation why mongoDB uses the QueryOptimizerCursor cursor.
nscannedObjectsAllPlans / nscannedAllPlans drop down to 29
You should not be worrying about these two parameters, they are a representation of the combined scans made by all the plans that mongodb has executed to select the appropriate index.
nscannedObjectsAllPlans is a number that reflects the total number of
documents scanned for all query plans during the database operation
nscannedAllPlans is a number that reflects the total number of
documents or index entries scanned for all query plans during the
database operation.
These lines are from the docs.
So what exactly is causing my documents to be scanned twice when limit() and sort() are both used in a find()?
As said, the documents are not scanned twice, they are scanned in parallel by two different plans executed by mongodb to select the appropriate index. If you have two different indexes, two plans may be run in parallel., and so on.

Why indexOnly==false

I have a collection with index:
{
"UserId" : 1,
"ShareId" : 1,
"ParentId" : 1,
"DeletedDate" : 1
}
If I making query:
db.Files.find({ "UserId" : ObjectId("5450d837f32a1e098c844e2a"),
"ShareId" : ObjectId("5450d879f32a1e098c844e94"),
"ParentId" : ObjectId("5450d8af6a092a0b74a44026"),
"DeletedDate":null},
{_id:0, ShareId:1}).explain()
output says that "indexOnly" : false:
{
"cursor" : "BtreeCursor UserId_1_ShareId_1_ParentId_1_DeletedDate_1",
"isMultiKey" : false,
"n" : 2120,
"nscannedObjects" : 2120,
"nscanned" : 2120,
"nscannedObjectsAllPlans" : 2318,
"nscannedAllPlans" : 2320,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 21,
"nChunkSkips" : 0,
"millis" : 42,
"indexBounds" : {
"UserId" : [
[
ObjectId("5450d837f32a1e098c844e2a"),
ObjectId("5450d837f32a1e098c844e2a")
]
],
"ShareId" : [
[
ObjectId("5450d879f32a1e098c844e94"),
ObjectId("5450d879f32a1e098c844e94")
]
],
"ParentId" : [
[
ObjectId("5450d8af6a092a0b74a44026"),
ObjectId("5450d8af6a092a0b74a44026")
]
],
"DeletedDate" : [
[
null,
null
]
]
},
"server" : "mongowecntprod:27017",
"filterSet" : false,
"stats" : {
"type" : "PROJECTION",
"works" : 2124,
"yields" : 21,
"unyields" : 21,
"invalidates" : 0,
"advanced" : 2120,
"needTime" : 0,
"needFetch" : 2,
"isEOF" : 1,
"children" : [
{
"type" : "KEEP_MUTATIONS",
"works" : 2124,
"yields" : 21,
"unyields" : 21,
"invalidates" : 0,
"advanced" : 2120,
"needTime" : 1,
"needFetch" : 2,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 2124,
"yields" : 21,
"unyields" : 21,
"invalidates" : 0,
"advanced" : 2120,
"needTime" : 1,
"needFetch" : 2,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 2120,
"children" : [
{
"type" : "IXSCAN",
"works" : 2121,
"yields" : 21,
"unyields" : 21,
"invalidates" : 0,
"advanced" : 2120,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ UserId: 1, ShareId: 1, ParentId: 1, DeletedDate: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['UserId']: [ObjectId('5450d837f32a1e098c844e2a'), ObjectId('5450d837f32a1e098c844e2a')], field #1['ShareId']: [ObjectId('5450d879f32a1e098c844e94'), ObjectId('5450d879f32a1e098c844e94')], field #2['ParentId']: [ObjectId('5450d8af6a092a0b74a44026'), ObjectId('5450d8af6a092a0b74a44026')], field #3['DeletedDate']: [null, null]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 2120,
"children" : []
}
]
}
]
}
]
}
}
but if I making query without DeletedDate:
db.Files.find({ "UserId" : ObjectId("5450d837f32a1e098c844e2a"),
"ShareId" : ObjectId("5450d879f32a1e098c844e94"),
"ParentId" : ObjectId("5450d8af6a092a0b74a44026")},
{_id:0, ShareId:1}).explain()
then "indexOnly" is true.
How I can change first query to making indexOnly=true?
Let me give you a simple example that will hopefully demonstrate what you're seeing when you are querying for a field being null:
db.nullexplain.find()
{ "_id" : ObjectId("5456759f51a9d5271dc55bba"), "a" : 1 }
{ "_id" : ObjectId("545675a251a9d5271dc55bbb"), "a" : null }
{ "_id" : ObjectId("545675a551a9d5271dc55bbc") }
db.nullexplain.ensureIndex({a:1})
db,nullexplain.count({a:1}).count()
1
db.nullexplain.count({a:null}).count()
2
Do you see the issue? When "a" is present and explicitly set to null, it's indexed as null.
When "a" is not present in the document, it's also indexed as null.
When you query:
db.nullexplain.find({a:null},{_id:0,a:1})
{ "a" : null }
{ }
How can we derive from the index only whether the return document should have the field "a" set to null or if the field should not be present at all?
The answer is we cannot and therefore we must examine the document itself.
db.nullexplain.find({a:null},{_id:0,a:1}).explain()
{
"cursor" : "BasicCursor",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 3,
"nscanned" : 3,
"nscannedObjectsAllPlans" : 3,
"nscannedAllPlans" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 3,
"server" : "Asyas-MacBook-Pro.local:27017",
"filterSet" : false
}
Hope this helps you understand why querying for DeletedDate:null has to check the document and cannot be answered from the index.

MongoDB refuses to use index intersection

I use MongoDB 2.6.4.
My indexes looks like this:
{
"v" : 1,
"key" : {
"isFolder" : 1
},
"name" : "isFolder_1",
"ns" : "Tenant_51.files",
"background" : true
},
{
"v" : 1,
"key" : {
"isForeign" : 1
},
"name" : "isForeign_1",
"ns" : "Tenant_51.files",
"background" : true
},
My query looks like this:
db.files.find({ isFolder: true, isForeign: false }).explain(true)
For some reason, it chooses to use only 1 index (VERY SLOW: 680 seconds!!)
It looks like it does calculate the Complex Plan, however, decides not to use it, and I don't understand why.
Here is the execution plan:
{
"cursor" : "BtreeCursor isFolder_1",
"isMultiKey" : false,
"n" : 107441,
"nscannedObjects" : 110580,
"nscanned" : 110580,
"nscannedObjectsAllPlans" : 110689,
"nscannedAllPlans" : 110801,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 21056,
"nChunkSkips" : 0,
"millis" : 679121,
"indexBounds" : {
"isFolder" : [
[
true,
true
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor isFolder_1",
"isMultiKey" : false,
"n" : 107441,
"nscannedObjects" : 110580,
"nscanned" : 110580,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"isFolder" : [
[
true,
true
]
]
}
},
{
"cursor" : "BtreeCursor isForeign_1",
"isMultiKey" : false,
"n" : 68,
"nscannedObjects" : 109,
"nscanned" : 110,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"isForeign" : [
[
false,
false
]
]
}
},
{
"cursor" : "Complex Plan",
"n" : 42,
"nscannedObjects" : 0,
"nscanned" : 111,
"nChunkSkips" : 0
}
],
"server" : "XXX",
"filterSet" : false,
"stats" : {
"type" : "KEEP_MUTATIONS",
"works" : 128743,
"yields" : 21056,
"unyields" : 21056,
"invalidates" : 13834,
"advanced" : 107441,
"needTime" : 3140,
"needFetch" : 18161,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 128743,
"yields" : 21056,
"unyields" : 21056,
"invalidates" : 13834,
"advanced" : 107441,
"needTime" : 3140,
"needFetch" : 18161,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 107441,
"children" : [
{
"type" : "IXSCAN",
"works" : 110581,
"yields" : 21056,
"unyields" : 21056,
"invalidates" : 13834,
"advanced" : 110580,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ isFolder: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['isFolder']: [true, true]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 110580,
"children" : [ ]
}
]
}
]
}
}
From the MongoDB docs about indexing:
MongoDB can only use one index to support any given operation.
The solution, however, is as easy as the explanation: Use a compound index.
db.files.ensureIndex({isFolder:1,isForeign:1})