query wouldn't use suitable index - mongodb

Question

I have a query
db.Product.find({
CategoryPath: /^399-305-352(-\d+)*$/,
"Availability.Status": {
$lt: 4
},
$or: [{
_id: {
$lt: 331000000
}
}, {
_id: {
$gt: 852000000,
$lt: 853000000
}
}, {
_id: {
$gt: 972000000,
$lt: 973000000
}
}]
}).sort({
"Availability.Status": 1,
Popularity: -1
});
with explain I find it uses index Availability.Status_1_Popularity_-1:
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 913,
"nscannedObjects" : 470239,
"nscanned" : 470239,
"nscannedObjectsAllPlans" : 1387264,
"nscannedAllPlans" : 1387264,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 10838,
"nChunkSkips" : 0,
"millis" : 10117,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
},
"server" : "dal05mgo13.sl.dx:27017",
"filterSet" : false
}
this is pretty slow for me. I actually have another index CategoryPath_1_Availability.Status_1 which I think is a better choice. but when I force mongodb to use it with hint, I get an error:
{
"$err" : "Runner error: Overflow sort stage buffered data usage of 33581891 bytes exceeds internal limit of 33554432 bytes",
"code" : 17144
}
Now what I don't understand is, with the conditions specified in find, there are only 913 results selected, even without index, sort shouldn't have used up 32MB memory to sort the 913 records. Can anyone tell me what's happening?
I'm using MongoDB 2.6.10 x86_64
EDIT: my colleague just created a new index Availability.Status_1_Popularity_-1_CategoryPath_1 which is now winning from other plans. I still don't understand why though. Here are the detail explain info:
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1_CategoryPath_1",
"isMultiKey" : true,
"n" : 913,
"nscannedObjects" : 1325,
"nscanned" : 1930,
"nscannedObjectsAllPlans" : 7729,
"nscannedAllPlans" : 8334,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 64,
"nChunkSkips" : 0,
"millis" : 45,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"CategoryPath" : [
[
"399-305-352",
"399-305-353"
],
[
/^399-305-352(-\d+)*$/,
/^399-305-352(-\d+)*$/
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1_CategoryPath_1",
"isMultiKey" : true,
"n" : 913,
"nscannedObjects" : 1325,
"nscanned" : 1930,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"CategoryPath" : [
[
"399-305-352",
"399-305-353"
],
[
/^399-305-352(-\d+)*$/,
/^399-305-352(-\d+)*$/
]
]
}
},
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 306,
"nscanned" : 306,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 7,
"nscannedObjects" : 7,
"nscanned" : 7,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 297,
"nscannedObjects" : 297,
"nscanned" : 297,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 304,
"nscanned" : 304,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
}
],
"server" : "dal05mgo12.sl.dx:27017",
"filterSet" : false,
"stats" : {
"type" : "FETCH",
"works" : 1931,
"yields" : 64,
"unyields" : 64,
"invalidates" : 0,
"advanced" : 913,
"needTime" : 1017,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 913,
"children" : [
{
"type" : "IXSCAN",
"works" : 1931,
"yields" : 64,
"unyields" : 64,
"invalidates" : 0,
"advanced" : 1325,
"needTime" : 605,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ Availability.Status: 1.0, Popularity: -1.0, CategoryPath: 1.0 }",
"isMultiKey" : 1,
"boundsVerbose" : "field #0['Availability.Status']: [-inf.0, 4.0), field #1['Popularity']: [MaxKey, MinKey], field #2['CategoryPath']: [\"399-305-352\", \"399-305-353\"), [/^399-305-352(-\\d+)*$/, /^399-305-352(-\\d+)*$/]",
"yieldMovedCursor" : 0,
"dupsTested" : 1325,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 1930,
"children" : [ ]
}
]
}
}

Answer 1

That's because the order of the fields in the index matters.
CategoryPath_1_Availability.Status_1 translates for the following indexing order: index by CategoryPath then by Availability.Status.
But you want the index to start with Availability.Status, and mongo checks to see if there is any index that starts with Availability.Status (ascending), and there isn't (before your friend created the new index). Order matters :)
As a general rule: lets say you have the following fields in a collection: a, b and c. Let's set an index on a and b (a ascending, b ascending). You will be able to use this index in a sort only if the sort starts with a (ascending), or a (ascending) and b (ascending). Hope you understand :)