Mongodb log show they're using IXScan but db.currentOp() no - mongodb

On mongodb log shows they're using IXScan
[conn37225] command hone.$cmd command: count { count: "events", query:
{ eventName: "Started", createdAt: { $gt: new Date(1449789260690),
$lte: new Date(1450221260693) }, data.quizId:
"54fe05f26aa23aeb72e3fdbb" }, fields: null } planSummary: IXSCAN {
createdAt: 1, data.quizId: 1, eventName: 1 } keyUpdates:0
numYields:4279 locks(micros) r:53800992 reslen:48 38679ms
but on db.currentOp() don't
{
"opid" : 4719819,
"active" : true,
"secs_running" : 4,
"microsecs_running" : NumberLong(4221829),
"op" : "query",
"ns" : "hone.events",
"query" : {
"count" : "events",
"query" : {
"eventName" : "Started",
"createdAt" : {
"$gt" : ISODate("2015-12-10T23:14:20.690Z"),
"$lte" : ISODate("2015-12-15T23:14:20.693Z")
},
"data.quizId" : "54fe05f26aa23aeb72e3fdbb"
},
"fields" : null
},
"client" : "10.8.0.7:43823",
"desc" : "conn37225",
"threadId" : "0x7f60aee9a700",
"connectionId" : 37225,
"locks" : {
"^" : "r",
"^hone" : "R"
},
"waitingForLock" : false,
"numYields" : 384,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(6417513),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(800491),
"w" : NumberLong(0)
}
}
}
Is the mongodb using properly the { createdAt: 1, data.quizId: 1, eventName: 1 } index?
using hint() with explain() is possible see the difference.
with hint() use specifically that index and works fast.
I'm wondering know what is causing it to be slower.
Without hint
> db.events.find({ eventName: "Completed", createdAt: { $gt: new Date(1449945185127), $lte: new Date(1450377185140) }, "data.quizId": "55ce6dde5239249a70f9ef54" }).explain("executionStats")
{
"cursor" : "BtreeCursor createdAt_1_data.quizId_1_eventName_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 2519326,
"nscannedObjectsAllPlans" : 2543445,
"nscannedAllPlans" : 7536363,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 79008,
"nChunkSkips" : 0,
"millis" : 74443,
"indexBounds" : {
"createdAt" : [
[
ISODate("2015-12-12T18:33:05.127Z"),
ISODate("2015-12-17T18:33:05.140Z")
]
],
"data.quizId" : [
[
"55ce6dde5239249a70f9ef54",
"55ce6dde5239249a70f9ef54"
]
],
"eventName" : [
[
"Completed",
"Completed"
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor createdAt_1_data.quizId_1_eventName_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 2519326,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"createdAt" : [
[
ISODate("2015-12-12T18:33:05.127Z"),
ISODate("2015-12-17T18:33:05.140Z")
]
],
"data.quizId" : [
[
"55ce6dde5239249a70f9ef54",
"55ce6dde5239249a70f9ef54"
]
],
"eventName" : [
[
"Completed",
"Completed"
]
]
}
},
{
"cursor" : "BtreeCursor eventName_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2497710,
"nscanned" : 2497710,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"eventName" : [
[
"Completed",
"Completed"
]
]
}
},
{
"cursor" : "BtreeCursor createdAt_1_data.questionId_1_eventName_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 45735,
"nscanned" : 2519327,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"createdAt" : [
[
ISODate("2015-12-12T18:33:05.127Z"),
ISODate("2015-12-17T18:33:05.140Z")
]
],
"data.questionId" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"eventName" : [
[
"Completed",
"Completed"
]
]
}
}
],
"server" : "mongodb-05:27017",
"filterSet" : false,
"stats" : {
"type" : "FETCH",
"works" : 2519328,
"yields" : 79008,
"unyields" : 79008,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 2519326,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 2519327,
"yields" : 79008,
"unyields" : 79008,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 2519326,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ createdAt: 1, data.quizId: 1, eventName: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['createdAt']: (new Date(1449945185127), new Date(1450377185140)], field #1['data.quizId']: [\"55ce6dde5239249a70f9ef54\", \"55ce6dde5239249a70f9ef54\"], field #2['eventName']: [\"Completed\", \"Completed\"]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 2519326,
"children" : [ ]
}
]
}
}
With hint
> db.events.find({ eventName: "Completed", createdAt: { $gt: new Date(1449945185127), $lte: new Date(1450377185140) }, "data.quizId": "55ce6dde5239249a70f9ef54" }).hint('createdAt_1_data.quizId_1_eventName_1').explain("executionStats")
{
"cursor" : "BtreeCursor createdAt_1_data.quizId_1_eventName_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 2519326,
"nscannedObjectsAllPlans" : 0,
"nscannedAllPlans" : 2519326,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 19682,
"nChunkSkips" : 0,
"millis" : 8211,
"indexBounds" : {
"createdAt" : [
[
ISODate("2015-12-12T18:33:05.127Z"),
ISODate("2015-12-17T18:33:05.140Z")
]
],
"data.quizId" : [
[
"55ce6dde5239249a70f9ef54",
"55ce6dde5239249a70f9ef54"
]
],
"eventName" : [
[
"Completed",
"Completed"
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor createdAt_1_data.quizId_1_eventName_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 2519326,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"createdAt" : [
[
ISODate("2015-12-12T18:33:05.127Z"),
ISODate("2015-12-17T18:33:05.140Z")
]
],
"data.quizId" : [
[
"55ce6dde5239249a70f9ef54",
"55ce6dde5239249a70f9ef54"
]
],
"eventName" : [
[
"Completed",
"Completed"
]
]
}
}
],
"server" : "mongodb-05:27017",
"filterSet" : false,
"stats" : {
"type" : "FETCH",
"works" : 2519327,
"yields" : 19682,
"unyields" : 19682,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 2519326,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 2519327,
"yields" : 19682,
"unyields" : 19682,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 2519326,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ createdAt: 1, data.quizId: 1, eventName: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['createdAt']: (new Date(1449945185127), new Date(1450377185140)], field #1['data.quizId']: [\"55ce6dde5239249a70f9ef54\", \"55ce6dde5239249a70f9ef54\"], field #2['eventName']: [\"Completed\", \"Completed\"]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 2519326,
"children" : [ ]
}
]
}
}

Related

MongoDB efficient way to search for objectId field with null using Index?

I am trying to find an efficient way to search for items in which specific field is null.
In the MongoDB, I have folder schema which has parent field of its parent folder's ObjectId and parent is indexed. For root folders, parent fields are null.
When I try to find all the root folders with parent:null, explain displays indexOnly: false
db.folders.find({parent: null}, {parent: 1, _id: 0}).explain()
{
"cursor" : "BtreeCursor parent_1",
"isMultiKey" : false,
"n" : 126,
"nscannedObjects" : 126,
"nscanned" : 126,
"nscannedObjectsAllPlans" : 126,
"nscannedAllPlans" : 126,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 1,
"nChunkSkips" : 0,
"millis" : 0,
"indexBounds" : {
"parent" : [
[
null,
null
]
]
},
"server" : "c268.candidate.36:10268",
"filterSet" : false,
"stats" : {
"type" : "PROJECTION",
"works" : 128,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "KEEP_MUTATIONS",
"works" : 128,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 128,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 126,
"children" : [
{
"type" : "IXSCAN",
"works" : 127,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 126,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ parent: 1.0 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['parent']: [null, null]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 126,
"children" : []
}
]
}
]
}
]
}
}
I tried to use $type to find any ones which parent folder is not ObjectId. But still indexOnly:false.
db.folders.find({parent: {$ne: {$type: 7}}}, {parent: 1, _id: 0}).explain()
"indexOnly": false
Is there a way to search null value only using index? If not, is there a better value to store instead of null to be able to search with index?
Additional
example of root folder
{
"_id" : ObjectId("55a04a2d754971030059b7ad"),
"active" : true,
"modified" : ISODate("2016-02-05T22:30:08.053Z"),
"created" : ISODate("2015-07-10T22:41:49.009Z"),
"user" : ObjectId("54d3ae187a738c0300f59e61"),
"name" : "2nd Grade",
"parent" : null,
"clientModified" : ISODate("2016-02-05T22:30:07.872Z"),
"userCreated" : ISODate("2015-07-10T22:41:48.328Z"),
"ancestors" : [],
"__v" : 2
}
example of child folder
{
"_id" : ObjectId("56d0b4edb6f05e03009bcabc"),
"active" : true,
"modified" : ISODate("2016-02-26T20:26:21.328Z"),
"created" : ISODate("2016-02-26T20:26:21.328Z"),
"user" : ObjectId("54d3ae187a738c0300f59e61"),
"name" : "music",
"parent" : ObjectId("55a04a2d754971030059b7ad"),
"clientModified" : ISODate("2016-02-26T20:26:20.398Z"),
"userCreated" : ISODate("2016-02-26T20:26:20.398Z"),
"ancestors" : [
ObjectId("55a04a2d754971030059b7ad")
],
"__v" : 0
}
db.folders.getIndexes()
{
"0" : {
"v" : 1,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "app29099188.folders"
},
"1" : {
"v" : 1,
"key" : {
"user" : 1,
"_fts" : "text",
"_ftsx" : 1
},
"name" : "user_1_name_text",
"ns" : "app29099188.folders",
"background" : true,
"safe" : null,
"weights" : {
"name" : 1
},
"default_language" : "english",
"language_override" : "language",
"textIndexVersion" : 2
},
"2" : {
"v" : 1,
"key" : {
"user" : 1,
"parent" : 1
},
"name" : "user_1_parent_1",
"ns" : "app29099188.folders",
"background" : true,
"safe" : null
},
"3" : {
"v" : 1,
"key" : {
"parent" : 1.0000000000000000
},
"name" : "parent_1",
"ns" : "app29099188.folders"
}
}
After comment - update:
The way to eliminate docScan is to have a value in parent field. It can be zeroed objectId or just "/" as a root.
db.satoko.insert({"test":"sdsf", parent: "/"})
db.satoko.insert({"test":"sds33f", parent: "/"})
db.satoko.insert({"parent":ObjectId("56d8b2879bd059e7247a6096"), "test":"sdsf"})
explain results:
db.satoko.find({parent:{$eq:"/"} }, {parent: 1, _id: 0}).explain("allPlansExec
ution")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.satoko",
"indexFilterSet" : false,
"parsedQuery" : {
"parent" : {
"$eq" : "/"
}
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"parent" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"parent" : 1
},
"indexName" : "parent_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"parent" : [
"[\"/\", \"/\"]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 2,
"executionTimeMillis" : 0,
"totalKeysExamined" : 2,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 3,
"advanced" : 2,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"parent" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 3,
"advanced" : 2,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"parent" : 1
},
"indexName" : "parent_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"parent" : [
"[\"/\", \"/\"]"
]
},
"keysExamined" : 2,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [ ]
},
"serverInfo" : {
"host" : "greg",
"port" : 27017,
"version" : "3.2.3",
"gitVersion" : "b326ba837cf6f49d65c2f85e1b70f6f31ece7937"
},
"ok" : 1
}

query wouldn't use suitable index

I have a query
db.Product.find({
CategoryPath: /^399-305-352(-\d+)*$/,
"Availability.Status": {
$lt: 4
},
$or: [{
_id: {
$lt: 331000000
}
}, {
_id: {
$gt: 852000000,
$lt: 853000000
}
}, {
_id: {
$gt: 972000000,
$lt: 973000000
}
}]
}).sort({
"Availability.Status": 1,
Popularity: -1
});
with explain I find it uses index Availability.Status_1_Popularity_-1:
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 913,
"nscannedObjects" : 470239,
"nscanned" : 470239,
"nscannedObjectsAllPlans" : 1387264,
"nscannedAllPlans" : 1387264,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 10838,
"nChunkSkips" : 0,
"millis" : 10117,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
},
"server" : "dal05mgo13.sl.dx:27017",
"filterSet" : false
}
this is pretty slow for me. I actually have another index CategoryPath_1_Availability.Status_1 which I think is a better choice. but when I force mongodb to use it with hint, I get an error:
{
"$err" : "Runner error: Overflow sort stage buffered data usage of 33581891 bytes exceeds internal limit of 33554432 bytes",
"code" : 17144
}
Now what I don't understand is, with the conditions specified in find, there are only 913 results selected, even without index, sort shouldn't have used up 32MB memory to sort the 913 records. Can anyone tell me what's happening?
I'm using MongoDB 2.6.10 x86_64
EDIT: my colleague just created a new index Availability.Status_1_Popularity_-1_CategoryPath_1 which is now winning from other plans. I still don't understand why though. Here are the detail explain info:
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1_CategoryPath_1",
"isMultiKey" : true,
"n" : 913,
"nscannedObjects" : 1325,
"nscanned" : 1930,
"nscannedObjectsAllPlans" : 7729,
"nscannedAllPlans" : 8334,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 64,
"nChunkSkips" : 0,
"millis" : 45,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"CategoryPath" : [
[
"399-305-352",
"399-305-353"
],
[
/^399-305-352(-\d+)*$/,
/^399-305-352(-\d+)*$/
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1_CategoryPath_1",
"isMultiKey" : true,
"n" : 913,
"nscannedObjects" : 1325,
"nscanned" : 1930,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"CategoryPath" : [
[
"399-305-352",
"399-305-353"
],
[
/^399-305-352(-\d+)*$/,
/^399-305-352(-\d+)*$/
]
]
}
},
{
"cursor" : "BtreeCursor Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 306,
"nscanned" : 306,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"Availability.Status" : [
[
-Infinity,
4
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 7,
"nscannedObjects" : 7,
"nscanned" : 7,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 297,
"nscannedObjects" : 297,
"nscanned" : 297,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 304,
"nscanned" : 304,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
},
{
"clauses" : [
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 305,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
852000000,
853000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
972000000,
973000000
]
]
}
},
{
"cursor" : "BtreeCursor _id_1_Availability.Status_1_Popularity_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"_id" : [
[
-Infinity,
331000000
]
],
"Availability.Status" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"Popularity" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
}
],
"cursor" : "QueryOptimizerCursor",
"n" : 0,
"nscannedObjects" : 305,
"nscanned" : 305,
"scanAndOrder" : true,
"nChunkSkips" : 0
}
],
"server" : "dal05mgo12.sl.dx:27017",
"filterSet" : false,
"stats" : {
"type" : "FETCH",
"works" : 1931,
"yields" : 64,
"unyields" : 64,
"invalidates" : 0,
"advanced" : 913,
"needTime" : 1017,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 913,
"children" : [
{
"type" : "IXSCAN",
"works" : 1931,
"yields" : 64,
"unyields" : 64,
"invalidates" : 0,
"advanced" : 1325,
"needTime" : 605,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ Availability.Status: 1.0, Popularity: -1.0, CategoryPath: 1.0 }",
"isMultiKey" : 1,
"boundsVerbose" : "field #0['Availability.Status']: [-inf.0, 4.0), field #1['Popularity']: [MaxKey, MinKey], field #2['CategoryPath']: [\"399-305-352\", \"399-305-353\"), [/^399-305-352(-\\d+)*$/, /^399-305-352(-\\d+)*$/]",
"yieldMovedCursor" : 0,
"dupsTested" : 1325,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 1930,
"children" : [ ]
}
]
}
}
That's because the order of the fields in the index matters.
CategoryPath_1_Availability.Status_1 translates for the following indexing order: index by CategoryPath then by Availability.Status.
But you want the index to start with Availability.Status, and mongo checks to see if there is any index that starts with Availability.Status (ascending), and there isn't (before your friend created the new index). Order matters :)
As a general rule: lets say you have the following fields in a collection: a, b and c. Let's set an index on a and b (a ascending, b ascending). You will be able to use this index in a sort only if the sort starts with a (ascending), or a (ascending) and b (ascending). Hope you understand :)

Mongo selecting the wrong index

I'm using mongo db 2.6.9.
I created 2 indexes on a collection and i don`t understand the reson that the query planer allways selects the wrong index.
I would like to understand what I'm missing.
First Index:
{
"TimeStamp":1,
"A":1,
"B":1,
}
Second Index:
{
"TimeStamp":1,
"A":1,
"C":1,
}
When I'm useing the following query the planner selects the first index:
db.collection.find({"TimeStamp":{ "$gte" : ISODate("2015-04-14T00:00:00Z"), "$lt" : ISODate("2015-04-15T00:00:00Z") },"C":2137,"A":1}).explain()
explain({verbose:1}) results:
{
"cursor" : "BtreeCursor IX_First",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"nscannedObjectsAllPlans" : 0,
"nscannedAllPlans" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 0,
"indexBounds" : {
"TimeStamp" : [
[
ISODate("2015-04-14T00:00:00Z"),
ISODate("2015-04-15T00:00:00Z")
]
],
"A" : [
[
1,
1
]
],
"B" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
},
"allPlans" : [
{
"cursor" : "BtreeCursor IX_First",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"TimeStamp" : [
[
ISODate("2015-04-14T00:00:00Z"),
ISODate("2015-04-15T00:00:00Z")
]
],
"A" : [
[
1,
1
]
],
"B" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
},
{
"cursor" : "BtreeCursor IIX_Second",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 0,
"nscanned" : 0,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"TimeStamp" : [
[
ISODate("2015-04-14T00:00:00Z"),
ISODate("2015-04-15T00:00:00Z")
]
],
"A" : [
[
1,
1
]
],
"C" : [
[
2137,
2137
]
]
}
}
],
"server" : "mongo2:27017",
"filterSet" : false,
"stats" : {
"type" : "KEEP_MUTATIONS",
"works" : 2,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 1,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 1,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ TimeStamp: 1, A: 1, B: 1}",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['TimeStamp']: [new Date(1428969600000), new Date(1429056000000)), field #1['A']: [1.0, 1.0], field #2['B']: [MinKey, MaxKey]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 0,
"children" : [ ]
}
]
}
]
}
}
That's because the order of the index fields matters in this case. Try swapping A and C:
db.collection.find({"TimeStamp":{ "$gte" : ISODate("2015-04-14T00:00:00Z"), "$lt" : ISODate("2015-04-15T00:00:00Z") },"A":1,"C":2137}).explain()

MongoDB refuses to use index intersection

I use MongoDB 2.6.4.
My indexes looks like this:
{
"v" : 1,
"key" : {
"isFolder" : 1
},
"name" : "isFolder_1",
"ns" : "Tenant_51.files",
"background" : true
},
{
"v" : 1,
"key" : {
"isForeign" : 1
},
"name" : "isForeign_1",
"ns" : "Tenant_51.files",
"background" : true
},
My query looks like this:
db.files.find({ isFolder: true, isForeign: false }).explain(true)
For some reason, it chooses to use only 1 index (VERY SLOW: 680 seconds!!)
It looks like it does calculate the Complex Plan, however, decides not to use it, and I don't understand why.
Here is the execution plan:
{
"cursor" : "BtreeCursor isFolder_1",
"isMultiKey" : false,
"n" : 107441,
"nscannedObjects" : 110580,
"nscanned" : 110580,
"nscannedObjectsAllPlans" : 110689,
"nscannedAllPlans" : 110801,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 21056,
"nChunkSkips" : 0,
"millis" : 679121,
"indexBounds" : {
"isFolder" : [
[
true,
true
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor isFolder_1",
"isMultiKey" : false,
"n" : 107441,
"nscannedObjects" : 110580,
"nscanned" : 110580,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"isFolder" : [
[
true,
true
]
]
}
},
{
"cursor" : "BtreeCursor isForeign_1",
"isMultiKey" : false,
"n" : 68,
"nscannedObjects" : 109,
"nscanned" : 110,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"isForeign" : [
[
false,
false
]
]
}
},
{
"cursor" : "Complex Plan",
"n" : 42,
"nscannedObjects" : 0,
"nscanned" : 111,
"nChunkSkips" : 0
}
],
"server" : "XXX",
"filterSet" : false,
"stats" : {
"type" : "KEEP_MUTATIONS",
"works" : 128743,
"yields" : 21056,
"unyields" : 21056,
"invalidates" : 13834,
"advanced" : 107441,
"needTime" : 3140,
"needFetch" : 18161,
"isEOF" : 1,
"children" : [
{
"type" : "FETCH",
"works" : 128743,
"yields" : 21056,
"unyields" : 21056,
"invalidates" : 13834,
"advanced" : 107441,
"needTime" : 3140,
"needFetch" : 18161,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 107441,
"children" : [
{
"type" : "IXSCAN",
"works" : 110581,
"yields" : 21056,
"unyields" : 21056,
"invalidates" : 13834,
"advanced" : 110580,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ isFolder: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['isFolder']: [true, true]",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 110580,
"children" : [ ]
}
]
}
]
}
}
From the MongoDB docs about indexing:
MongoDB can only use one index to support any given operation.
The solution, however, is as easy as the explanation: Use a compound index.
db.files.ensureIndex({isFolder:1,isForeign:1})

Why so many scanned Objects?

I have a table with the following objects:
> db.shapes.find()
{ "_id" : "P1", "amenity" : "restaurant", "shape" : { "type" : "Point",
"coordinates" : [ 2, 2 ] } }
{ "_id" : "P2", "amenity" : "restaurant", "shape" : { "type" : "Point",
"coordinates" : [ 2, 4 ] } }
{ "_id" : "P3", "amenity" : "police", "shape" : { "type" : "Point",
"coordinates" : [ 4, 2 ] } }
{ "_id" : "P4", "amenity" : "police", "shape" : { "type" : "Point",
"coordinates" : [ 4, 4 ] } }
The explain() on the following query gives a strange (in my opinion) result:
> db.shapes.find({shape:{$nearSphere:{$geometry:{type: "Point", coordinates:
[0,0]}}}}, {id:1, amenity:1}).limit(2).explain()
{
"cursor" : "S2NearCursor",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 22,
"nscanned" : 22,
"nscannedObjectsAllPlans" : 22,
"nscannedAllPlans" : 22,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 1,
"indexBounds" : {
},
"server" : "DBTest:27017",
"filterSet" : false
}
Why there are this much objects scanned ? I mean, the table only has 4
objects and mongodb scans 22 ?
I am thankful for any explanation.
Bye, Andre
> db.shapes.find({shape:{$nearSphere:{$geometry:{type: "Point", coordinates:
... [0,0]}}}}, {id:1, amenity:1}).limit(2).explain(1)
{
"cursor" : "S2NearCursor",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 22,
"nscanned" : 22,
"nscannedObjectsAllPlans" : 22,
"nscannedAllPlans" : 22,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 0,
"nChunkSkips" : 0,
"millis" : 1,
"indexBounds" : {
},
"allPlans" : [
{
"cursor" : "S2NearCursor",
"isMultiKey" : false,
"n" : 2,
"nscannedObjects" : 22,
"nscanned" : 22,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
}
}
],
"server" : "DBTest:27017",
"filterSet" : false,
"stats" : {
"type" : "LIMIT",
"works" : 22,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 2,
"needTime" : 20,
"needFetch" : 0,
"isEOF" : 1,
"children" : [
{
"type" : "PROJECTION",
"works" : 22,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 2,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 0,
"children" : [
{
"type" : "GEO_NEAR_2DSPHERE",
"works" : 22,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 2,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 0,
"children" : [ ]
}
]
}
]
}
}
It looks like you are using version 2.4 and are running into https://jira.mongodb.org/browse/SERVER-12231 which was fixed for version 2.6. If you upgrade, it's likely you will no longer see the bogus nscanned numbers in explain() output.