MongoDB executionStats understanding - mongodb

I am currently trying to integrate indexes in one of my collection. All of my tests worked and I can really see the improvments (number of scanned document etc) however one of my result is a bit weird (or maybe I did not understand it well).
Could you please explain me?
I have a collection of owner/holder/product (here is an example and by the same the concerned document):
{
"_id": ObjectId("59ef7c84f545c8278c8cd967"),
"owner": "59ef7c83f545c8278c8cd965",
"holder": "59ef7c84f545c8278c8cd967",
"product": 1,
"date" : ISODate("2017-10-24T17:46:44.367Z")
}
Here are the indexes:
{
"v" : 1,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "mydb.relation"
},
{
"v" : 1,
"key" : {
"owner" : 1
},
"name" : "onlyOwner",
"ns" : "mydb.relation"
},
{
"v" : 1,
"key" : {
"holder" : 1
},
"name" : "onlyHolder",
"ns" : "mydb.relation"
},
{
"v" : 1,
"key" : {
"owner" : 1,
"holder" : 1
},
"name" : "ownerAndHolder",
"ns" : "mydb.relation"
},
{
"v" : 1,
"key" : {
"owner" : 1,
"product" : 1
},
"name" : "ownerAndProduct",
"ns" : "mydb.relation"
}
I executed the following query:
db.relation.find({ $and : [{"owner" : "59ef7c83f545c8278c8cd965"}, {"holder" : "59ef7c84f545c8278c8cd967"}]})
And my relation is well returned (the one mentionned earlier). So I decided to check executionStats to see the status and here is the result:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mydb.relation",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"holder" : {
"$eq" : "59ef7c84f545c8278c8cd967"
}
},
{
"owner" : {
"$eq" : "59ef7c83f545c8278c8cd965"
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"holder" : {
"$eq" : "59ef7c84f545c8278c8cd967"
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"owner" : 1,
"product" : 1
},
"indexName" : "ownerAndProduct",
"isMultiKey" : false,
"multiKeyPaths" : {
"owner" : [ ],
"product" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"owner" : [
"[\"59ef7c83f545c8278c8cd965\", \"59ef7c83f545c8278c8cd965\"]"
],
"product" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"owner" : 1,
"holder" : 1
},
"indexName" : "ownerAndHolder",
"isMultiKey" : false,
"multiKeyPaths" : {
"owner" : [ ],
"holder" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"owner" : [
"[\"59ef7c83f545c8278c8cd965\", \"59ef7c83f545c8278c8cd
],
"holder" : [
"[\"59ef7c84f545c8278c8cd967\", \"59ef7c84f545c8278c8cd
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"holder" : {
"$eq" : "59ef7c84f545c8278c8cd967"
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"owner" : 1
},
"indexName" : "onlyOwner",
"isMultiKey" : false,
"multiKeyPaths" : {
"owner" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"owner" : [
"[\"59ef7c83f545c8278c8cd965\", \"59ef7c83f545c8278c8cd
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"owner" : {
"$eq" : "59ef7c83f545c8278c8cd965"
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"holder" : 1
},
"indexName" : "onlyHolder",
"isMultiKey" : false,
"multiKeyPaths" : {
"holder" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"holder" : [
"[\"59ef7c84f545c8278c8cd967\", \"59ef7c84f545c8278c8cd
]
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 0,
"totalKeysExamined" : 1,
"totalDocsExamined" : 1,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"holder" : {
"$eq" : "59ef7c84f545c8278c8cd967"
}
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 3,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 1,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 2,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"owner" : 1,
"product" : 1
},
"indexName" : "ownerAndProduct",
"isMultiKey" : false,
"multiKeyPaths" : {
"owner" : [ ],
"product" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"owner" : [
"[\"59ef7c83f545c8278c8cd965\", \"59ef7c83f545c8278c8cd965\"]"
],
"product" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 1,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "NCEL15605",
"port" : 27017,
"version" : "3.4.6",
"gitVersion" : "c55eb86ef46ee7aede3b1e2a5d184a7df4bfb5b5"
},
"ok" : 1
}
So I understand that onlyHolder onlyOwner are rejected but I don't understand why ownerAndHolder is rejected and ownerAndProduct is used as I am not using it as variable.
Can someone help me to understand this result (the response is correct from the find, however this is the choice of MongoDB I did not understand) ?
Thanks in advance
Regards

Look at these two fields:
"nReturned" : 1,
"totalDocsExamined" : 1,
It means that Mongo had to examine only one document to return one. Then the chosen query plan, although counter-intuitively, achieved the best result possible. It looks like at least one other plan could have achieved the same result (the owner holder index). In such cases Mongo is free to choose a plan.
If you want to know the details of query optimizations read this material.

Related

MongoDb C# query performance

I have the MongoDB C# Driver 2.14.1 and a Mongo Server v3.6 running on a Docker container. I'm running an easy (I think) query over 3 collections. The biggest collection has 400k documents, each document averaged 64Kb. I'm filtering by two fields and returning the results in a cursor to reduce overhead. The results are sorted by descending by a field that is configured as an index as well. This is happening on my production server which is taking nearly 1GB of memory (not being released afterwards). The following queries explained (run on the server):
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "table1",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "Location1"
}
},
{
"Date" : {
"$gt" : ISODate("2021-01-18T23:00:00Z")
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "Location1"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 2,
"totalKeysExamined" : 7,
"totalDocsExamined" : 7,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "Location1"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 8,
"advanced" : 1,
"needTime" : 6,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 7,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 7,
"executionTimeMillisEstimate" : 0,
"works" : 8,
"advanced" : 7,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
},
"keysExamined" : 7,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "7aa23bd7fe20",
"port" : 27018,
"version" : "3.6.18",
"gitVersion" : "2005f25eed7ed88fa698d9b800fe536bb0410ba4"
},
"ok" : 1
}
The second query:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "table2",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "02"
}
},
{
"Date" : {
"$gt" : ISODate("2021-01-18T23:00:00Z")
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "02"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 63,
"totalKeysExamined" : 9896,
"totalDocsExamined" : 9896,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "02"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 50,
"works" : 9897,
"advanced" : 1,
"needTime" : 9895,
"needYield" : 0,
"saveState" : 77,
"restoreState" : 77,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 9896,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 9896,
"executionTimeMillisEstimate" : 0,
"works" : 9897,
"advanced" : 9896,
"needTime" : 0,
"needYield" : 0,
"saveState" : 77,
"restoreState" : 77,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
},
"keysExamined" : 9896,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "7aa23bd7fe20",
"port" : 27018,
"version" : "3.6.18",
"gitVersion" : "2005f25eed7ed88fa698d9b800fe536bb0410ba4"
},
"ok" : 1
}
And the third:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "table3",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "52"
}
},
{
"Date" : {
"$gt" : ISODate("2021-01-18T23:00:00Z")
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "52"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 30,
"executionTimeMillis" : 934,
"totalKeysExamined" : 366370,
"totalDocsExamined" : 366370,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "52"
}
}
]
},
"nReturned" : 30,
"executionTimeMillisEstimate" : 889,
"works" : 366371,
"advanced" : 30,
"needTime" : 366340,
"needYield" : 0,
"saveState" : 2862,
"restoreState" : 2862,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 366370,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 366370,
"executionTimeMillisEstimate" : 160,
"works" : 366371,
"advanced" : 366370,
"needTime" : 0,
"needYield" : 0,
"saveState" : 2862,
"restoreState" : 2862,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
},
"keysExamined" : 366370,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "7aa23bd7fe20",
"port" : 27018,
"version" : "3.6.18",
"gitVersion" : "2005f25eed7ed88fa698d9b800fe536bb0410ba4"
},
"ok" : 1
}
Now with my little knowledge of the "explainPlan" I would have imagined that the performance is OK, and not many records get returned on each query. Having said that I can see a pattern when I run the code via the .NET application (C# code) that makes the CPU go through the roof and right after the memory follows in the Mongo server Docker container.
I don't know if there is misconfigured or whether I can do something to improve the performance of it because I'm quite puzzle with the performance on just that query - I think is this query because the rest of the queries with similar amount of data on different parts of the system work quite well. Besides that, is there any mechanism or tool I can use to diagnose the memory consumption in my Mongo server?

MongoDB index doing a collection scan instead index scan [duplicate]

Here are the compound index and single index I have for this Collection:
///db.Collection.getIndexes()
/* 1 */
{
"v" : 2,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "service.Collection"
},
/* 2 */
{
"v" : 2,
"key" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"name" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"ns" : "service.Collection"
},
/* 3 */
{
"v" : 2,
"key" : {
"CreationTime" : 1
},
"name" : "CreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"ns" : "service.Collection"
}
The expected result is an IXSCAN using the FirstIdSecondIdCreationTime index:
///service.Collection.find({ FirstId: "771367b7-4bef-49ab-bda1-6230254c6349", ///SecondId: "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f" })
/// .projection({})
/// .sort({_id:-1}).hint("FirstIdSecondIdCreationTime").explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "service.Collection",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"indexName" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"FirstId" : [
"[MinKey, MaxKey]"
],
"SecondId" : [
"[MinKey, MaxKey]"
],
"CreationTime" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 5491,
"totalKeysExamined" : 856730,
"totalDocsExamined" : 856730,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1,
"executionTimeMillisEstimate" : 5261,
"works" : 856734,
"advanced" : 1,
"needTime" : 856732,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"_id" : -1
},
"memUsage" : 432,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 1,
"executionTimeMillisEstimate" : 5201,
"works" : 856732,
"advanced" : 1,
"needTime" : 856730,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 5131,
"works" : 856731,
"advanced" : 1,
"needTime" : 856729,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 856730,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 856730,
"executionTimeMillisEstimate" : 820,
"works" : 856731,
"advanced" : 856730,
"needTime" : 0,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"indexName" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"FirstId" : [
"[MinKey, MaxKey]"
],
"SecondId" : [
"[MinKey, MaxKey]"
],
"CreationTime" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 856730,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"indexDef" : {
"indexName" : "FirstIdSecondIdCreationTime",
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"direction" : "forward"
}
}
}
}
}
but the actual result is a COLLSCAN that takes over 8000ms:
"event": {
"dataset": "mongodb.log",
"module": "mongodb"
},
"service": {
"type": "mongodb"
},
"message": "command service.Collection command: find { find: \"Collection\",
filter: { FirstId: \"771367b7-4bef-49ab-bda1-6230254c6349\", SecondId: \"3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f\" }, sort: { CreationTime: -1 }, limit: 1,
planSummary: COLLSCAN keysExamined:0 docsExamined:784787 hasSortStage:1 cursorExhausted:1 numYields:6175 nreturned:1 reslen:677
locks:{ Global: { acquireCount: { r: 12352 } }, Database: { acquireCount: { r: 6176 } }, Collection: { acquireCount: { r: 6176 } } } protocol:op_msg 8441ms",
"mongodb.docsExamined": 784787,
"fileset": {
"name": "log"
},
Why am I COLLSCANing instead of IXSCANing with the FirstIdSecondIDCreationTime compound index? Is there a way to change my index/ my query to speed up the query?
Per a suggestion in the comments, I've run explain("allPlansExecution").
///db.Collection.find({ FirstId: "771367b7-4bef-49ab-bda1-6230254c6349", ///SecondId: "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f" })
/// .projection({})
/// .sort({_id:-1}).explain('allPlansExecution')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "service.Collection",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 5408,
"totalKeysExamined" : 856748,
"totalDocsExamined" : 856748,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 4862,
"works" : 856749,
"advanced" : 1,
"needTime" : 856747,
"needYield" : 0,
"saveState" : 6694,
"restoreState" : 6694,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 856748,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 856748,
"executionTimeMillisEstimate" : 1220,
"works" : 856749,
"advanced" : 856748,
"needTime" : 0,
"needYield" : 0,
"saveState" : 6694,
"restoreState" : 6694,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 856748,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [ ]
}
}
The "FirstIdSecondIdCreationTime" index was not automatically considered because it was created with a collation, and the query is being run without a collation.
Use the .collation() cursor method to specify the same collation for the query that was used for the index.
The 5.5 second run time using that index is pretty slow as well.
You may see some improvement in that query if you create an index on {FirstId: 1, SecondId: 1, _id: 1} so that they query executor can use the index to meet the sort instead of an in-memory sort.
Can you please sort by leading indexes i.e firstId, secondId and creationTime in the same sequence and see index is used. it will give an idea whether leading indexes fields should be there in sort as well.

MongoDB currentOp() and Cursor.explain() shows different result for the same query

I've been using MongoDB 3.2 for years, and I found that currentOp() and Cursor.explain() shows different result for the same query.
I found that several queries are being executed very long time(20+ sec), I thought it's not possible because I tested a lot and have index on it. Queries are generally same, as far as I saw. I think they're causing entire database locks because when some queries get slower, almost 40-50 queries are stuck in currentOp() .
But when I executed same read operation in shell, it ran very quickly, as I intended. I'd taken the same query from currentOp and executed.
When the database locks (I think it's locked), CPU utilization hits 100% for hours, and my application is going to be really slow. I'm monitoring currentOp every 1 minute and when it doesn't end for seconds, I had to restart the application then it goes normal.
Here's the one of query that takes very long time. Once it happens, 40-50 other but similar queries are also getting stuck in currentOp.
{
"desc" : "conn32882",
"threadId" : "140677207643904",
"connectionId" : 32882,
"client" : "client",
"active" : true,
"opid" : 1374027609,
"secs_running" : 20,
"microsecs_running" : NumberLong(20560351),
"op" : "query",
"ns" : "db.collection",
"query" : {
"find" : "collection",
"filter" : {
"p" : {
"$gt" : 0
},
"type" : "canvas",
"id" : {
"$in" : [
576391,
570391,
767422
]
}
},
"sort" : {
"_id" : -1
},
"projection" : {
},
"limit" : 5000,
"returnKey" : false,
"showRecordId" : false
},
"numYields" : 2761,
"locks" : {
"Global" : "r",
"Database" : "r",
"Collection" : "r"
},
"waitingForLock" : false,
"lockStats" : {
"Global" : {
"acquireCount" : {
"r" : NumberLong(5524)
},
"acquireWaitCount" : {
"r" : NumberLong(349)
},
"timeAcquiringMicros" : {
"r" : NumberLong(6613952)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(2762)
}
},
"Collection" : {
"acquireCount" : {
"r" : NumberLong(2762)
}
}
}
}
And here's an output of same query in shell with executionStats option.
Command :
db.canvasdatas.find({
"p" : {
"$gt": 0
},
"type": "canvas",
"id" : {
"$in": [
576391,
570391,
767422
]
}
}, {}).sort({ _id: -1 }).limit(5000).explain('executionStats');
Output :
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "db.collection",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"type" : {
"$eq" : "canvas"
}
},
{
"p" : {
"$gt" : 0
}
},
{
"id" : {
"$in" : [
570391,
576391,
767422
]
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 5000,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type" : {
"$eq" : "canvas"
}
},
{
"p" : {
"$gt" : 0
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"id" : 1
},
"indexName" : "id_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"id" : [
"[570391.0, 570391.0]",
"[576391.0, 576391.0]",
"[767422.0, 767422.0]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 5000,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"id" : {
"$in" : [
570391,
576391,
767422
]
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"p" : 1,
"type" : 1
},
"indexName" : "p_1_type_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"p" : [
"(0.0, inf.0]"
],
"type" : [
"[\"canvas\", \"canvas\"]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 5000,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type" : {
"$eq" : "canvas"
}
},
{
"id" : {
"$in" : [
570391,
576391,
767422
]
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"p" : 1
},
"indexName" : "p_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"p" : [
"(0.0, inf.0]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 5000,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"p" : {
"$gt" : 0
}
},
{
"id" : {
"$in" : [
570391,
576391,
767422
]
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"type" : 1
},
"indexName" : "type_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"type" : [
"[\"canvas\", \"canvas\"]"
]
}
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 5000,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type" : {
"$eq" : "canvas"
}
},
{
"p" : {
"$gt" : 0
}
},
{
"id" : {
"$in" : [
570391,
576391,
767422
]
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 2,
"executionTimeMillis" : 0,
"totalKeysExamined" : 5,
"totalDocsExamined" : 2,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 10,
"advanced" : 2,
"needTime" : 6,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"_id" : -1
},
"memUsage" : 906,
"memLimit" : 33554432,
"limitAmount" : 5000,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 6,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type" : {
"$eq" : "canvas"
}
},
{
"p" : {
"$gt" : 0
}
}
]
},
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 5,
"advanced" : 2,
"needTime" : 2,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 2,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 2,
"executionTimeMillisEstimate" : 0,
"works" : 5,
"advanced" : 2,
"needTime" : 2,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"id" : 1
},
"indexName" : "id_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"id" : [
"[570391.0, 570391.0]",
"[576391.0, 576391.0]",
"[767422.0, 767422.0]"
]
},
"keysExamined" : 5,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "host",
"port" : 27017,
"version" : "3.2.21",
"gitVersion" : ""
},
"ok" : 1
}
I googled it with this unexpected behavior but I didn't find any solution. So I had to restart server when it hangs..
To help understanding, here's my case :
I'm using MongoDB Cloud manager and DB instances are hosted on AWS EC2
I'm using ReplicaSet and my read preference is secondaryPreferred. So all read operations are going toward the secondary node.
MongoDB version is 3.2
I created index for every fields used in the query (per field)
I executed same query both in Primary Node and Secondary Node (with slaveOk)
The collection has 20M objects.
It isn't happening every time for the same query. I think that there's something else that take effect on the performance (such as replicating?)
But I don't know how to debug this case. Is there any better idea for this issue or way to debug?
Thanks,
Edit : I still don't get what's the reason but tried to solve it by making any changes. I removed $gt and it seems to work. But $gt has no problem in my previous execution, and I think it's because there's few user at this moment.
Edit : I have MongoDB cloud manager so I could do some metrics change, I think Query targeting is increased while I still don't know. Normally it's about 100 per 1 document, but today it hits over 2K. May be related?
Query targeting for 2month
I think this explains what you see:
"lockStats" : {
"Global" : {
"acquireCount" : {
"r" : NumberLong(5524)
},
"acquireWaitCount" : {
"r" : NumberLong(349)
},
"timeAcquiringMicros" : {
"r" : NumberLong(6613952)
}
},
It appears that the stalled read takes its time trying to acquire the read intent lock. This is expected in older version of MongoDB (pre-4.0) since reading from a secondary would wait while oplog apply operation is in progress. This is done so that the secondary read would not read data in its inconsistent form while oplog is being applied.
This is a longstanding secondary read behaviour dating from the earliest MongoDB versions, and I guess you're seeing this now because your database has reached a point where it's busy enough for this to be an issue.
This situation was improved in MongoDB 4.0 and newer via SERVER-34192, allowing secondary reads to proceed while oplog apply is in progress.

MongoDB - Index not being used when sorting and limiting on ranged query

I'm trying to get a sorted list of items using a ranged query on a collection containing bulletin-board data. The data structure of a "thread" document is:
{
"_id" : ObjectId("5a779b47f4fa72412126526a"),
"title" : "necessitatibus tincidunt libris assueverit",
"content" : "Corrumpitvenenatis cubilia adipiscing sollicitudin",
"flagged" : false,
"locked" : false,
"sticky" : false,
"lastPostAt" : ISODate("2018-02-05T06:35:24.656Z"),
"postCount" : 42,
"user" : ObjectId("5a779b46f4fa72412126525a"),
"category" : ObjectId("5a779b31f4fa724121265164"),
"createdAt" : ISODate("2018-02-04T23:46:15.852Z"),
"updatedAt" : ISODate("2018-02-05T06:35:24.656Z")
}
The query is:
db.threads.find({
category: ObjectId('5a779b31f4fa724121265142'),
_id : { $gt: ObjectId('5a779b5cf4fa724121269be8') }
}).sort({ sticky: -1, lastPostAt: -1, _id: 1 }).limit(25)
I set up the following indexes to support it:
{ category: 1, _id: 1 }
{ category: 1, _id: 1, sticky: 1, lastPostAt: 1 }
{ sticky: 1, lastPostAt: 1, _id: 1 }
In spite of this, it's still scanning hundreds of documents/keys according to execution stats:
{
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 772,
"executionTimeMillis" : 17,
"totalKeysExamined" : 772,
"totalDocsExamined" : 772,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 772,
"executionTimeMillisEstimate" : 0,
"works" : 1547,
"advanced" : 772,
"needTime" : 774,
"needYield" : 0,
"saveState" : 33,
"restoreState" : 33,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"sticky" : -1,
"lastPostAt" : -1,
"_id" : 1
},
"memUsage" : 1482601,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 772,
"executionTimeMillisEstimate" : 0,
"works" : 774,
"advanced" : 772,
"needTime" : 1,
"needYield" : 0,
"saveState" : 33,
"restoreState" : 33,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 772,
"executionTimeMillisEstimate" : 0,
"works" : 773,
"advanced" : 772,
"needTime" : 0,
"needYield" : 0,
"saveState" : 33,
"restoreState" : 33,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 772,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 772,
"executionTimeMillisEstimate" : 0,
"works" : 773,
"advanced" : 772,
"needTime" : 0,
"needYield" : 0,
"saveState" : 33,
"restoreState" : 33,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"category" : 1,
"_id" : 1,
"sticky" : 1,
"lastPostAt" : 1
},
"indexName" : "category_1__id_1_sticky_1_lastPostAt_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"category" : [ ],
"_id" : [ ],
"sticky" : [ ],
"lastPostAt" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
],
"_id" : [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
],
"sticky" : [
"[MinKey, MaxKey]"
],
"lastPostAt" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 772,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
}
When I take out the sorting stage, it correctly scans only 25 documents. And the keys examined (772) remains the same no matter which fields I place in the sort function.
Here is the full explain() for the sorted query:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "database.threads",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"category" : {
"$eq" : ObjectId("5a779b31f4fa724121265142")
}
},
{
"_id" : {
"$gt" : ObjectId("5a779b5cf4fa724121269be8")
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"sticky" : -1,
"lastPostAt" : -1,
"_id" : 1
},
"limitAmount" : 25,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"category" : 1,
"_id" : 1,
"sticky" : 1,
"lastPostAt" : 1
},
"indexName" : "category_1__id_1_sticky_1_lastPostAt_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"category" : [ ],
"_id" : [ ],
"sticky" : [ ],
"lastPostAt" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
],
"_id" : [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
],
"sticky" : [
"[MinKey, MaxKey]"
],
"lastPostAt" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"sticky" : -1,
"lastPostAt" : -1,
"_id" : 1
},
"limitAmount" : 25,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"_id" : {
"$gt" : ObjectId("5a779b5cf4fa724121269be8")
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"category" : 1
},
"indexName" : "category_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"category" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"sticky" : -1,
"lastPostAt" : -1,
"_id" : 1
},
"limitAmount" : 25,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"category" : 1,
"_id" : 1
},
"indexName" : "category_1__id_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"category" : [ ],
"_id" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
],
"_id" : [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"sticky" : -1,
"lastPostAt" : -1,
"_id" : 1
},
"limitAmount" : 25,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"category" : {
"$eq" : ObjectId("5a779b31f4fa724121265142")
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
]
}
}
}
}
}
]
},
"serverInfo" : {
"host" : "CRF-MBP.local",
"port" : 27017,
"version" : "3.6.2",
"gitVersion" : "489d177dbd0f0420a8ca04d39fd78d0a2c539420"
},
"ok" : 1
}
And here is the full explain() for the non-sorted query:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "database.threads",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"category" : {
"$eq" : ObjectId("5a779b31f4fa724121265142")
}
},
{
"_id" : {
"$gt" : ObjectId("5a779b5cf4fa724121269be8")
}
}
]
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 25,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"category" : 1,
"_id" : 1,
"sticky" : 1,
"lastPostAt" : 1
},
"indexName" : "category_1__id_1_sticky_1_lastPostAt_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"category" : [ ],
"_id" : [ ],
"sticky" : [ ],
"lastPostAt" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
],
"_id" : [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
],
"sticky" : [
"[MinKey, MaxKey]"
],
"lastPostAt" : [
"[MinKey, MaxKey]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 25,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"_id" : {
"$gt" : ObjectId("5a779b5cf4fa724121269be8")
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"category" : 1
},
"indexName" : "category_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"category" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
]
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 25,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"category" : 1,
"_id" : 1
},
"indexName" : "category_1__id_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"category" : [ ],
"_id" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
],
"_id" : [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
]
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 25,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"category" : {
"$eq" : ObjectId("5a779b31f4fa724121265142")
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
]
}
}
}
}
]
},
"serverInfo" : {
"host" : "CRF-MBP.local",
"port" : 27017,
"version" : "3.6.2",
"gitVersion" : "489d177dbd0f0420a8ca04d39fd78d0a2c539420"
},
"ok" : 1
}
Does anyone have any idea why this might not fully use an index?
The issue is that none of your indexes actually help with the sorted query. This is the reason for the high number of scanned objects and the presence of SORT_KEY_GENERATOR stage (in-memory sort, limited to 32MB).
The non-sorted query, on the other hand, can use either the { category: 1, _id: 1 } or { category: 1, _id: 1, sticky: 1, lastPostAt: 1 } indexes. Note that it's perfectly valid to use either one, since one contains the prefix of the other. See Prefixes for more details.
MongoDB find() queries typically uses only one index, so a single compound index should cater for all the parameters of your query. This would include both the parameters of find() and sort().
A good writeup of how your index should be created is available in Optimizing MongoDB Compound Indexes. Let's take the main point of the article, where the compound index ordering should be equality --> sort --> range:
Your query "shape" is:
db.collection.find({category:..., _id: {$gt:...}})
.sort({sticky:-1, lastPostAt:-1, _id:1})
.limit(25)
We see that:
category:... is equality
sticky:-1, lastPostAt:-1, _id:1 is sort
_id: {$gt:...} is range
So the compound index you need is:
{category:1, sticky:-1, lastPostAt:-1, _id:1}
Where the winning plan of the explain() output of your query with the above index shows:
"winningPlan": {
"stage": "LIMIT",
"limitAmount": 25,
"inputStage": {
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"category": 1,
"sticky": -1,
"lastPostAt": -1,
"_id": 1
},
"indexName": "category_1_sticky_-1_lastPostAt_-1__id_1",
"isMultiKey": false,
"multiKeyPaths": {
"category": [ ],
"sticky": [ ],
"lastPostAt": [ ],
"_id": [ ]
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"category": [
"[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
],
"sticky": [
"[MaxKey, MinKey]"
],
"lastPostAt": [
"[MaxKey, MinKey]"
],
"_id": [
"(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
]
}
}
}
}
Note that the winning plan doesn't contain a SORT_KEY_GENERATOR stage. This means that the index can be fully utilized to respond to the sorted query.
I believe there are 2 problems both having to do with your sort. These problems come straight from the documentations but if you would comment I'll help explain (and might possibly learn something myself)
The first and biggest problem is that you must sort in the order given by the index. From docs:
You can specify a sort on all the keys of the index or on a subset;
however, the sort keys must be listed in the same order as they appear
in the index. For example, an index key pattern { a: 1, b: 1 } can
support a sort on { a: 1, b: 1 } but not on { b: 1, a: 1 }.
This means that you must sort in the order given by your winning plan: category, _id, sticky, lastPostAt (or any prefix of that order such as category, _id, sticky or category _id). If not mongodb will identify the 772 docs which are indexed using your winning plan, but will then have to comb through each key in order to assess values and provide the desired sort order. If you want to sort by the order you are curenttly querying must provide a index in that order:
The second problem is that you must sort in the direction that you provided by the index (or the inverse direction).
For a query to use a compound index for a sort, the specified sort
direction for all keys in the cursor.sort() document must match the
index key pattern or match the inverse of the index key pattern. For
example, an index key pattern { a: 1, b: -1 } can support a sort on {
a: 1, b: -1 } and { a: -1, b: 1 } but not on { a: -1, b: -1 } or {a:
1, b: 1}.
Because your indexes are all in ascending order, you would have to either sort in ascending order for all indexes, or descending order for all indexes. If not we run into the same problem in which mongo finds all the relevant docs, but has to comb through them to provide the desired order.
I believe you would get imporved functionality by providing an additional index of:
{ sticky: -1, lastPostAt: -1, _id: 1 }
or its inverse:
{ sticky: 1, lastPostAt: 1, _id: -1 }
This would create a situation where mongo uses your first index
{ category: 1, _id: 1 }
To identify potential unsorted documents, then uses the one of the new index (provided above) since they would already be sorted. Then the limit would take care of giving you your 25 docs.
I'm pretty sure this would created a covered query (a query with no docs examined). Let me know how it goes, cheers!

mongodb index find index sort

I would like to ask about mongodb indexes. Can I use a different index in the find and the sort. By example I have two indexes:
(a:-1)
(b:1,c:1)
What indexes uses this sentence?
({a:[$gt30}},{a:[$lt50}}]}.sort({c:1})
Can I use a different index in the find and the sort.
After reading some more into this you will see at the bottom of the documentation page on index intersectioning: http://docs.mongodb.org/manual/core/index-intersection/#index-intersection-and-sort
Index intersection does not apply when the sort() operation requires an index completely separate from the query predicate.
So no, even if ypou created an index of {c:1} it could not be used independantly to intersect {a:1}
What indexes uses this sentence?
In this case only {a:1} will be used.
Creating an index on a single field called Single Field Index.
Creating multiple Single Field indexes to boost your query and the sort performance won't help much!. You should use Compound Indexes instead.
Check the documentation on MongoDB: https://docs.mongodb.com/manual/core/index-compound/
If you want to learn how to index your fields and how to measure the performance of your queries.
And Check this tutorial on Youtube: https://dplink.app/nxLgvk7lR
Suppose I am having persons collection having documents like below :
{
dob:
{ age : 50} ,
gender : "male" ,
phone : ""
}
Now i create indexes as below .
1 : db.persons.createIndex({"dob.age" : -1})
2 : db.persons.createIndex({phone : 1 , gender : 1})
Now If i execute below query like yours
db.persons.explain("executionStats").find({$and : [ {"dob.age" : {$lt : 50} } ,
{"dob.age" : {$gt : 30} } ] } ).sort({gender : 1 })
I will get below execution stats :
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "college.persons",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"queryHash" : "22FEA299",
"planCacheKey" : "5E8F38C1",
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"phone" : 1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dob.age" : -1
},
"indexName" : "dob.age_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dob.age" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dob.age" : [
"(50.0, 30.0)"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1734,
"executionTimeMillis" : 10,
"totalKeysExamined" : 1734,
"totalDocsExamined" : 1734,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 3471,
"advanced" : 1734,
"needTime" : 1736,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"sortPattern" : {
"phone" : 1
},
"memUsage" : 1914799,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 1736,
"advanced" : 1734,
"needTime" : 1,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 1735,
"advanced" : 1734,
"needTime" : 0,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"docsExamined" : 1734,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 1735,
"advanced" : 1734,
"needTime" : 0,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"keyPattern" : {
"dob.age" : -1
},
"indexName" : "dob.age_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dob.age" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dob.age" : [
"(50.0, 30.0)"
]
},
"keysExamined" : 1734,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "RGGYSLT-0483",
"port" : 27017,
"version" : "4.2.0",
"gitVersion" : "a4b751dcf51dd249c5865812b390cfd1c0129c30"
},
"ok" : 1
}
This Means Data was fetched with IXScan on single field first and then sorted on
third field separately .
But the moment i change the query to sort on fields for which the index is already created things change . Now {"dob.age" : -1} index gets rejected .
In Mongo db Winning plan is the one for which 100 docs can be fetched early and Mongo db caches that plan for a query . Now this cache will be purged after 1000 docs insertions , index rebuild , server restarts or new index insertions.
Hence which index will be used depends upon winning plans .
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "college.persons",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"queryHash" : "DA8248FA",
"planCacheKey" : "E779554F",
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"gender" : 1,
"phone" : 1
},
"indexName" : "gender_1_phone_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [ ],
"phone" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[MinKey, MaxKey]"
],
"phone" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"gender" : 1,
"phone" : 1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dob.age" : -1
},
"indexName" : "dob.age_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dob.age" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dob.age" : [
"(50.0, 30.0)"
]
}
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1734,
"executionTimeMillis" : 12,
"totalKeysExamined" : 5002,
"totalDocsExamined" : 5002,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 5003,
"advanced" : 1734,
"needTime" : 3268,
"needYield" : 0,
"saveState" : 41,
"restoreState" : 41,
"isEOF" : 1,
"docsExamined" : 5002,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 5002,
"executionTimeMillisEstimate" : 0,
"works" : 5003,
"advanced" : 5002,
"needTime" : 0,
"needYield" : 0,
"saveState" : 41,
"restoreState" : 41,
"isEOF" : 1,
"keyPattern" : {
"gender" : 1,
"phone" : 1
},
"indexName" : "gender_1_phone_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [ ],
"phone" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[MinKey, MaxKey]"
],
"phone" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 5002,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"serverInfo" : {
"host" : "RGGYSLT-0483",
"port" : 27017,
"version" : "4.2.0",
"gitVersion" : "a4b751dcf51dd249c5865812b390cfd1c0129c30"
},
"ok" : 1
}