mongodb index find index sort - mongodb

I would like to ask about mongodb indexes. Can I use a different index in the find and the sort. By example I have two indexes:
(a:-1)
(b:1,c:1)
What indexes uses this sentence?
({a:[$gt30}},{a:[$lt50}}]}.sort({c:1})

Can I use a different index in the find and the sort.
After reading some more into this you will see at the bottom of the documentation page on index intersectioning: http://docs.mongodb.org/manual/core/index-intersection/#index-intersection-and-sort
Index intersection does not apply when the sort() operation requires an index completely separate from the query predicate.
So no, even if ypou created an index of {c:1} it could not be used independantly to intersect {a:1}
What indexes uses this sentence?
In this case only {a:1} will be used.

Creating an index on a single field called Single Field Index.
Creating multiple Single Field indexes to boost your query and the sort performance won't help much!. You should use Compound Indexes instead.
Check the documentation on MongoDB: https://docs.mongodb.com/manual/core/index-compound/
If you want to learn how to index your fields and how to measure the performance of your queries.
And Check this tutorial on Youtube: https://dplink.app/nxLgvk7lR

Suppose I am having persons collection having documents like below :
{
dob:
{ age : 50} ,
gender : "male" ,
phone : ""
}
Now i create indexes as below .
1 : db.persons.createIndex({"dob.age" : -1})
2 : db.persons.createIndex({phone : 1 , gender : 1})
Now If i execute below query like yours
db.persons.explain("executionStats").find({$and : [ {"dob.age" : {$lt : 50} } ,
{"dob.age" : {$gt : 30} } ] } ).sort({gender : 1 })
I will get below execution stats :
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "college.persons",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"queryHash" : "22FEA299",
"planCacheKey" : "5E8F38C1",
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"phone" : 1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dob.age" : -1
},
"indexName" : "dob.age_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dob.age" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dob.age" : [
"(50.0, 30.0)"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1734,
"executionTimeMillis" : 10,
"totalKeysExamined" : 1734,
"totalDocsExamined" : 1734,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 3471,
"advanced" : 1734,
"needTime" : 1736,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"sortPattern" : {
"phone" : 1
},
"memUsage" : 1914799,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 1736,
"advanced" : 1734,
"needTime" : 1,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 1735,
"advanced" : 1734,
"needTime" : 0,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"docsExamined" : 1734,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 1735,
"advanced" : 1734,
"needTime" : 0,
"needYield" : 0,
"saveState" : 27,
"restoreState" : 27,
"isEOF" : 1,
"keyPattern" : {
"dob.age" : -1
},
"indexName" : "dob.age_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dob.age" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dob.age" : [
"(50.0, 30.0)"
]
},
"keysExamined" : 1734,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "RGGYSLT-0483",
"port" : 27017,
"version" : "4.2.0",
"gitVersion" : "a4b751dcf51dd249c5865812b390cfd1c0129c30"
},
"ok" : 1
}
This Means Data was fetched with IXScan on single field first and then sorted on
third field separately .
But the moment i change the query to sort on fields for which the index is already created things change . Now {"dob.age" : -1} index gets rejected .
In Mongo db Winning plan is the one for which 100 docs can be fetched early and Mongo db caches that plan for a query . Now this cache will be purged after 1000 docs insertions , index rebuild , server restarts or new index insertions.
Hence which index will be used depends upon winning plans .
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "college.persons",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"queryHash" : "DA8248FA",
"planCacheKey" : "E779554F",
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"gender" : 1,
"phone" : 1
},
"indexName" : "gender_1_phone_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [ ],
"phone" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[MinKey, MaxKey]"
],
"phone" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"gender" : 1,
"phone" : 1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dob.age" : -1
},
"indexName" : "dob.age_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dob.age" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dob.age" : [
"(50.0, 30.0)"
]
}
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1734,
"executionTimeMillis" : 12,
"totalKeysExamined" : 5002,
"totalDocsExamined" : 5002,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"dob.age" : {
"$lt" : 50
}
},
{
"dob.age" : {
"$gt" : 30
}
}
]
},
"nReturned" : 1734,
"executionTimeMillisEstimate" : 0,
"works" : 5003,
"advanced" : 1734,
"needTime" : 3268,
"needYield" : 0,
"saveState" : 41,
"restoreState" : 41,
"isEOF" : 1,
"docsExamined" : 5002,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 5002,
"executionTimeMillisEstimate" : 0,
"works" : 5003,
"advanced" : 5002,
"needTime" : 0,
"needYield" : 0,
"saveState" : 41,
"restoreState" : 41,
"isEOF" : 1,
"keyPattern" : {
"gender" : 1,
"phone" : 1
},
"indexName" : "gender_1_phone_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"gender" : [ ],
"phone" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"gender" : [
"[MinKey, MaxKey]"
],
"phone" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 5002,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"serverInfo" : {
"host" : "RGGYSLT-0483",
"port" : 27017,
"version" : "4.2.0",
"gitVersion" : "a4b751dcf51dd249c5865812b390cfd1c0129c30"
},
"ok" : 1
}

Related

MongoDb C# query performance

I have the MongoDB C# Driver 2.14.1 and a Mongo Server v3.6 running on a Docker container. I'm running an easy (I think) query over 3 collections. The biggest collection has 400k documents, each document averaged 64Kb. I'm filtering by two fields and returning the results in a cursor to reduce overhead. The results are sorted by descending by a field that is configured as an index as well. This is happening on my production server which is taking nearly 1GB of memory (not being released afterwards). The following queries explained (run on the server):
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "table1",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "Location1"
}
},
{
"Date" : {
"$gt" : ISODate("2021-01-18T23:00:00Z")
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "Location1"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 2,
"totalKeysExamined" : 7,
"totalDocsExamined" : 7,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "Location1"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 8,
"advanced" : 1,
"needTime" : 6,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 7,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 7,
"executionTimeMillisEstimate" : 0,
"works" : 8,
"advanced" : 7,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
},
"keysExamined" : 7,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "7aa23bd7fe20",
"port" : 27018,
"version" : "3.6.18",
"gitVersion" : "2005f25eed7ed88fa698d9b800fe536bb0410ba4"
},
"ok" : 1
}
The second query:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "table2",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "02"
}
},
{
"Date" : {
"$gt" : ISODate("2021-01-18T23:00:00Z")
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "02"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 63,
"totalKeysExamined" : 9896,
"totalDocsExamined" : 9896,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "02"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 50,
"works" : 9897,
"advanced" : 1,
"needTime" : 9895,
"needYield" : 0,
"saveState" : 77,
"restoreState" : 77,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 9896,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 9896,
"executionTimeMillisEstimate" : 0,
"works" : 9897,
"advanced" : 9896,
"needTime" : 0,
"needYield" : 0,
"saveState" : 77,
"restoreState" : 77,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
},
"keysExamined" : 9896,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "7aa23bd7fe20",
"port" : 27018,
"version" : "3.6.18",
"gitVersion" : "2005f25eed7ed88fa698d9b800fe536bb0410ba4"
},
"ok" : 1
}
And the third:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "table3",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "52"
}
},
{
"Date" : {
"$gt" : ISODate("2021-01-18T23:00:00Z")
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "52"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 30,
"executionTimeMillis" : 934,
"totalKeysExamined" : 366370,
"totalDocsExamined" : 366370,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"Type" : {
"$eq" : "Type1"
}
},
{
"LocationId" : {
"$eq" : "52"
}
}
]
},
"nReturned" : 30,
"executionTimeMillisEstimate" : 889,
"works" : 366371,
"advanced" : 30,
"needTime" : 366340,
"needYield" : 0,
"saveState" : 2862,
"restoreState" : 2862,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 366370,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 366370,
"executionTimeMillisEstimate" : 160,
"works" : 366371,
"advanced" : 366370,
"needTime" : 0,
"needYield" : 0,
"saveState" : 2862,
"restoreState" : 2862,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"Date" : -1
},
"indexName" : "Date_-1",
"isMultiKey" : false,
"multiKeyPaths" : {
"Date" : [ ]
},
"isUnique" : false,
"isSparse" : true,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"Date" : [
"[new Date(9223372036854775807), new Date(1611010800000))"
]
},
"keysExamined" : 366370,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "7aa23bd7fe20",
"port" : 27018,
"version" : "3.6.18",
"gitVersion" : "2005f25eed7ed88fa698d9b800fe536bb0410ba4"
},
"ok" : 1
}
Now with my little knowledge of the "explainPlan" I would have imagined that the performance is OK, and not many records get returned on each query. Having said that I can see a pattern when I run the code via the .NET application (C# code) that makes the CPU go through the roof and right after the memory follows in the Mongo server Docker container.
I don't know if there is misconfigured or whether I can do something to improve the performance of it because I'm quite puzzle with the performance on just that query - I think is this query because the rest of the queries with similar amount of data on different parts of the system work quite well. Besides that, is there any mechanism or tool I can use to diagnose the memory consumption in my Mongo server?

mongodb query slow in different version and env

I have two env mongodbs,
the difference between them is:
test mongodb version: 3.2.20 , prod mongodb version : 4.0.18
and test env query plan first stage is Limit , however the other is Sort.
in my test env, it's very quick and totalDocsExamined == limit
they both hit the index:
{
"v" : 1,
"key" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"name" : "appIds_1_ctime_-1_background_1",
"ns" : "newsmine.newstoapp"
}
query: db.newstoapp.find({"appIds":{"$in":[999]}}).sort({"ctime":-1}).limit(10).explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "newsmine.newstoapp",
"indexFilterSet" : false,
"parsedQuery" : {
"appIds" : {
"$in" : [
999
]
}
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[999.0, 999.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 0,
"totalKeysExamined" : 10,
"totalDocsExamined" : 10,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 0,
"works" : 11,
"advanced" : 10,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 10,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 10,
"executionTimeMillisEstimate" : 0,
"works" : 10,
"advanced" : 10,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 10,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 10,
"executionTimeMillisEstimate" : 0,
"works" : 10,
"advanced" : 10,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[999.0, 999.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 10,
"dupsTested" : 10,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
},
"serverInfo" : {
"host" : "",
"port" : ,
"version" : "3.2.20",
"gitVersion" : "a7a144f40b70bfe290906eb33ff2714933544af8"
},
"ok" : 1
}
in my prod env, it's getting slow query
query: datamongo:PRIMARY> db.newstoapp.find({"appIds":{"$in":[1460]}}).sort({"ctime":-1}).limit(10).explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "newsmine.newstoapp",
"indexFilterSet" : false,
"parsedQuery" : {
"appIds" : {
"$eq" : 1460
}
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"ctime" : -1
},
"limitAmount" : 10,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[1460.0, 1460.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"ctime" : -1
},
"limitAmount" : 10,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"appIds" : 1
},
"indexName" : "appIds_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[1460.0, 1460.0]"
]
}
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"appIds" : {
"$eq" : 1460
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"ctime" : 1
},
"indexName" : "ctime_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"ctime" : [
"[MaxKey, MinKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 40,
"totalKeysExamined" : 405,
"totalDocsExamined" : 405,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 3,
"works" : 418,
"advanced" : 10,
"needTime" : 407,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"ctime" : -1
},
"memUsage" : 8471,
"memLimit" : 33554432,
"limitAmount" : 10,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 405,
"executionTimeMillisEstimate" : 3,
"works" : 407,
"advanced" : 405,
"needTime" : 1,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 405,
"executionTimeMillisEstimate" : 3,
"works" : 406,
"advanced" : 405,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 405,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 405,
"executionTimeMillisEstimate" : 1,
"works" : 406,
"advanced" : 405,
"needTime" : 0,
"needYield" : 0,
"saveState" : 9,
"restoreState" : 9,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"appIds" : 1,
"ctime" : -1,
"background" : 1
},
"indexName" : "appIds_1_ctime_-1_background_1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"appIds" : [
"[1460.0, 1460.0]"
],
"ctime" : [
"[MaxKey, MinKey]"
],
"background" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 405,
"seeks" : 1,
"dupsTested" : 405,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "",
"port" : ,
"version" : "4.0.18",
"gitVersion" : "6883bdfb8b8cff32176b1fd176df04da9165fd67"
},
"ok" : 1,
"operationTime" : Timestamp(1629988625, 146),
"$clusterTime" : {
"clusterTime" : Timestamp(1629988625, 146),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
i followed Joe, changed my index, ctime before appIds.
but it does not work very well. there still be slow log for my new sql. it's hard to find out why it is slow
2021-10-19T22:38:16.918+0800 I COMMAND [conn2434281164] command newsmine.newstoapp command: find { find: "newstoapp", filter: { appIds: { $elemMatch: { $in: [ 2433 ] } }, ctime: { $gte: 0 } }, sort: { ctime: -1 }, hint: { ctime: -1, appIds: 1 }, skip: 0, limit: 50, batchSize: 50, $readPreference: { mode: "secondaryPreferred" }, $db: "newsmine" } planSummary: IXSCAN { ctime: -1, appIds: 1 } keysExamined:1471582 docsExamined:50 cursorExhausted:1 numYields:11496 nreturned:50 reslen:34043 locks:{ Global: { acquireCount: { r: 11497 } }, Database: { acquireCount: { r: 11497 } }, Collection: { acquireCount: { r: 11497 } } } storage:{ data: { bytesRead: 44958, timeReadingMicros: 618 } } protocol:op_query 7038ms
The cause of the slowness is that MongoDB 4.0.18 hash a blocking sort stage, so all matching documents must be found, retrieved, and sorted in memory before returning the requested batch.
In prior versions of MongoDB it was found that under certain conditions using a multi-key index to support a sort would provide incorrect result.
I never fully understood these conditions or why the results were incorrect, so if you are able to find those details, please edit or comment.
Prior to MongoDB 3.4 the index metadata contained a boolean value to indicate whether or not the index was multi-key (indexed a field that contained an array for at least one document).
MongoDB 3.4 introduced a new index version that also keeps track of which fields in the index are multi-key.
MongoDB 3.6 introduced a change to sorting to avoid the situations where results would be incorrect. This is why your query has a sort stage and is taking longer.
There are a couple things you could try to get back to the previous behavior without a blocking sort:
Drop and rebuild the index.
The existing index is version 1, which does not track multi-key paths. When rebuilding, the index should be created at version 2, which does track these, and may permit the query executor to use the index for sorting.
Create a new index with ctime before appIds.
A multi-key index has an entry in the index for each value in the indexed array. This may cause the query planner to assume it will disrupt sorting on a following key.
An index on {ctime:-1, appIds:1, background:1} would place the sort key ahead of the multi-key field, and while this may require reading more of the index, it may also permit the query executor to use the index for sorting.

query text search index on compound text search and timestamp mongodb

I have a collection, with a total of documents about 65 million records like this
{
"_id" : ObjectId("5e0b814660da38d499ecf178"),
"brands" : null,
"client_id" : null,
"code_co_owner" : ",7359562, ",
"code_segment" : "7359562",
"core" : "",
"created" : "01-01-2020",
"created_full" : "01-01-2020 00:00:27",
"created_int" : NumberLong(1577811627),
"email" : ",phamthanhlam17_gmail_com, "
.....
}
I made an compound index on (email, created_int): {"email": text, created_int: -1} for search and filter which name in a range of created_int
But I see that it's poor performance on search.
I tried to use explain on query:
db.getCollection('profile_20201').explain().find({"$text":{"$search":"phamthanhlam17_gmail_com"},
"created_int":{"$lte":1585627013, "$gte":1583035013}}).count()
The explain result is:
{
"queryPlanner" : {
"plannerVersion" : 1,
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
},
{
"$text" : {
"$search" : "phamthanhlam17_gmail_com",
"$language" : "english",
"$caseSensitive" : false,
"$diacriticSensitive" : false
}
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "TEXT",
"indexPrefix" : {},
"indexName" : "email_text_created_int_-1",
"parsedTextQuery" : {
"terms" : [
"phamthanhlam17_gmail_com"
],
"negatedTerms" : [],
"phrases" : [],
"negatedPhrases" : []
},
"textIndexVersion" : 3,
"inputStage" : {
"stage" : "TEXT_MATCH",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "OR",
"filter" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1,
"created_int" : -1.0
},
"indexName" : "email_text_created_int_-1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {}
}
}
}
}
}
},
"rejectedPlans" : []
},
"serverInfo" : {
},
"ok" : 1.0
}
It's explain stats:
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "namespace",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
},
{
"$text" : {
"$search" : "phamthanhlam17_gmail_com",
"$language" : "english",
"$caseSensitive" : false,
"$diacriticSensitive" : false
}
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "TEXT",
"indexPrefix" : {},
"indexName" : "email_text_created_int_-1",
"parsedTextQuery" : {
"terms" : [
"phamthanhlam17_gmail_com"
],
"negatedTerms" : [],
"phrases" : [],
"negatedPhrases" : []
},
"textIndexVersion" : 3,
"inputStage" : {
"stage" : "TEXT_MATCH",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "OR",
"filter" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1,
"created_int" : -1.0
},
"indexName" : "email_text_created_int_-1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {}
}
}
}
}
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 1499057,
"totalKeysExamined" : 72544123,
"totalDocsExamined" : 39448083,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 1483861,
"works" : 72544124,
"advanced" : 0,
"needTime" : 72544123,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 39448083,
"nSkipped" : 0,
"inputStage" : {
"stage" : "TEXT",
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 1475831,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"indexPrefix" : {},
"indexName" : "email_text_created_int_-1",
"parsedTextQuery" : {
"terms" : [
"phamthanhlam17_gmail_com"
],
"negatedTerms" : [],
"phrases" : [],
"negatedPhrases" : []
},
"textIndexVersion" : 3,
"inputStage" : {
"stage" : "TEXT_MATCH",
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 1473041,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"docsRejected" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 1465951,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 39448083,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "OR",
"filter" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
}
]
},
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 439664,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"dupsTested" : 72544123,
"dupsDropped" : 0,
"recordIdsForgotten" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 72544123,
"executionTimeMillisEstimate" : 291188,
"works" : 72544124,
"advanced" : 72544123,
"needTime" : 0,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1,
"created_int" : -1.0
},
"indexName" : "email_text_created_int_-1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {},
"keysExamined" : 72544123,
"seeks" : 1,
"dupsTested" : 72544123,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
}
}
},
"serverInfo" : {
},
"ok" : 1.0
}```
So, is the index is cover the query?
Or which index will give me better performance for this problem?
Thank you.
Well, it seems like you have created the compound index with text. But in official MongoDB Documentation, it says that:
A compound index can include a text index key in combination with ascending/descending index keys. However, these compound indexes have the following restrictions:
A compound text index cannot include any other special index types, such as multi-key or geospatial index fields.
If the compound text index includes keys preceding the text index key, to perform a $text search, the query predicate must include equality match conditions on the preceding keys. (You are using the range query here)
When creating a compound text index, all text index keys must be listed adjacently in the index specification document.
So, that's the first issue.
Next, I want you to have a look at prefixes, it will help you understand how the compound indexes will be used in your query.
Hope this will help you understand the problem :)

What index to use for a Mongo request with distinct and query?

I have a Mongo collection which hold millions of IoT device data.
The structure of the document is like this :
{ ObjectID:"...", device:"DEVICE3", topic:"TEMP", vhost:"client1", date:ISODate("2017-08-23T08:00:00.000Z"), value:23.5 }
I have a Rest API with a request that finds all the devices for one specific vhost.
The request looks like that : db.data.distinct("device", { vhost:"client1" })
I added an index on vhost and device : db.data.createIndex( { vhost:1, device:1 }) but it is still a lot of examined documents. What kind of index can I use to optimize the request ?
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 15848881,
"executionTimeMillis" : 42425,
"totalKeysExamined" : 15848881,
"totalDocsExamined" : 15848881,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 15848881,
"executionTimeMillisEstimate" : 36240,
"works" : 15848882,
"advanced" : 15848881,
"needTime" : 0,
"needYield" : 0,
"saveState" : 123949,
"restoreState" : 123949,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 15848881,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 15848881,
"executionTimeMillisEstimate" : 9890,
"works" : 15848882,
"advanced" : 15848881,
"needTime" : 0,
"needYield" : 0,
"saveState" : 123949,
"restoreState" : 123949,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"vhost" : 1,
"device" : 1
},
"indexName" : "vhost_1_device_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"vhost" : [
"[\"client1\", \"client1\"]"
],
"device" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 15848881,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
In final, there is about 30 distinct device.
EDIT :
Here is the queryPlanner as asked :
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "thingsplay.data",
"indexFilterSet" : false,
"parsedQuery" : {
"vhost" : {
"$eq" : "client1"
}
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"vhost" : 1,
"device" : 1
},
"indexName" : "vhost_1_device_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"vhost" : [
"[\"client1\", \"client1\"]"
],
"device" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
The result :
[
"F000105",
"F000107",
"F000109",
"F000110",
"F000113",
"F000119",
"F000121",
"F000124",
"F000128",
"F000131",
"F000134",
"F000138",
"F000144",
"F000146",
"F000147",
"F000148",
"F000149",
"F000150",
"F000153",
"F000155",
"F000156",
"F000159",
"F000161",
"F000164",
"F000166",
"F000167",
"F000168",
"F000169",
"F000170",
"F000171",
"F000172",
"F000181",
"F000183",
"F000184",
"F000187",
"F000190",
"F000192",
"F000193",
"F000203",
"F000204",
"F000205",
"F000208",
"F000209",
"F000215",
"F000221",
"F000223",
"F000243",
"F000249",
"F000250",
"F000251",
"F000253",
"F000255",
"S0E190E",
"S0E1A45",
"S0E1AC0",
"SYS_STATUS_ID",
"TS4D9292",
"TS4D9294",
"TS4D9296",
"TS4D9297",
"TS4D9298",
"TS4D9299",
"TS4D929B",
"TS4D929D",
"TS4D929F",
"TS4D92A0",
"TS4D92A2",
"TS4D92A6",
"TS4D92AA",
"TS4D92B1",
"TS4D92B2",
"TS4D92B3",
"TS4D92B4",
"TS4D92C2"
]

Mongo query is taking a while even with index

I have been running a mongo test server for a few weeks now and everything has been going good. I had a query to get a count of deleted files and it was running in the sub 1 second range for 140 million documents (44 gb of data). Today, all of the sudden that same query is taking a while. When running explain on the find query it shows it is using the index that is in place but it still takes a long time. The weird thing is, is that this isn't the only query that appears to do this. If I run a query that returns 1 item, it's almost instantaneous. When its returning a set of documents it takes forever (multiple minutes). Below is the query and the explain output from Mongo shell. I've removed the rejected plan to save space, but can put it back if needed. I'm new to mongo so any help is greatly appreciated. I have another query that I've been waiting on for 24 minutes that is just a search on an EventID field which is an indexed field.
db.EventEntries.find({IsDelete:true, TimeCreated:{$gte:ISODate('2017-06-30')}}).explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "Events.EventEntries",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"IsDelete" : {
"$eq" : true
}
},
{
"TimeCreated" : {
"$gte" : ISODate("2017-06-30T00:00:00Z")
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"IsDelete" : 1,
"TimeCreated" : 1
},
"indexName" : "IsDelete_1_TimeCreated_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"IsDelete" : [ ],
"TimeCreated" : [
"TimeCreated"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"IsDelete" : [
"[true, true]"
],
"TimeCreated" : [
"[new Date(1498780800000), new Date(9223372036854775807)]"
]
}
}
},
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 138570,
"executionTimeMillis" : 177933,
"totalKeysExamined" : 138570,
"totalDocsExamined" : 138570,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 138570,
"executionTimeMillisEstimate" : 177834,
"works" : 138571,
"advanced" : 138570,
"needTime" : 0,
"needYield" : 0,
"saveState" : 8887,
"restoreState" : 8887,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 138570,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 138570,
"executionTimeMillisEstimate" : 561,
"works" : 138571,
"advanced" : 138570,
"needTime" : 0,
"needYield" : 0,
"saveState" : 8887,
"restoreState" : 8887,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"IsDelete" : 1,
"TimeCreated" : 1
},
"indexName" : "IsDelete_1_TimeCreated_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"IsDelete" : [ ],
"TimeCreated" : [
"TimeCreated"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"IsDelete" : [
"[true, true]"
],
"TimeCreated" : [
"[new Date(1498780800000), new Date(9223372036854775807)]"
]
},
"keysExamined" : 138570,
"seeks" : 1,
"dupsTested" : 138570,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "",
"port" : ,
"version" : "3.4.4",
"gitVersion" : "888390515874a9debd1b6c5d36559ca86b44babd"
},
"ok" : 1
}