Why does this explain indicate an index was not used? - mongodb

Given the following query:
var exp = db.messages.explain('executionStats')
exp.find( { 'headers.Date' : { '$gt' : new Date(2001,3,1) } }, { 'headers.From' : 1, '_id' : 0 } ).sort( { 'headers.From' : 1 } )
That produces the following output:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "enron.messages",
"indexFilterSet" : false,
"parsedQuery" : {
"headers.Date" : {
"$gt" : ISODate("2001-04-01T05:00:00Z")
}
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"headers.From" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"headers.Date" : {
"$gt" : ISODate("2001-04-01T05:00:00Z")
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"headers.From" : 1
},
"indexName" : "headers.From_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"headers.From" : [
"[MinKey, MaxKey]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 83057,
"executionTimeMillis" : 726,
"totalKeysExamined" : 120477,
"totalDocsExamined" : 120477,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 83057,
"executionTimeMillisEstimate" : 690,
"works" : 120478,
"advanced" : 83057,
"needTime" : 37420,
"needFetch" : 0,
"saveState" : 941,
"restoreState" : 941,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"headers.From" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"headers.Date" : {
"$gt" : ISODate("2001-04-01T05:00:00Z")
}
},
"nReturned" : 83057,
"executionTimeMillisEstimate" : 350,
"works" : 120478,
"advanced" : 83057,
"needTime" : 37420,
"needFetch" : 0,
"saveState" : 941,
"restoreState" : 941,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 120477,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 120477,
"executionTimeMillisEstimate" : 60,
"works" : 120477,
"advanced" : 120477,
"needTime" : 0,
"needFetch" : 0,
"saveState" : 941,
"restoreState" : 941,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"headers.From" : 1
},
"indexName" : "headers.From_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"headers.From" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 120477,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
}
},
"serverInfo" : {
"host" : "dpercy-mac-air.local",
"port" : 27017,
"version" : "3.0.1",
"gitVersion" : "534b5a3f9d10f00cd27737fbcd951032248b5952"
},
"ok" : 1
}
What in the explain says that the query did NOT use an index? From my understanding the following section implies that an index was used because the winning plan has an IXSCAN stage in it:
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"headers.From" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"headers.Date" : {
"$gt" : ISODate("2001-04-01T05:00:00Z")
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"headers.From" : 1
},
"indexName" : "headers.From_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"headers.From" : [
"[MinKey, MaxKey]"
]
}
}
}
But I'm being told that this query did not use an index. Why is that?

Related

query text search index on compound text search and timestamp mongodb

I have a collection, with a total of documents about 65 million records like this
{
"_id" : ObjectId("5e0b814660da38d499ecf178"),
"brands" : null,
"client_id" : null,
"code_co_owner" : ",7359562, ",
"code_segment" : "7359562",
"core" : "",
"created" : "01-01-2020",
"created_full" : "01-01-2020 00:00:27",
"created_int" : NumberLong(1577811627),
"email" : ",phamthanhlam17_gmail_com, "
.....
}
I made an compound index on (email, created_int): {"email": text, created_int: -1} for search and filter which name in a range of created_int
But I see that it's poor performance on search.
I tried to use explain on query:
db.getCollection('profile_20201').explain().find({"$text":{"$search":"phamthanhlam17_gmail_com"},
"created_int":{"$lte":1585627013, "$gte":1583035013}}).count()
The explain result is:
{
"queryPlanner" : {
"plannerVersion" : 1,
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
},
{
"$text" : {
"$search" : "phamthanhlam17_gmail_com",
"$language" : "english",
"$caseSensitive" : false,
"$diacriticSensitive" : false
}
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "TEXT",
"indexPrefix" : {},
"indexName" : "email_text_created_int_-1",
"parsedTextQuery" : {
"terms" : [
"phamthanhlam17_gmail_com"
],
"negatedTerms" : [],
"phrases" : [],
"negatedPhrases" : []
},
"textIndexVersion" : 3,
"inputStage" : {
"stage" : "TEXT_MATCH",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "OR",
"filter" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1,
"created_int" : -1.0
},
"indexName" : "email_text_created_int_-1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {}
}
}
}
}
}
},
"rejectedPlans" : []
},
"serverInfo" : {
},
"ok" : 1.0
}
It's explain stats:
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "namespace",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
},
{
"$text" : {
"$search" : "phamthanhlam17_gmail_com",
"$language" : "english",
"$caseSensitive" : false,
"$diacriticSensitive" : false
}
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "TEXT",
"indexPrefix" : {},
"indexName" : "email_text_created_int_-1",
"parsedTextQuery" : {
"terms" : [
"phamthanhlam17_gmail_com"
],
"negatedTerms" : [],
"phrases" : [],
"negatedPhrases" : []
},
"textIndexVersion" : 3,
"inputStage" : {
"stage" : "TEXT_MATCH",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "OR",
"filter" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1,
"created_int" : -1.0
},
"indexName" : "email_text_created_int_-1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {}
}
}
}
}
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 1499057,
"totalKeysExamined" : 72544123,
"totalDocsExamined" : 39448083,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 1483861,
"works" : 72544124,
"advanced" : 0,
"needTime" : 72544123,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 39448083,
"nSkipped" : 0,
"inputStage" : {
"stage" : "TEXT",
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 1475831,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"indexPrefix" : {},
"indexName" : "email_text_created_int_-1",
"parsedTextQuery" : {
"terms" : [
"phamthanhlam17_gmail_com"
],
"negatedTerms" : [],
"phrases" : [],
"negatedPhrases" : []
},
"textIndexVersion" : 3,
"inputStage" : {
"stage" : "TEXT_MATCH",
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 1473041,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"docsRejected" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 1465951,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 39448083,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "OR",
"filter" : {
"$and" : [
{
"created_int" : {
"$lte" : 1585627013.0
}
},
{
"created_int" : {
"$gte" : 1583035013.0
}
}
]
},
"nReturned" : 39448083,
"executionTimeMillisEstimate" : 439664,
"works" : 72544124,
"advanced" : 39448083,
"needTime" : 33096040,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"dupsTested" : 72544123,
"dupsDropped" : 0,
"recordIdsForgotten" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 72544123,
"executionTimeMillisEstimate" : 291188,
"works" : 72544124,
"advanced" : 72544123,
"needTime" : 0,
"needYield" : 0,
"saveState" : 578233,
"restoreState" : 578233,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1,
"created_int" : -1.0
},
"indexName" : "email_text_created_int_-1",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {},
"keysExamined" : 72544123,
"seeks" : 1,
"dupsTested" : 72544123,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
}
}
},
"serverInfo" : {
},
"ok" : 1.0
}```
So, is the index is cover the query?
Or which index will give me better performance for this problem?
Thank you.
Well, it seems like you have created the compound index with text. But in official MongoDB Documentation, it says that:
A compound index can include a text index key in combination with ascending/descending index keys. However, these compound indexes have the following restrictions:
A compound text index cannot include any other special index types, such as multi-key or geospatial index fields.
If the compound text index includes keys preceding the text index key, to perform a $text search, the query predicate must include equality match conditions on the preceding keys. (You are using the range query here)
When creating a compound text index, all text index keys must be listed adjacently in the index specification document.
So, that's the first issue.
Next, I want you to have a look at prefixes, it will help you understand how the compound indexes will be used in your query.
Hope this will help you understand the problem :)

mongodb 3.2.7 sorting not using compound index

I'm using mongodb 3.2.7, docs like:
{
"finished" : true,
"buildNo": 1,
"tryTimes" : 1,
"createdTime" : ISODate("2019-05-16T19:00:50.604+08:00"),
"modifiedTime" : ISODate("2019-12-27T18:33:25.682+08:00"),
}
I created my index using:
db.getCollection('builds').createIndex({"createdTime": 1})
db.getCollection('builds').createIndex({"tryTimes": 1, "createdTime": -1})
and my query is :
db.getCollection('builds').find({
"createdTime": {$lte: new Date("2020-04-15T00:00:00.000Z"), $gte: new Date("2020-04-01T00:00:00.000Z")},
"buildNo": 1,
"finished": {$ne: true}
})
.sort({"tryTimes": 1, "createdTime": -1})
.limit(200)
But this query doesn't use the index I created before. the explain() output:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "builds",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"buildNo" : {
"$eq" : 1.0
}
},
{
"createdTime" : {
"$lte" : ISODate("2020-04-15T08:00:00.000+08:00")
}
},
{
"createdTime" : {
"$gte" : ISODate("2020-04-01T08:00:00.000+08:00")
}
},
{
"$not" : {
"finished" : {
"$eq" : true
}
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"tryTimes" : 1.0,
"createdTime" : -1.0
},
"limitAmount" : 200,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"buildNo" : {
"$eq" : 1.0
}
},
{
"$not" : {
"finished" : {
"$eq" : true
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"createdTime" : 1.0
},
"indexName" : "createdTime_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"createdTime" : [
"[new Date(1585699200000), new Date(1586908800000)]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 200,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"buildNo" : {
"$eq" : 1.0
}
},
{
"createdTime" : {
"$lte" : ISODate("2020-04-15T08:00:00.000+08:00")
}
},
{
"createdTime" : {
"$gte" : ISODate("2020-04-01T08:00:00.000+08:00")
}
},
{
"$not" : {
"finished" : {
"$eq" : true
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"tryTimes" : 1.0,
"createdTime" : -1.0
},
"indexName" : "tryTimes_1_createdTime_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"tryTimes" : [
"[MinKey, MaxKey]"
],
"createdTime" : [
"[MaxKey, MinKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 200,
"executionTimeMillis" : 352,
"totalKeysExamined" : 48256,
"totalDocsExamined" : 48256,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 200,
"executionTimeMillisEstimate" : 200,
"works" : 48459,
"advanced" : 200,
"needTime" : 48258,
"needYield" : 0,
"saveState" : 757,
"restoreState" : 757,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"tryTimes" : 1.0,
"createdTime" : -1.0
},
"memUsage" : 198706,
"memLimit" : 33554432,
"limitAmount" : 200,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 0,
"executionTimeMillisEstimate" : 180,
"works" : 48258,
"advanced" : 0,
"needTime" : 24539,
"needYield" : 0,
"saveState" : 757,
"restoreState" : 757,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"buildNo" : {
"$eq" : 1.0
}
},
{
"$not" : {
"finished" : {
"$eq" : true
}
}
}
]
},
"nReturned" : 23718,
"executionTimeMillisEstimate" : 160,
"works" : 48257,
"advanced" : 23718,
"needTime" : 24538,
"needYield" : 0,
"saveState" : 757,
"restoreState" : 757,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 48256,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 48256,
"executionTimeMillisEstimate" : 20,
"works" : 48257,
"advanced" : 48256,
"needTime" : 0,
"needYield" : 0,
"saveState" : 757,
"restoreState" : 757,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"createdTime" : 1.0
},
"indexName" : "createdTime_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"createdTime" : [
"[new Date(1585699200000), new Date(1586908800000)]"
]
},
"keysExamined" : 48256,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "...",
"port" : 9999,
"version" : "3.2.7",
"gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2"
},
"ok" : 1.0}
even if I try to simplify the query to
db.getCollection('builds')
.find({ "buildNo": 1, "finished": true })
.sort({"tryTimes": 1, "createdTime": -1})
.limit(200)
, the index still doesn't work.
The wierd things is: I use the same index and query on another mongondb ver. 3.4.14.3. The index works...
Any ideas is welcomed and appreciated.
Thanks a lot!
If a query can be satisfied by multiple indexes (satisfied is used losely as Mongo actually chooses all possibly relevant indexes) defined in the collection, MongoDB will then test all the applicable indexes in parallel (meaning Mongo's performs a "race"). The first index that can returns 101 results will be selected by the query planner.
Meaning that for that certain query your using the "wrong" index wins.
What can you do?:
You can use $hint, hint basically forces Mongo to use a specific index, however Mongo this is not recommended because if changes occur Mongo will not adapt to those.
Like so:
db.getCollection('builds')
.find({ "buildNo": 1, "finished": true })
.sort({"tryTimes": 1, "createdTime": -1})
.limit(200)
.hint({tryTimes: 1, createdTime: -1})

mongodb 4.2 sort by id slow with compound index

I have docs in the fasion
{
_id: ...,
p: [
{
k: 'stringvalue',
v: 'stringvalue'
},
...
]
}
_id is provided to drivers as a sequence number of insertion, e.g. ...0001, ....0002, etc.
That works extermely well for find queries, now I have added sort condition and added _id field to the compound index (see below).
That works extermely bad around x100 slower, what could be the reason? The server has plenty of RAM.
db.getCollection('mycoll').find({ p: { '$elemMatch': { k: 'd', v: { '$gt': '2019-10-16T08:01:39.741' } } } }).sort({_id: 1}).explain()
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mydb.mycoll",
"indexFilterSet" : false,
"parsedQuery" : {
"p" : {
"$elemMatch" : {
"$and" : [
{
"k" : {
"$eq" : "d"
}
},
{
"v" : {
"$gt" : "2019-10-16T08:01:39.741"
}
}
]
}
}
},
"queryHash" : "E28632C1",
"planCacheKey" : "E28632C1",
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"p" : {
"$elemMatch" : {
"$and" : [
{
"k" : {
"$eq" : "d"
}
},
{
"v" : {
"$gt" : "2019-10-16T08:01:39.741"
}
}
]
}
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"p.k" : 1,
"p.v" : 1,
"_id" : 1
},
"indexName" : "p.k_1_p.v_1__id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"p.k" : [
"p"
],
"p.v" : [
"p"
],
"_id" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"p.k" : [
"[\"d\", \"d\"]"
],
"p.v" : [
"(\"2019-10-16T08:01:39.741\", {})"
],
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"p" : {
"$elemMatch" : {
"$and" : [
{
"k" : {
"$eq" : "d"
}
},
{
"v" : {
"$gt" : "2019-10-16T08:01:39.741"
}
}
]
}
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
]
},
"serverInfo" : {
"host" : "dev-10-178-3-247",
"port" : 27017,
"version" : "4.2.0",
"gitVersion" : "a4b751dcf51dd249c5865812b390cfd1c0129c30"
},
"ok" : 1
}
> db.getCollection('mycoll').getIndexes()
[
{
"v" : 2,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "mydb.mycoll"
},
{
"v" : 2,
"key" : {
"p.k" : 1,
"p.v" : 1,
"_id" : 1
},
"name" : "p.k_1_p.v_1__id_1",
"ns" : "mydb.mycoll"
}
]
{
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1015268,
"executionTimeMillis" : 77032,
"totalKeysExamined" : 3122749,
"totalDocsExamined" : 3122749,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"p" : {
"$elemMatch" : {
"$and" : [
{
"k" : {
"$eq" : "d"
}
},
{
"v" : {
"$gt" : "2019-10-16T08:01:39.741"
}
}
]
}
}
},
"nReturned" : 1015268,
"executionTimeMillisEstimate" : 16092,
"works" : 3122750,
"advanced" : 1015268,
"needTime" : 2107481,
"needYield" : 0,
"saveState" : 24469,
"restoreState" : 24469,
"isEOF" : 1,
"docsExamined" : 3122749,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 3122749,
"executionTimeMillisEstimate" : 492,
"works" : 3122750,
"advanced" : 3122749,
"needTime" : 0,
"needYield" : 0,
"saveState" : 24469,
"restoreState" : 24469,
"isEOF" : 1,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 3122749,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
},
"allPlansExecution" : [
{
"nReturned" : 0,
"executionTimeMillisEstimate" : 85,
"totalKeysExamined" : 9298,
"totalDocsExamined" : 9298,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 85,
"works" : 9300,
"advanced" : 0,
"needTime" : 9299,
"needYield" : 0,
"saveState" : 7391,
"restoreState" : 7391,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memUsage" : 33559558,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 9298,
"executionTimeMillisEstimate" : 71,
"works" : 9299,
"advanced" : 9298,
"needTime" : 1,
"needYield" : 0,
"saveState" : 7391,
"restoreState" : 7391,
"isEOF" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"p" : {
"$elemMatch" : {
"$and" : [
{
"k" : {
"$eq" : "d"
}
},
{
"v" : {
"$gt" : "2019-10-16T08:01:39.741"
}
}
]
}
}
},
"nReturned" : 9298,
"executionTimeMillisEstimate" : 58,
"works" : 9298,
"advanced" : 9298,
"needTime" : 0,
"needYield" : 0,
"saveState" : 7391,
"restoreState" : 7391,
"isEOF" : 0,
"docsExamined" : 9298,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 9298,
"executionTimeMillisEstimate" : 0,
"works" : 9298,
"advanced" : 9298,
"needTime" : 0,
"needYield" : 0,
"saveState" : 7391,
"restoreState" : 7391,
"isEOF" : 0,
"keyPattern" : {
"p.k" : 1,
"p.v" : 1,
"_id" : 1
},
"indexName" : "p.k_1_p.v_1__id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"p.k" : [
"p"
],
"p.v" : [
"p"
],
"_id" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"p.k" : [
"[\"d\", \"d\"]"
],
"p.v" : [
"(\"2019-10-16T08:01:39.741\", {})"
],
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 9298,
"seeks" : 1,
"dupsTested" : 9298,
"dupsDropped" : 0
}
}
}
}
},
{
"nReturned" : 0,
"executionTimeMillisEstimate" : 85,
"totalKeysExamined" : 9298,
"totalDocsExamined" : 9298,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 85,
"works" : 9301,
"advanced" : 0,
"needTime" : 9299,
"needYield" : 0,
"saveState" : 24469,
"restoreState" : 24469,
"isEOF" : 0,
"sortPattern" : {
"_id" : 1
},
"memUsage" : 33559558,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 9298,
"executionTimeMillisEstimate" : 71,
"works" : 9299,
"advanced" : 9298,
"needTime" : 1,
"needYield" : 0,
"saveState" : 24469,
"restoreState" : 24469,
"isEOF" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"p" : {
"$elemMatch" : {
"$and" : [
{
"k" : {
"$eq" : "d"
}
},
{
"v" : {
"$gt" : "2019-10-16T08:01:39.741"
}
}
]
}
}
},
"nReturned" : 9298,
"executionTimeMillisEstimate" : 58,
"works" : 9298,
"advanced" : 9298,
"needTime" : 0,
"needYield" : 0,
"saveState" : 24469,
"restoreState" : 24469,
"isEOF" : 0,
"docsExamined" : 9298,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 9298,
"executionTimeMillisEstimate" : 0,
"works" : 9298,
"advanced" : 9298,
"needTime" : 0,
"needYield" : 0,
"saveState" : 24469,
"restoreState" : 24469,
"isEOF" : 0,
"keyPattern" : {
"p.k" : 1,
"p.v" : 1,
"_id" : 1
},
"indexName" : "p.k_1_p.v_1__id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"p.k" : [
"p"
],
"p.v" : [
"p"
],
"_id" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"p.k" : [
"[\"d\", \"d\"]"
],
"p.v" : [
"(\"2019-10-16T08:01:39.741\", {})"
],
"_id" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 9298,
"seeks" : 1,
"dupsTested" : 9298,
"dupsDropped" : 0
}
}
}
}
}
]
},
"ok" : 1
}

Sort of MongoDB using index

Below is the status of the index status of the collection that I want to let you see.
> db.histories.getIndexes();
[
{
"v" : 1,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "development.histories"
},
{
"v" : 1,
"key" : {
"hoge_id" : 1,
"created_at" : 1
},
"name" : "hoge_id_1_created_at_1",
"ns" : "development.histories",
"background" : true
},
{
"v" : 1,
"key" : {
"created_at" : 1
},
"name" : "created_at_1",
"ns" : "development.histories",
"background" : true
}
]
And, I executed the following query.
> db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { created_at: -1 } ).limit(1).explain("executionStats");
And, the result was below.
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "development.histories",
"indexFilterSet" : false,
"parsedQuery" : {
"hoge_id" : {
"$eq" : ObjectId("5a5c171010ebfb1a2c901008")
}
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 1,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MaxKey, MinKey]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 0,
"totalKeysExamined" : 1,
"totalDocsExamined" : 1,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 2,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 1,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 1,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
},
"serverInfo" : {
"host" : "b9cb1b8d1fc1",
"port" : 27017,
"version" : "3.2.18",
"gitVersion" : "4c1bae566c0c00f996a2feb16febf84936ecaf6f"
},
"ok" : 1
}
The result is fast, I guess it's due to creating index on created_at.
ref. "totalDocsExamined" : 1, "executionTimeMillis" : 0
Then, I did exection the following query. The difference of previous is the field used for sort.
> db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { id: -1 } ).limit(1).explain("executionStats");
And, the result was below.
> db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { id: -1 } ).limit(1).explain("executionStats");
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "development.histories",
"indexFilterSet" : false,
"parsedQuery" : {
"hoge_id" : {
"$eq" : ObjectId("5a5c171010ebfb1a2c901008")
}
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"id" : -1
},
"limitAmount" : 1,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 1215,
"totalKeysExamined" : 1034353,
"totalDocsExamined" : 1034353,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1,
"executionTimeMillisEstimate" : 1120,
"works" : 1034357,
"advanced" : 1,
"needTime" : 1034355,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"id" : -1
},
"memUsage" : 297,
"memLimit" : 33554432,
"limitAmount" : 1,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 0,
"executionTimeMillisEstimate" : 950,
"works" : 1034355,
"advanced" : 0,
"needTime" : 1,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 1034353,
"executionTimeMillisEstimate" : 650,
"works" : 1034354,
"advanced" : 1034353,
"needTime" : 0,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 1034353,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1034353,
"executionTimeMillisEstimate" : 330,
"works" : 1034354,
"advanced" : 1034353,
"needTime" : 0,
"needYield" : 0,
"saveState" : 8080,
"restoreState" : 8080,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"hoge_id" : 1,
"created_at" : 1
},
"indexName" : "hoge_id_1_created_at_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hoge_id" : [
"[ObjectId('5a5c171010ebfb1a2c901008'), ObjectId('5a5c171010ebfb1a2c901008')]"
],
"created_at" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 1034353,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "b9cb1b8d1fc1",
"port" : 27017,
"version" : "3.2.18",
"gitVersion" : "4c1bae566c0c00f996a2feb16febf84936ecaf6f"
},
"ok" : 1
}
>
The result is late this time.
ref. "totalDocsExamined" : 1034353, "executionTimeMillis" : 1215
About totalDocsExamined, That's all in all documents.
Regardress that id is enable for index as created_at, but, when it is sorted using id, the result is late?
For your 1st query:
db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { created_at: -1 } ).limit(1).explain("executionStats");
MongoDB is optimizing the performance by using the compound index on hoge_id and created_at. It firstly looks at the hoge_id and then it uses the index of created_at to sort the query results. In this way, the sort operation can be very fast because of efficient usage of compound index.
However, for your 2nd query:
db.histories.find({hoge_id: ObjectId("5a5c171010ebfb1a2c901008")}).sort( { id: -1 } ).limit(1).explain("executionStats");
Since there is no compound index on hoge_id and id(you only have a single index on id), MongoDB is actually manually sorting results by id.
More info on sorting with compound index can be found here.

$elemMatch queries in mongodb return different result when used with emended documents

I have collection of visitors. Each visitor contains brand id and trans array that has documents like {ct: Date, total: Number}. I try to get visitors that have transaction between two dates... Also I have index on {brandid:1, trans.ct:1} Here are my queries:
/*1*/
db.visitors.find({
brandid:12,
'trans':{
$elemMatch:{
ct:{
$gte:ISODate("2015-01-17"),
$lt:ISODate("2015-01-18")
}
}
}
})
.hint("transct")
/*2*/
db.visitors.find({
brandid:12,
'trans.ct':{
$elemMatch:{
$gte:ISODate("2015-01-17"),
$lt:ISODate("2015-01-18")
}
}
})
.hint("transct")
The first query returns 42 documents and the second one 0
Here are explains to above queries
/* 1 */
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "m.visitors",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"trans" : {
"$elemMatch" : {
"$and" : [
{
"ct" : {
"$lt" : ISODate("2015-01-18T00:00:00.000Z")
}
},
{
"ct" : {
"$gte" : ISODate("2015-01-17T00:00:00.000Z")
}
}
]
}
}
},
{
"brandid" : {
"$eq" : 12
}
}
]
},
"winningPlan" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"trans" : {
"$elemMatch" : {
"$and" : [
{
"ct" : {
"$lt" : ISODate("2015-01-18T00:00:00.000Z")
}
},
{
"ct" : {
"$gte" : ISODate("2015-01-17T00:00:00.000Z")
}
}
]
}
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"brandid" : 1,
"trans.ct" : 1
},
"indexName" : "transct",
"isMultiKey" : true,
"direction" : "forward",
"indexBounds" : {
"brandid" : [
"[12.0, 12.0]"
],
"trans.ct" : [
"(true, new Date(1421539200000))"
]
}
}
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 42,
"executionTimeMillis" : 99,
"totalKeysExamined" : 78999,
"totalDocsExamined" : 38063,
"executionStages" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 42,
"executionTimeMillisEstimate" : 100,
"works" : 79000,
"advanced" : 42,
"needTime" : 78957,
"needFetch" : 0,
"saveState" : 617,
"restoreState" : 617,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"trans" : {
"$elemMatch" : {
"$and" : [
{
"ct" : {
"$lt" : ISODate("2015-01-18T00:00:00.000Z")
}
},
{
"ct" : {
"$gte" : ISODate("2015-01-17T00:00:00.000Z")
}
}
]
}
}
},
"nReturned" : 42,
"executionTimeMillisEstimate" : 100,
"works" : 79000,
"advanced" : 42,
"needTime" : 78957,
"needFetch" : 0,
"saveState" : 617,
"restoreState" : 617,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 38063,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 38063,
"executionTimeMillisEstimate" : 60,
"works" : 79000,
"advanced" : 38063,
"needTime" : 40936,
"needFetch" : 0,
"saveState" : 617,
"restoreState" : 617,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"brandid" : 1,
"trans.ct" : 1
},
"indexName" : "transct",
"isMultiKey" : true,
"direction" : "forward",
"indexBounds" : {
"brandid" : [
"[12.0, 12.0]"
],
"trans.ct" : [
"(true, new Date(1421539200000))"
]
},
"keysExamined" : 78999,
"dupsTested" : 78999,
"dupsDropped" : 40936,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
},
"allPlansExecution" : []
},
"serverInfo" : {
"host" : "linux-jxch",
"port" : 27017,
"version" : "3.0.0",
"gitVersion" : "a841fd6394365954886924a35076691b4d149168"
}
}
/* 2 */
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "m.visitors",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"trans.ct" : {
"$elemMatch" : {
"" : {
"$lt" : ISODate("2015-01-18T00:00:00.000Z")
},
"" : {
"$gte" : ISODate("2015-01-17T00:00:00.000Z")
}
}
}
},
{
"brandid" : {
"$eq" : 12
}
}
]
},
"winningPlan" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"trans.ct" : {
"$elemMatch" : {
"" : {
"$lt" : ISODate("2015-01-18T00:00:00.000Z")
},
"" : {
"$gte" : ISODate("2015-01-17T00:00:00.000Z")
}
}
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"brandid" : 1,
"trans.ct" : 1
},
"indexName" : "transct",
"isMultiKey" : true,
"direction" : "forward",
"indexBounds" : {
"brandid" : [
"[12.0, 12.0]"
],
"trans.ct" : [
"[new Date(1421452800000), new Date(1421539200000))"
]
}
}
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 42,
"totalDocsExamined" : 42,
"executionStages" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 43,
"advanced" : 0,
"needTime" : 42,
"needFetch" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"trans.ct" : {
"$elemMatch" : {
"" : {
"$lt" : ISODate("2015-01-18T00:00:00.000Z")
},
"" : {
"$gte" : ISODate("2015-01-17T00:00:00.000Z")
}
}
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 43,
"advanced" : 0,
"needTime" : 42,
"needFetch" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 42,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 42,
"executionTimeMillisEstimate" : 0,
"works" : 43,
"advanced" : 42,
"needTime" : 0,
"needFetch" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"brandid" : 1,
"trans.ct" : 1
},
"indexName" : "transct",
"isMultiKey" : true,
"direction" : "forward",
"indexBounds" : {
"brandid" : [
"[12.0, 12.0]"
],
"trans.ct" : [
"[new Date(1421452800000), new Date(1421539200000))"
]
},
"keysExamined" : 42,
"dupsTested" : 42,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
},
"allPlansExecution" : []
},
"serverInfo" : {
"host" : "linux-jxch",
"port" : 27017,
"version" : "3.0.0",
"gitVersion" : "a841fd6394365954886924a35076691b4d149168"
}
}
How to force the first query to use index, or how to force second query to return right data?
Here is example of data returned by the 1st query:
/* 40 */
{
"_id" : ObjectId("54cbfbe06f352a17778b483d"),
"brandid" : NumberLong(12),
"trans" : [
{
"id" : "54cbfbcd6f352a09778b481e",
"ct" : ISODate("2015-01-17T18:55:29.000Z"),
"t" : 891.09
}
]
}
/* 41 */
{
"_id" : ObjectId("54cbfbe16f352a17778b4b8b"),
"brandid" : NumberLong(12),
"trans" : [
{
"id" : "54cbfbcd6f352a09778b481f",
"ct" : ISODate("2015-01-17T18:59:24.000Z"),
"t" : 689.45
}
]
}
The 2nd query does not return any documents.
Also "transct" index looks like:
{
"brandid" : 1,
"trans.ct" : 1
}
The visitor collection has over 900K documents. Most of them look the same as from above example, but have different brand id and of course "ct" dates in trans array.