Same mongodb query gives different explain plan - mongodb

I have two mongo queries, the only change in a query is merchantId field still both queries giving me different winning plan.
First Query
db.transactions.find({"created":{"$gte":1527465600000,"$lte":1527551999000},"merchantId":940,"additionalInformation.REQUESTOR":{"$ne":"MOTO"},"$or":[{"paymentMode":{"$ne":"UPI"}},{"bankCode":{"$ne":"GTEZ"}}]}).sort({ _id: -1 }).limit(200).explain()
Output of above query
{
"queryPlanner" : {
"plannerVersion" : 1,
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"$or" : [
{
"$not" : {
"bankCode" : {
"$eq" : "GTEZ"
}
}
},
{
"$not" : {
"paymentMode" : {
"$eq" : "UPI"
}
}
}
]
},
{
"merchantId" : {
"$eq" : 940
}
},
{
"created" : {
"$lte" : 1527551999000
}
},
{
"created" : {
"$gte" : 1527465600000
}
},
{
"$not" : {
"additionalInformation.REQUESTOR" : {
"$eq" : "MOTO"
}
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 200,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$or" : [
{
"$not" : {
"bankCode" : {
"$eq" : "GTEZ"
}
}
},
{
"$not" : {
"paymentMode" : {
"$eq" : "UPI"
}
}
}
]
},
{
"$not" : {
"additionalInformation.REQUESTOR" : {
"$eq" : "MOTO"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"created" : 1,
"merchantId" : 1
},
"indexName" : "created_1_merchantId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"created" : [
"[1527465600000.0, 1527551999000.0]"
],
"merchantId" : [
"[940.0, 940.0]"
]
}
}
}
}
}
},
"serverInfo" : {
},
"ok" : 1
}
Second Query
db.transactions.find({"created":{"$gte":1527465600000,"$lte":1527551999000},"merchantId":1429,"additionalInformation.REQUESTOR":{"$ne":"MOTO"},"$or":[{"paymentMode":{"$ne":"UPI"}},{"bankCode":{"$ne":"GTEZ"}}]}).sort({ _id: -1 }).limit(200).explain()
Output of above query
{
"queryPlanner" : {
"plannerVersion" : 1,
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"$or" : [
{
"$not" : {
"bankCode" : {
"$eq" : "GTEZ"
}
}
},
{
"$not" : {
"paymentMode" : {
"$eq" : "UPI"
}
}
}
]
},
{
"merchantId" : {
"$eq" : 1429
}
},
{
"created" : {
"$lte" : 1527551999000
}
},
{
"created" : {
"$gte" : 1527465600000
}
},
{
"$not" : {
"additionalInformation.REQUESTOR" : {
"$eq" : "MOTO"
}
}
}
]
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 200,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$or" : [
{
"$not" : {
"bankCode" : {
"$eq" : "GTEZ"
}
}
},
{
"$not" : {
"paymentMode" : {
"$eq" : "UPI"
}
}
}
]
},
{
"merchantId" : {
"$eq" : 1429
}
},
{
"created" : {
"$lte" : 1527551999000
}
},
{
"created" : {
"$gte" : 1527465600000
}
},
{
"$not" : {
"additionalInformation.REQUESTOR" : {
"$eq" : "MOTO"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
"serverInfo" : {
},
"ok" : 1
}
As you can see only param difference is merchantId, still explain gives different winning plan also IXSCAN also shows different indexes used. In first query created_1_merchantId_1 index is used and in second query id index is used. First query takes 40 seconds to get results while second query gives 1 sec. Quick will be highly appreciated.

Related

MongoDB Query plan not using compound index

I am trying MongoDB with a dataset about the company profile margin for learning purpose. Here is the sample document
{
"parent_comp" : 1
"child_comp" : 101
"profit" : NumberLong(70320020)
}
I have created two indexes i.e one on child_comp field and the other one is a compound index with parent_comp, child_comp, and last_outage_timestamp.
For the below query, I executed the explain command to see the query plan.
MongoDB Enterprise > db.data.find({ "$and" : [{ "parent_comp" : 951, "child_comp" : 9351, "profit" : { "$gte" : { "$numberLong" : "500000000" } } }, { "profit" : { "$lte" : { "$numberLong" : "1000000000" } } }] }).sort({"profit" : 1}).limit(3).explain();
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.data",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"child_comp" : {
"$eq" : 9351
}
},
{
"parent_comp" : {
"$eq" : 951
}
},
{
"profit" : {
"$lte" : {
"$numberLong" : "1000000000"
}
}
},
{
"profit" : {
"$gte" : {
"$numberLong" : "500000000"
}
}
}
]
},
"queryHash" : "B570EF0C",
"planCacheKey" : "187EF74B",
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 3,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"child_comp" : {
"$eq" : 9351
}
},
{
"parent_comp" : {
"$eq" : 951
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"profit" : 1
},
"indexName" : "profit_index",
"isMultiKey" : false,
"multiKeyPaths" : {
"profit" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"profit" : [
"[{ $numberLong: \"500000000\" }, { $numberLong: \"1000000000\" }]"
]
}
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"profit" : 1
},
"limitAmount" : 3,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"parent_comp" : {
"$eq" : 951
}
},
{
"profit" : {
"$lte" : {
"$numberLong" : "1000000000"
}
}
},
{
"profit" : {
"$gte" : {
"$numberLong" : "500000000"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"child_comp" : 1
},
"indexName" : "child_comp_index",
"isMultiKey" : false,
"multiKeyPaths" : {
"child_comp" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"child_comp" : [
"[9351.0, 9351.0]"
]
}
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 3,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"parent_comp" : 1,
"child_comp" : 1,
"profit" : 1
},
"indexName" : "parent_comp_1_child_comp_1_profit_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"parent_comp" : [ ],
"child_comp" : [ ],
"profit" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"parent_comp" : [
"[951.0, 951.0]"
],
"child_comp" : [
"[9351.0, 9351.0]"
],
"profit" : [
"[{ $numberLong: \"500000000\" }, { $numberLong: \"1000000000\" }]"
]
}
}
}
}
]
},
"serverInfo" : {
"host" : "localhost",
"port" : 27017,
"version" : "4.2.8",
"gitVersion" : "43d25888249164d76d5e04dd6cf38f6111e21f5f"
},
"ok" : 1
}
As you can see winning plan used single index instead of compound index. So could you please let me know why compound index was not used.
Your query is sorting on profit, and the compound index does not include the field you are sorting on hence using the compound index would necessitate an additional sort stage.
The trade-offs and reasoning is further explained in the docs.
See also https://www.alexbevi.com/blog/2020/05/16/optimizing-mongodb-compound-indexes-the-equality-sort-range-esr-rule/.

MongoDB (text search with the relevant field) aggregation problem

I have the MongoDB aggregation query
db.data.aggregate([{ "$match" : { "$text" : { "$search" : "STORAGE TYPE" } } },
{ "$group" :
{ "_id" :{"doc_type": "$doc_type" ,"title" : "$title", "player_name" : "$player_name", "player_type" : "INSTITUTION", "country_code" :"$country_code" },
"number_records" : { "$sum" : 1}
}
},
{"$match" : {"doc_type": "PATENT"} },
{"$sort":{"number_records" : -1}},
{"$limit" : 10}],
{"allowDiskuse" : true}
)
When I tried to execute the above code, it keeps on buffering for a long time, I am not getting any output. Can anyone help me?
When I used command explain(), it shows the following code:
{
"stages" : [
{
"$cursor" : {
"query" : {
"$and" : [
{
"$text" : {
"$search" : "STORAGE TYPE"
}
},
{
"doc_type" : "PATENT"
}
]
},
"fields" : {
"country_code" : 1,
"doc_type" : 1,
"player_name" : 1,
"title" : 1,
"_id" : 0
},
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "datadocuments.data",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"doc_type" : {
"$eq" : "PATENT"
}
},
{
"$text" : {
"$search" : "STORAGE TYPE",
"$language" : "english",
"$caseSensitive" : false,
"$diacriticSensitive" : false
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"doc_type" : {
"$eq" : "PATENT"
}
},
"inputStage" : {
"stage" : "TEXT",
"indexPrefix" : {
},
"indexName" : "title",
"parsedTextQuery" : {
"terms" : [
"storag",
"type"
],
"negatedTerms" : [ ],
"phrases" : [ ],
"negatedPhrases" : [ ]
},
"textIndexVersion" : 3,
"inputStage" : {
"stage" : "TEXT_MATCH",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1
},
"indexName" : "title",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"_fts" : "text",
"_ftsx" : 1
},
"indexName" : "title",
"isMultiKey" : true,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
}
}
]
}
}
}
}
},
"rejectedPlans" : [ ]
}
}
},
{
"$group" : {
"_id" : {
"doc_type" : "$doc_type",
"title" : "$title",
"player_name" : "$player_name",
"player_type" : {
"$const" : "INSTITUTION"
},
"country_code" : "$country_code"
},
"number_records" : {
"$sum" : {
"$const" : 1
}
}
}
},
{
"$sort" : {
"sortKey" : {
"number_records" : -1
},
"limit" : NumberLong("10")
}
}
],
"ok" : 1
}
I couldn't figure out the mistake; is there any problem in aggregation, if not, how to increase the performance?
Your error comes from your second match stage : at this point, doc_type doesn't exist, but _id.doc_type instead. But you bettermerge this stage with the first one, to improve performance by reducing number of documents passed to the $group stage.
Your improved query will be :
db.data.aggregate([
{"$match" : { "$text" : { "$search" : "STORAGE TYPE" `},"doc_type": "PATENT" } },`
{ "$group" :
{ "_id" :{"doc_type": "$doc_type" ,"title" : "$title", "player_name" : "$player_name", "player_type" : "INSTITUTION", "country_code" :"$country_code" },
"number_records" : { "$sum" : 1}
}
},
{"$sort":{"number_records" : -1}},
{"$limit" : 10}],
{"allowDiskuse" : true}
)

Mongo Find complete at 0.105 sec, But count takes 1.0059 sec: Total 30024 records

My Count query is very slow. I have made the indexes and believe hence the Find query return records in 0.105 sec.
db.collectionname.find({}) => 0.105 sec
When do count with the same query it takes 1.0059 sec.
db.collectionname.count({}) => 1.0059 sec
There are 2L records in total in my collection and search result is 30024 records.
Please guide me what are the steps I need to take?
Mongo Version is : 3.2
db.getCollection('beta').find({"status":"1",
"type.und.value":"beta","state":{"$in":["a","b"]},
"changed":{"$gte":1463804682},"uid":{"$ne":"333"},
"price":{"$ne":[]}}).explain()
/* 1 */ {
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "beta",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"changed" : 1.0
},
"indexName" : "changed_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"changed" : [
"[1463804682.0, inf.0]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"uid" : 1.0
},
"indexName" : "uid_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"uid" : [
"[MinKey, \"333\")",
"(\"333\", MaxKey]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"price" : 1.0
},
"indexName" : "price_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"price" : [
"[MinKey, undefined)",
"(undefined, [])",
"([], MaxKey]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"type.und.value" : 1.0
},
"indexName" : "type.und.value_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"type.und.value" : [
"[\"beta\", \"beta\"]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"changed" : -1
},
"indexName" : "_changed",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"changed" : [
"[inf.0, 1463804682.0]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"status" : 1.0
},
"indexName" : "status_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"status" : [
"[\"1\", \"1\"]"
]
}
}
}
]
},
"serverInfo" : {
"host" : "xxx",
"port" : xxx,
"version" : "3.2.13",
"gitVersion" : "xxx"
},
"ok" : 1.0 }
==========================================
db.getCollection('beta').explain().count({"status":"1",
"type.und.value":"beta","state":{"$in":["a","b"]},
"changed":{"$gte":1463804682},"uid":{"$ne":"333"},
"price":{"$ne":[]}})
/* 1 */ {
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "property.beta",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"changed" : 1.0
},
"indexName" : "changed_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"changed" : [
"[1463804682.0, inf.0]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"uid" : 1.0
},
"indexName" : "uid_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"uid" : [
"[MinKey, \"333\")",
"(\"333\", MaxKey]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"price" : 1.0
},
"indexName" : "price_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"price" : [
"[MinKey, undefined)",
"(undefined, [])",
"([], MaxKey]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"status" : {
"$eq" : "1"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"type.und.value" : 1.0
},
"indexName" : "type.und.value_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"type.und.value" : [
"[\"beta\", \"beta\"]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"status" : {
"$eq" : "1"
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"changed" : -1
},
"indexName" : "_changed",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"changed" : [
"[inf.0, 1463804682.0]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"type.und.value" : {
"$eq" : "beta"
}
},
{
"changed" : {
"$gte" : 1463804682.0
}
},
{
"state" : {
"$in" : [
"a",
"b"
]
}
},
{
"$not" : {
"price" : {
"$eq" : []
}
}
},
{
"$not" : {
"uid" : {
"$eq" : "333"
}
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"status" : 1.0
},
"indexName" : "status_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"status" : [
"[\"1\", \"1\"]"
]
}
}
}
]
},
"serverInfo" : {
"host" : "xxx",
"port" : xxx,
"version" : "3.2.13",
"gitVersion" : "xxx"
},
"ok" : 1.0 }

Mongo aggregation performance

I am new to mongo and below query performs really slow with record set over 2 Million records
Query
db.testCollection.aggregate({
$match: {
active: {
$ne: false
}
}
}, {
$group: {
_id: {
productName: "$productName",
model: "$model",
version: "$version",
uid: "$uid"
},
total: {
$sum: 1
}
}
}, {
$project: {
total: 1,
model: "$_id.model",
version: "$_id.version",
uid: "$_id.uid",
productName: "$_id.productName"
}
}, {
$sort: {
model: 1
}
})
explain()
{
"stages" : [
{
"$cursor" : {
"query" : {
"active" : {
"$ne" : false
}
},
"fields" : {
"version" : 1,
"productName" : 1,
"model" : 1,
"uid" : 1,
"_id" : 0
},
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "fms2.device",
"indexFilterSet" : false,
"parsedQuery" : {
"$nor" : [
{
"active" : {
"$eq" : false
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1
},
"indexName" : "active",
"isMultiKey" : false,
"multiKeyPaths" : {
"active" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[MinKey, false)",
"(false, MaxKey]"
]
}
}
},
"rejectedPlans" : [ ]
}
}
},
{
"$group" : {
"_id" : {
"productName" : "$productName",
"model" : "$model",
"version" : "$version",
"uid" : "$uid"
},
"total" : {
"$sum" : {
"$const" : 1
}
}
}
},
{
"$project" : {
"_id" : true,
"total" : true,
"model" : "$_id.model",
"version" : "$_id.version",
"uid" : "$_id.uid",
"productName" : "$_id.productName"
}
},
{
"$sort" : {
"sortKey" : {
"model" : 1
}
}
}
],
"ok" : 1
}
Is there a way to optimize this query more ? I had a look into https://docs.mongodb.com/manual/core/aggregation-pipeline-optimization/ as well but most of the stated suggestions are not applicable for this query.
Not sure if it matters, result of this aggregation ends up with only 20-30 records.

MongoDB query not using intersection indexes

I have MongoDB 3.28 with a collection and few indexes on different fields. I'm executing a query with many filters expecting the intersection indexes to kick in, but for some reason it is not. Additionally it is not showing up in the explain(true) allPlans
The query
{
"$and": [
{
"category": {
"$in": [
1,
5
]
}
},
{
"city": {
"$in": [
"y"
]
}
},
{
"neighbourhood": {
"$in": [
"x",
null
]
}
},
{
"$or": [
{
"price": {
"$lte": 3
}
},
{
"price": null
}
]
}
]
}
db.items.find(query).explain(true)
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "xxx.items",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"$or" : [
{
"price" : {
"$eq" : null
}
},
{
"price" : {
"$lte" : 3
}
}
]
},
{
"category" : {
"$in" : [
1,
5
]
}
},
{
"city" : {
"$in" : [
"y"
]
}
},
{
"neighbourhood" : {
"$in" : [
null,
"x"
]
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$or" : [
{
"price" : {
"$eq" : null
}
},
{
"price" : {
"$lte" : 3
}
}
]
},
{
"category" : {
"$in" : [
1,
5
]
}
},
{
"neighbourhood" : {
"$in" : [
null,
"x"
]
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"city" : 1,
"updatedAt" : 1
},
"indexName" : "city_updatedAt",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"city" : [
"[\"y\", \"y\"]"
],
"updatedAt" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"category" : {
"$in" : [
1,
5
]
}
},
{
"city" : {
"$in" : [
"y"
]
}
},
{
"neighbourhood" : {
"$in" : [
null,
"x"
]
}
}
]
},
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$or" : [
{
"price" : {
"$eq" : null
}
},
{
"price" : {
"$lte" : 3
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"price" : 1
},
"indexName" : "price_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"price" : [
"[null, null]",
"[-inf.0, 3.0]"
]
}
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$or" : [
{
"price" : {
"$eq" : null
}
},
{
"price" : {
"$lte" : 3
}
}
]
},
{
"category" : {
"$in" : [
1,
5
]
}
},
{
"neighbourhood" : {
"$in" : [
null,
"x"
]
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"city" : 1
},
"indexName" : "city_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"city" : [
"[\"y\", \"y\"]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$or" : [
{
"price" : {
"$eq" : null
}
},
{
"price" : {
"$lte" : 3
}
}
]
},
{
"city" : {
"$in" : [
"y"
]
}
},
{
"neighbourhood" : {
"$in" : [
null,
"x"
]
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"category" : 1,
"updatedAt" : -1
},
"indexName" : "category",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"category" : [
"[1.0, 1.0]",
"[5.0, 5.0]"
],
"updatedAt" : [
"[MaxKey, MinKey]"
]
}
}
}
]
},
"serverInfo" : {
"host" : "xxx",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
db.items.getIndexes()
[
{
"v" : 1,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "xxx.items"
},
{
"v" : 1,
"unique" : true,
"key" : {
"RecordID" : 1
},
"name" : "RecordID",
"ns" : "xxx.items"
},
{
"v" : 1,
"key" : {
"SubCatID" : 1,
"updatedAt" : -1
},
"name" : "category",
"ns" : "xxx.items"
},
{
"v" : 1,
"key" : {
"updatedAt" : -1
},
"name" : "updatedAt_-1",
"ns" : "xxx.items",
"background" : true
},
{
"v" : 1,
"key" : {
"city" : 1
},
"name" : "city_1",
"ns" : "xxx.items"
},
{
"v" : 1,
"key" : {
"search" : 1
},
"name" : "search_1",
"ns" : "xxx.items",
"background" : true
},
{
"v" : 1,
"key" : {
"rooms" : 1
},
"name" : "rooms_1",
"ns" : "xxx.items",
"background" : true
},
{
"v" : 1,
"key" : {
"price" : 1
},
"name" : "price_1",
"background" : true,
"ns" : "xxx.items"
},
{
"v" : 1,
"key" : {
"city" : 1,
"updatedAt" : 1
},
"name" : "city_updatedAt",
"ns" : "xxx.items",
"background" : true
}
]
Sample record: db.items.find().limit(1)[0]
{
"_id" : ObjectId("568ee714578df40300ac65b0"),
"type" : "z",
"CatID" : 2,
"category" : 5,
"RecordID" : "1469882",
"title" : "x - x x",
"subtitle" : "x/x' - 7 x",
"subtitle2" : "3,700,000",
"someProps1" : false,
"someProps2" : 14,
"subtite3" : "some title",
"type" : 3,
"img" : "x",
"URL" : "y",
"someProps" : 0,
"latitude" : 31.809843,
"longitude" : 35.191562,
"Map_address" : {
"adress" : "x - x x",
"lat" : 31.809843,
"long" : 35.191562
},
"created_at" : ISODate("2016-01-07T22:30:44.425Z"),
"updated_at" : ISODate("2016-01-14T15:53:43.110Z"),
"price" : 3700000,
"rooms" : 7,
"updatedAt" : 1452292244,
"search" : "x - x x 14-01-2016",
"street" : null
}