Mongodb index not used with aggregate query - mongodb

Question

Context
=> Check edit !
I'm running some queries in which sorting is applied and that are supposed to take advantage of indexes. For some indexes, for certain queries (aggregate) on a given environment, queries are by-passing the index and I haven't yet been able to figure out why.
Setup
I have a mongo collection that contains 3 indexes:
_id
definition.name
definition.financial.profitability.highlights.prof_net
I'm using 2 queries to test usage of each indexes:
B. Find and sort query
db.getCollection('properties_po').find().sort({"definition.financial.profitability.highlights.prof_net" : 1.0 }).limit(1);
db.getCollection('properties_po').find().sort({"definition.name" : 1.0}).limit(1);
B. Aggregate with sort query
db.getCollection("properties_po").aggregate([{"$sort":{"definition.name":1.0}},{"$limit":1}])
db.getCollection("properties_po").aggregate([{"$sort":{"definition.financial.profitability.highlights.prof_net":1.0}},{"$limit":1}])
I use db.properties_po.aggregate( [ { $indexStats: { } } ] ) to check if index usage has been incremented after each queries.
Results
Environment 1 (local mongodb 3.4):
Both queries on both indexes works as expected, incrementing each index usages after each query.
Environment 2 (prod monhodb 3.6):
Queries A works as expected and index usage is incremented after each query and for each indexes.
Query B works for the index "definition.name" where index usage is incremented properly.
Query B doesn't work for index "definition.financial.profitability.highlights.prof_net" where index is not used (very slow query) as it it reflected in the index usage not being incremented.
I'm not sure where to look at, it might be some mongodb configuration that I'm missing, some nested document attribute limitation or maybe my index is not created properly.
Thanks for your help
EDIT 18/03
It is not related to mongodb version as I've just tested creating the same collection on the prod server and I have 2 different results there too.
Collection 1: 15026 records
Output of the aggregate query with explain:
db.getCollection("properties").explain("executionStats").aggregate([{"$sort":{"definition.financial.profitability.highlights.prof_net":1.0}},{"$limit":1}])
-
{
"stages" : [
{
"$cursor" : {
"query" : {
},
"queryPlanner" : {
"plannerVersion" : 1.0,
"namespace" : "collec.properties",
"indexFilterSet" : false,
"parsedQuery" : {
},
"winningPlan" : {
"stage" : "COLLSCAN",
"direction" : "forward"
},
"rejectedPlans" : [
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 15026.0,
"executionTimeMillis" : 1522.0,
"totalKeysExamined" : 0.0,
"totalDocsExamined" : 15026.0,
"executionStages" : {
"stage" : "COLLSCAN",
"nReturned" : 15026.0,
"executionTimeMillisEstimate" : 1506.0,
"works" : 15028.0,
"advanced" : 15026.0,
"needTime" : 1.0,
"needYield" : 0.0,
"saveState" : 128.0,
"restoreState" : 128.0,
"isEOF" : 1.0,
"invalidates" : 0.0,
"direction" : "forward",
"docsExamined" : 15026.0
}
}
}
},
{
"$sort" : {
"sortKey" : {
"definition.financial.profitability.highlights.prof_net" : 1.0
},
"limit" : NumberLong(1)
}
}
],
"ok" : 1.0
}
Collection 2: 2 records
Same query output:
db.getCollection("properties_pl").explain("executionStats").aggregate([{"$sort":{"definition.financial.profitability.highlights.prof_net":1.0}},{"$limit":1}])
-
{
"stages" : [
{
"$cursor" : {
"query" : {
},
"sort" : {
"definition.financial.profitability.highlights.prof_net" : 1.0
},
"limit" : NumberLong(1),
"queryPlanner" : {
"plannerVersion" : 1.0,
"namespace" : "patrimmoine.properties_pl",
"indexFilterSet" : false,
"parsedQuery" : {
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"definition.financial.profitability.highlights.prof_net" : 1.0
},
"indexName" : "definition.financial.profitability.highlights.prof_net_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"definition.financial.profitability.highlights.prof_net" : [
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2.0,
"direction" : "forward",
"indexBounds" : {
"definition.financial.profitability.highlights.prof_net" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 2.0,
"executionTimeMillis" : 0.0,
"totalKeysExamined" : 2.0,
"totalDocsExamined" : 2.0,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 2.0,
"executionTimeMillisEstimate" : 0.0,
"works" : 3.0,
"advanced" : 2.0,
"needTime" : 0.0,
"needYield" : 0.0,
"saveState" : 1.0,
"restoreState" : 1.0,
"isEOF" : 1.0,
"invalidates" : 0.0,
"docsExamined" : 2.0,
"alreadyHasObj" : 0.0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 2.0,
"executionTimeMillisEstimate" : 0.0,
"works" : 3.0,
"advanced" : 2.0,
"needTime" : 0.0,
"needYield" : 0.0,
"saveState" : 1.0,
"restoreState" : 1.0,
"isEOF" : 1.0,
"invalidates" : 0.0,
"keyPattern" : {
"definition.financial.profitability.highlights.prof_net" : 1.0
},
"indexName" : "definition.financial.profitability.highlights.prof_net_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"definition.financial.profitability.highlights.prof_net" : [
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2.0,
"direction" : "forward",
"indexBounds" : {
"definition.financial.profitability.highlights.prof_net" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 2.0,
"seeks" : 1.0,
"dupsTested" : 0.0,
"dupsDropped" : 0.0,
"seenInvalidated" : 0.0
}
}
}
}
}
],
"ok" : 1.0
}
So one query is indeed not using any index and doing a "COLLSCAN" where the other is doing a "FETCH" with a "IXSCAN".
So my query is now, how could content of a collection change the behavior of a query?