MongoDB $or query against itself much faster - mongodb

Question

I have a mongo instance with 16m documents in a collection. I was writing a query to search for one of the (indexed) fields and I'm getting some weird results, which I cannot explain.
If I execute a query directly like:
find({ "$and" : [ { "ipAddr" : { "$regex" : "^01:172"}} , { "active" : true}]}).limit(100).sort({ "_id" : 1})
or even adding a pointless $or to the query:
find({ "$and" : [ { "$or" : [ { "ipAddr" : { "$regex" : "^01:172"}}]} , { "active" : true}]}).limit(100).sort({ "_id" : 1})
It returns
Fetched 3 record(s) in 71673ms
However, if I use an $or against itself like:
find({ "$and" : [ { "$or" : [ { "ipAddr" : { "$regex" : "^01:172"}} , { "ipAddr" : { "$regex" : "^01:172"}}]} , { "active" : true}]}).limit(100).sort({ "_id" : 1})
It returns:
Fetched 3 record(s) in 4ms
So a big performance difference. From inspecting the explain() on the queries, I could not determine why such a large performance difference exists. Can anyone shed light on what I'm missing or what mongo is doing differently between these?
Explain() on single $or which takes >60000ms
find({ "$and" : [ { "$or" : [ { "ipAddr" : { "$regex" : "^01:172"}}]} , { "active" : true}]}).limit(100).sort({ "_id" : 1}).explain()
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "CLS-TEST.Leases",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"active" : {
"$eq" : true
}
},
{
"ipAddr" : /^01:172/
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"active" : {
"$eq" : true
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"ipAddr" : 1
},
"indexName" : "ipAddr_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"ipAddr" : [
"[\"01:172\", \"01:173\")",
"[/^01:172/, /^01:172/]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"ipAddr" : /^01:172/
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"sessionId" : 1,
"updateTime" : 1
},
"indexName" : "active_1_sessionId_1_updateTime_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"sessionId" : [
"[MinKey, MaxKey]"
],
"updateTime" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"ipAddr" : /^01:172/
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"clientId" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_clientId_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"clientId" : [
"[MinKey, MaxKey]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"ipAddr" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_ipAddr_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"ipAddr" : [
"[\"01:172\", \"01:173\")",
"[/^01:172/, /^01:172/]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"ipAddr" : /^01:172/
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"macAddress" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_macAddress_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"macAddress" : [
"[MinKey, MaxKey]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"ipAddr" : /^01:172/
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"remoteId" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_remoteId_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"remoteId" : [
"[MinKey, MaxKey]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"active" : {
"$eq" : true
}
},
{
"ipAddr" : /^01:172/
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"serverInfo" : {
"host" : "",
"port" : 27017,
"version" : "3.2.3",
"gitVersion" : "b326ba837cf6f49d65c2f85e1b70f6f31ece7937"
},
"ok" : 1
}
Explain() on $or against itself which takes <50ms
find({ "$and" : [ { "$or" : [ { "ipAddr" : { "$regex" : "^01:172"}} , { "ipAddr" : { "$regex" : "^01:172"}}]} , { "active" : true}]}).limit(100).sort({ "_id" : 1}).explain()
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "CLS-TEST.Leases",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"$or" : [
{
"ipAddr" : /^01:172/
},
{
"ipAddr" : /^01:172/
}
]
},
{
"active" : {
"$eq" : true
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"active" : {
"$eq" : true
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"ipAddr" : 1
},
"indexName" : "ipAddr_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"ipAddr" : [
"[\"01:172\", \"01:173\")",
"[/^01:172/, /^01:172/]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$or" : [
{
"ipAddr" : /^01:172/
},
{
"ipAddr" : /^01:172/
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"sessionId" : 1,
"updateTime" : 1
},
"indexName" : "active_1_sessionId_1_updateTime_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"sessionId" : [
"[MinKey, MaxKey]"
],
"updateTime" : [
"[MinKey, MaxKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$or" : [
{
"ipAddr" : /^01:172/
},
{
"ipAddr" : /^01:172/
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"clientId" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_clientId_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"clientId" : [
"[MinKey, MaxKey]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$or" : [
{
"ipAddr" : /^01:172/
},
{
"ipAddr" : /^01:172/
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"ipAddr" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_ipAddr_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"ipAddr" : [
"[MinKey, MaxKey]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$or" : [
{
"ipAddr" : /^01:172/
},
{
"ipAddr" : /^01:172/
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"macAddress" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_macAddress_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"macAddress" : [
"[MinKey, MaxKey]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "SORT",
"sortPattern" : {
"_id" : 1
},
"limitAmount" : 100,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$or" : [
{
"ipAddr" : /^01:172/
},
{
"ipAddr" : /^01:172/
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"active" : 1,
"remoteId" : 1,
"startTime" : -1,
"_id" : -1
},
"indexName" : "active_1_remoteId_1_startTime_-1__id_-1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"active" : [
"[true, true]"
],
"remoteId" : [
"[MinKey, MaxKey]"
],
"startTime" : [
"[MaxKey, MinKey]"
],
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
{
"stage" : "LIMIT",
"limitAmount" : 100,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$or" : [
{
"ipAddr" : /^01:172/
},
{
"ipAddr" : /^01:172/
}
]
},
{
"active" : {
"$eq" : true
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"serverInfo" : {
"host" : "",
"port" : 27017,
"version" : "3.2.3",
"gitVersion" : "b326ba837cf6f49d65c2f85e1b70f6f31ece7937"
},
"ok" : 1
}

Answer 1

You might note that none of the index selections include the combination of "active" and "ipAddr", which would be the useful index to define here.
In short, the "slower" query is using just the index for "ipAddr" and therefore it's taking more work to then "filter" out the { "active": true } entries.
Clearly when the other index selection(s) are using the "active" key with those bounds there are less results being passed to the subsequent filter on the regex pattern. You seem to have quite a few indexes here, and none of them are really optimal for the query.
I'll give you props for at least running the "explain" output on either query, but if you look closely then you should see that the "slow" query "incorrectly" chooses the "ipAddr" index thinking it to be optimal. It might not be, but it's a reasonable assumption for the optimiser to make considering an "anchored" regex in use.
The $or forces "index intersection", and it's smart enough to not do that when there is only "one" argument in the $or. "Two" arguments makes that happen, and the optimiser takes another "guess" by looking for an index preceeding with your other query condition ( the "active" value ).
That makes sense, since this is now running "two" queries from which it wil "intersect" the results, and therefore any conditions outside of the $or statement would be the logical choice to optimally choose an index for.
Since the returned results from these is likely smaller, it's therefore faster to "filter" out the regex match than to look at all the regex results and filter out the "active" values.
Therefore the "best" index to define for this query would be:
.createIndex({ "active": 1, "ipAddr": 1 })
And then the results from either query would be consistent, providing of course that the optimiser does not get confused by other indexes present and chooses that one. To force the index selection, use .hint()