Related
I used Query below its take to much time
Query
db.saleOrder.find({"currentStatus._id":"147"},{"_id":1}).limit(10).explain("executionStats")
ExecutionStats result
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "db_erp_tube.saleOrder",
"indexFilterSet" : false,
"parsedQuery" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "PROJECTION",
"transformBy" : {
"_id" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"direction" : "forward"
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 8673,
"totalKeysExamined" : 0,
"totalDocsExamined" : 3458482,
"executionStages" : {
"stage" : "LIMIT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 8460,
"works" : 3458484,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 1,
"invalidates" : 0,
"limitAmount" : 10,
"inputStage" : {
"stage" : "PROJECTION",
"nReturned" : 10,
"executionTimeMillisEstimate" : 8450,
"works" : 3458483,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 0,
"invalidates" : 0,
"transformBy" : {
"_id" : 1
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
"nReturned" : 10,
"executionTimeMillisEstimate" : 8400,
"works" : 3458483,
"advanced" : 10,
"needTime" : 3458473,
"needYield" : 0,
"saveState" : 27019,
"restoreState" : 27019,
"isEOF" : 0,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 3458482
}
}
}
},
"serverInfo" : {
"host" : "172.16.109",
"port" : 27017,
"version" : "4.0.0",
"gitVersion" : "3b07af3d4f471ae89e8186d33bbb1d5259597d51"
},
"ok" : 1,
"operationTime" : Timestamp(1556365275, 114),
"$clusterTime" : {
"clusterTime" : Timestamp(1556365275, 114),
"signature" : {
"hash" : BinData(0,"ppu91nKmeiC//+UvdsEbjrBTDLU="),
"keyId" : NumberLong("6633468944474701825")
}
}
}
The query took more than 8 seconds(8673ms) to execute because it had to scan all 3458482 documents in the saleOrder collection before returning 10 documents.
This is stated in the filter stage of the explain output.
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"currentStatus._id" : {
"$eq" : "147"
}
},
COLLSCAN indicate full collection scan,
"totalDocsExamined" : 3458482,
This is the number of document scanned before an result is returned.
"executionTimeMillis" : 8673,
This is the total time taken for the query to run.
As #the_mahasagar suggested, build a index on currentStatus._id can speed up the query a lot, use the command below.
db.saleOrder.createIndex({ "currentStatus._id": 1})
I don't understand why the following query is so slow, the explain shows it will take 74+ seconds to finish, in spite of the presence of index.
both dev_id and _id are indexed I can assure you, it seems just not helpful at all.
db.DeviceLoginLog.find({"dev_id": "xxx"}).skip(0).limit(10).sort({"_id": -1}).explain("executionStats");
Can anyone help me to interpret explain output and advise me how to speed up?
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "example.DeviceLoginLog",
"indexFilterSet" : false,
"parsedQuery" : {
"dev_id" : {
"$eq" : "xxx"
}
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"limitAmount" : 10,
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"dev_id" : 1
},
"indexName" : "dev_id_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"dev_id" : [
"[\"xxx\", \"xxx\"]"
]
}
}
}
}
},
"rejectedPlans" : [
{
"stage" : "LIMIT",
"limitAmount" : 4,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"dev_id" : {
"$eq" : "xxx"
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 10,
"executionTimeMillis" : 74867,
"totalKeysExamined" : 9203,
"totalDocsExamined" : 9203,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 10,
"executionTimeMillisEstimate" : 49860,
"works" : 9537,
"advanced" : 10,
"needTime" : 9204,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"sortPattern" : {
"_id" : -1
},
"memUsage" : 1890,
"memLimit" : 33554432,
"limitAmount" : 10,
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 49820,
"works" : 9525,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"inputStage" : {
"stage" : "FETCH",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 49820,
"works" : 9525,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 321,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"docsExamined" : 9203,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 9203,
"executionTimeMillisEstimate" : 10,
"works" : 9204,
"advanced" : 9203,
"needTime" : 0,
"needFetch" : 0,
"saveState" : 1864,
"restoreState" : 1864,
"isEOF" : 1,
"invalidates" : 1064,
"keyPattern" : {
"dev_id" : 1
},
"indexName" : "dev_id_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"dev_id" : [
"[\"xxx\", \"xxx\"]"
]
},
"keysExamined" : 9203,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
}
}
},
"serverInfo" : {
"host" : "iZ231ear7c9Z",
"port" : 27017,
"version" : "3.0.3",
"gitVersion" : "b40106b36eecd1b4407eb1ad1af6bc60593c6105"
},
"ok" : 1
}
Thanks in advanceļ¼
The query used index {dev_id: 1} to find and then had to fetch matched docs into memory to sort without using an index. Create a compound index as {dev_id: 1, _id: -1} will improve performance.
See sort-and-non-prefix-subset-of-an-index.
"totalKeysExamined" : 9203,
"totalDocsExamined" : 9203,
That shows that the query had to examine each individual document. You aren't using an index on the _id field. Also depending on which fields you are returning, consider creating an index that allows for a covered query.
A covered query is a query that can be satisfied entirely using an
index and does not have to examine any documents. An index covers a
query when both of the following apply:
all the fields in the query are part of an index, and
all the fields returned in the results are in the same index.
https://docs.mongodb.com/manual/core/query-optimization/
I'm trying to analyze the following query performance. Executing the query with 50k entries in a collection it takes 23 sec to respond.
db.getCollection('collection').find({ipAddress: "127.0.0.1"}).sort({"_id": -1})
In this query ipAddress is indexed and id is indexed by default. I checked with explain command and the response is
"totalKeysExamined" : 49827,
"totalDocsExamined" : 49827,
But when i removed the sorting and the response is
"totalKeysExamined" : 23877,
"totalDocsExamined" : 23877,
I would like to know Mongodb sort with id examine all the records?
EDITED:
Adding the entire response of explain query
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "collection",
"indexFilterSet" : false,
"parsedQuery" : {
"ipAddress" : {
"$eq" : "127.0.0.1"
}
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"ipAddress" : {
"$eq" : "127.0.0.1"
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "SORT",
"sortPattern" : {
"_id" : -1.0
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"ipAddress" : 1.0
},
"indexName" : "ipAddress_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"ipAddress" : [
"[\"127.0.0.1\", \"127.0.0.1\"]"
]
}
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 33809,
"executionTimeMillis" : 25572,
"totalKeysExamined" : 105260,
"totalDocsExamined" : 105260,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"ipAddress" : {
"$eq" : "127.0.0.1"
}
},
"nReturned" : 33809,
"executionTimeMillisEstimate" : 24894,
"works" : 105261,
"advanced" : 33809,
"needTime" : 71451,
"needYield" : 0,
"saveState" : 1376,
"restoreState" : 1376,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 105260,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 105260,
"executionTimeMillisEstimate" : 2020,
"works" : 105261,
"advanced" : 105260,
"needTime" : 0,
"needYield" : 0,
"saveState" : 1376,
"restoreState" : 1376,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 105260,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "client",
"port" : 27017,
"version" : "3.2.16",
"gitVersion" : "1"
},
"ok" : 1.0
}
1/3 of your collection matches the filter {ipAddress: "127.0.0.1"}.
It is quicker to fetch all 100k the documents in the correct order
"stage" : "IXSCAN", "keyPattern" : {"_id" : 1}
and then filter out the ones that don't match:
"stage" : "FETCH", "filter" : {"ipAddress" : {"$eq" : "127.0.0.1"}}
rather than fetch by filter
"stage" : "IXSCAN", "keyPattern" : {"ipAddress" : 1.0}
and then sort 40k documents in memory
"stage" : "SORT","sortPattern" : {"_id" : -1.0}
When you remove sorting, it is quicker to use the {"ipAddress" : 1.0} index to fetch only matched documents.
You may find an index {ipAddress: 1, _id: 1} more efficient, specially if you combine it with same projection option.
As a side note, it doesn't look quite healthy. I can imagine the FETCH stage so slow because of heavy documents being loaded from disk, swapped back, et. But I see no valid reason why the IXSCAN stage
"stage" : "IXSCAN",
"nReturned" : 105260,
"executionTimeMillisEstimate" : 2020,
takes more than 2s. IXSCAN of 100k _ids should be 10 times quicker.
Background
I have a collection of users with structure of documents like this:
{
"_id" : ObjectId("54e61137cca5d2ff0a8b4567"),
"login" : "test1",
"emails" : [
{
"email" : "test1#example.com",
"is_primary" : true,
"_id" : ObjectId("57baf3e97323afb2688e639c")
},
{
"email" : "test1_1#example.com",
"is_primary" : false,
"_id" : ObjectId("57baf3e97323afb2688e639d")
}
]
}
Indexes:
{
"v" : 1,
"key" : {
"login" : 1
},
"name" : "login_1",
"ns" : "mydb.users",
"background" : true
},
{
"v" : 1,
"key" : {
"emails.email" : 1
},
"name" : "emails.email_1",
"ns" : "mydb.users"
}
Count of documents is ~700000
Scenario
To explain the search of users by login, I make this:
rs0:PRIMARY> db.users.explain('executionStats').find({'login' : /test123123123/})
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mydb.users",
"indexFilterSet" : false,
"parsedQuery" : {
"login" : /test123123123/
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"filter" : {
"login" : /test123123123/
},
"keyPattern" : {
"login" : 1
},
"indexName" : "login_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"login" : [
"[\"\", {})",
"[/test123123123/, /test123123123/]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 1040,
"totalKeysExamined" : 698993,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 0,
"executionTimeMillisEstimate" : 930,
"works" : 698994,
"advanced" : 0,
"needTime" : 698993,
"needFetch" : 0,
"saveState" : 5460,
"restoreState" : 5460,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"filter" : {
"login" : /test123123123/
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 920,
"works" : 698993,
"advanced" : 0,
"needTime" : 698993,
"needFetch" : 0,
"saveState" : 5460,
"restoreState" : 5460,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"login" : 1
},
"indexName" : "login_1",
"isMultiKey" : false,
"direction" : "forward",
"indexBounds" : {
"login" : [
"[\"\", {})",
"[/test123123123/, /test123123123/]"
]
},
"keysExamined" : 698993,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
},
"serverInfo" : {
"host" : "myhost",
"port" : 27017,
"version" : "3.0.12",
"gitVersion" : "33934938e0e95d534cebbaff656cde916b9c3573"
},
"ok" : 1
}
As you can see executionStats.executionStages.inputStage.nReturned is 0 and executionStats.totalDocsExamined is so 0. It's ok, I guess there is no documents with login like entered. But if I want search users by email I'll do next:
rs0:PRIMARY> db.users.explain('executionStats').find({'emails.email' : /test123123123/})
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mydb.users",
"indexFilterSet" : false,
"parsedQuery" : {
"emails.email" : /test123123123/
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"emails.email" : /test123123123/
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"emails.email" : 1
},
"indexName" : "emails.email_1",
"isMultiKey" : true,
"direction" : "forward",
"indexBounds" : {
"emails.email" : [
"[\"\", {})",
"[/test123123123/, /test123123123/]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 7666,
"totalKeysExamined" : 699016,
"totalDocsExamined" : 698993,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"emails.email" : /test123123123/
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 7355,
"works" : 699017,
"advanced" : 0,
"needTime" : 699016,
"needFetch" : 0,
"saveState" : 5462,
"restoreState" : 5462,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 698993,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 698993,
"executionTimeMillisEstimate" : 1630,
"works" : 699016,
"advanced" : 698993,
"needTime" : 23,
"needFetch" : 0,
"saveState" : 5462,
"restoreState" : 5462,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"emails.email" : 1
},
"indexName" : "emails.email_1",
"isMultiKey" : true,
"direction" : "forward",
"indexBounds" : {
"emails.email" : [
"[\"\", {})",
"[/test123123123/, /test123123123/]"
]
},
"keysExamined" : 699016,
"dupsTested" : 699016,
"dupsDropped" : 23,
"seenInvalidated" : 0,
"matchTested" : 0
}
}
},
"serverInfo" : {
"host" : "myhost",
"port" : 27017,
"version" : "3.0.12",
"gitVersion" : "33934938e0e95d534cebbaff656cde916b9c3573"
},
"ok" : 1
}
And here executionStats.executionStages.inputStage.nReturned (and executionStats.totalDocsExamined) is equal 698993 (executionStats.nReturned is 0 like in first query)
Question
Why when I use search with multikey index (users.user) on the ixscan stage returns all my collection and fetch stage occurs all collection. But If I use search by non-multikey index (login) ixscan stage scans expected values and on the fetch stage I give what I want.
UPD: when I use regular expression not like /smth/, but /^smth/ then scan by emails.email field returns also 0 elements. Why multikey and ordinary index give me different results for regular expression like /smth/ ?
Because it is multikey index.
explained here
When a query filter specifies an exact match for an array as a whole, MongoDB can use the multikey index to look up the first element of the query array but cannot use the multikey index scan to find the whole array. Instead, after using the multikey index to look up the first element of the query array, MongoDB retrieves the associated documents and filters for documents whose array matches the array in the query.
Assuming I have only male and females in my user collection. Is the following :
User.find({ gender: { $in: ['male','female'] }})
slower than this one :
User.find()
I feel like it would be, but I don't really know how MongoDB works internally. Both requests return the entire collection. I'm building a filter feature and I'd like to simplify my api code by considering that every call is filtered somehow.
it is a good question as it touches basic query planning capabilites.
Comparing explain results we can see that using IN invokes collection scan by specified query parameter - which is more expensive than basic document dump, when querying without parameters.
db.User.find({ gender: { $in: ['male','female'] }}).explain("executionStats")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.User",
"indexFilterSet" : false,
"parsedQuery" : {
"gender" : {
"$in" : [
"female",
"male"
]
}
},
"winningPlan" : {
"stage" : "COLLSCAN",
"filter" : {
"gender" : {
"$in" : [
"female",
"male"
]
}
},
"direction" : "forward"
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 24,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 24,
"executionStages" : {
"stage" : "COLLSCAN",
"filter" : {
"gender" : {
"$in" : [
"female",
"male"
]
}
},
"nReturned" : 24,
"executionTimeMillisEstimate" : 0,
"works" : 26,
"advanced" : 24,
"needTime" : 1,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 24
}
},
"serverInfo" : {
"host" : "greg",
"port" : 27017,
"version" : "3.2.3",
"gitVersion" : "b326ba837cf6f49d65c2f85e1b70f6f31ece7937"
},
"ok" : 1
}
db.User.find().explain("executionStats")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.User",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : []
},
"winningPlan" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : []
},
"direction" : "forward"
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 24,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 24,
"executionStages" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : []
},
"nReturned" : 24,
"executionTimeMillisEstimate" : 0,
"works" : 26,
"advanced" : 24,
"needTime" : 1,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 24
}
},
"serverInfo" : {
"host" : "greg",
"port" : 27017,
"version" : "3.2.3",
"gitVersion" : "b326ba837cf6f49d65c2f85e1b70f6f31ece7937"
},
"ok" : 1
}
When querying without a condition, it return all the documents without checking. But if you and a condition. Simply it compile the condition into BSON and match with the data in the database, Which is slower. But if you create an index on gender. You can not see any difference in time (in both cases)