a mongodb shard shows strange slow behavior - mongodb

Question

I'm testing below aggregate
db.getCollection('token_transfer').explain("executionStats").aggregate([
{$match: {
$and: [{"contract_address":"0xTTTTTTTTTTT"}, {"timestamp": {$gte: ISODate("2019-12-13T00:45:05.000Z")}}]
}},
{$group: {
_id: null,
transfer_count: {$sum: 1},
transfer_volume_sum: {$sum: "$value"},
}}
])
indexes of the token_transfer collection is like below
{
"contract_address" : 1.0,
"value" : -1.0,
"timestamp" : 1.0
},
{
"tx_hash" : "hashed"
}
...
token_transfer collection has below fields
[contract_address, from, to, value, tx_hash, timestamp, ...]
shard key is like below
shard key: { "tx_hash" : "hashed" }
Key point is that it tooks 42706 executionTimeMillis at only shd2.
The other shards show under 5000 executionTimeMillis.
Why does slowness occur on certain shard which results in total query slow?
Below is my executionStats of aggregate.
/* 1 */
{
"mergeType" : "mongos",
"splitPipeline" : {
"shardsPart" : [
{
"$match" : {
"$and" : [
{
"contract_address" : {
"$eq" : "0xTTTTTTTTTTT"
}
},
{
"timestamp" : {
"$gte" : ISODate("2019-12-13T00:45:05.000Z")
}
}
]
}
},
{
"$group" : {
"_id" : {
"$const" : null
},
"transfer_count" : {
"$sum" : {
"$const" : 1.0
}
},
"transfer_volume_sum" : {
"$sum" : "$value"
}
}
}
],
"mergerPart" : [
{
"$group" : {
"_id" : "$$ROOT._id",
"transfer_count" : {
"$sum" : "$$ROOT.transfer_count"
},
"transfer_volume_sum" : {
"$sum" : "$$ROOT.transfer_volume_sum"
},
"$doingMerge" : true
}
}
]
},
"shards" : {
"shd3" : {
"host" : "x-db1:33333",
"stages" : [
{
"$cursor" : {
"query" : {
"$and" : [
{
"contract_address" : "0xTTTTTTTTTTT"
},
{
"timestamp" : {
"$gte" : ISODate("2019-12-13T00:45:05.000Z")
}
}
]
},
"fields" : {
"value" : 1,
"_id" : 0
},
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.token_transfer",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"contract_address" : {
"$eq" : "0xTTTTTTTTTTT"
}
},
{
"timestamp" : {
"$gte" : ISODate("2019-12-13T00:45:05.000Z")
}
}
]
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"value" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"contract_address" : 1.0,
"value" : -1.0,
"timestamp" : 1.0
},
"indexName" : "contract_address_1_value_-1_timestamp_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"contract_address" : [],
"value" : [],
"timestamp" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"contract_address" : [
"[\"0xTTTTTTTTTTT\", \"0xTTTTTTTTTTT\"]"
],
"value" : [
"[MaxKey, MinKey]"
],
"timestamp" : [
"[new Date(1576197905000), new Date(9223372036854775807)]"
]
}
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 671,
"executionTimeMillis" : 1384,
"totalKeysExamined" : 1231507,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 671,
"executionTimeMillisEstimate" : 1370,
"works" : 1231507,
"advanced" : 671,
"needTime" : 1230835,
"needYield" : 0,
"saveState" : 9622,
"restoreState" : 9622,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"value" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 671,
"executionTimeMillisEstimate" : 1340,
"works" : 1231507,
"advanced" : 671,
"needTime" : 1230835,
"needYield" : 0,
"saveState" : 9622,
"restoreState" : 9622,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"contract_address" : 1.0,
"value" : -1.0,
"timestamp" : 1.0
},
"indexName" : "contract_address_1_value_-1_timestamp_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"contract_address" : [],
"value" : [],
"timestamp" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"contract_address" : [
"[\"0xTTTTTTTTTTT\", \"0xTTTTTTTTTTT\"]"
],
"value" : [
"[MaxKey, MinKey]"
],
"timestamp" : [
"[new Date(1576197905000), new Date(9223372036854775807)]"
]
},
"keysExamined" : 1231507,
"seeks" : 1230836,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
{
"$group" : {
"_id" : {
"$const" : null
},
"transfer_count" : {
"$sum" : {
"$const" : 1.0
}
},
"transfer_volume_sum" : {
"$sum" : "$value"
}
}
}
]
},
"shd1" : {
"host" : "x-db2:11111",
"stages" : [
{
"$cursor" : {
"query" : {
"$and" : [
{
"contract_address" : "0xTTTTTTTTTTT"
},
{
"timestamp" : {
"$gte" : ISODate("2019-12-13T00:45:05.000Z")
}
}
]
},
"fields" : {
"value" : 1,
"_id" : 0
},
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.token_transfer",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"contract_address" : {
"$eq" : "0xTTTTTTTTTTT"
}
},
{
"timestamp" : {
"$gte" : ISODate("2019-12-13T00:45:05.000Z")
}
}
]
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"value" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"contract_address" : 1.0,
"value" : -1.0,
"timestamp" : 1.0
},
"indexName" : "contract_address_1_value_-1_timestamp_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"contract_address" : [],
"value" : [],
"timestamp" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"contract_address" : [
"[\"0xTTTTTTTTTTT\", \"0xTTTTTTTTTTT\"]"
],
"value" : [
"[MaxKey, MinKey]"
],
"timestamp" : [
"[new Date(1576197905000), new Date(9223372036854775807)]"
]
}
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 680,
"executionTimeMillis" : 4545,
"totalKeysExamined" : 1234518,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 680,
"executionTimeMillisEstimate" : 3685,
"works" : 1234518,
"advanced" : 680,
"needTime" : 1233837,
"needYield" : 0,
"saveState" : 9656,
"restoreState" : 9656,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"value" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 680,
"executionTimeMillisEstimate" : 3685,
"works" : 1234518,
"advanced" : 680,
"needTime" : 1233837,
"needYield" : 0,
"saveState" : 9656,
"restoreState" : 9656,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"contract_address" : 1.0,
"value" : -1.0,
"timestamp" : 1.0
},
"indexName" : "contract_address_1_value_-1_timestamp_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"contract_address" : [],
"value" : [],
"timestamp" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"contract_address" : [
"[\"0xTTTTTTTTTTT\", \"0xTTTTTTTTTTT\"]"
],
"value" : [
"[MaxKey, MinKey]"
],
"timestamp" : [
"[new Date(1576197905000), new Date(9223372036854775807)]"
]
},
"keysExamined" : 1234518,
"seeks" : 1233838,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
{
"$group" : {
"_id" : {
"$const" : null
},
"transfer_count" : {
"$sum" : {
"$const" : 1.0
}
},
"transfer_volume_sum" : {
"$sum" : "$value"
}
}
}
]
},
"shd2" : {
"host" : "x-db3:22222",
"stages" : [
{
"$cursor" : {
"query" : {
"$and" : [
{
"contract_address" : "0xTTTTTTTTTTT"
},
{
"timestamp" : {
"$gte" : ISODate("2019-12-13T00:45:05.000Z")
}
}
]
},
"fields" : {
"value" : 1,
"_id" : 0
},
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.token_transfer",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"contract_address" : {
"$eq" : "0xTTTTTTTTTTT"
}
},
{
"timestamp" : {
"$gte" : ISODate("2019-12-13T00:45:05.000Z")
}
}
]
},
"winningPlan" : {
"stage" : "PROJECTION",
"transformBy" : {
"value" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"contract_address" : 1.0,
"value" : -1.0,
"timestamp" : 1.0
},
"indexName" : "contract_address_1_value_-1_timestamp_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"contract_address" : [],
"value" : [],
"timestamp" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"contract_address" : [
"[\"0xTTTTTTTTTTT\", \"0xTTTTTTTTTTT\"]"
],
"value" : [
"[MaxKey, MinKey]"
],
"timestamp" : [
"[new Date(1576197905000), new Date(9223372036854775807)]"
]
}
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 720,
"executionTimeMillis" : 42706,
"totalKeysExamined" : 1181569,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION",
"nReturned" : 720,
"executionTimeMillisEstimate" : 4284,
"works" : 1181569,
"advanced" : 720,
"needTime" : 1180848,
"needYield" : 0,
"saveState" : 9249,
"restoreState" : 9249,
"isEOF" : 1,
"invalidates" : 0,
"transformBy" : {
"value" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 720,
"executionTimeMillisEstimate" : 4244,
"works" : 1181569,
"advanced" : 720,
"needTime" : 1180848,
"needYield" : 0,
"saveState" : 9249,
"restoreState" : 9249,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"contract_address" : 1.0,
"value" : -1.0,
"timestamp" : 1.0
},
"indexName" : "contract_address_1_value_-1_timestamp_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"contract_address" : [],
"value" : [],
"timestamp" : []
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"contract_address" : [
"[\"0xTTTTTTTTTTT\", \"0xTTTTTTTTTTT\"]"
],
"value" : [
"[MaxKey, MinKey]"
],
"timestamp" : [
"[new Date(1576197905000), new Date(9223372036854775807)]"
]
},
"keysExamined" : 1181569,
"seeks" : 1180849,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
},
{
"$group" : {
"_id" : {
"$const" : null
},
"transfer_count" : {
"$sum" : {
"$const" : 1.0
}
},
"transfer_volume_sum" : {
"$sum" : "$value"
}
}
}
]
}
},
"ok" : 1.0,
"$clusterTime" : {
"clusterTime" : Timestamp(1576201630, 116),
"signature" : {
"hash" : { "$binary" : "AAAAAAAAAAAAAAAAAAAAAAAAAAA=", "$type" : "00" },
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1576201630, 115)
}