Match Distinct Count from Multiple Arrays - mongodb

Question

Now I have a collection col including docs like follows:
{
"_id": 1,
"shares": [{
"fundcode": "000001",
"lastshares": 1230.20,
"agencyno": "260",
"netno": "260"
},{
"fundcode": "000002",
"lastshares": 213124.00,
"agencyno": "469",
"netno": "001"
},{
"fundcode": "000003",
"lastshares": 10000.80,
"agencyno": "469",
"netno": "002"
}
],
"trade": [{
"fundcode": "000001",
"c_date": "20160412",
"agencyno": "260",
"netno": "260",
"bk_tradetype": "122",
"confirmbalance": 1230.20,
"cserialno": "10110000119601",
"status": "1"
},{
"fundcode": "000002",
"c_date": "20160506",
"agencyno": "469",
"netno": "001",
"bk_tradetype": "122",
"confirmbalance": 213124.00,
"cserialno": "10110000119602",
"status": "1"
},{
"fundcode": "000003",
"c_date": "20170507",
"agencyno": "469",
"netno": "002",
"bk_tradetype": "122",
"confirmbalance": 10000.80,
"netvalue": 1.0000,
"cserialno": "10110000119602",
"status": "1"
}
]
}
how can I realize a selection like the following sql using mongodb query?:
SELECT _id
FROM col
WHERE col.shares.lastshares > 1000
AND col.trade.agencyno = '469'
GROUP BY _id
HAVING COUNT(DISTINCT col.shares.fundcode) > 2
AND COUNT(DISTINCT col.trade.fundcode) > 2
I tried the $unwind, $groupby, $match aggregate pipeline twice, but I did not get the right answer. Thanks for help.

Answer 1

It doesn't really help that the supplied sample does not meet the conditions, but of course only because the "trade" array would only produce 2 distinct matches which is not enough to meet the *"greater than 2"` constraint in the query.
The structure is certainly different than in an RDBMS, so "sub-queries" do not apply, but at least you made these array. But ideally we would not use $unwind here at all.
And therefore all we need to do is "count" the "distinct" matches from within the arrays. This can basically be applied within a $redact using $map, $setDifference and $size as the main operations:
db.getCollection('collection').aggregate([
{ "$match": {
"shares.lastshares": { "$gt": 1000 },
"trade.agencyno": "469"
}},
{ "$redact": {
"$cond": {
"if": {
"$and": [
{ "$gt": [
{ "$size": {
"$setDifference": [
{ "$map": {
"input": "$shares",
"as": "el",
"in": {
"$cond": {
"if": { "$gt": [ "$$el.lastshares", 1000 ] },
"then": "$$el.fundcode",
"else": false
}
}
}},
[false]
]
}},
2
]},
{ "$gt": [
{ "$size": {
"$setDifference": [
{ "$map": {
"input": "$trade",
"as": "el",
"in": {
"$cond": {
"if": { "$eq": [ "$$el.agencyno", "469" ] },
"then": "$$el.fundcode",
"else": false
}
}
}},
[false]
]
}},
2
]}
]
},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
/*
{ "$addFields": {
"shares": {
"$filter": {
"input": "$shares",
"as": "el",
"cond": { "$gt": [ "$$el.lastshares", 1000 ] }
}
},
"trade": {
"$filter": {
"input": "$trade",
"as": "el",
"cond": { "$eq": [ "$$el.agencyno", "469" ] }
}
}
}}
*/
])
That makes it basically compatible with MongoDB 2.6 and upwards, and only adding the $addFields in there so you could see the results of the "filter" at least, but it's not needed since that is not what the query in the question asks for, which is in fact "just the document _id", but just returning the whole document takes less work. Add a $project for just the _id on the end if you really want that.
Also, to taste you can use $filter instead with MongoDB 3.x releases, but the syntax in this case is actually a little longer:
db.getCollection('collection').aggregate([
{ "$match": {
"shares.lastshares": { "$gt": 1000 },
"trade.agencyno": "469"
}},
{ "$redact": {
"$cond": {
"if": {
"$and": [
{ "$gt": [
{ "$size": {
"$setDifference": [
{ "$map": {
"input": {
"$filter": {
"input": "$shares",
"as": "el",
"cond": { "$gt": [ "$$el.lastshares", 1000 ] }
}
},
"as": "el",
"in": "$$el.fundcode"
}},
[]
]
}},
2
]},
{ "$gt": [
{ "$size": {
"$setDifference": [
{ "$map": {
"input": {
"$filter": {
"input": "$trade",
"as": "el",
"cond": { "$eq": [ "$$el.agencyno", "469" ] }
}
},
"as": "el",
"in": "$$el.fundcode"
}},
[]
]
}},
2
]}
]
},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
/*
{ "$addFields": {
"shares": {
"$filter": {
"input": "$shares",
"as": "el",
"cond": { "$gt": [ "$$el.lastshares", 1000 ] }
}
},
"trade": {
"$filter": {
"input": "$trade",
"as": "el",
"cond": { "$eq": [ "$$el.agencyno", "469" ] }
}
}
}}
*/
])
The basic principles here is the part with:
having (count(distinct fundcode))...
Is being achieved by $size and $setDifference to the "filtered" array content by the conditions. And in fact the "GROUP BY"parts are not even required, since the "array" represents the relationship in a "grouped" form already. Think of the overall $redact statement as the "HAVING" here.
If your MongoDB is truly ancient and you cannot use those forms, then it still is possible with $unwind. And this time we $addToSet to get the "distinct" entries:
db.getCollection('collection').aggregate([
{ "$match": {
"shares.lastshares": { "$gt": 1000 },
"trade.agencyno": "469"
}},
{ "$unwind": "$shares" },
{ "$match": {
"shares.lastshares": { "$gt": 1000 },
}},
{ "$group": {
"_id": "$_id",
"shares": { "$addToSet": "$shares.fundcode" },
"trade": { "$first": "$trade" }
}},
{ "$unwind": "$trade" },
{ "$match": {
"trade.agencyno": "469"
}},
{ "$group": {
"_id": "$_id",
"shares": { "$first": "$shares" },
"trade": { "$addToSet": "$trade.fundcode" }
}},
{ "$match": {
"shares.2": { "$exists": true },
"trade.2": { "$exists": true }
}}
])
Where in this case the "HAVING" is represented by the $match clause, where the notations such as "shares.2": { "$exists": true } actually ask if the array being tested actually has a "third index", which in turn means it has "greater than two", which is the point of the condition.
But the document only has "two" matches
As noted it would have helped your question if you actually supplied a document that matched the conditions you asked for. Unfortunately the supplied document falls short of the required number of matches for the "trade" array in the document.
Fixing your condition to match the supplied document we make it rather $gte for greater than or equal to 2 on the "trade" conditions:
db.getCollection('collection').aggregate([
{ "$match": {
"shares.lastshares": { "$gt": 1000 },
"trade.agencyno": "469"
}},
{ "$redact": {
"$cond": {
"if": {
"$and": [
{ "$gt": [
{ "$size": {
"$setDifference": [
{ "$map": {
"input": "$shares",
"as": "el",
"in": {
"$cond": {
"if": { "$gt": [ "$$el.lastshares", 1000 ] },
"then": "$$el.fundcode",
"else": false
}
}
}},
[false]
]
}},
2
]},
{ "$gte": [
{ "$size": {
"$setDifference": [
{ "$map": {
"input": "$trade",
"as": "el",
"in": {
"$cond": {
"if": { "$eq": [ "$$el.agencyno", "469" ] },
"then": "$$el.fundcode",
"else": false
}
}
}},
[false]
]
}},
2
]}
]
},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
{ "$addFields": {
"shares": {
"$filter": {
"input": "$shares",
"as": "el",
"cond": { "$gt": [ "$$el.lastshares", 1000 ] }
}
},
"trade": {
"$filter": {
"input": "$trade",
"as": "el",
"cond": { "$eq": [ "$$el.agencyno", "469" ] }
}
}
}}
])
Which outputs in that form as:
{
"_id" : 1.0,
"shares" : [
{
"fundcode" : "000001",
"lastshares" : 1230.2,
"agencyno" : "260",
"netno" : "260"
},
{
"fundcode" : "000002",
"lastshares" : 213124.0,
"agencyno" : "469",
"netno" : "001"
},
{
"fundcode" : "000003",
"lastshares" : 10000.8,
"agencyno" : "469",
"netno" : "002"
}
],
"trade" : [
{
"fundcode" : "000002",
"c_date" : "20160506",
"agencyno" : "469",
"netno" : "001",
"bk_tradetype" : "122",
"confirmbalance" : 213124.0,
"cserialno" : "10110000119602",
"status" : "1"
},
{
"fundcode" : "000003",
"c_date" : "20170507",
"agencyno" : "469",
"netno" : "002",
"bk_tradetype" : "122",
"confirmbalance" : 10000.8,
"netvalue" : 1.0,
"cserialno" : "10110000119602",
"status" : "1"
}
]
}
Or with the $unwind, relax the length to test for 2 positions:
db.getCollection('collection').aggregate([
{ "$match": {
"shares.lastshares": { "$gt": 1000 },
"trade.agencyno": "469"
}},
{ "$unwind": "$shares" },
{ "$match": {
"shares.lastshares": { "$gt": 1000 },
}},
{ "$group": {
"_id": "$_id",
"shares": { "$addToSet": "$shares.fundcode" },
"trade": { "$first": "$trade" }
}},
{ "$unwind": "$trade" },
{ "$match": {
"trade.agencyno": "469"
}},
{ "$group": {
"_id": "$_id",
"shares": { "$first": "$shares" },
"trade": { "$addToSet": "$trade.fundcode" }
}},
{ "$match": {
"shares.2": { "$exists": true },
"trade.1": { "$exists": true }
}}
])
Which returns:
{
"_id" : 1.0,
"shares" : [
"000003",
"000002",
"000001"
],
"trade" : [
"000003",
"000002"
]
}
But of course both identify the "document" to the conditions which is what the original query asks, and therefore it's the same basic result regardless of content returned. Which you can alwauys $project as just the _id if you must.