Filter Array Content to a Query containing $concatArrays - mongodb

Given this function, I have a data set that I am querying. The data looks like this:
db.activity.insert(
{
"_id" : ObjectId("5908e64e3b03ca372dc945d5"),
"startDate" : ISODate("2017-05-06T00:00:00Z"),
"details" : [
{
"code" : "2",
"_id" : ObjectId("5908ebf96ae5003a4471c9b2"),
"walkDistance" : "03",
"jogDistance" : "01",
"runDistance" : "08",
"sprintDistance" : "01"
}
]
}
)
db.activity.insert(
{
"_id" : ObjectId("58f79163bebac50d5b2ae760"),
"startDate" : ISODate("2017-05-07T00:00:00Z"),
"details" : [
{
"code" : "2",
"_id" : ObjectId("58f7948fbebac50d5b2ae7f2"),
"walkDistance" : "01",
"jogDistance" : "02",
"runDistance" : "09",
"sprintDistance" : ""
}
]
}
)
Using this function, thanks to Neil Lunn, I am able to get my desired output:
db.activity.aggregate([
{ "$project": {
"_id": 0,
"unique": {
"$filter": {
"input": {
"$setDifference": [
{ "$concatArrays": [
"$details.walkDistance",
"$details.jogDistance",
"$details.runDistance",
"$details.sprintDistance"
]},
[]
]
},
"cond": { "$ne": [ "$$this", "" ] }
}
}
}},
{ "$unwind": "$unique" },
{ "$group": {
"_id": null,
"uniqueArray": { "$addToSet": "$unique" }
}}
])
However, I cannot add a match statement to the beginning.
db.activity.aggregate([
{$match: {"startDate" : ISODate("2017-05-06T00:00:00Z"), "details.code": "2" },
{$unwind: '$details'},
{$match: {"startDate" : ISODate("2017-05-06T00:00:00Z"), "details.code": "2" },
{ "$project": {
"_id": 0,
"unique": {
"$filter": {
"input": {
"$setDifference": [
{ "$concatArrays": [
"$details.walkDistance",
"$details.jogDistance",
"$details.runDistance",
"$details.sprintDistance"
]},
[]
]
},
"cond": { "$ne": [ "$$this", "" ] }
}
}
}},
{ "$unwind": "$unique" },
{ "$group": {
"_id": null,
"uniqueArray": { "$addToSet": "$unique" }
}}
])
Because it gives an error message of:
> $concatArrays only supports arrays, not string
How can I modify this query so that a $match statement can be added?

Don't $unwind the array you are feeding to $concatArrays. Instead apply $filter to only extract the matching values. And as stated, we can just use $setUnion for the 'unique concatenation' instead:
db.activity.aggregate([
{ "$match": { "startDate" : ISODate("2017-05-06T00:00:00Z"), "details.code": "2" } },
{ "$project": {
"_id": 0,
"unique": {
"$let": {
"vars": {
"filtered": {
"$filter": {
"input": "$details",
"cond": { "$eq": [ "$$this.code", "2" ] }
}
}
},
"in": {
"$setDifference": [
{ "$setUnion": [
"$$filtered.walkDistance",
"$$filtered.jogDistance",
"$$filtered.runDistance",
"$$filtered.sprintDistance"
]},
[""]
]
}
}
}
}},
{ "$unwind": "$unique" },
{ "$group": {
"_id": null,
"uniqueArray": { "$addToSet": "$unique" }
}}
])
Using $let makes things a bit cleaner syntax wise since you don't need to specify multiple $map and $filter statements "inline" as the source for $setUnion

Related

MongoDB: Get all $matched elements individually from an array

I'm trying to get all matched elements individually, here is the sample data and the query.
// json
[
{
"name": "Mr Cool",
"ican": [
{
"subcategory": [
{
"id": "5bffdba824488b182ec86f8d", "name": "Cricket"
},
{
"id": "5bffdba824488b182ec86f8c", "name": "Footbal"
}
],
"category": "5bffdba824488b182ec86f88",
"name": "Sports"
}
]
}
]
// query
db.collection.aggregate([
{
"$match": {
"ican.subcategory.name": { $in: ["Cricket","Football"] }
}
},
{
"$project": { "_id": 1, "name": 1, }
}
])
I'm getting the combined result, I need the individual match record. I tried $all and $elementMatch but getting the same response. how can I get the results as below. I'm using $aggregate because I will be using $geoNear pipeline for getting the nearby users.
// current result
[
{
"_id": ObjectId("5a934e000102030405000000"),
"name": "Mr Cool"
}
]
// expected result
[
{
"_id": ObjectId("5a934e000102030405000000"),
"name": "Mr Cool",
"subcategory: "Cricket"
},
{
"_id": ObjectId("5a934e000102030405000000"),
"name": "Mr Cool",
"subcategory: "Footbal"
}
]
Thank you
Try this Mongo Playground
db.col.aggregate([
{"$unwind" : "$ican"},
{"$unwind" : "$ican.subcategory"},
{"$match" : {"ican.subcategory.name": { "$in": ["Cricket","Football"] }}},
{"$group" : {"_id" : null,"data" : {"$push" : {"_id" : "$_id","name" : "$name","subcategory" : "$ican.subcategory.name"}}}},
{"$unwind" : "$data"},
{"$replaceRoot" : {"newRoot" : "$data"}}
])
You can use below aggregation without the $unwind and for better performance
db.collection.aggregate([
{ "$match": { "ican.subcategory.name": { "$in": ["Cricket","Football"] }}},
{ "$project": {
"ican": {
"$reduce": {
"input": "$ican",
"initialValue": [],
"in": {
"$concatArrays": [
{ "$filter": {
"input": {
"$map": {
"input": "$$this.subcategory",
"as": "s",
"in": { "name": "$name", "subcategory": "$$s.name" }
}
},
"as": "fil",
"cond": { "$in": ["$$fil.subcategory", ["Football"]] }
}},
"$$value"
]
}
}
}
}},
{ "$unwind": "$ican" },
{ "$replaceRoot": { "newRoot": "$ican" }}
])

Nested filters: $filter array, then $filter child array

Essentially I'm trying to filter OUT subdocuments and sub-subdocuments that have been "trashed". Here's a stripped-down version of my schema:
permitSchema = {
_id,
name,
...
feeClassifications: [
new Schema({
_id,
_trashed,
name,
fees: [
new Schema({
_id,
_trashed,
name,
amount
})
]
})
],
...
}
So I'm able to get the effect I want with feeClassifications. But I'm struggling to find a way to have the same effect for feeClassifications.fees as well.
So, this works as desired:
Permit.aggregate([
{ $match: { _id: mongoose.Types.ObjectId(req.params.id) }},
{ $project: {
_id: 1,
_name: 1,
feeClassifications: {
$filter: {
input: '$feeClassifications',
as: 'item',
cond: { $not: {$gt: ['$$item._trashed', null] } }
}
}
}}
])
But I also want to filter the nested array fees. I've tried a few things including:
Permit.aggregate([
{ $match: { _id: mongoose.Types.ObjectId(req.params.id) }},
{ $project: {
_id: 1,
_name: 1,
feeClassifications: {
$filter: {
input: '$feeClassifications',
as: 'item',
cond: { $not: {$gt: ['$$item._trashed', null] } }
},
fees: {
$filter: {
input: '$fees',
as: 'fee',
cond: { $not: {$gt: ['$$fee._trashed', null] } }
}
}
}
}}
])
Which seems to follow the mongodb docs the closest. But I get the error:
this object is already an operator expression, and can't be used as a document expression (at 'fees')
Update: -----------
As requested, here's a sample document:
{
"_id" : ObjectId("57803fcd982971e403e3e879"),
"_updated" : ISODate("2016-07-11T19:24:27.204Z"),
"_created" : ISODate("2016-07-09T00:05:33.274Z"),
"name" : "Single Event",
"feeClassifications" : [
{
"_updated" : ISODate("2016-07-11T19:05:52.418Z"),
"_created" : ISODate("2016-07-11T17:49:12.247Z"),
"name" : "Event Type 1",
"_id" : ObjectId("5783dc18e09be99840fad29f"),
"fees" : [
{
"_updated" : ISODate("2016-07-11T18:51:10.259Z"),
"_created" : ISODate("2016-07-11T18:41:16.110Z"),
"name" : "Basic Fee",
"amount" : 156.5,
"_id" : ObjectId("5783e84cc46a883349bb2339")
},
{
"_updated" : ISODate("2016-07-11T19:05:52.419Z"),
"_created" : ISODate("2016-07-11T19:05:47.340Z"),
"name" : "Secondary Fee",
"amount" : 50,
"_id" : ObjectId("5783ee0bad7bf8774f6f9b5f"),
"_trashed" : ISODate("2016-07-11T19:05:52.410Z")
}
]
},
{
"_updated" : ISODate("2016-07-11T18:22:21.567Z"),
"_created" : ISODate("2016-07-11T18:22:21.567Z"),
"name" : "Event Type 2",
"_id" : ObjectId("5783e3dd540078de45bbbfaf"),
"_trashed" : ISODate("2016-07-11T19:24:27.203Z")
}
]
}
And here's the desired output ("trashed" subdocuments are excluded from BOTH feeClassifications AND fees):
{
"_id" : ObjectId("57803fcd982971e403e3e879"),
"_updated" : ISODate("2016-07-11T19:24:27.204Z"),
"_created" : ISODate("2016-07-09T00:05:33.274Z"),
"name" : "Single Event",
"feeClassifications" : [
{
"_updated" : ISODate("2016-07-11T19:05:52.418Z"),
"_created" : ISODate("2016-07-11T17:49:12.247Z"),
"name" : "Event Type 1",
"_id" : ObjectId("5783dc18e09be99840fad29f"),
"fees" : [
{
"_updated" : ISODate("2016-07-11T18:51:10.259Z"),
"_created" : ISODate("2016-07-11T18:41:16.110Z"),
"name" : "Basic Fee",
"amount" : 156.5,
"_id" : ObjectId("5783e84cc46a883349bb2339")
}
]
}
]
}
Since we want to filter both the outer and inner array fields, we can use the $map variable operator which return an array with the "values" we want.
In the $map expression, we provide a logical $conditional $filter to remove the non matching documents from both the document and subdocument array field.
The conditions are $lt which return true when the field "_trashed" is absent in the sub-document and or in the sub-document array field.
Note that in the $cond expression we also return false for the <false case>. Of course we need to apply filter to the $map result to remove all false.
Permit.aggregate(
[
{ "$match": { "_id": mongoose.Types.ObjectId(req.params.id) } },
{ "$project": {
"_updated": 1,
"_created": 1,
"name": 1,
"feeClassifications": {
"$filter": {
"input": {
"$map": {
"input": "$feeClassifications",
"as": "fclass",
"in": {
"$cond": [
{ "$lt": [ "$$fclass._trashed", 0 ] },
{
"_updated": "$$fclass._updated",
"_created": "$$fclass._created",
"name": "$$fclass.name",
"_id": "$$fclass._id",
"fees": {
"$filter": {
"input": "$$fclass.fees",
"as": "fees",
"cond": { "$lt": [ "$$fees._trashed", 0 ] }
}
}
},
false
]
}
}
},
"as": "cls",
"cond": "$$cls"
}
}
}}
]
)
In the upcoming MongoDB release (as of this writing and since MongoDB 3.3.5), You can replace the $cond expression in the the $map expression with a $switch expression:
Permit.aggregate(
[
{ "$match": { "_id": mongoose.Types.ObjectId(req.params.id) } },
{ "$project": {
"_updated": 1,
"_created": 1,
"name": 1,
"feeClassifications": {
"$filter": {
"input": {
"$map": {
"input": "$feeClassifications",
"as": "fclass",
"in": {
"$switch": {
"branches": [
{
"case": { "$lt": [ "$$fclass._trashed", 0 ] },
"then": {
"_updated": "$$fclass._updated",
"_created": "$$fclass._created",
"name": "$$fclass.name",
"_id": "$$fclass._id",
"fees": {
"$filter": {
"input": "$$fclass.fees",
"as": "fees",
"cond": { "$lt": [ "$$fees._trashed", 0 ] }
}
}
}
}
],
"default": false
}
}
}
},
"as": "cls",
"cond": "$$cls"
}
}
}}
]
)
For more complicated bigdats, it would be unnecessarily difficult.
Just edit it in $filter input by adding a dotted annotation field.You can search the document to any depth of JSON by dotted annotation without further complicated $filter mapping.
"$filter":{
"input": "$feeClassifications._trashed",
"as": "trashed",
"cond": { "$lt": [ "$$trashed._trashed", 0 ] }
}

MongoDB aggregate/grouping by key-value pairs

My data looks something like this:
{
"_id" : "9aa072e4-b706-47e6-9607-1a39e904a05a",
"customerId" : "2164289-4",
"channelStatuses" : {
"FOO" : {
"status" : "done"
},
"BAR" : {
"status" : "error"
}
},
"channel" : "BAR",
}
My aggregate/group looks like this:
{
"_id" : {
"customerId" : "$customerId",
"channel" : "$channel",
"status" : "$channelStatuses[$channel].status"
},
"count" : {
"$sum" : 1
}
}
So basically with the example data the group should give me a group grouped by:
{"customerId": "2164289-4", "channel": "BAR", "status": "error"}
But I cannot use []-indexing in a aggregate/group. What should I do instead?
You cannot get the result you want with the current structure using .aggregate(). You "could" change the structure to use an array rather than named keys, and the operation is actually quite simple.
So with a document like:
{
"_id" : "9aa072e4-b706-47e6-9607-1a39e904a05a",
"customerId" : "2164289-4",
"channelStatuses" : [
{
"channel": "FOO",
"status" : "done"
},
{
"channel": "BAR",
"status" : "error"
}
],
"channel" : "BAR",
}
You can then do in modern releases with $filter, $map and $arrayElemAt:
{ "$group": {
"_id": {
"customerId" : "$customerId",
"channel" : "$channel",
"status": {
"$arrayElemAt": [
{ "$map": {
"input": { "$filter": {
"input": "$chanelStatuses",
"as": "el",
"cond": { "$eq": [ "$$el.channel", "$channel" ] }
}},
"as": "el",
"in": "$$el.status"
}},
0
]
}
},
"count": { "$sum": 1 }
}}
Older versions of MongoDB are going to going to require $unwind to access the matched array element.
In MongoDB 2.6 then you can still "pre-filter" the array before unwind:
[
{ "$project": {
"customerId": 1,
"channel": 1,
"status": {
"$setDifference": [
{ "$map": {
"input": "$channelStatuses",
"as": "el",
"in": {
"$cond": [
{ "$eq": [ "$$el.channel", "$channel" ] },
"$$el.status",
false
]
}
}},
[false]
]
}
}},
{ "$unwind": "$status" },
{ "$group": {
"_id": {
"customerId": "$customerId",
"channel": "$channel",
"status": "$status"
},
"count": { "$sum": 1 }
}}
]
And anything prior to that you "filter" after $unwind instead:
[
{ "$unwind": "$channelStatuses" },
{ "$project": {
"customerId": 1,
"channel": 1,
"status": "$channelStatuses.status",
"same": { "$eq": [ "$channelStatuses.status", "$channel" ] }
}},
{ "$match": { "same": true } },
{ "$group": {
"_id": "$_id",
"customerId": { "$first": "$customerId" },
"channel": { "$first": "$channel" },
"status": { "$first": "$status" }
}},
{ "$group": {
"_id": {
"customerId": "$customerId",
"channel": "$channel",
"status": "$status"
},
"count": { "$sum": 1 }
}}
]
In a lesser version than MongoDB 2.6 you also need to $project the result of the equality test between the two fields and then $match on the result in a seperate stage. You might also note the "two" $group stages, since the first one removes any possible duplicates of the "channel" values after the filter via the $first accumulators. The following $group is exactly the same as in the previous listing.
But if you cannot change the structure and need "flexible" matching of keys where you cannot supply every name, then you must use mapReduce:
db.collection.mapReduce(
function() {
emit({
"customerId": this.customerId,
"channel": this.channel,
"status": this.channelStatuses[this.channel].status
},1);
},
function(key,values) {
return Array.sum(values);
},
{ "out": { "inline": 1 } }
)
Where of course you can use that sort of notation

Mongodb count all array elements in all objects matching by criteria

I have a collection that is log of activity on objects like this:
{
"_id" : ObjectId("55e3fd1d7cb5ac9a458b4567"),
"object_id" : "1",
"activity" : [
{
"action" : "test_action",
"time" : ISODate("2015-08-31T00:00:00.000Z")
},
{
"action" : "test_action",
"time" : ISODate("2015-08-31T00:00:22.000Z")
}
]
}
{
"_id" : ObjectId("55e3fd127cb5ac77478b4567"),
"object_id" : "2",
"activity" : [
{
"action" : "test_action",
"time" : ISODate("2015-08-31T00:00:00.000Z")
}
]
}
{
"_id" : ObjectId("55e3fd0f7cb5ac9f458b4567"),
"object_id" : "1",
"activity" : [
{
"action" : "test_action",
"time" : ISODate("2015-08-30T00:00:00.000Z")
}
]
}
If i do followoing query:
db.objects.find({
"createddate": {$gte : ISODate("2015-08-30T00:00:00.000Z")},
"activity.action" : "test_action"}
}).count()
it returns count of documents containing "test_action" (3 in this set), but i need to get count of all test_actions (4 on this set). How do i do that?
The most "performant" way to do this is to skip the $unwind altogther and simply $group to count. Essentially "filter" arrays get the $size of the results to $sum:
db.objects.aggregate([
{ "$match": {
"createddate": {
"$gte": ISODate("2015-08-30T00:00:00.000Z")
},
"activity.action": "test_action"
}},
{ "$group": {
"_id": null,
"count": {
"$sum": {
"$size": {
"$setDifference": [
{ "$map": {
"input": "$activity",
"as": "el",
"in": {
"$cond": [
{ "$eq": [ "$$el.action", "test_action" ] },
"$$el",
false
]
}
}},
[false]
]
}
}
}
}}
])
Since MongoDB version 3.2 we can use $filter, which makes this much more simple:
db.objects.aggregate([
{ "$match": {
"createddate": {
"$gte": ISODate("2015-08-30T00:00:00.000Z")
},
"activity.action": "test_action"
}},
{ "$group": {
"_id": null,
"count": {
"$sum": {
"$size": {
"$filter": {
"input": "$activity",
"as": "el",
"cond": {
"$eq": [ "$$el.action", "test_action" ]
}
}
}
}
}
}}
])
Using $unwind causes the documents to de-normalize and effectively creates a copy per array entry. Where possible you should avoid this due the the often extreme cost. Filtering and counting array entries per document is much faster by comparison. As is a simple $match and $group pipeline compared to many stages.
You can do so by using aggregation:
db.objects.aggregate([
{$match: {"createddate": {$gte : ISODate("2015-08-30T00:00:00.000Z")}, {"activity.action" : "test_action"}}},
{$unwind: "$activity"},
{$match: {"activity.action" : "test_action"}}},
{$group: {_id: null, count: {$sum: 1}}}
])
This will produce a result like:
{
count: 4
}

How to find document and single subdocument matching given criterias in MongoDB collection

I have collection of products. Each product contains array of items.
> db.products.find().pretty()
{
"_id" : ObjectId("54023e8bcef998273f36041d"),
"shop" : "shop1",
"name" : "product1",
"items" : [
{
"date" : "01.02.2100",
"purchasePrice" : 1,
"sellingPrice" : 10,
"count" : 15
},
{
"date" : "31.08.2014",
"purchasePrice" : 10,
"sellingPrice" : 1,
"count" : 5
}
]
}
So, can you please give me an advice, how I can query MongoDB to retrieve all products with only single item which date is equals to the date I pass to query as parameter.
The result for "31.08.2014" must be:
{
"_id" : ObjectId("54023e8bcef998273f36041d"),
"shop" : "shop1",
"name" : "product1",
"items" : [
{
"date" : "31.08.2014",
"purchasePrice" : 10,
"sellingPrice" : 1,
"count" : 5
}
]
}
What you are looking for is the positional $ operator and "projection". For a single field you need to match the required array element using "dot notation", for more than one field use $elemMatch:
db.products.find(
{ "items.date": "31.08.2014" },
{ "shop": 1, "name":1, "items.$": 1 }
)
Or the $elemMatch for more than one matching field:
db.products.find(
{ "items": {
"$elemMatch": { "date": "31.08.2014", "purchasePrice": 1 }
}},
{ "shop": 1, "name":1, "items.$": 1 }
)
These work for a single array element only though and only one will be returned. If you want more than one array element to be returned from your conditions then you need more advanced handling with the aggregation framework.
db.products.aggregate([
{ "$match": { "items.date": "31.08.2014" } },
{ "$unwind": "$items" },
{ "$match": { "items.date": "31.08.2014" } },
{ "$group": {
"_id": "$_id",
"shop": { "$first": "$shop" },
"name": { "$first": "$name" },
"items": { "$push": "$items" }
}}
])
Or possibly in shorter/faster form since MongoDB 2.6 where your array of items contains unique entries:
db.products.aggregate([
{ "$match": { "items.date": "31.08.2014" } },
{ "$project": {
"shop": 1,
"name": 1,
"items": {
"$setDifference": [
{ "$map": {
"input": "$items",
"as": "el",
"in": {
"$cond": [
{ "$eq": [ "$$el.date", "31.08.2014" ] },
"$$el",
false
]
}
}},
[false]
]
}
}}
])
Or possibly with $redact, but a little contrived:
db.products.aggregate([
{ "$match": { "items.date": "31.08.2014" } },
{ "$redact": {
"$cond": [
{ "$eq": [ { "$ifNull": [ "$date", "31.08.2014" ] }, "31.08.2014" ] },
"$$DESCEND",
"$$PRUNE"
]
}}
])
More modern, you would use $filter:
db.products.aggregate([
{ "$match": { "items.date": "31.08.2014" } },
{ "$addFields": {
"items": {
"input": "$items",
"cond": { "$eq": [ "$$this.date", "31.08.2014" ] }
}
}}
])
And with multiple conditions, the $elemMatch and $and within the $filter:
db.products.aggregate([
{ "$match": {
"$elemMatch": { "date": "31.08.2014", "purchasePrice": 1 }
}},
{ "$addFields": {
"items": {
"input": "$items",
"cond": {
"$and": [
{ "$eq": [ "$$this.date", "31.08.2014" ] },
{ "$eq": [ "$$this.purchasePrice", 1 ] }
]
}
}
}}
])
So it just depends on whether you always expect a single element to match or multiple elements, and then which approach is better. But where possible the .find() method will generally be faster since it lacks the overhead of the other operations, which in those last to forms does not lag that far behind at all.
As a side note, your "dates" are represented as strings which is not a very good idea going forward. Consider changing these to proper Date object types, which will greatly help you in the future.
Based on Neil Lunn's code I work with this solution, it includes automatically all first level keys (but you could also exclude keys if you want):
db.products.find(
{ "items.date": "31.08.2014" },
{ "shop": 1, "name":1, "items.$": 1 }
{ items: { $elemMatch: { date: "31.08.2014" } } },
)
With multiple requirements:
db.products.find(
{ "items": {
"$elemMatch": { "date": "31.08.2014", "purchasePrice": 1 }
}},
{ items: { $elemMatch: { "date": "31.08.2014", "purchasePrice": 1 } } },
)
Mongo supports dot notation for sub-queries.
See: http://docs.mongodb.org/manual/reference/glossary/#term-dot-notation
Depending on your driver, you want something like:
db.products.find({"items.date":"31.08.2014"});
Note that the attribute is in quotes for dot notation, even if usually your driver doesn't require this.