Concat and get all matching nested arrays across entire collection - mongodb

I have question, in database I have model :
board.model.js
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var BoardSchema = new Schema({
name: { type: String, maxlength: 20 },
lists : { type: Array },
users : [{ type : Schema.Types.ObjectId, ref: 'User' }],
});
module.exports = mongoose.model('Board', BoardSchema);
Inside lists is added array cards.
In practice it looks like this:
{
"_id" : ObjectId("59df60fb6fad6224f4f9f22a"),
"name" : "1",
"users" : [
ObjectId("59cd114cea98d9326ca1c421")
],
"lists" : [
{
"list" : "1",
"cards" : [
{
"name" : "1",
"Author" : [
ObjectId("59df60fb6fad6224f4f9f22a")
],
},
{
"name" : "2"
},
{
"name" : "3"
}
]
},
{
"list" : "2",
"cards" : [
{
"name" : "1",
"Author" : [
ObjectId("59df60fb6fad6224f4f9f22a")
],
},
{
"name" : "2",
"Author" : [
ObjectId("59df60fb6fad6224f4f9f22a")
],
},
{
"name" : "3"
}
]
}
],
"__v" : 0
}
Okay this is an example of one board, the question is can I pull only the cards in which the author is "Author": ObjectId ("59df60fb6fad6224f4f9f22a") of all boards?

You can try below aggregation in 3.4.
Concat all the matching ($filter) cards in each board followed by $unwind and push to collect all cards across boards.
Breakdown:
$reduce to concat all the cards arrays in a single document.
$filter operates on $$this ( cards array ) arguments and filters for matching cards while $$value keeps the previous values.
$concatArrays to merge the previous values with the current filtered values.
$unwind to deconstruct cards arrays and $group with $push to get all cards arrays across boards.
db.collection.aggregate([
{
"$project": {
"boardcards": {
"$reduce": {
"input": "$lists.cards",
"initialValue": [],
"in": {
"$concatArrays": [
"$$value",
{
"$filter": {
"input": "$$this",
"as": "result",
"cond": {
"$eq": [
"$$result.Author",
ObjectId("59df60fb6fad6224f4f9f22a")
]
}
}
}
]
}
}
}
}
},
{
"$unwind": "$boardcards"
},
{
"$group": {
"_id": null,
"allboardcards": {
"$push": "$boardcards"
}
}
}
])
As asked in the comment to get just the names of cards you have to wrap the $filter in $map operator to only map card names.
So replace $filter with $map + $filter
{
"$map": {
"input": {
"$filter": {
"input": "$$this",
"as": "result",
"cond": {
"$eq": [
"$$result.Author",
ObjectId("59df60fb6fad6224f4f9f22a")
]
}
}
},
"as": "result",
"in": {
"name": "$$result.name"
}
}
}

Related

How do I fetch only the first element from the array?

How do I fetch only the first element from the "topicsName" array?
Data I have input:
{
"_id" : ObjectId("606b7046a0ccf72222c00c2f"),
"groupId" : ObjectId("5f06cca74e51ba15f5167b86"),
"insertedAt" : "2021-04-05T20:17:10.144521Z",
"isActive" : true,
"staffId" : [
"606b6c34a0ccf72222c5a4df",
"606b6c48a0ccf722228aa035"
],
"subjectName" : "Maths",
"teamId" : ObjectId("6069a6a9a0ccf704e7f4b537"),
"updatedAt" : "2022-04-29T07:57:31.072067Z",
"syllabus" : [
{
"chapterId" : "626b9b94ae6cd2092024f3ee",
"chapterName" : "chap1",
"topicsName" : [
{
"topicId" : "626b9b94ae6cd2092024f3ef",
"topicName" : "1.1"
},
{
"topicId" : "626b9b94ae6cd2092024f3f0",
"topicName" : "1.2"
}
]
},
{
"chapterId" : "626b9b94ae6cd2092024f3f1",
"chapterName" : "chap2",
"topicsName" : [
{
"topicId" : "626b9b94ae6cd2092024f3f2",
"topicName" : "2.1"
},
{
"topicId" : "626b9b94ae6cd2092024f3f3",
"topicName" : "2.2"
}
]
}
]
}
The Query I used to try to fetch the element:- "topicId" : "626b9b94ae6cd2092024f3ef" from the
"topicsName" array.
db.subject_staff_database
.find(
{ _id: ObjectId("606b7046a0ccf72222c00c2f") },
{
syllabus: {
$elemMatch: {
chapterId: "626b9b94ae6cd2092024f3f1",
topicsName: { $elemMatch: { topicId: "626b9b94ae6cd2092024f3f2" } },
},
},
}
)
.pretty();
I was trying to fetch only the first element from the "topicsName" array, but it fetched both the elements in that array.
You can do the followings in an aggregation pipeline.
$match with your given id locate documents
$reduce to flatten the syllabus and topicsName arrays
$filter to get the expected element
db.collection.aggregate([
{
$match: {
"syllabus.topicsName.topicId": "626b9b94ae6cd2092024f3ef"
}
},
{
"$project": {
result: {
"$reduce": {
"input": "$syllabus.topicsName",
"initialValue": [],
"in": {
"$concatArrays": [
"$$value",
"$$this"
]
}
}
}
}
},
{
"$project": {
result: {
"$filter": {
"input": "$result",
"as": "r",
"cond": {
$eq: [
"$$r.topicId",
"626b9b94ae6cd2092024f3ef"
]
}
}
}
}
}
])
Here is the Mongo playground for your reference.
Welcome Ganesh Sowdepalli,
You are not only asking to "fetch only the first element from the array", but to fetch only the matching element of a nested array property of an object item in array.
Edit: (according to #ray's comment)
One way to do it is using an aggregation pipeline:
db.subject_staff_database.aggregate([
{
$match: {"_id": ObjectId("606b7046a0ccf72222c00c2f")}
},
{
$project: {
syllabus: {
$filter: {
input: "$syllabus",
as: "item",
cond: {$eq: ["$$item.chapterId", "626b9b94ae6cd2092024f3f1"
]
}
}
}
}
},
{
$unwind: "$syllabus"
},
{
$project: {
"syllabus.topicsName": {
$filter: {
input: "$syllabus.topicsName",
as: "item",
cond: {$eq: ["$$item.topicId", "626b9b94ae6cd2092024f3f2"]}
}
},
"syllabus.chapterId": 1,
"syllabus.chapterName": 1,
_id: 0
}
}
])
As you can see on this playground example.
If you want the actual first element, not by _id, look here on my first understanding to your question.
The aggregation pipeline allows us to do several operation on the results.
Since syllabus is an array that may contain more than one matching chapterId, we need to $filter it for the items we want.

merge records in to one using mongodb

Here is my collection structure, tried $mergeObjects , but did not know how to use in right way, please help :
{
"_id" : ObjectId("5e39b407eb2b5e4c3c80c5b0"),
"groupId":"1",
"emp" : {
"roles" : [
{
"roleId" : "role1"
}
],
"designation" : [
"Manager"
],
"dept" : [
{
"deptId" : "dept1"
}
]
}
},
{
"_id" : ObjectId("5e39b435eb2b5e4c3c80c5b1"),
"groupId":"1",
"emp" : {
"roles" : [
{
"roleId" : "role2"
}
],
"designation" : [
"Developer"
],
"dept" : [
{
"deptId" : "dept2"
}
]
}
}
I want an aggregate query which merges the two documents w.r.t. "groupId" like this using mongodb
{
"_id" : <some id>,
"groupId":"1",
"emp" : {
"roles" : [
{
"roleId" : "role1"
},
{
"roleId" : "role2"
}
],
"designation" : [
"Manager","Developer"
],
"dept" : [
{
"deptId" : "dept1"
},
{
"deptId" : "dept2"
}
]
}
}
There are around 200 to 300 fields like arrays, array of an array , array of an array of an array and so on.
I don't think the $mergeObject operator would work in this given it overrides fields, for example the roleId in role would always be role2 in this case if $mergeObject is used. I are going to need a multistage solution combining $group to combine the record together and a $project to restructure the fields inside emp into what you want.
Try this:
db.collection.aggregate([
{ "$group": {
"_id": "$groupId",
"roles": { "$push": "$emp.roles" },
"designation": { "$push": "$emp.designation" },
"dept": { "$push": "$emp.dept" },
}},
{ "$project": {
"_id": 0,
"groupId": "$_id",
"roles": {
"$reduce": {
"input": "$roles",
"initialValue": [],
"in": { "$concatArrays": [ "$$value", "$$this" ] }
}
},
"designation": {
"$reduce": {
"input": "$designation",
"initialValue": [],
"in": { "$concatArrays": [ "$$value", "$$this" ] }
}
},
"dept": {
"$reduce": {
"input": "$dept",
"initialValue": [],
"in": { "$concatArrays": [ "$$value", "$$this" ] }
}
}
}}
]);
merging arrays together without repeating values is done using operator $concatArrays and this is done by pushing all arrays together in $group then using $reduce like in previous answer by Josh Balcitis
another operator that gives unique result isr $addToSet instead of $push but you have to $unwind all array records before $group to add unique single records.
ex:
db.collection.aggregate([{
$unwind: {
path: "$emp.roles",
preserveNullAndEmptyArrays: true
}
},
{
$unwind: {
path: "$emp.designation",
preserveNullAndEmptyArrays: true
}
},
{
$unwind: {
path: "$emp.dept",
preserveNullAndEmptyArrays: true
}
},
{
"$group": {
"_id": "$groupId",
"roles": {
$addToSet: "$emp.roles"
},
"designation": {
$addToSet: "$emp.designation"
},
"dept": {
$addToSet: "$emp.dept"
},
}
},
]);
which approach is better? that depends on your data and $group vs $reduce perfomance
note: preserveNullAndEmptyArrays option is used to $prevent empty arrays from affecting whole result, its optional

$elemMatch against two Array elements if one fails

A bit odd but this is what I am looking for.
I have an array as follow:
Document 1:
Items: [
{
"ZipCode": "11111",
"ZipCode4" "1234"
}
Document 2:
Items: [
{
"ZipCode": "11111",
"ZipCode4" "0000"
}
I would like to use a single query, and send a filter on ZipCode = 1111 && ZipCode4 = 4321, if this fails, the query should look for ZipCode = 1111 && ZipCode4: 0000
Is there a way to do this in a single query ? or do I need to make 2 calls to my database ?
For matching both data set (11111/4321) and (11111/0000), you can use $or and $and with $elemMatch like the following :
db.test.find({
$or: [{
$and: [{
"Items": {
$elemMatch: { "ZipCode": "11111" }
}
}, {
"Items": {
$elemMatch: { "ZipCode4": "4321" }
}
}]
}, {
$and: [{
"Items": {
$elemMatch: { "ZipCode": "11111" }
}
}, {
"Items": {
$elemMatch: { "ZipCode4": "0000" }
}
}]
}]
})
As you want conditional staging, this is not possible but we can get closer to it like this :
db.test.aggregate([{
$match: {
$or: [{
$and: [{ "Items.ZipCode": "11111" }, { "Items.ZipCode4": "4321" }]
}, {
$and: [{ "Items.ZipCode": "11111" }, { "Items.ZipCode4": "0000" }]
}]
}
}, {
$project: {
Items: 1,
match: {
"$map": {
"input": "$Items",
"as": "val",
"in": {
"$cond": [
{ $and: [{ "$eq": ["$$val.ZipCode", "11111"] }, { "$eq": ["$$val.ZipCode4", "4321"] }] },
true,
false
]
}
}
}
}
}, {
$unwind: "$match"
}, {
$group: {
_id: "$match",
data: {
$push: {
_id: "$_id",
Items: "$Items"
}
}
}
}])
The first $match is for selecting only the items we need
The $project will build a new field that check if this items is from the 1st set of data (11111/4321) or the 2nd set of data (11111/0000).
The $unwind is used to remove the array generated by $map.
The $group group by set of data
So in the end you will have an output like the following :
{ "_id" : true, "data" : [ { "_id" : ObjectId("58af69ac594b51730a394972"), "Items" : [ { "ZipCode" : "11111", "ZipCode4" : "4321" } ] }, { "_id" : ObjectId("58af69ac594b51730a394974"), "Items" : [ { "ZipCode" : "11111", "ZipCode4" : "4321" } ] } ] }
{ "_id" : false, "data" : [ { "_id" : ObjectId("58af69ac594b51730a394971"), "Items" : [ { "ZipCode" : "11111", "ZipCode4" : "0000" } ] } ] }
Your application logic can check if there is _id:true in this output array, just take the corresponding data field for _id:true. If there is _id:false in this object take the corresponding data field for _id:false.
In the last $group, you can also use $addToSet to builds 2 field data1 & data2 for both type of data set but this will be painful to use as it will add null object to the array for each one of the opposite type :
"$addToSet": {
"$cond": [
{ "$eq": ["$_id", true] },
"$data",
null
]
}
Here is a gist

Filter array in subdocument array field

I am trying to fetch an element from an array in the MongoDB. I think the aggregation filter is the right one to apply. But I tried million times already, I still cannot find where is the problem. Could you give me hand?
MongoDB sample data:
{
"_id" : 12,
"items" : [
{
"columns" : [
{
"title" : "hhh",
"value" : 10
},
{
"title" : "hahaha",
"value" : 20
}
]
},
{
"columns" : [
{
"title" : "hiii",
"value" : 50
}
]
}
]
}
My solution:
db.myCollection.aggregate([
{
$project: {
items: {
$filter: {
input: "$items",
as: "item",
cond: { $eq: [ "$$item.columns.title", "hahaha" ]}
}
}
}
}
]).pretty()
My result:
{
"_id" : 15,
"items" : [
{
"columns" : [ ]
},
{
"columns" : [ ]
}
]
}
Expected result:
{
"_id" : 15,
"items" : [
{
"columns" : [
{
"title" : "hahaha",
"value" : 20
}
]
},
{
"columns" : []
}
]
}
I have checked the Mongo reference:
https://docs.mongodb.com/manual/reference/operator/aggregation/filter/#example
MongoDB version:3.4.1
Testing environment: Mongo Shell
You need to use the $map array operator to $filter the sub array in your subdocument. Also you should do this in the $addFields aggregation pipeline stage to automatically include all others fields in the query result if you need them.
You can also replace the $addFields stage with $project as you were doing but in this case, you will need to explicitly include all other fields.
let value = "hahaha";
db.coll.aggregate([
{
"$addFields": {
"items": {
"$map": {
"input": "$items",
"as": "item",
"in": {
"columns": {
"$filter": {
"input": "$$item.columns",
"as": "elt",
"cond": { "$eq": [ "$$elt.title", value ] }
}
}
}
}
}
}
}
])

Nested filters: $filter array, then $filter child array

Essentially I'm trying to filter OUT subdocuments and sub-subdocuments that have been "trashed". Here's a stripped-down version of my schema:
permitSchema = {
_id,
name,
...
feeClassifications: [
new Schema({
_id,
_trashed,
name,
fees: [
new Schema({
_id,
_trashed,
name,
amount
})
]
})
],
...
}
So I'm able to get the effect I want with feeClassifications. But I'm struggling to find a way to have the same effect for feeClassifications.fees as well.
So, this works as desired:
Permit.aggregate([
{ $match: { _id: mongoose.Types.ObjectId(req.params.id) }},
{ $project: {
_id: 1,
_name: 1,
feeClassifications: {
$filter: {
input: '$feeClassifications',
as: 'item',
cond: { $not: {$gt: ['$$item._trashed', null] } }
}
}
}}
])
But I also want to filter the nested array fees. I've tried a few things including:
Permit.aggregate([
{ $match: { _id: mongoose.Types.ObjectId(req.params.id) }},
{ $project: {
_id: 1,
_name: 1,
feeClassifications: {
$filter: {
input: '$feeClassifications',
as: 'item',
cond: { $not: {$gt: ['$$item._trashed', null] } }
},
fees: {
$filter: {
input: '$fees',
as: 'fee',
cond: { $not: {$gt: ['$$fee._trashed', null] } }
}
}
}
}}
])
Which seems to follow the mongodb docs the closest. But I get the error:
this object is already an operator expression, and can't be used as a document expression (at 'fees')
Update: -----------
As requested, here's a sample document:
{
"_id" : ObjectId("57803fcd982971e403e3e879"),
"_updated" : ISODate("2016-07-11T19:24:27.204Z"),
"_created" : ISODate("2016-07-09T00:05:33.274Z"),
"name" : "Single Event",
"feeClassifications" : [
{
"_updated" : ISODate("2016-07-11T19:05:52.418Z"),
"_created" : ISODate("2016-07-11T17:49:12.247Z"),
"name" : "Event Type 1",
"_id" : ObjectId("5783dc18e09be99840fad29f"),
"fees" : [
{
"_updated" : ISODate("2016-07-11T18:51:10.259Z"),
"_created" : ISODate("2016-07-11T18:41:16.110Z"),
"name" : "Basic Fee",
"amount" : 156.5,
"_id" : ObjectId("5783e84cc46a883349bb2339")
},
{
"_updated" : ISODate("2016-07-11T19:05:52.419Z"),
"_created" : ISODate("2016-07-11T19:05:47.340Z"),
"name" : "Secondary Fee",
"amount" : 50,
"_id" : ObjectId("5783ee0bad7bf8774f6f9b5f"),
"_trashed" : ISODate("2016-07-11T19:05:52.410Z")
}
]
},
{
"_updated" : ISODate("2016-07-11T18:22:21.567Z"),
"_created" : ISODate("2016-07-11T18:22:21.567Z"),
"name" : "Event Type 2",
"_id" : ObjectId("5783e3dd540078de45bbbfaf"),
"_trashed" : ISODate("2016-07-11T19:24:27.203Z")
}
]
}
And here's the desired output ("trashed" subdocuments are excluded from BOTH feeClassifications AND fees):
{
"_id" : ObjectId("57803fcd982971e403e3e879"),
"_updated" : ISODate("2016-07-11T19:24:27.204Z"),
"_created" : ISODate("2016-07-09T00:05:33.274Z"),
"name" : "Single Event",
"feeClassifications" : [
{
"_updated" : ISODate("2016-07-11T19:05:52.418Z"),
"_created" : ISODate("2016-07-11T17:49:12.247Z"),
"name" : "Event Type 1",
"_id" : ObjectId("5783dc18e09be99840fad29f"),
"fees" : [
{
"_updated" : ISODate("2016-07-11T18:51:10.259Z"),
"_created" : ISODate("2016-07-11T18:41:16.110Z"),
"name" : "Basic Fee",
"amount" : 156.5,
"_id" : ObjectId("5783e84cc46a883349bb2339")
}
]
}
]
}
Since we want to filter both the outer and inner array fields, we can use the $map variable operator which return an array with the "values" we want.
In the $map expression, we provide a logical $conditional $filter to remove the non matching documents from both the document and subdocument array field.
The conditions are $lt which return true when the field "_trashed" is absent in the sub-document and or in the sub-document array field.
Note that in the $cond expression we also return false for the <false case>. Of course we need to apply filter to the $map result to remove all false.
Permit.aggregate(
[
{ "$match": { "_id": mongoose.Types.ObjectId(req.params.id) } },
{ "$project": {
"_updated": 1,
"_created": 1,
"name": 1,
"feeClassifications": {
"$filter": {
"input": {
"$map": {
"input": "$feeClassifications",
"as": "fclass",
"in": {
"$cond": [
{ "$lt": [ "$$fclass._trashed", 0 ] },
{
"_updated": "$$fclass._updated",
"_created": "$$fclass._created",
"name": "$$fclass.name",
"_id": "$$fclass._id",
"fees": {
"$filter": {
"input": "$$fclass.fees",
"as": "fees",
"cond": { "$lt": [ "$$fees._trashed", 0 ] }
}
}
},
false
]
}
}
},
"as": "cls",
"cond": "$$cls"
}
}
}}
]
)
In the upcoming MongoDB release (as of this writing and since MongoDB 3.3.5), You can replace the $cond expression in the the $map expression with a $switch expression:
Permit.aggregate(
[
{ "$match": { "_id": mongoose.Types.ObjectId(req.params.id) } },
{ "$project": {
"_updated": 1,
"_created": 1,
"name": 1,
"feeClassifications": {
"$filter": {
"input": {
"$map": {
"input": "$feeClassifications",
"as": "fclass",
"in": {
"$switch": {
"branches": [
{
"case": { "$lt": [ "$$fclass._trashed", 0 ] },
"then": {
"_updated": "$$fclass._updated",
"_created": "$$fclass._created",
"name": "$$fclass.name",
"_id": "$$fclass._id",
"fees": {
"$filter": {
"input": "$$fclass.fees",
"as": "fees",
"cond": { "$lt": [ "$$fees._trashed", 0 ] }
}
}
}
}
],
"default": false
}
}
}
},
"as": "cls",
"cond": "$$cls"
}
}
}}
]
)
For more complicated bigdats, it would be unnecessarily difficult.
Just edit it in $filter input by adding a dotted annotation field.You can search the document to any depth of JSON by dotted annotation without further complicated $filter mapping.
"$filter":{
"input": "$feeClassifications._trashed",
"as": "trashed",
"cond": { "$lt": [ "$$trashed._trashed", 0 ] }
}