mongoDB updateMany based on nested array condition - mongodb

I have following structure in users collection:
[
{ "name": "Ivan",
"payments": [
{"date": new Date("2019-01-01"), "details": [{"payment_system": "A", "spent": 95},
{"payment_system": "B", "spent": 123}]},
{"date": new Date("2019-01-03"), "details": [{"payment_system": "A", "spent": 12},
{"payment_system": "B", "spent": 11}]}]},
{ "name": "Mark",
"payments": [
{"date": new Date("2019-01-01"), "details": [{"payment_system": "D", "spent": 456},
{"payment_system": "B", "spent": 123}]},
{"date": new Date("2019-01-02"), "details": [{"payment_system": "A", "spent": 98},
{"payment_system": "C", "spent": 4}]}]}
]
Is it any way to add a field to users who spent more than, lets say 100 during the specific date range in specific payment system?
I tried updateMany, but have no idea how to filter "details" array element based on payment_system field.
For payment_system IN ("A", "C"), date >= "2019-01-02", spent_total >= 100 update should return
[
{ "name": "Ivan", ...},
{ "name": "Mark", "filter_passed": true, ... }
]

This this one:
db.collection.aggregate([
{
$set: {
payments: {
$filter: {
input: "$payments",
cond: { $gte: ["$$this.date", new Date("2019-01-02")] }
}
}
}
},
{
$set: {
spent_total: {
$reduce: {
input: "$payments.details.spent",
initialValue: [],
in: { $concatArrays: ["$$value", "$$this"] }
}
}
}
},
{ $set: { spent_total: { $sum: "$spent_total" } } },
{ $match: { "spent_total": { $gte: 100 } } }
])
Mongo Playground
Update:
Filter by payment_system is a bit longer. You have to $unwind and $group:
db.collection.aggregate([
{
$set: {
payments: {
$filter: {
input: "$payments",
cond: { $gte: ["$$this.date", new Date("2019-01-02")] }
}
}
}
},
{ $unwind: "$payments" },
{
$set: {
"payments.details": {
$filter: {
input: "$payments.details",
cond: { $in: ["$$this.payment_system", ["A", "C"]] }
},
},
}
},
{
$group: {
_id: { _id: "$_id", name: "$name", },
payments: { $push: "$payments" }
}
},
{
$set: {
spent_total: {
$reduce: {
input: "$payments.details.spent",
initialValue: [],
in: { $concatArrays: ["$$value", "$$this"] }
}
}
}
},
{ $set: { spent_total: { $sum: "$spent_total" } } },
{ $match: { "spent_total": { $gte: 100 } } },
{ // just some cosmetic
$project: {
_id: "$_id._id",
name: "$_id.name",
payments: 1
}
}
])
You cannot update your collection like db.collection.updateMany({}, [<the aggregation pipeline from above>]) because it contains $unwind and $group.
However, you can make $lookup or $out to save entire result into new collection.
If you need to sum up for each payment_system individually then try:
db.collection.aggregate([
{
$set: {
payments: {
$filter: {
input: "$payments",
cond: { $gte: ["$$this.date", new Date("2019-01-01")] }
}
}
}
},
{ $unwind: "$payments" },
{
$set: {
"payments.details": {
$filter: {
input: "$payments.details",
cond: { $in: ["$$this.payment_system", ["A", "B","C"]] }
},
},
}
},
{ $unwind: "$payments.details" },
{
$group: {
_id: {
_id: "$_id",
name: "$name",
payments: "$payments.details.payment_system"
},
spent_total: { $sum: "$payments.details.spent" }
}
},
{ $match: { "spent_total": { $gte: 100 } } },
{
$project: {
_id: "$_id._id",
name: "$_id.name",
payments: "$_id.payments",
spent_total: 1
}
}
])

Related

MongoDB - Aggregate get specific objects in an array

How can I get only objects in the sales array matching with 2021-10-14 date ?
My aggregate query currently returns all objects of the sales array if at least one is matching.
Dataset Documents
{
"name": "#0",
"sales": [{
"date": "2021-10-14",
"price": 3.69,
},{
"date": "2021-10-15",
"price": 2.79,
}]
},
{
"name": "#1",
"sales": [{
"date": "2021-10-14",
"price": 1.5,
}]
}
Aggregate
{
$match: {
sales: {
$elemMatch: {
date: '2021-10-14',
},
},
},
},
{
$group: {
_id: 0,
data: {
$push: '$sales',
},
},
},
{
$project: {
data: {
$reduce: {
input: '$data',
initialValue: [],
in: {
$setUnion: ['$$value', '$$this'],
},
},
},
},
}
Result
{"date": "2021-10-14","price": 3.69},
{"date": "2021-10-15","price": 2.79},
{"date": "2021-10-14","price": 1.5}
Result Expected
{"date": "2021-10-14","price": 3.69},
{"date": "2021-10-14","price": 1.5}
You actually need to use a $replaceRoot or $replaceWith pipeline which takes in an expression that gives you the resulting document filtered using $arrayElemAt (or $first) and $filter from the sales array:
[
{ $match: { 'sales.date': '2021-10-14' } },
{ $replaceWith: {
$arrayElemAt: [
{
$filter: {
input: '$sales',
cond: { $eq: ['$$this.date', '2021-10-14'] }
}
},
0
]
} }
]
OR
[
{ $match: { 'sales.date': '2021-10-14' } },
{ $replaceRoot: {
newRoot: {
$arrayElemAt: [
{
$filter: {
input: '$sales',
cond: { $eq: ['$$this.date', '2021-10-14'] }
}
},
0
]
}
} }
]
Mongo Playground
In $project stage, you need $filter operator with input as $reduce operator to filter the documents.
{
$project: {
data: {
$filter: {
input: {
$reduce: {
input: "$data",
initialValue: [],
in: {
$setUnion: [
"$$value",
"$$this"
],
}
}
},
cond: {
$eq: [
"$$this.date",
"2021-10-14"
]
}
}
}
}
}
Sample Mongo Playground
How about using $unwind:
.aggregate([
{$match: { sales: {$elemMatch: {date: '2021-10-14'} } }},
{$unwind: '$sales'},
{$match: {'sales.date': '2021-10-14'}},
{$project: {date: '$sales.date', price: '$sales.price', _id: 0}}
])
This will separate the sales into different documents, each containing only one sale, and allow you to match conditions easily.
See: https://docs.mongodb.com/manual/reference/operator/aggregation/unwind/

MongoDB $lookup on nested document, limit and count the retrieved data

I would like to get a count of all notifications that aren't read by an User ("A", "B", "C", etc) for each subRoom. Taking into account that it could be millions of notifications documents and hundreds of subrooms elements in Rooms Collections, i need to limit it. For that reason I've limited the $lookup for first 100 elements and then check if that notifications have been read or not by an User. I did it using documents (roomId) in $lookup but I cant do it using subdocuments (subRoom.id).
Notifications Collection is indexed using a Compound of (roomId: 1, timestamp: -1)
Notifications Collection: (id corresponds to notification id and roomId is the link to Rooms collection)
[{
"_id": "XXX",
"id": "1",
"read": ["A", "B", "C"],
"roomId": "c1d87a4c-231d-4cc8-8438-35cf21ed7fc5",
"content": "XXX",
"timestamp": { "$date": "2021-12-31T22:50:53.000Z" }
},{
"_id": "XXX",
"id": "2",
"read": ["C"],
"roomId": "c1d87a4c-231d-4cc8-8438-35cf21ed7fc5",
"content": "XXX",
"timestamp": { "$date": "2021-12-31T22:50:53.000Z" }
},
...
]
Rooms Collection:
[{
"_id": "XXX"
"subRoom": [{
"id": "c1d87a4c-231d-4cc8-8438-35cf21ed7fc5",
"image": "XXX",
"name": "XXX"
}, {
"id": "c2d5081e-0cf1-4e69-937d-be357da1d104",
"image": "XXX",
"name": "XXX"
}, {
"id": "530c2c02-26e8-441c-af39-c5232dfe1f73",
"image": "XXX",
"name": "XXX"
}],
"id": "453a6458-6545-4842-8946-05f49efea216",
"name": "XXX",
},
...
]
Code working using roomId instead subRoom.id:
{ $lookup: {
from: "notifications",
let: { "id": "$id" },
pipeline: [
{ $match: {
$expr:
{ $eq: [ "$roomId", "$$id" ] }
}},
{ $limit: 100},
{ $project: {_id: 0, read: 1}}
],
as: "messages"
}},
{ $project: {_id: 0, id: 1, notRead: {
$size: {
$filter: {
input: "$notifications",
cond: {
$not: {
$in: [
"A",
"$$this.read"
]
}
}
}
}
},
}
Code NOT WORKING using subRoom.id:
{ $lookup: {
from: "notifications",
let: { "id": "$subRoom.id" },
pipeline: [
{ $match: {
$expr:
{ $eq: [ "$roomId", "$$id" ] }
}},
{ $limit: 100},
{ $project: {_id: 0, read: 1}}
],
as: "messages"
}},
{
$addFields: {
items: {
$map: {
input: { $zip: { inputs: ["$subRoom", "$messages"] } },
in: { $mergeObjects: "$$this" },
},
},
},
},
.
. projection
.
Expected Result:
[{
"_id": "XXX"
"subRoom": [{
"id": "c1d87a4c-231d-4cc8-8438-35cf21ed7fc5",
"notRead": 50 //e.g
}, {
"id": "c2d5081e-0cf1-4e69-937d-be357da1d104",
"notRead": 35 //e.g
}, {
"id": "530c2c02-26e8-441c-af39-c5232dfe1f73",
"image": "XXX",
"notRead": 5 //e.g
}],
"id": "453a6458-6545-4842-8946-05f49efea216",
"name": "XXX",
},
...
]
Finally and very importantly, I want an scalable solution that can be done with big data.
Thank you very much in advance.
$unwind deconstruct subRoom array with preserve null and empty array property
$lookup with notification collection using pipeline, let to pass id to pipeline, check condition for roomId and user should not read notification
$group by null and count total unread notifications
$addFields to get count to notifications using $sum
$group by _id and reconstruct the subRoom array with required fields in result
db.rooms.aggregate([
{
$unwind: {
path: "$subRoom",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: "nitifications",
let: { id: "$subRoom.id" },
pipeline: [
{
$match: {
$and: [
{ $expr: { $eq: ["$$id", "$roomId"] } },
{ read: { $ne: "A" } }
]
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
}
],
as: "subRoom.notRead"
}
},
{
$addFields: {
"subRoom.notRead": { $sum: "$subRoom.notRead.count" }
}
},
{
$group: {
_id: "$_id",
name: { $first: "$name" },
id: { $first: "$id" },
subRoom: { $push: "$subRoom" }
}
}
])
Playground
Second option without using $unwind stage,
$lookup with notification collection using pipeline, let to pass id to pipeline, check condition for roomId and user should not read notification
$group by null and count total unread notifications
$map to iterate loop of subRoom array
$filter to iterate loop of return result from lookup notifications count and get current subRoom document
$let to declare a variable n and assign above filtered result to it and return $sum from count
$mergeObjects to merge current object of subRoom and new field notRead
db.rooms.aggregate([
{
$lookup: {
from: "nitifications",
let: { id: "$subRoom.id" },
pipeline: [
{
$match: {
$and: [
{ $expr: { $in: ["$roomId", "$$id"] } },
{ read: { $ne: "A" } }
]
}
},
{
$group: {
_id: "$roomId",
count: { $sum: 1 }
}
}
],
as: "notRead"
}
},
{
$project: {
id: 1,
name: 1,
subRoom: {
$map: {
input: "$subRoom",
as: "s",
in: {
$mergeObjects: [
"$$s",
{
notRead: {
$let: {
vars: {
n: {
$filter: {
input: "$notRead",
cond: { $eq: ["$$this._id", "$$s.id"] }
}
}
},
in: { $sum: "$$n.count" }
}
}
}
]
}
}
}
}
}
])
Playground

Mongodb loop through every distinct values and select tags using aggregate (facet)

I have collection like this:
{
"labels": [{
"description": "Dog"
}, {
"description": "Red"
}, {
"description": "XXX"
}]
}
{
"labels": [{
"description": "Cat"
}, {
"description": "XXX"
}, {
"description": "Yellow"
}]
}
{
"labels": [{
"description": "Dog"
}, {
"description": "Red"
}, {
"description": "Yellow"
}]
}
{
"labels": [{
"description": "Bird"
}, {
"description": "XXX"
}, {
"description": "XXX"
}]
}
I want to filter for example only "Red" and "Yellow" colors from ALL elements and output document like this:
// because "Dog" appears 2 times so total = 2
{
description: "Dog",
total: 2,
colors: [
{ "_id": "Red", total: 2 },
{ "_id": "Yellow", total: 1 }
]
}
{
description: "Cat",
total: 1,
colors: [
{ "_id": "Yellow", total: 1 }
]
}
{
description: "Bird",
total: 1,
colors: []
}
{
description: "Red",
total: 2,
colors: [
{ _id: "Yellow", total: 1 }
]
}
{
description: "XXX",
total: 4,
colors: [
{ _id: "Yellow", total: 1 }
]
}
I can do this by using collection.distinct('labels.description') and then iterating through every single element + make a separate collection.count({ 'labels.description': 'Dog' }) like this:
for (...)
db.collection.aggregate([
{
"$match": {
"labels.description": valueFromLoop // (e.g. Dog)
}
},
{ $unwind : "$labels" },
{
"$group": {
"_id": "$labels.description",
"count": { "$sum": 1 }
}
},
{
"$match": {
"$or": [
{ "_id": "Red" },
{ "_id": "Yellow" }
]
}
},
{
"$sort": {
"count": -1
}
}
])
I want to do this in a single aggregation or mapReduce so that I could easily output it to new collection using $out instead of using Bulk operations separately, however I don't know if it's possible.
Try this:
let filter = ["Red", "Yellow"];
db.testcollection.aggregate([
{
$addFields: { bkp: "$labels" }
},
{ $unwind: "$labels" },
{
$addFields: {
bkp: {
$filter: {
input: "$bkp",
as: "item",
cond: {
$and: [
{ $ne: ["$$item.description", "$labels.description"] },
{ $in: ["$$item.description", filter] }
]
}
}
}
}
},
{
$unwind: {
path: "$bkp",
preserveNullAndEmptyArrays: true
}
},
{
$group: {
_id: {
key1: "$labels.description",
key2: { $ifNull: ["$bkp.description", false] }
},
total: { $sum: 1 }
}
},
{
$group: {
_id: "$_id.key1",
description: { $first: "$_id.key1" },
total: {
$sum: {
$cond: {
if: { $first: [["$_id.key2"]] },
then: 1,
else: "$total"
}
}
},
colors: {
$push: {
$cond: {
if: { $first: [["$_id.key2"]] },
then: {
_id: "$_id.key2",
total: "$total"
},
else: "$$REMOVE"
}
}
}
}
},
{ $project: { _id: 0 } }
]);
For some reason with code from both answers it does not count all tags properly.
I'm posting what works:
db.collection.aggregate([
{
$project: {
labels: 1,
result: {
$filter: {
input: "$labels",
as: "label",
cond: {
$or: [
{ $eq: ["$$label.description", "Blue"] },
{ $eq: ["$$label.description", "Red"] },
{ $eq: ["$$label.description", "Black-and-white"] },
{ $eq: ["$$label.description", "Purple"] },
{ $eq: ["$$label.description", "Orange"] },
{ $eq: ["$$label.description", "Yellow"] },
{ $eq: ["$$label.description", "Green"] },
{ $eq: ["$$label.description", "Teal"] }
]
}
}
}
}
},
{
$unwind: "$labels"
},
{
"$group": {
_id: "$labels.description",
x: {
$push: "$result.description"
},
total: { "$sum": 1 }
}
},
{
$project: {
x: {
$reduce: {
input: '$x',
initialValue: [],
in: {$concatArrays: ['$$value', '$$this']}
}
},
total: 1
}
},
{
$project: {
x: 1,
y: { $setUnion: "$x" },
total: 1
}
},
{
$project: {
_id: 0,
description: "$_id",
"colors": {
$map: {
input: "$y",
as: "item",
in: {
_id: "$$item",
count: {
$size: {
$filter: {
input: "$x",
as: "itemx",
cond: {
$eq: ["$$item", "$$itemx"]
}
}
}
}
}
}
},
total: 1
}
},
{
$out: "backgrounds_meta"
}
])
db.test2.aggregate([
{
$project: {
labels:1,
colours: {
$filter: {
input: "$labels",
as: "label",
cond: {
$or: [
{$eq:["Yellow","$$label.description"]},
{$eq:["Red", "$$label.description"]}
]
}
}
}
}
},
{$unwind:"$labels"},
{$group:{
_id: "$labels.description",
total: {$sum:1},
colours: {$addToSet:"$colours.description"}
}},
{
$project:{
_id:0,
description:"$_id",
total:1,
colours: {
$reduce:{
input: "$colours",
initialValue: [],
in: {$concatArrays: ["$$value", "$$this"]}
}
}
}
},
{
$unwind: {
path:"$colours",preserveNullAndEmptyArrays: true
}
},
{
$group:{
_id:{
description:"$description",
total:"$total",
colour:"$colours"
},
count: {
$sum: {$cond:[{$ifNull:["$colours",false]},1,0]}
}
}
},
{
$group:{
_id:{
description:"$_id.description",
total:"$_id.total"
},
colours: {
$push: {
$cond: [{$gt:["$count",0]},
{
"_id":"$_id.colour",
total:"$count"
},
"$$REMOVE"
]
}
}
}
},
{
$project: {
_id:0,
description: "$_id.description",
total: "$_id.total",
colours: 1
}
}
]);
**Edit In your answer, you are missing the Yellows for Red and Dog because you are taking the first item from $result with $arrayElemAt: ["$result.description", 0].
If description is a colour, do you also want to include the counts for itself in colours?
Never mind, you've updated the answer

How to filter string arrays within documents in mongodb?

I have multiple mongodb documents which looks like this
{
"_id": "001",
"car_description": "Audi",
"sales": [
"India/Mumbai",
"India/Delhi",
"India/Chennai",
"India/Kolkata",
"US/NYC",
"US/SF"]
},
{
"_id": "002",
"car_description": "BMW",
"sales": [
"India/Mumbai",
"India/Delhi",
"India/Chennai",
"India/Kolkata",
"US/NYC",
"US/SF"]
}
I am trying to get the car_description and sales which happened in India.
Final output should be something like this.
{
{
"car_description": "Audi",
"sales": [
"India/Mumbai",
"India/Delhi",
"India/Chennai",
"India/Kolkata"]
},
{
"car_description": "BMW",
"sales": [
"India/Mumbai",
"India/Delhi",
"India/Chennai",
"India/Kolkata"]
}
}
EDIT : I tried using this but this does not filter out the sales. Instead it gives an error saying "Unrecognized expression '$regexMatch"
db.collection.aggregate([
{
"$project": {
"car_description": 1,
"sales": 1,
"sales": {
"$filter": {
"input": "$sales",
"as": "sale",
"cond": {
$regexMatch: {
input: "$$sale",
regex: "India",
options: "i"
}
}
}
}
}
}
])
Check out this snippet to see if it fulfills your need:
db.sales.find({
"sales": {
$elemMatch: {
$regex: "^India\/",
$options: "i"
}
}
},
{
"car_description": 1,
"sales": 1
})
https://mongoplayground.net/p/61sqeMXU_yg
EDIT: If you'd like to filter out all regions which are not matched by the regex expression you could try the following aggregate query:
MongoDB >= 4.1.11
db.sales.aggregate([
{
$match: {
"sales": {
$elemMatch: {
$regex: "^India\/",
$options: "i"
}
}
}
},
{
$set: {
sales: {
$filter: {
input: "$sales",
as: "sale",
cond: {
$regexMatch: {
input: "$$sale",
regex: "^India\/",
options: "i"
}
}
}
}
}
}
])
https://mongoplayground.net/p/NGDSVQeXtRI
Older MongoDB releases
db.collection.aggregate([
{
$match: {
"sales": {
$elemMatch: {
$regex: "^India\/",
$options: "i"
}
}
}
},
{
$set: {
sales: {
$filter: {
input: "$sales",
as: "sale",
cond: {
$eq: [
{
$toLower: {
$arrayElemAt: [
{
$split: [
"$$sale",
"/"
]
},
0
]
}
},
"india"
]
}
}
}
}
}
])
https://mongoplayground.net/p/19TARSVTaMN

$group inner array values without $unwind

I want to group objects in the array by same value for specified field and produce a count.
I have the following mongodb document (non-relevant fields are not present).
{
arrayField: [
{ fieldA: value1, ...otherFields },
{ fieldA: value2, ...otherFields },
{ fieldA: value2, ...otherFields }
],
...otherFields
}
The following is what I want.
{
arrayField: [
{ fieldA: value1, ...otherFields },
{ fieldA: value2, ...otherFields },
{ fieldA: value2, ...otherFields }
],
newArrayField: [
{ fieldA: value1, count: 1 },
{ fieldA: value2, count: 2 },
],
...otherFields
}
Here I grouped embedded documents by fieldA.
I know how to do it with unwind and 2 group stages the following way. (irrelevant stages are ommited)
Concrete example
// document structure
{
_id: ObjectId(...),
type: "test",
results: [
{ choice: "a" },
{ choice: "b" },
{ choice: "a" }
]
}
db.test.aggregate([
{ $match: {} },
{
$unwind: {
path: "$results",
preserveNullAndEmptyArrays: true
}
},
{
$group: {
_id: {
_id: "$_id",
type: "$type",
choice: "$results.choice",
},
count: { $sum: 1 }
}
},
{
$group: {
_id: {
_id: "$_id._id",
type: "$_id.type",
result: "$results.choice",
},
groupedResults: { $push: { count: "$count", choice: "$_id.choice" } }
}
}
])
You can use below aggregation
db.test.aggregate([
{ "$addFields": {
"newArrayField": {
"$map": {
"input": { "$setUnion": ["$arrayField.fieldA"] },
"as": "m",
"in": {
"fieldA": "$$m",
"count": {
"$size": {
"$filter": {
"input": "$arrayField",
"as": "d",
"cond": { "$eq": ["$$d.fieldA", "$$m"] }
}
}
}
}
}
}
}}
])
The below adds a new array field, which is generated by:
Using $setUnion to get unique set of array items, with inner $map to
extract only the choice field
Using $map on the unique set of items,
with inner $reduce on the original array, to sum all items where
choice matches
Pipeline:
db.test.aggregate([{
$addFields: {
newArrayField: {
$map: {
input: {
$setUnion: [{
$map: {
input: "$results",
in: { choice: "$$this.choice" }
}
}
]
},
as: "i",
in: {
choice: '$$i.choice',
count: {
$reduce: {
input: "$results",
initialValue: 0,
in: {
$sum: ["$$value", { $cond: [ { $eq: [ "$$this.choice", "$$i.choice" ] }, 1, 0 ] }]
}
}
}
}
}
}
}
}])
The $reduce will iterate over the results array n times, where n is the number of unique values of choice, so the performance will depend on that.