Mongo Query to fetch distinct nested documents - mongodb

I need to fetch distinct nested documents.
Please find the sample document:
{
"propertyId": 1001820437,
"date": ISODate("2020-07-17T00:00:00.000Z"),
"HList":[
{
"productId": 123,
"name": "Dubai",
"tsh": true
}
],
"PList":[
{
"productId": 123,
"name": "Dubai",
"tsh": false
},
{
"productId": 234,
"name": "India",
"tsh": true
}
],
"CList":[
{
"productId": 234,
"name": "India",
"tsh": false
}
]
}
Expected result is:
{
"produts":[
{
"productId": 123,
"name": "Dubai"
},
{
"productId": 234,
"name": "India"
}
]
}
I tried with this query:
db.property.aggregate([
{
$match: {
"propertyId": 1001820437,
"date": ISODate("2020-07-17T00:00:00.000Z")
}
},
{
"$project": {
"_id": 0,
"unique": {
"$filter": {
"input": {
"$setDifference": [
{
"$concatArrays": [
"$HList.productId",
"$PList.productId",
"$CList.productId"
]
},
[]
]
},
"cond": {
"$ne": [ "$$this", "" ]
}
}
}
}
}
]);
Is $setDifference aggregation is correct choice here?
My query returns only unique product ids but i need a productId with name.
Could someone help me to solve this?
Thanks in advance

You can use $projectfirst to get rid of tsh field and then run $setUnion which ignores duplicated entries:
db.collection.aggregate([
{
$project: {
"HList.tsh": 0,
"PList.tsh": 0,
"CList.tsh": 0,
}
},
{
$project: {
products: {
$setUnion: [ "$HList", "$PList", "$CList" ]
}
}
}
])
Mongo Playground

The following two aggregations return the expected and same result (you can use any of the two):
db.collection.aggregate( [
{
$project: {
_id: 0,
products: {
$reduce: {
input: { $setUnion: [ "$HList", "$PList", "$CList" ] },
initialValue: [],
in: {
$setUnion: [ "$$value", [ { productId: "$$this.productId", name: "$$this.name" } ] ]
}
}
}
}
}
] )
This one is little verbose:
db.collection.aggregate( [
{
$project: { list: { $setUnion: [ "$HList", "$PList", "$CList" ] } }
},
{
$unwind: "$list"
},
{
$group: {
_id: null,
products: { $addToSet: { "productId": "$list.productId", "name": "$list.name" } }
}
},
{
$project: { _id: 0 }
}
] )

db.collection.aggregate([
{
$match: {
"propertyId": 1001820437,
"date": ISODate("2020-07-17T00:00:00.000Z")
}
},
{
$project: {
products: {
$filter: {
input: { "$setUnion" : ["$CList", "$HList", "$PList"] },
as: 'product',
cond: {}
}
}
}
},
{
$project: {
"_id":0,
"products.tsh": 1,
"products.name": 1,
}
},
])

Related

Mongodb Aggregate Filter Array Of Array Of Array

We would like to filter SKU's List which has verificationData data and differenceInStock difference greater than or Less than 0
Here is an example Data Set.
[
{
"_id": "636e0beaa13ef73324e613f0",
"status": "ACTIVE",
"inventory": 132,
"parentCategory": [
"Salt"
],
"title": "Aashirvaad MRP: 28Rs Salt 27 kg Bopp Bag (Set of 1 kg x 27)",
"createdAt": "2022-11-11T08:46:34.950Z",
"updatedAt": "2022-11-24T17:43:27.361Z",
"__v": 3,
"verificationData": [
{
"_id": "637c57ebbe783a9a138fc2d3",
"verificationDate": "2022-11-22T05:02:35.155Z",
"items": {
"listingId": "636e0beaa13ef73324e613f0",
"phyiscalVerification": [
{
"verifiedBy": "634534e72ef6462fcb681a39",
"closingStock": 178,
"phyiscalStock": 178,
"differenceInStock": 0,
"verifiedAt": "2022-11-22T10:19:38.388Z",
"_id": "637ca23abe783a9a1394f402"
}
],
"_id": "637ca23abe783a9a1394f401"
},
"yearMonthDayUTC": "2022-11-22"
},
{
"_id": "637d9b65be783a9a13998726",
"verificationDate": "2022-11-23T04:02:45.804Z",
"items": {
"listingId": "636e0beaa13ef73324e613f0",
"phyiscalVerification": [
{
"verifiedBy": "634534e72ef6462fcb681a39",
"closingStock": 161,
"phyiscalStock": 167,
"differenceInStock": 6,
"verifiedAt": "2022-11-23T09:52:36.815Z",
"_id": "637ded64be783a9a13a29d55"
}
],
"_id": "637ded64be783a9a13a29d54"
},
"yearMonthDayUTC": "2022-11-23"
},
{
"_id": "637f0254be783a9a13a94354",
"verificationDate": "2022-11-24T05:34:12.995Z",
"items": {
"listingId": "636e0beaa13ef73324e613f0",
"phyiscalVerification": [
{
"verifiedBy": "634534e72ef6462fcb681a39",
"closingStock": 144,
"phyiscalStock": 146,
"differenceInStock": 2,
"verifiedAt": "2022-11-24T12:02:28.123Z",
"_id": "637f5d54be783a9a13b1039a"
}
],
"_id": "637f5d54be783a9a13b10399"
},
"yearMonthDayUTC": "2022-11-24"
},
{
"_id": "2022-11-25",
"yearMonthDayUTC": "2022-11-25",
"items": null
}
]
},
{
"_id": "62b5c39062ddb963fc64c42d",
"status": "ACTIVE",
"inventory": 10,
"parentCategory": [
"Salt"
],
"finalMeasurementUnit": "kg",
"finalMeasure": "1 kg",
"title": "Marvella Citric Acid Lemon Salt 1 kg Pouch (Set of 500 gm x 2)",
"createdAt": "2022-06-24T14:00:49.052Z",
"updatedAt": "2022-11-21T11:04:21.643Z",
"__v": 2,
"verificationData": [
{
"_id": "2022-11-22",
"yearMonthDayUTC": "2022-11-22",
"items": null
},
{
"_id": "2022-11-23",
"yearMonthDayUTC": "2022-11-23",
"items": null
},
{
"_id": "2022-11-24",
"yearMonthDayUTC": "2022-11-24",
"items": null
},
{
"_id": "2022-11-25",
"yearMonthDayUTC": "2022-11-25",
"items": null
}
]
}
]
This could have array of 100+ SKU's
Our Aggregate Functions is as Follows
let reqData = await userListing.aggregate([
{
$match: {
warehouseId: { $eq: ObjectId(warehouseId) },
parentCategory: { $in: catList },
isWarehouseListing: { $eq: true },
isBlocked: { $ne: true },
isArchived: { $ne: true },
},
},
{ $sort: { whAddedAt: -1 } },
{
$lookup: {
from: "listingstockverifications",
let: { listId: "$_id" },
pipeline: [
{
$match: {
verificationDate: {
$gte: newFromDate,
$lt: newToDate,
},
},
},
{
$project: {
verificationDate: 1,
items: {
$filter: {
input: "$items",
cond: {
$and: [
/* {
"$$this.phyiscalVerification": {
$filter: {
input: "$$this.phyiscalVerification",
as: "psitem",
cond: { $gt: [ "$$psitem.differenceInStock", 0 ] },
},
},
}, */
{
$eq: ["$$this.listingId", "$$listId"],
},
],
},
},
},
yearMonthDayUTC: {
$dateToString: {
format: "%Y-%m-%d",
date: "$verificationDate",
},
},
},
},
{ $unwind: "$items" },
],
as: "stockVerification",
},
},
{
$addFields: {
verificationData: {
$map: {
input: dummyArray,
as: "date",
in: {
$let: {
vars: {
dateIndex: {
$indexOfArray: [
"$stockVerification.yearMonthDayUTC",
"$$date",
],
},
},
in: {
$cond: {
if: { $ne: ["$$dateIndex", -1] },
then: {
$arrayElemAt: ["$stockVerification", "$$dateIndex"],
},
else: {
_id: "$$date",
yearMonthDayUTC: "$$date",
items: null,
},
},
},
},
},
},
},
},
},
{
$project: {
stockVerification: 0,
},
},
]);
At Last now we would like to filter the SKU List the which has following Data
verificationData[].items.phyiscalVerification[].differenceInStock is Greater than or Less than 0
Expected Output in the following Exmaple would be 1st SKUs
as 2nd SKU does not have any Item Data
and even if in 3rd SKU if we got Item Data but should match the following condition
verificationData[].items.phyiscalVerification[].differenceInStock is Greater than or Less than 0
Thank you for taking your time to read and support.
You can add these two following stages to your aggregation, The idea is simple - just filter out all subdocuments that do not match the condition.
Because of the nested structure it's just not the sexiest of pipelines but it will suffice.
db.collection.aggregate([
{
$match: {
$or: [
{
"verificationData.items.phyiscalVerification.differenceInStock": {
$gt: 0
}
},
{
"verificationData.items.phyiscalVerification.differenceInStock": {
$lt: 0
}
}
]
}
},
{
$addFields: {
verificationData: {
$filter: {
input: {
$map: {
input: {
$filter: {
input: "$verificationData",
as: "verification",
cond: {
$ne: [
"$$verification.items",
null
]
}
}
},
as: "top",
in: {
$mergeObjects: [
"$$top",
{
"items": {
"$mergeObjects": [
"$$top.items",
{
phyiscalVerification: {
$filter: {
input: "$$top.items.phyiscalVerification",
as: "pshycical",
cond: {
$ne: [
"$$pshycical.differenceInStock",
0
]
}
}
}
}
]
}
}
]
}
}
},
cond: {
$gt: [
{
$size: "$$this.items.phyiscalVerification"
},
0
]
}
}
}
}
}
])
Mongo Playground

MongoDB - Lookup match with condition array of object with string

I have two collections "datasets" and "users".
I tried to lookup datasets.assignedTo = users.id that's working fine. Also, I want to match the field of datasets.firstBillable >= users.prices.beginDate date field are matched to get the current index price value. And also check users.prices.endDate is less than or equal to users.prices.beginDate.
For example:
cgPrices: 45
https://mongoplayground.net/p/YQps9EozlAL
Collections:
db={
users: [
{
id: 1,
name: "Aravinth",
prices: [
{
beginDate: "2022-08-24T07:29:01.639Z",
endDate: "2022-08-31T07:29:01.639Z",
price: 45
}
]
},
{
id: 2,
name: "Raja",
prices: [
{
beginDate: "2022-07-25T07:29:01.639Z",
endDate: "2022-07-30T07:29:01.639Z",
price: 55
}
]
}
],
datasets: [
{
color: "braun, rose gold",
firstBillable: "2022-08-24T07:29:01.639Z",
assignedTo: 1
},
{
color: "beige, silber",
firstBillable: "2022-07-25T07:29:01.639Z",
assignedTo: 2
}
]
}
My current implementation:
db.datasets.aggregate([
{
"$lookup": {
"from": "users",
"as": "details",
let: {
assigned_to: "$assignedTo",
first_billable: "$firstBillable"
},
pipeline: [
{
"$match": {
$expr: {
"$and": [
{
"$eq": [
"$id",
"$$assigned_to"
]
},
{
"$gte": [
"$first_billable",
"$details.prices.beginDate"
]
},
{
"$lte": [
"$first_billable",
"$details.prices.endDate"
]
}
]
}
}
}
]
}
},
{
"$addFields": {
"details": 0,
"cg": {
$first: {
"$first": "$details.prices.price"
}
}
}
}
])
Output i needed:
[
{
"_id": ObjectId("5a934e000102030405000000"),
"assignedTo": 1,
"cg": 45,
"color": "braun, rose gold",
"details": 0,
"firstBillable": "2022-08-24T07:29:01.639Z"
},
{
"_id": ObjectId("5a934e000102030405000001"),
"assignedTo": 2,
"cg": 55,
"color": "beige, silber",
"details": 0,
"firstBillable": "2022-07-25T07:29:01.639Z"
}
]
https://mongoplayground.net/p/YQps9EozlAL
Concerns:
You should compare the date as Date instead of string, hence you are required to convert the date strings to Date before comparing.
In users collection, prices is an array. You need to deconstruct the array to multiple documents first before compare the date fields in price.
The query should be:
db.datasets.aggregate([
{
"$lookup": {
"from": "users",
"as": "details",
let: {
assigned_to: "$assignedTo",
first_billable: {
$toDate: "$firstBillable"
}
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$id",
"$$assigned_to"
]
}
}
},
{
$unwind: "$prices"
},
{
"$match": {
$expr: {
"$and": [
{
"$gte": [
"$$first_billable",
{
$toDate: "$prices.beginDate"
}
]
},
{
"$lte": [
"$$first_billable",
{
$toDate: "$prices.endDate"
}
]
}
]
}
}
}
]
}
},
{
"$addFields": {
"details": 0,
"cg": {
$first: "$details.prices.price"
}
}
}
])
Demo # Mongo Playground

MongoDB : group and count users by gender, civilStatus and professionalCategory

I have a collection of users, each user has a profile. I want to implement a query to make statistics on users.
This is my collection.
[
{
"_id": ObjectId("61d2db0d273a9076d630697b"),
"state": "VALIDATED",
"phone": "xxx",
"civilStatus": "SINGLE",
"gender": "MALE",
"professionalCategory": "STUDENT"
}
]
I want the result to contain an array of all genders of users in the database, and the number of users with each gender. same for civilStatus and professionalCategories
This is the result i am looking for :
{
"total": 2000
"validated": 1800,
"genders": [
{
"value": "MALE",
"count": 1200
},
{
"value": "FEMALE",
"count": 600
}
],
"civilStatus": [
{
"value": "SINGLE",
"count": "300"
}
...
],
"professionalCategories": [
{
"value": "STUDENT",
"count": "250"
}
...
]
}
I implemented the query, but I still have a few things that I don't know how to do.
db.getCollection("users").aggregate([
{
$group: {
_id: null,
validated: {
$sum: {
$cond: {
if: { $eq: ["$state", "VALIDATED"] },
then: 1,
else: 0
}
}
},
genders: {
$push: "$gender"
},
civilStatus: {
$push: "$civilStatus"
},
professionalCategories: {
$push: "$professionalCategory"
}
}
}
])
This is the result of this query :
{
"total": 2000
"validated": 1800,
"genders": [
"MALE",
"MALE",
"FEMALE",
"MALE",
"FEMALE",
"FEMALE"
...
],
"civilStatus": [
"SINGLE",
"MARIED",
"SINGLE",
...
],
"professionalCategories": [
"STUDENT",
"WORKER",
"RETIRED"
...
]
}
I miss how to group each gender, civil Status and professional Category and calculate the number of users for each one.
I also tried this query, but I don't know how to complete the "count" field for each item of the array :
db.getCollection("users").aggregate([
{
$group: {
_id: null,
validated: {
$sum: {
$cond: {
if: { $eq: ["$state", "VALIDATED"] },
then: 1,
else: 0
}
}
},
genders: {
$addToSet: {
value: "$gender",
count: {
//
}
}
},
civilStatus: {
$addToSet: {
value: "$civilStatus",
count: {
//
}
}
},
professionalCategories: {
$addToSet: {
value: "$professionalCategory",
count: {
//
}
}
},
}
}
])
if the query was to treat only one field, for example gender. it would have been easier with "unwind". but here I have 3 fields.
can someone help me please?
You can use following aggregation
Here is the code
db.collection.aggregate([
{
"$facet": {
"genders": [
{
"$group": {
"_id": "$gender",
"total": { $sum: 1 }
}
}
],
"civilStatus": [
{
"$group": {
"_id": "$civilStatus",
"total": { $sum: 1 }
}
}
],
"professionalCategory": [
{
"$group": {
"_id": "$professionalCategory",
"total": { $sum: 1 }
}
}
],
"validated": [
{
"$group": {
"_id": "$state",
"total": { "$sum": 1 }
}
}
]
}
},
{
$set: {
validated: {
"$filter": {
"input": "$validated",
"cond": {
"$eq": [ "$$this._id", "VALIDATED" ]
}
}
}
}
},
{
$set: {
validated: {
"$ifNull": [
{
"$arrayElemAt": [ "$validated", 0 ]
},
0
]
}
}
},
{
$set: { validated: "$validated.total" }
}
])
Working Mongo playground

How to combine mongodb original output of query with some new fields?

Collection:
[
{
"name": "device1",
"type": "a",
"para": {
"number": 3
}
},
{
"name": "device2",
"type": "b",
"additional": "c",
"para": {
"number": 1
}
}
]
My query:
db.collection.aggregate([
{
"$addFields": {
"arrayofkeyvalue": {
"$objectToArray": "$$ROOT"
}
}
},
{
"$unwind": "$arrayofkeyvalue"
},
{
"$group": {
"_id": null,
"allkeys": {
"$addToSet": "$arrayofkeyvalue.k"
}
}
}
])
The output currently:
[
{
"_id": null,
"allkeys": [
"additional",
"_id",
"para",
"type",
"name"
]
}
]
Detail see Playground
What I want to do is add a new column which includes all of top key of the mongodb query output, exclude "para". And then combine it with the old collection to form a new json.
Is it possible?
The expected result:
{
"column": [{"prop": "name"}, {"prop": "type"}, {"prop": "additional"}],
"columnData": [
{
"name": "device1",
"type": "a",
"para": {
"number": 3
}
},
{
"name": "device2",
"type": "b",
"additional": "c",
"para": {
"number": 1
}
}
]
}
You have the right general idea in mind, here's how I would do it by utilizing operators like $filter, $map and $reduce to manipulate the objects structure.
I separated the aggregation into 3 parts for readability but you can just merge stage 2 and 3 if you wish.
db.collection.aggregate([
{
"$group": {
"_id": null,
columnData: {
$push: "$$ROOT"
},
"keys": {
"$push": {
$map: {
input: {
"$objectToArray": "$$ROOT"
},
as: "field",
in: "$$field.k"
}
}
}
}
},
{
"$addFields": {
unionedKeys: {
$filter: {
input: {
$reduce: {
input: "$keys",
initialValue: [],
in: {
"$setUnion": [
"$$this",
"$$value"
]
}
}
},
as: "item",
cond: {
$not: {
"$setIsSubset": [
[
"$$item"
],
[
"_id",
"para"
]
]
}
}
}
}
}
},
{
$project: {
_id: 0,
columnData: 1,
column: {
$map: {
input: "$unionedKeys",
as: "key",
in: {
prop: "$$key"
}
}
}
}
}
])
Mongo Playground

How to use $mergeObjects to merge arrays corresponding to existing groups?

I'm trying to merge two arrays in my aggregation pipeline. After performing $facet, my MongoDB document has this format:
{
"final": [
{
"key": "TP-1",
"status_map": [
{ "status": "Closed", "final": [ "a", "b"]},
{ "status": "Done", "final": ["c","d" ] }
]
},
{
"key": "TP-2",
"status_map": [
{ "status": "Closed", "final": [ "x","y"] }
]
}
],
"start": [
{
"key": "TP-1",
"status_map": [
{ "status": "Closed", "start": [ "h"]},
{ "status": "Done", "start": ["a"]}
]
},
{
"key": "TP-2",
"status_map": [{ "status": "Done", "start": ["l","m"]}
]
}
]
}
Expected Output:
I need to merge final and start array corresponding to two groups:
Based on key and then
Based on status
{
"data": [
{
"key": "TP-1",
"status_map": [
{ "status": "Closed","final": ["a","b"],"start":["h"]},
{ "status": "Done","final": ["c","d"],"start":["a"]}
]
},
{
"key": "TP-2",
"status_map": [
{ "status": "Closed", "final":[ "x","y"],"start": []},
{ "status": "Done", "final": [ ],"start": [ "l","m"]}
]
}
]
}
How to achieve this use case?
There are several ways to approach this, not necessarily with $mergeObjects. But since you mentioned $mergeObjects this is one that uses it:
Note that, with this approach, we are merging objects of the same key and status, the values in the arrays will not get concatenated if the same key exists for multiple documents, The arrays will get replaced instead.
db.collection.aggregate([
{
$project: {
all: { $concatArrays: ["$final","$start"] }
}
},
{
$unwind: "$all"
},
{
$unwind: "$all.status_map"
},
{
$group: {
_id: {
_id: "$_id", // keep _id in $group to apply the group for each document, otherwise if you want to apply group on all documents, omit this
key: "$all.key",
status: "$all.status_map.status"
},
status_map: { $mergeObjects: "$$ROOT.all.status_map" }
}
},
{ // some data don't have start or end at all, we have to set a default empty array
$addFields: { // you can skip this stage if you allow data without start and final keys
"status_map.start": { $ifNull: ["$status_map.start", []] },
"status_map.final": { $ifNull: ["$status_map.final", []] }
}
},
{
$group: {
_id: { _id: "$_id._id", key: "$_id.key" },
key: { $first: "$_id.key" },
status_map: { $push: "$status_map" }
}
}
])
Mongo Playground
With no assumptions (for example for both keys to always appear) my strategy was to concat both arrays, unwind and finally group by the key.
db.collection.aggregate([
{
$project: {
concat: {
$concatArrays: [
"$final",
"$start"
]
}
}
},
{
$unwind: "$concat"
},
{
$unwind: "$concat.status_map"
},
{
$group: {
_id: {
k: "$concat.key",
status: "$concat.status_map.status"
},
final: {
$push: "$concat.status_map.final"
},
start: {
$push: "$concat.status_map.start"
}
}
},
{
$group: {
_id: "$_id.k",
status_map: {
$push: {
status: "$_id.status",
final: "$final",
start: "$start"
}
}
}
},
{
$project: {
key: "$_id",
status_map: 1,
_id: 0
}
}
])
Mongo Playground
Adding to #Tom Slabbaert's answer,
Mongo Playground
Here, final and start array is of format array of array. But It has to be simply an array.
It can be achieved by using $unwind on status_map and $reduce on status_map.final and status_map.start arrays.
Final query:
db.collection.aggregate([
{
$project: {
concat: {
$concatArrays: [
"$final",
"$start"
]
}
}
},
{
$unwind: "$concat"
},
{
$unwind: "$concat.status_map"
},
{
$group: {
_id: {
k: "$concat.key",
status: "$concat.status_map.status"
},
final: {
$push: "$concat.status_map.final"
},
start: {
$push: "$concat.status_map.start"
}
}
},
{
$group: {
_id: "$_id.k",
status_map: {
$push: {
status: "$_id.status",
final: "$final",
start: "$start"
}
}
}
},
{
$project: {
key: "$_id",
status_map: 1,
_id: 0
}
},
{
$unwind: "$status_map"
},
{
$project: {
key: 1,
"status_map.status": 1,
final: {
$reduce: {
input: "$status_map.final",
initialValue: [],
in: {
$concatArrays: [
"$$value",
"$$this"
]
}
}
},
start: {
$reduce: {
input: "$status_map.start",
initialValue: [],
in: {
$concatArrays: [
"$$value",
"$$this"
]
}
}
}
}
},
{
$group: {
_id: "$key",
status_map: {
$push: {
status: "$status_map.status",
final: "$final",
start: "$start"
}
}
}
}
])
Mongo Playground