I trying to build an aggregation quarry in MoongoDb that will merge arrays from
2 different collection (one of the collections is of type TTL). And I facing with 2 issues that I can’t resolve.
First Issue:
I would like to merge the TakenSeats fields of my temp collations and permanent collection and set the result instead of my correct TakenSeats field, Using my aggregation in the bottom i manage to merge the arrays with the $push operator, But I cant replace the result field with the TakenSeats field that is in my permanent document.
Second Issue:
In case that I don’t have any documents in my temp collection, how can I still receive the document from the permanent one?
Sample of document in the permanent collection: (extracting data from one document)
{
"_id" : ObjectId("5b6b656818883ec018d1542d"),
"showsHall" : [
ObjectId("5b64cb758ad5f81a6cb7e6ae")
],
"movie" : [
ObjectId("5b6b614218883ec018d15428")
],
"takenSeats" : [
{
"id" : 11
},
{
"id" : 12
}
],
"showDate" : "8/14/2018",
"showStartTime" : "3:00 PM",
"showEndTime" : "5:00 PM",
"creteDate" : ISODate("2018-08-08T21:49:28.020Z"),
"__v" : 0
}
From the TTL collection: (extracting data from multiple documents)
{
"_id" : ObjectId("5b6f35023f64851baa70c61b"),
"createdAt" : ISODate("2018-08-11T19:12:02.951Z"),
"showId" : [
ObjectId("5b6b656818883ec018d1542d")
],
"takenSeats" : [
{
"id" : 22
},
{
"id" : 25
}
]
}
This is the aggregation that I used:
db.getCollection('shows').aggregate([
{ $match: { _id: ObjectId("5b6b656818883ec018d1542d") } },
{
$lookup: {
from: "temp",
localField: "_id",
foreignField: "showId",
as: "fromItems"
}
},
{ $unwind: "$fromItems" },
{ "$project": {"takenSeats": { "$setUnion": ["$takenSeats", "$fromItems.takenSeats"]}, _id: 1, showsHall: 1, movie: 1, takenSeats: 1 , showDate: 1, showStartTime: 1, showEndTime: 1 }},
{$unwind:"$takenSeats"},
{$group:{_id: "$_id", takenSeats: {$push : "$takenSeats"} }},
])
Result:
[Edit]
I manage to maintain my original data with $first operator.
But now i cant resolve issue no 2 (prevent result if null), I tried to use preserveNullAndEmptyArrays
in both of the unwind stages but the result is that it pushes an empty array.
My wanted result is that it should push to a new array only if there is values to push
This is my aggregation :
db.getCollection('shows').aggregate([
{ $match: { _id: ObjectId("5b6b656818883ec018d1542d") } },
{
$lookup: {
from: "temp",
localField: "_id",
foreignField: "showId",
as: "fromItems"
}
},
{ $unwind:{path:"$fromItems" ,preserveNullAndEmptyArrays:true}},
{ "$project": {"takenSeats": { "$setUnion": ["$takenSeats", "$fromItems.takenSeats"]}, _id: 1, showsHall: 1, movie: 1, showDate: 1, showStartTime: 1, showEndTime: 1 }},
{$unwind:{path:"$takenSeats" ,preserveNullAndEmptyArrays:true}},
,
{$group:{
_id: "$_id",
showsHall : { $first: '$showsHall' },
movie : { $first: '$movie' },
showDate : { $first: '$showDate' },
showStartTime : { $first: '$showStartTime' },
showEndTime : { $first: '$showEndTime' },
takenSeats: {$push : "$takenSeats"}
}
}
])
This is the result that i getting if there is no documents in the temp collection
{
"_id" : ObjectId("5b6b656818883ec018d1542d"),
"showsHall" : [
ObjectId("5b64cb758ad5f81a6cb7e6ae")
],
"movie" : [
ObjectId("5b6b614218883ec018d15428")
],
"showDate" : "8/14/2018",
"showStartTime" : "3:00 PM",
"showEndTime" : "5:00 PM",
"takenSeats" : [
null
]
}
Here Please add ifNull Condition for solution 2
db.getCollection('shows').aggregate([
{ $match: { _id: ObjectId("5b6b656818883ec018d1542d") } },
{
$lookup: {
from: "tempShows",
localField: "_id",
foreignField: "showId",
as: "fromItems"
}
},
{ $unwind:{path:"$fromItems" ,preserveNullAndEmptyArrays:true}},
{ "$project": {"takenSeats": { $ifNull: [{ "$setUnion": ["$takenSeats", "$fromItems.takenSeats"]}, '$takenSeats'] } ,_id: 1, showsHall: 1, movie: 1, showDate: 1, showStartTime: 1, showEndTime: 1 }},
{$unwind:{path:"$takenSeats" ,preserveNullAndEmptyArrays:true}},
{$group:{
_id: "$_id",
showsHall : { $first: '$showsHall' },
movie : { $first: '$movie' },
showDate : { $first: '$showDate' },
showStartTime : { $first: '$showStartTime' },
showEndTime : { $first: '$showEndTime' },
takenSeats: {$push : "$takenSeats"}
}
}
])
Related
In here i'm using $lookup to to a left join from other collections, the query works fine but when some records missing values it returns
errmsg : $in requires an array as a second argument, found: null
Heres the querying document structure :
{
"no" : "2020921008981",
"sale" : {
"soldItems" : [
{
"itemId" : "5b55ac7f0550de00210a3b24",
},
{
"itemId" : "5b55ac7f0550de00215584re",
}
],
"bills" : [
{
"billNo" : "2020921053467",
"insurancePlanId" : "160",
},
{
"billNo" : "2020921053467",
"insurancePlanId" : "170",
}
],
"visitIds" : [
5b55ac7f0550de00210a3b24, 5b55ac7f0550de00210a3b24
]
}
}
the query :
db.case.aggregate([
{
$lookup: {
from: "insurance",
let: { ipids: "$sale.bill.insurancePlanId" },
pipeline: [
{
$unwind: "$coveragePlans"
},
{
$match: { $expr: { $in: ["$coveragePlans._id", "$$ipids"] } }
},
{
$project: { _id: 0, name: 1 }
}
],
as: "insurances"
}
},
{
$lookup: {
from: "item",
let: { iid: "$salesOrder.purchaseItems.itemRefId" },
pipeline: [
{
$match: {
$expr: {
$in: ["$_id", {
$map: {
input: "$$iid",
in: { $toObjectId: "$$this" }
}
}
]
}
}
}
],
as: "items"
}
}
])
insurance collection :
{
"_id" : ObjectId("5b55aca20550de00210a6d25"),
"name" : "HIJKL"
"coveragePlans" : [
{
"_id" : "160",
"name" : "UVWZ",
},
{
"_id" : "161",
"name" : "LMNO",
}
]
},
{
"_id" : ObjectId("5b55aca20550de00210a6d25"),
"name" : "WXYZ"
"coveragePlans" : [
{
"_id" : "169",
"name" : "5ABC",
},
{
"_id" : "170",
"name" : "4XYZ",
}
]
}
item collection :
{
"_id" : ObjectId("5b55ac7f0550de00210a3b24"),
"code" : "ABCDE"
},
{
"_id" : ObjectId("5b55ac7f0550de00215584re"),
"code" : "PQRST"
}
How to avoid this and do null checks effectively before pipe-lining into the next stages? Tried with { $match: { "fieldName": { $exists: true, $ne: null } } } but it returns mongo error regarding the format. If its the way to go please mention the stage i should put that.. Thanks in advance
You can use $ifNull operator
let: { ipids: {$ifNull:["$sale.bill.insurancePlanId", [] ]} },
EDIT: To skip empty "$salesOrder.purchaseItems.itemRefId" values
let: { iid: {$filter: {input:"$salesOrder.purchaseItems.itemRefId", cond:{$ne:["$$this", ""]}}} },
You can get around that by not using $in.
It looks like this $map is executed separately for every document in the items collection. If you were to run the map in an $addFields stage, you could used the simple form of lookup to match the added field to _id, which would automagically handle missing, null, and array.
Remove the added field with a $project stage if necessary.
db.case.aggregate([
{$lookup: {
from: "insurance",
let: { ipids: "$sale.bill.insurancePlanId" },
pipeline: [
{$unwind: "$coveragePlans"},
{$match: { $expr: { $in: ["$coveragePlans._id", "$$ipids"] } }},
{$project: { _id: 0, name: 1 }}
],
as: "insurances"
}}
{$addFields:{
matchArray:{$map: {
input: "$$iid",
in: { $toObjectId: "$$this" }
}}
}},
{$lookup: {
from: "item",
localField: "matchArray",
foreignField:"_id",
as: "items"
}},
{$project:{
arrayField: 0
}}
])
Suppose I have multiple documents like these in a collection
Parent document:
{
"_id" : ObjectId("5e86ebd6c2d28863e4e2c920"),
"users" : [],
"name" : "Annual",
"days" : 18,
},
Child documents:
{
"_id" : ObjectId("5e86ec22c2d28863e4e2c921"),
"users" : [
ObjectId("5e58fa20f3bea73c3cb07713"),
ObjectId("5e58fab5f3bea73c3cb07715")
],
"leaveTypeId" : ObjectId("5e86ebd6c2d28863e4e2c920"),
"name" : "Personal",
"days" : 5,
},
{
"_id" : ObjectId("5e86ec22c2d28863e4e2c921"),
"users" : [],
"leaveTypeId" : ObjectId("5e86ebd6c2d28863e4e2c920"),
"name" : "Personal",
"days" : 5,
}
Now I want to build a query like if user found in users array then return name and days from child document otherwise it should return name and days from parent document.
If user_id = ObjectId("5e58fa20f3bea73c3cb07713") then the output should be
{
name: 'Personal',
days: 5
}
If user_id = ObjectId("52fff32rax823vnvy3234es12") then the output should be
{
name: 'Annual',
days: 18
}
Try these aggregation queries :
When it's done on Parent Collection :
db.parent.aggregate([
{
$lookup: {
from: "child",
localField: "_id",
foreignField: "leaveTypeId",
as: "child_docs"
}
},
{
$unwind: "$child_docs"
},
{
$project: {
name: {
$cond: [
{ $in: [ObjectId("5e58fa20f3bea73c3cb07713"), "$child_docs.users"] },
"$child_docs.name",
"$name"
]
},
days: {
$cond: [
{ $in: [ObjectId("5e58fa20f3bea73c3cb07713"), "$child_docs.users"] },
"$child_docs.days",
"$days"
]
}
}
}
]);
Test : MongoDB-Playground
When it's done on Child Collection :
db.child.aggregate([
{
$lookup: {
from: "parent",
localField: "leaveTypeId",
foreignField: "_id",
as: "parent_docs"
}
},
{
$unwind: "$parent_docs"
},
{
$project: {
name: {
$cond: [
{ $in: [ObjectId("5e58fa20f3bea73c3cb07713"), "$users"] },
"$name",
"$parent_docs.name"
]
},
days: {
$cond: [
{ $in: [ObjectId("5e58fa20f3bea73c3cb07713"), "$users"] },
"$days",
"$parent_docs.days"
]
}
}
}
]);
Test : MongoDB-Playground
The aggregate was executed.
I got the results using lookup, but I need a sort.
In addition, I want to assign an index to the result value.
CollectionA :
{
"_id" : ObjectId("5a6cf47415621604942386cd"),
"contents" : [
ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA"),
ObjectId("BBBBBBBBBBBBBBBBBBBBBBBB")
],
"name" : "jason"
}
CollectionB :
{
"_id" : ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA")
"title" : "a title",
"date" : 2018-01-02
},
{
"_id" : ObjectId("BBBBBBBBBBBBBBBBBBBBBBBB")
"title" : "a title",
"date" : 2018-01-01
}
Query:
db.getCollection('A').aggregate([
{
$match : { "_id" : ObjectId("5a6cf47415621604942386cd") }
},
{
$lookup : {
from: "B",
localField: "contents",
foreignField: "_id",
as: "item"
}
},
{ $sort: { "item.date" : -1 } }
]);
Want Result:
{
"_id" : ObjectId("5a6cf47415621604942386cd"),
"contents" : [
{
"_id" : ObjectId("BBBBBBBBBBBBBBBBBBBBBBBB")
"title" : "a title",
"date" : 2018-01-01,
"index" : 0
},
{
"_id" : ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA")
"title" : "a title",
"date" : 2018-01-02,
"index" : 1
}],
"name" : "jason"
}
The current problem does not apply to the sort.
And I don't know how to designate an index.
Below Aggregation may you. For your desire result.
db.CollectionA.aggregate([
{
$match: { "_id": ObjectId("5a6cf47415621604942386cd") }
},
{
$lookup: {
from: "CollectionB",
let: { contents: "$contents" },
pipeline: [
{
$match: { $expr: { $in: ["$_id", "$$contents"] } }
},
{ $sort: { date: 1 } }
],
as: "contents"
}
},
{
$project: {
contents: {
$map: {
input: { $range: [0, { $size: "$contents" }, 1 ] },
as: "element",
in: {
$mergeObjects: [
{ index: "$$element" },
{ $arrayElemAt: [ "$contents", "$$element" ]}
]
}
}
}
}
}
])
One way to go about it would be to unwind the array, sort it and then group it back
db.A.aggregate([
{
$match: {
"_id": ObjectId("5a6cf47415621604942386cd")
}
},
{
$lookup: {
from: "B",
localField: "contents",
foreignField: "_id",
as: "item"
}
},
{
$unwind: "$item"
},
{
$sort: {
"item.date": -1
}
},
{
$group: {
_id: "$_id",
contents: {
$push: "$item"
}
}
}
])
Another method is, (this is applicable only if the date field corresponds to the document creation date),
db.A.aggregate([
{
$match: {
"_id": ObjectId("5a6cf47415621604942386cd")
}
},
{
$lookup: {
from: "B",
localField: "contents",
foreignField: "_id",
as: "item"
}
},
{
$sort: {
"item": -1
}
}
])
Basically, this sorts on the basis of _id, and since _id is created using the creation date, it should sort accordingly.
I want to get a list of my second level affiliate users, I've designed the collection like
user
{
"_id" : ObjectId("5a1b9df7bfdbfef2d4f1e9f3"),
"name" : "name 1",
"affKey" : "H1g-CfFxG",
}
{
"_id" : ObjectId("5a1bce5e9a2918f71a9ac4fb"),
"name" : "name 2",
"affKey" : "K1gKJfFxG",
}
affilites
{
"affKey" : "H1g-CfFxG",
"affUsers" : [
ObjectId("5a1bce5e9a2918f71a9ac4fb")
],
}
{
"affKey" : "K1gKJfFxG",
"affUsers" : [
ObjectId("5a1b9e43bfdbfef2d4f1e9f8"),
ObjectId("5a1b9e43bfdbfef2d4f1e911"),
],
}
Here I am saving a new set in affiliate.affKey is based on Users.affKey
Now I want to get a list of my first and second level affiliates, i.e. the list of affiliates of 5a1b9e43bfdbfef2d4f1e9f6 and 5a1bce5e9a2918f71a9ac4fb along with first level affiliate.
expecting the results like
{
first: [first level affiliates] // 1 result
second: [second level affiliates] // 2 results
}
For this situation you have to some steps to get expected result. can follow bellow steps
$lookup the affiliets collection for affKey
Then you have to $lookup the users collection to get the reference user information
After that you have to $lookup again the affiliets collection for second level user affKey
so query can be like bellow
db.users.aggregate([
{
$lookup: {
from: "affilites",
localField: "affKey",
foreignField: "affKey",
as: "affUsers"
}
},
{
$project: {
name: 1,
affKey: 1,
first: {$arrayElemAt: ["$affUsers.affUsers", 0]},
secondLevelUserId: {$arrayElemAt: ["$affUsers.affUsers", 0]}
}
},
{ $unwind: { path: "$secondLevelUserId", "preserveNullAndEmptyArrays": true }},
{
$lookup: {
from: "users",
localField: "secondLevelUserId",
foreignField: "_id",
as: "secondLevelUser"
}
},
{
$project: {
name: 1,
affKey: 1,
first: 1,
secondLevelUser: {$arrayElemAt: ["$secondLevelUser", 0]}
}
},
{
$lookup: {
from: "affilites",
localField: "secondLevelUser.affKey",
foreignField: "affKey",
as: "secondLevelUser"
}
},
{
$project: {
name: 1,
affKey: 1,
first: 1,
second: {$arrayElemAt: ["$secondLevelUser.affUsers", 0]}
}
},
{
$unwind: {
"path": "$second",
"preserveNullAndEmptyArrays": true
}
},
{
$group: {
_id: "$_id",
name: {$first: "$name"},
affKey: {$first: "$affKey"},
first: {$first: "$first"},
second: {$addToSet: "$second"}
}
}
]);
After execute the query you will get result like bellow
first document:
{
"_id" : ObjectId("5a1b9df7bfdbfef2d4f1e9f3"),
"name" : "name 1",
"affKey" : "H1g-CfFxG",
"first" : [
ObjectId("5a1bce5e9a2918f71a9ac4fb")
],
"second" : [
ObjectId("5a1b9e43bfdbfef2d4f1e911"),
ObjectId("5a1b9e43bfdbfef2d4f1e9f8")
]
}
second document
{
"_id" : ObjectId("5a1bce5e9a2918f71a9ac4fb"),
"name" : "name 2",
"affKey" : "K1gKJfFxG",
"first" : [
ObjectId("5a1b9e43bfdbfef2d4f1e9f8"),
ObjectId("5a1b9e43bfdbfef2d4f1e911")
],
"second" : []
}
Consider below Mongo DB 3.2 Document,
Clients Collection Document
{
"_id" : "gcJk4eRRo2WCbgWSL",
"services" : [
"2tLX8ALYfRvbgiurZ",
"wfE5MqgHfu9QKtK7d",
"MEZtABSEeskuRivXJ"
]
}
Also, Services Collection Documents
{ "_id" : "2tLX8ALYfRvbgiurZ", "name" : "GSTR 1" }
{ "_id" : "wfE5MqgHfu9QKtK7d", "name" : "GSTR 2" }
{ "_id" : "MEZtABSEeskuRivXJ", "name" : "GSTR 3" }
Now, the values in services array field in Clients is associated to _id of Services Collection.
Below is the code that I am currently executing,
db.getCollection('Clients').aggregate(
[
{ "$unwind" : { path: "$services"}},
{
"$lookup" : { from: "Services", localField: "services", foreignField: "_id", as: "services" }
},
{ "$unwind" : { path: "$services", preserveNullAndEmptyArrays: true }},
{ "$project" : {
_id : 1, services: '$services.name'
}
}
]
);
Output of Above code execution is,
{ "_id" : "gcJk4eRRo2WCbgWSL", "services" : "GSTR 1" }
{ "_id" : "gcJk4eRRo2WCbgWSL", "services" : "GSTR 2" }
{ "_id" : "gcJk4eRRo2WCbgWSL", "services" : "GSTR 3" }
But Expected output is as below,
{
"_id" : "gcJk4eRRo2WCbgWSL",
"services" : "GSTR 1, GSTR 2, GSTR 3"
}
Any help is highly appreciated.
You can add additional $group by your _id with $push to merge your services into one array.
db.Clients.aggregate(
[
{ "$unwind" : { path: "$services"} },
{
"$lookup" : { from: "Services", localField: "services", foreignField: "_id", as: "services" }
},
{ "$unwind" : { path: "$services", preserveNullAndEmptyArrays: true }},
{ "$project" : {
_id : 1, services: '$services.name'
}
},
{
"$group": {
_id: "$_id",
"services": { "$push": "$services"}
}
}
]
);
Here's how you do that - there is no need for any $unwinds as long as you do not care about the item order inside your string.
db.getCollection('Clients').aggregate([
{
"$lookup": {
from: "Services",
localField: "services",
foreignField: "_id",
as: "services"
}
}, {
"$project": {
"_id" : 1,
"services": {
$substr:
[
{
$reduce: { // transform array of strings into concatenated string
input: '$services.name',
initialValue: '',
in: {
$concat: ['$$value', ', ', '$$this']
}
}
},
2, // remove first two characters as they will be ', '
-1
]
}
}
}])
This, however, can potentially return something like the following document (note the order of the entries in the string):
{
"_id" : "gcJk4eRRo2WCbgWSL",
"services" : "GSTR 1, GSTR 3, GSTR 2"
}
If you need the items inside the string to be sorted you can do it this way:
db.getCollection('Clients').aggregate([
{
"$lookup": {
from: "Services",
localField: "services",
foreignField: "_id",
as: "services"
}
}, {
$unwind: "$services" // flatten the services array
}, {
$sort: {
"services.name": 1 // sort all documents by service name
}
}, {
$group: { // group the everything back into the original structure again
"_id": "$_id", // we want one group per document id
"services": {
$push: "$services" // and all its services in an array - this time reliably sorted!
}
}
}, {
"$project": {
"_id": 1,
"services": {
$substr:
[
{
$reduce: { // transform array of strings into concatenated string
input: '$services.name',
initialValue: '',
in: {
$concat: ['$$value', ', ', '$$this']
}
}
},
2, // remove first two characters as they will be ', '
-1
]
}
}
}])
This way you reliably get the entries sorted by name:
{
"_id" : "gcJk4eRRo2WCbgWSL",
"services" : "GSTR 1, GSTR 2, GSTR 3"
}
You can try using concatArrays.
Like what mickl answered, you can do grouping first and project it to concat the array value.
//previous code
{
"$group": {
_id: "$_id",
"services": { "$push": "$services"}
}
},{ $project: {
_id : "$_id",
"services": { $concatArrays: [ "$services" ] } }
}
hope it helps..