Full Outer join in MongoDB - mongodb

I want to do a Full Outer Join in MongoDB by lookup mongoDB query. Is this possible? Is a Full Outer Join supported by MongoDB by any other alternative?
[Update:]
I want to achieve result from Collection1 & Collection2 as following attachment:
Example: Result Required
In above result column there may be different arithmetic operations and will be further used in calculations.

You can use $unionWith (starting 4.4)
Something like this:
db.c1.aggregate([
{$set: {
mark1: "$marks"
}},
{$unionWith: {
coll: 'c2',
pipeline: [{$set: {mark2: "$marks"}}]
}},
{$group: {
_id: "$name",
result: {
$sum: "$marks"
},
mark1: {$first: {$ifNull: ["$mark1", 0]}},
mark2: {$first: {$ifNull: ["$mark2", 0]}}
}}])

I have named the collections as coll1 and coll2 then just use this query it will give you the required output.
db.getCollection('coll1').aggregate([
{
$facet: {
commonRecords: [{
$lookup: {
from: "coll2",
localField: 'name',
foreignField: 'name',
as: "coll2"
}
},
{
$unwind: {
path: '$coll2',
preserveNullAndEmptyArrays: true
}
}
]
}
},
{
$lookup: {
from: "coll2",
let: {
names: {
$map: {
input: '$commonRecords',
as: 'commonRecord',
in: '$$commonRecord.name'
}
}
},
pipeline: [{
$match: {
$expr: {
$eq: [{
$indexOfArray: ['$$names', '$name']
}, -1]
}
}
}, ],
as: "coll2"
}
},
{
$addFields: {
coll2: {
$map: {
input: '$coll2',
as: 'doc',
in: {
coll2: '$$doc'
}
}
}
}
},
{
$project: {
records: {
$concatArrays: ['$commonRecords', '$coll2']
}
}
},
{
$unwind: '$records'
},
{
$replaceRoot: {
newRoot: '$records'
}
},
{
$project: {
_id: 0,
name: {
$ifNull: ['$name', '$coll2.name']
},
marks1: {
$ifNull: ['$marks', 0]
},
marks2: {
$ifNull: ['$coll2.marks', 0]
}
}
},
{
$addFields: {
result: {
$add: ['$marks1', '$marks2']
}
}
}
])

This is a sample:
{
$lookup:
{
from: [collection to join],
local_Field: [field from the input documents],
foreign_Field: [field from the documents of the "from" collection],
as: [output field]
}
}
show this link

Related

Mongo db transaction query

I'm trying to create a mongodb query to see which invoices are paid or not. I would like to add few things to the outcome like:
virtualAmount : original amount - (sum of all creditnotes)
total paid amount : sum of all transactions where delete is false
paid (true/false) : if virtualAmount - total paid amount is 0
I have created mongo playground:
https://mongoplayground.net/p/0OyK_bOZu9X
Anyone know if this is possible?
How to still have the original object when using group?
Is it also possible to create this result with mongoose?
by $unwind array and $group it.
db.collection.aggregate(
[{
$match: {
_id: '62b46391be7c618aa5c9bf86'
}
}, {
$set: {
'transactions': {
$filter: {
'input': '$transactions',
'as': 'item',
'cond': { $eq: ['$$item.deleted', false] }
}
},
}
}, {
$unwind: {
path: '$transactions'
}
}, {
$group: {
_id: '_id',
'total-paid-amount': { $sum: '$transactions.amount.value'},
//keep to next stage
'creditnote': {$first: '$creditnote'},
'original-amount': {$first: '$amount.value'}
}
}, {
$unwind: {
path: '$creditnote'
}
}, {
$group: {
_id: '_id',
'sum-all-creditnotes': {$sum: '$creditnote.amount.value'},
//keep to next stage
'total-paid-amount': {$first: '$total-paid-amount'},
'original-amount': {$first: '$original-amount'}
}
}, {
$addFields: {
'virtual-amount': {$subtract: ['$original-amount','$sum-all-creditnotes']}
}
}, {
$addFields: {
paid: {$eq: [{$subtract: ['$virtual-amount','$total-paid-mount']},0]}
}
}]
result
{
"_id" : "_id",
"sum-all-creditnotes" : 1000,
"total-paid-amount" : 1000,
"original-amount" : 3370,
"virtual-amount" : 2370,
"paid" : false
}
EDIT
or easy way without group
db.collection.aggregate(
[{
$match: {
_id: '62b46391be7c618aa5c9bf86'
}
}, {
$set: {
'transactions': {
$filter: {
'input': '$transactions',
'as': 'item',
'cond': { $eq: ['$$item.deleted', false] }
}
},
}
}, {
$project:
{
'total-paid-amount': { $sum: '$transactions.amount.value' },
'sum-all-creditnotes': { $sum: '$creditnote.amount.value' },
'original-amount': '$amount.value'
}
}, {
$addFields: {
'virtual-amount': { $subtract: ['$original-amount', '$sum-all-creditnotes'] }
}
}, {
$addFields: {
'paid': { $eq: [{ $subtract: ['$virtual-amount', '$total-paid-mount'] }, 0] }
}
}])

Filter nested array using projection without using unwind

I have my collection1 which holds the _ids of collection2 in projects field as follows:
{
"name": "adafd",
"employeeId": "employeeId",
"locations": [
"ObjectId(adfaldjf)",
"ObjectId(adfaldjf)",
"ObjectId(adfaldjf)",
"ObjectId(adfaldjf)",
"ObjectId(adfaldjf)",
"ObjectId(adfaldjf)"
]
}
collection2 is as follows
"collection2": [
{
"location": "india",
"states": [
{
"stateCode": "TN",
"districts": {
"cities": [
{
"code": 1,
"name": "xxx"
},
{
"code": 4,
"name": "zzz"
},
{
"code": 6,
"name": "yyy"
}
]
}
}
]
}
]
I am trying to filter nested arrays inside collection2 after lookup as follows:
db.collection.aggregate([
{
$lookup: {
from: "collection2",
localField: "locations",
foreignField: "_id",
as: "locations"
}
},
{
$match: {
"name": "adafd",
},
},
{
$project: {
'details': {
$filter: {
input: "$locations",
as: "location",
cond: {
"$eq": ["$$location.states.stateCode", "TN" ]
}
}
}
}
}
]
)
It is returning an empty array for locations.
I modified the project as follows to even filter states inside collection2 array in the projection as follows, but filters are not applying. It is returning all the data inside the states array.
{
$project: {
'details': {
$filter: {
input: "$locations",
as: "location",
cond: {
$filter: {
input: "$location.states",
as: "state",
cond: {
"$eq": ["$$state.stateCode", "TN" ]
}
}
}
}
}
}
}
I have found several solutions regarding this but none worked for me. As I don't want to use unwind. Is there any way to achieve this..?
Note: I don't want to use pipeline inside $lookup as it is not supported by DocumentDB. And also there should be any $unwind and $group in the query.
$match your conditions
$lookup with collection2
$project to filter locations by location name
$unwind deconstruct the locations array
$project to filter states by state code
$unwind deconstruct the states array
$project to filter cities by city code
$unwind deconstruct the cities array
db.collection1.aggregate([
{ $match: { name: "adafd" } },
{
$lookup: {
from: "collection2",
localField: "locations",
foreignField: "_id",
as: "locations"
}
},
{
$project: {
locations: {
$filter: {
input: "$locations",
cond: { $eq: ["$$this.location", "india"] }
}
}
}
},
{ $unwind: "$locations" },
{
$project: {
locations: {
_id: "$locations._id",
location: "$locations.location",
states: {
$filter: {
input: "$locations.states",
cond: { $eq: ["$$this.stateCode", "TN"] }
}
}
}
}
},
{ $unwind: "$locations.states" },
{
$project: {
locations: {
_id: "$locations._id",
location: "$locations.location",
states: {
stateCode: "$locations.states.stateCode",
districts: {
cities: {
$filter: {
input: "$locations.states.districts.cities",
cond: { $eq: ["$$this.code", 1] }
}
}
}
}
}
}
},
{ $unwind: "$locations.states.districts.cities" }
])
Playground
Second option without using $unwind, instead of you can use $arrayElemAt,
db.collection1.aggregate([
{ $match: { name: "adafd" } },
{
$lookup: {
from: "collection2",
localField: "locations",
foreignField: "_id",
as: "locations"
}
},
{
$project: {
locations: {
$arrayElemAt: [
{
$filter: {
input: "$locations",
cond: { $eq: ["$$this.location", "india"] }
}
},
0
]
}
}
},
{
$project: {
locations: {
_id: "$locations._id",
location: "$locations.location",
states: {
$arrayElemAt: [
{
$filter: {
input: "$locations.states",
cond: { $eq: ["$$this.stateCode", "TN"] }
}
},
0
]
}
}
}
},
{
$project: {
locations: {
_id: "$locations._id",
location: "$locations.location",
states: {
stateCode: "$locations.states.stateCode",
districts: {
cities: {
$arrayElemAt: [
{
$filter: {
input: "$locations.states.districts.cities",
cond: { $eq: ["$$this.code", 1] }
}
},
0
]
}
}
}
}
}
}
])
Playground
You might use $map. There are several mistakes in your code. $match needs to be "locations.location":"india" and inside the filter input you need to use $$<var name>
db.collection1.aggregate([
{
$lookup: {
from: "collection2",
localField: "locations",
foreignField: "_id",
as: "locations"
}
},
{
$project: {
"details": {
input: {
$map: {
input: "$locations",
as: "location",
in: {
"_id": "$$location._id",
location: "$$location.location",
states: {
$filter: {
input: "$$location.states",
as: "state",
cond: {
"$eq": [
"$$state.stateCode",
"TN"
]
}
}
}
}
}
}
}
}
}
])
Working Mongo playground
Suppose if you need to eliminate documents which hold empty state, you can easily do with match
You can try it like this Its working
db.getCollection('col').aggregate([
{
$lookup: {
from: "col",
localField: "locations",
foreignField: "_id",
as: "locations"
}
},
{
$project: {
'locations': {
$filter: {
input: "$collection2",
as: "collection",
cond: {
"$eq": ["$$collection.location", "india" ],
}
}
}
}
},
{
$project: {
'details': {
$filter: {
input: "$locations",
as: "location",
cond: {
$filter: {
input: "$$location.states",
as: "state",
cond: {
"$eq": ["$$state.stateCode", "TN" ]
}
}
}
}
}
}
}
])

MongoDB $lookup and $map array of objects

I'm trying to do this for days, but can't find any success
I'm using MongoDB, and I tried to do it with many pipeline steps but I couldn't find a way.
I have a players collection, each player contains an items array
{
"_id": ObjectId("5fba17c1c4566e57fafdcd7e"),
"username": "moshe",
"items": [
{
"_id": ObjectId("5fbb5ac178045a985690b5fd"),
"equipped": false,
"itemId": "5fbb5ab778045a985690b5fc"
}
]
}
I have an items collection where there is more information about each item
in the player items array.
{
"_id": ObjectId("5fbb5ab778045a985690b5fc"),
"name": "Axe",
"damage": 4,
"defense": 6
}
My goal is to have a player document with all the information about the item inside his items array, so it will look like that:
{
"_id": ObjectId("5fba17c1c4566e57fafdcd7e"),
"username": "moshe",
"items": [
{
"_id": ObjectId("5fbb5ac178045a985690b5fd"),
"equipped": false,
"itemId": "5fbb5ab778045a985690b5fc",
"name": "Axe",
"damage": 4,
"defense": 6
}
]
}
$unwind deconstruct items array
$lookup to join items collection, pass itemsId into let after converting it to object id using $toObjectId and pass items object,
$match itemId condition
$mergeObject merge items object and $$ROOT object and replace to root using $replaceRoot
$group reconstruct items array again, group by _id and get first username and construct items array
db.players.aggregate([
{ $unwind: "$items" },
{
$lookup: {
from: "items",
let: {
itemId: { $toObjectId: "$items.itemId" },
items: "$items"
},
pipeline: [
{ $match: { $expr: { $eq: ["$_id", "$$itemId" ] } } },
{ $replaceRoot: { newRoot: { $mergeObjects: ["$$items", "$$ROOT"] } } }
],
as: "items"
}
},
{
$group: {
_id: "$_id",
username: { $first: "$username" },
items: { $push: { $first: "$items" } }
}
}
])
Playground
Second option using $map, and without $unwind,
$addFields for items convert itemId string to object type id using $toObjectId and $map
$lookup to join items collection
$project to show required fields, and merge items array and itemsCollection using $map to iterate loop of items array $filter to get matching itemId and $first to get first object from return result, $mergeObject to merge current object and returned object from $first
db.players.aggregate([
{
$addFields: {
items: {
$map: {
input: "$items",
in: {
$mergeObjects: ["$$this", { itemId: { $toObjectId: "$$this.itemId" } }]
}
}
}
}
},
{
$lookup: {
from: "items",
localField: "items.itemId",
foreignField: "_id",
as: "itemsCollection"
}
},
{
$project: {
username: 1,
items: {
$map: {
input: "$items",
as: "i",
in: {
$mergeObjects: [
"$$i",
{
$first: {
$filter: {
input: "$itemsCollection",
cond: { $eq: ["$$this._id", "$$i.itemId"] }
}
}
}
]
}
}
}
}
}
])
Playground
First I'd strongly suggest that you should store the items.itemId as ObjectId, not strings.
Then another simple solution can be:
db.players.aggregate([
{
$lookup: {
from: "items",
localField: "items.itemId",
foreignField: "_id",
as: "itemsDocuments",
},
},
{
$addFields: {
items: {
$map: {
input: { $zip: { inputs: ["$items", "$itemsDocuments"] } },
in: { $mergeObjects: "$$this" },
},
},
},
},
{ $unset: "itemsDocuments" },
])

How to write a $match on a $lookup document in MongoDB

I have two MongoDB collections that look like this:
Products Specials
---------- ----------
_id _id
name product_id
country zip
price percent_discount
out_of_stock
I'm using GraphQL as well so I wrote up an aggregate pipeline that returns the data in this structure:
specials {
_id
product {
_id
name
country
price
}
zip
percent_discount
out_of_stock
}
This aggregate pipeline I wrote works great and looks like this:
let response = await Specials.aggregate([
{
$lookup: {
from: 'products',
localField: 'product_id',
foreignField: '_id',
as: 'product'
}
},
{
$unwind: '$product'
},
{
$match: {
zip: zip
}
}
])
return response;
Now I'm trying to add a filter into this. The filter should match the name or country in the product collection preferably with regex. So I tried writing something like this but it's yielding over 8000 results when there should be only 2-3:
let response = await Specials.aggregate([
{
$match: { zip: zip }
},
{
$lookup: {
from: "products",
let: { product_id: "$product_id" },
pipeline: [
{
$match: {
$expr: {
$and: [
{
_id: "$$product_id"
},
{
$or: [
{
name: filter
},
{
country: filter
}
]
}
]
}
}
}
],
as: "product"
}
},
{
$unwind: "$product"
}
])
If you're using Mongo version 4.2+ you can use $regexMatch
let response = await Specials.aggregate([
{
$lookup: {
from: "products",
let: { product_id: "$product_id" },
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ["$$product_id", "$_id"]
},
{
$or: [
{
$regexMatch: { input: "$name", regex: filterRegex }
},
{
$regexMatch: { input: "$country", regex: filterRegex }
}
]
}
]
}
}
}
],
as: "product"
}
},
{
$unwind: "$product"
},
{
$match: {
$and: [
{
zip: zip
}
]
}
}
])
return response;
This is the following solution that I was able to get working:
let response = await Specials.aggregate([
{
$lookup: {
from: "products",
let: { product_id: "$product_id" },
pipeline: [
{
$match: {
$and: [
{
$or: [
{
name: { $regex: filter }
},
{
country: { $regex: filter }
}
]
},
{
$expr: {
$eq: ["$$product_id", "$_id"]
}
}
]
}
}
],
as: "product"
}
},
{
$unwind: "$product"
},
{
$match: {
$and: [
{
zip: zip
}
]
}
}
])
return response;
** Updated with the working $regex implementation

how to reduce unnecessary unwind stages from aggregation pipeline

Like if i'm applying many lookup stages in aggregation pipeline and each lookup is followed by an unwind(just to covert into object) first question does it affect query performance? and if yes how to do that in optimised manner
Note: all lookup's will return only one object
For Ex:
xyz.aggregate([
{ $lookup:{ ----}} //first lookup
{$unwind :{----}} //first unwind
{ $lookup:{ ----}} //second lookup
{$unwind :{----}} //second unwind
{ $lookup:{ ----}} //third lookup
{$unwind :{----}} //third unwind
{ $lookup:{ ----}} //fourth lookup
{$unwind :{----}} //fourth unwind
])
In reference to comments, here is advanced $lookup:
$lookup: {
from: 'accounts',
let: { "localAccountField": "$account" },
pipeline: [
{
$match: {
$expr: {
$eq: ["$_id", "$$localAccountField"]
}
}
},
{
$project: {
_id: 1,
user: 1
}
},
{
$lookup: {
from: 'users',
let: { 'localUserField': "$user" },
pipeline: [
{
$match: {
$expr: {
$eq: ["$_id", "$$localUserField"]
}
}
},
{
$project: {
_id: 1,
username: "$uid",
phone:"$phoneNumber",
email: "$email.add",
name: {
$concat: [
"$profile.name.first",
' ',
"$profile.name.last"
]
},
}
}
],
as: "users"
}
},
{
$lookup: {
from: 'documents',
let: { 'localDocumentField': "$user" },
pipeline: [
{
$match: {
$expr: {
$eq: ["$user", "$$localDocumentField"]
},
status:"verified",
"properties.expirydate": { $exists: true, $ne: "" },
name: "idcard"
}
},
{
$project: {
_id: 0,
cnic: "$properties.number"
}
}
],
as: "documents"
}
}
],
as: 'account'
}