MongoDB $lookup and $map array of objects - mongodb

I'm trying to do this for days, but can't find any success
I'm using MongoDB, and I tried to do it with many pipeline steps but I couldn't find a way.
I have a players collection, each player contains an items array
{
"_id": ObjectId("5fba17c1c4566e57fafdcd7e"),
"username": "moshe",
"items": [
{
"_id": ObjectId("5fbb5ac178045a985690b5fd"),
"equipped": false,
"itemId": "5fbb5ab778045a985690b5fc"
}
]
}
I have an items collection where there is more information about each item
in the player items array.
{
"_id": ObjectId("5fbb5ab778045a985690b5fc"),
"name": "Axe",
"damage": 4,
"defense": 6
}
My goal is to have a player document with all the information about the item inside his items array, so it will look like that:
{
"_id": ObjectId("5fba17c1c4566e57fafdcd7e"),
"username": "moshe",
"items": [
{
"_id": ObjectId("5fbb5ac178045a985690b5fd"),
"equipped": false,
"itemId": "5fbb5ab778045a985690b5fc",
"name": "Axe",
"damage": 4,
"defense": 6
}
]
}

$unwind deconstruct items array
$lookup to join items collection, pass itemsId into let after converting it to object id using $toObjectId and pass items object,
$match itemId condition
$mergeObject merge items object and $$ROOT object and replace to root using $replaceRoot
$group reconstruct items array again, group by _id and get first username and construct items array
db.players.aggregate([
{ $unwind: "$items" },
{
$lookup: {
from: "items",
let: {
itemId: { $toObjectId: "$items.itemId" },
items: "$items"
},
pipeline: [
{ $match: { $expr: { $eq: ["$_id", "$$itemId" ] } } },
{ $replaceRoot: { newRoot: { $mergeObjects: ["$$items", "$$ROOT"] } } }
],
as: "items"
}
},
{
$group: {
_id: "$_id",
username: { $first: "$username" },
items: { $push: { $first: "$items" } }
}
}
])
Playground
Second option using $map, and without $unwind,
$addFields for items convert itemId string to object type id using $toObjectId and $map
$lookup to join items collection
$project to show required fields, and merge items array and itemsCollection using $map to iterate loop of items array $filter to get matching itemId and $first to get first object from return result, $mergeObject to merge current object and returned object from $first
db.players.aggregate([
{
$addFields: {
items: {
$map: {
input: "$items",
in: {
$mergeObjects: ["$$this", { itemId: { $toObjectId: "$$this.itemId" } }]
}
}
}
}
},
{
$lookup: {
from: "items",
localField: "items.itemId",
foreignField: "_id",
as: "itemsCollection"
}
},
{
$project: {
username: 1,
items: {
$map: {
input: "$items",
as: "i",
in: {
$mergeObjects: [
"$$i",
{
$first: {
$filter: {
input: "$itemsCollection",
cond: { $eq: ["$$this._id", "$$i.itemId"] }
}
}
}
]
}
}
}
}
}
])
Playground

First I'd strongly suggest that you should store the items.itemId as ObjectId, not strings.
Then another simple solution can be:
db.players.aggregate([
{
$lookup: {
from: "items",
localField: "items.itemId",
foreignField: "_id",
as: "itemsDocuments",
},
},
{
$addFields: {
items: {
$map: {
input: { $zip: { inputs: ["$items", "$itemsDocuments"] } },
in: { $mergeObjects: "$$this" },
},
},
},
},
{ $unset: "itemsDocuments" },
])

Related

Change element name from the result set of Mongo DB Query

I have collection like below named as "FormData",
{
"_id": ObjectId("5e3c27bf1ef77236945ef07b"),
"eed12747-0923-4290-b09c-5a05107f5609": "20200206",
"bd637691-782d-4cfd-8624-feeedfe11b3e": "20200206_1#mail.com"
}
I have another collection named as "Form" which will have Title of Fields,
{
"_id": ObjectId("5e3c27bf1ef77236945ef07b"),
"Fields":[
{
"FieldID": "eed12747-0923-4290-b09c-5a05107f5609",
"Title": "Phone"
},
{
"FieldID": "bd637691-782d-4cfd-8624-feeedfe11b3e",
"Title": "Email"
}]
}
Now I have to map element name with Form field title and I need result like below,
{
"_id": ObjectId("5e3c27bf1ef77236945ef07b"),
"Phone": "20200206",
"Email": "20200206_1#mail.com"
}
Please help me to solve this.
Thanks in advance!
You can:
$objectToArray to convert the $$ROOT document into an array of k-v pairs for future lookups
use a sub-pipeline in $lookup to find the value by the uuid
use $mergeObject to combine the original values(i.e. "20200206"...) with the new field name looked up (i.e. "Phone"...)
wrangle the result back into original form using $arrayToObject and $replaceRoot
db.FormData.aggregate([
{
$match: {
"_id": ObjectId("5e3c27bf1ef77236945ef07b")
}
},
{
$project: {
kv: {
"$objectToArray": "$$ROOT"
}
}
},
{
$unwind: "$kv"
},
{
"$lookup": {
"from": "Form",
"let": {
uuid: "$kv.k"
},
"pipeline": [
{
$match: {
"_id": ObjectId("5e3c27bf1ef77236945ef07b")
}
},
{
"$unwind": "$Fields"
},
{
$match: {
$expr: {
$eq: [
"$$uuid",
"$Fields.FieldID"
]
}
}
},
{
$project: {
_id: false,
k: "$Fields.Title"
}
}
],
"as": "formLookup"
}
},
{
$unwind: "$formLookup"
},
{
$project: {
kv: {
"$mergeObjects": [
"$kv",
"$formLookup"
]
}
}
},
{
$group: {
_id: "$_id",
kv: {
$push: "$kv"
}
}
},
{
"$project": {
newDoc: {
"$arrayToObject": "$kv"
}
}
},
{
"$replaceRoot": {
"newRoot": {
"$mergeObjects": [
{
"_id": "$_id"
},
"$newDoc"
]
}
}
}
])
Mongo Playground
Another option is to start from Form collection and avoid $unwind:
$match and $lookup to get all needed data into one document
$objectToArray to get known keys for FormData
Match the items using $indexOfArray and $arrayElemAt and merge them using $mergeObjects. Then use arrayToObject to format the response
db.Form.aggregate([
{$match: {_id: ObjectId("5e3c27bf1ef77236945ef07b")}},
{$lookup: {
from: "FormData",
localField: "_id",
foreignField: "_id",
as: "formLookup",
pipeline: [{$project: {_id: 0}}]
}},
{$set: {formLookup: {$objectToArray: {$first: "$formLookup"}}}},
{$replaceRoot: {
newRoot: {
$mergeObjects: [
{$arrayToObject: {
$map: {
input: "$formLookup",
in: {$mergeObjects: [
{v: "$$this.v"},
{k: {$getField: {
input: {$arrayElemAt: [
"$Fields",
{$indexOfArray: ["$Fields.FieldID", "$$this.k"]}
]},
field: "Title"
}}}
]}
}
}},
{_id: "$_id"}
]
}
}}
])
See how it works on the playground example

MongoDB aggregation, use value from one document as key in another

So I’m trying to aggregate two documents matched on an id and based on the value of the first.
Document 1
{
“id”:3
“Whats for dinner”: “dinner”,
“What is for dinner tonight”: “dinner”,
“Whats for lunch”:“lunch”
}
Document 2
{
“Id”:3
“dinner” : “We are having roast!”,
“lunch” : “We are having sandwiches”
}
I’d like to start by matching the id and test if the question exists in doc1.
then return the question from doc1 and the answer from doc 2 . Like
{“Whats for dinner”:“We are having roast!”}
I’ve tried:
{ “$match”: { “id”: 3, “Whats for dinner”:{"$exists":True}} },
{
"$lookup": {
"from": "doc 2",
"localField": "id",
"foreignField": "id",
"as": "qa"
}
}
But from here I can’t figure out how to use the value from doc1 as key in doc2
It might be simple! but I’m a new to this, and just can’t get it to work!?
Crazy data model! This would be a solution:
db.doc1.aggregate([
{ $project: { data: { $objectToArray: "$$ROOT" } } },
{ $unwind: "$data" },
{
$lookup: {
from: "doc2",
pipeline: [
{ $project: { data: { $objectToArray: "$$ROOT" } } }
],
as: "answers"
}
},
{
$set: {
answers: {
$first: {
$filter: {
input: { $first: "$answers.data" },
cond: { $eq: [ "$$this.k", "$data.v" ] }
}
}
}
}
},
{ $match: { answers: { $exists: true } } },
{
$project: {
data: [
{
k: "$data.k",
v: "$answers.v"
}
]
}
},
{ $replaceWith: { $arrayToObject: "$data" } }
])
Mongo Playground
Better don't use any user data as key names, you will always have to juggle with $objectToArray and $arrayToObject
Maybe consider this:
questions: {
guildid: 3,
text: [
"Whats for dinner",
"What is for dinner tonight",
"Whats for lunch"
],
"nospace": 1
}

Use $lookup on a double nested array

I'm trying to use an _id saved in a double nested array to find and attach data from a different document. It essentially looks like this:
doc1 = {
_id: ObjectId,
...other stuff...,
firstArray: [
...other stuff...,
secondArray: [
other_id: ObjectId
]
]
}
doc2 = {
_id: ObjectId,
...the stuff I want...
}
Not every entry in secondArray is going to contain an other_id field, only some. This has gotten me close to the result I want except that the firstArray field contains an entry for the total number of entries in the secondArray field. I think I'm just missing one step to essentially undo the one of the two $unwinds.
doc1.aggregate([
{ $match: { _id: req.params._id }},
{ $unwind: "$firstArray" },
{ $unwind: "$firstArray.secondArray" },
{ $lookup: {
from: "doc2",
localField: "firstArray.secondArray.other_id",
foreignField: "_id",
as: "firstArray.secondArray.other",
}},
{ $addFields: {
"firstArray.secondArray.other": {
$arrayElemAt: ["$firstArray.secondArray.other", 0]
}
}},
{ $group: {
_id: "$_id",
...other stuff...,
firstArray: { $push: "$firstArray" },
}},
]);
You need to add another $group stage to reconstruct the "secondArray" structure, in general it's easy to remember that for each $unwind action you do, you'll need to counter that with another $group in order to restore the initial structure.
In order to do that you'll need some kind of a unique identifier for firstArray so you can group by it, if it doesn't exist you can add an _id that is the index of the element in the array like so:
{
"$addFields": {
"firstArray": {
$map: {
input: {
$zip: {
inputs: [
"$firstArray",
{
$range: [
0,
{
$size: "$firstArray"
}
]
}
]
}
},
as: "item",
in: {
"$mergeObjects": [
{
"$arrayElemAt": [
"$$item",
0
]
},
{
_id: {
"$arrayElemAt": [
"$$item",
1
]
}
}
]
}
}
}
}
}
Now firstArray has a unique field _id that we can later use to group on like so:
... the pipeline ...
{
$group: {
_id: {
id: "$_id",
first: "$firstArray._id"
},
secondArray: {
$push: "$firstArray.secondArray"
}
}
},
{
$group: {
_id: "$id._id",
firstArray: {
$push: {
secondArray: "$secondArray"
}
},
}
}
You will need to add support for all the "other stuff" you defined in your own pipeline, but this pipeline flow is the way to go.
Mongo Playground

How to use $lookup on array of subdocuments

I have these Schemas:
const chatbots = new Schema({
name: String,
campaigns: [{
name: String,
channels: [{
_id: String,
name: String,
budget: Number
}]
}]
});
const chatbotusers = new Schema({
name: String,
campaign_channel: String
})
And I need to get a list of Campaigns where, for each Channel, I have the total of ChatbotUsers. Something like this:
[
{
"name": "Campaign #1",
"channels": {
"_id": "eyRyZ1gD0",
"name": "Channel #1",
"users": 10
}
},
{
"name": "Campaign #1",
"channels": {
"_id": "tsKH7WxE",
"name": "Channel #2",
"users": 4
}
}
]
Any ideas?
The furthest I got was something like this:
{
$lookup: {
from: "chatbotusers",
localField: "channels._id",
foreignField: "campaign_channel",
as: "users",
}
},
{
$project: {
name: "$name",
channels: {
$map: {
input: "$channels",
as: "channel",
in: {
_id: "$$channel._id",
name: "$$channel.name",
users: { $size: "$users" },
}
}
}
}
}
But it sums the users for the Campaign, not the Channel.
(Sorry if the question title is not appropriate, I didn't even know how to ask this properly)
You can try this query :
db.chatbots.aggregate([
{
$lookup: {
from: "chatbotusers",
localField: "campaigns.channels._id",
foreignField: "campaign_channel",
as: "users"
}
},
{
$addFields: {
campaigns: {
$map: {
input: "$campaigns",
as: "eachCampaign",
in: {
$mergeObjects: ['$$eachCampaign', {
channels:
{
$reduce: {
input: "$$eachCampaign.channels",
initialValue: [],
in: {
$concatArrays: [
"$$value",
[
{
$mergeObjects: [
"$$this",
{
user: {
$size: {
$filter: {
input: "$users",
as: "e",
cond: {
$eq: [
"$$e.campaign_channel",
"$$this._id"
]
}
}
}
}
}
]
}
]
]
}
}
}
}]
}
}
}
}
},
{
$project: {
users: 0
}
}
])
Note : There can be multiple ways to do this, but this way we're working on same no.of docs from the chatbots collection rather than exploding docs by doing $unwind which may be helpful when you've huge dataset.
Test : MongoDB-Playground
This above query should get you what is needed, but in any case if it's slow or you think to enhance it then here :
{
user: {
$size: {
$filter: {
input: "$users", as: "e",
cond: {
$eq: [
"$$e.campaign_channel",
"$$this._id"
]
}
}
}
}
}
Where We're iterating thru users array for every channel in every campaign, So instead of iterating every time, right after lookup - You can iterate over users for once using reduce to get count of each unique campaign_channel replace this data as users array, that way you can get count of users directly. In general main intention of above query is to preserve original document structure with less stages being used.
Alternatively you can use this query, which doesn't preserve original doc structure (also no.of docs in output can be more than what you've in collection) but can do what you needed :
db.chatbots.aggregate([
{
$unwind: "$campaigns"
},
{
$unwind: "$campaigns.channels"
},
{
$lookup: {
from: "chatbotusers",
localField: "campaigns.channels._id",
foreignField: "campaign_channel",
as: "users"
}
},
{
$addFields: {
"channels": "$campaigns.channels",
campaigns: "$campaigns.name"
}
},
{
$addFields: {
"channels.users": {
$size: "$users"
}
}
},
{
$project: {
users: 0
}
}
])
Test : MongoDB-Playground

$match in $lookup result

I have next mongo code:
db.users.aggregate([
{
$match: {
$and: [
{ UserName: { $eq: 'administrator' } },
{ 'Company.CompanyName': { $eq: 'test' } }
]
}
},
{
$lookup: {
from: "companies",
localField: "CompanyID",
foreignField: "CompanyID",
as: "Company"
}
},
])
The $lookup part of the code working great. I got next result:
But if I add $match to the code, it brings nothing.
I found that the problem is in the second match: { 'Company.CompanyName': { $eq: 'test' } }, but I can not realize what is wrong with it.
Any ideas?
UPDATE:
I had also tried $unwind on the $lookup result, but no luck:
db.users.aggregate([
{
$match: {
$and: [
{ UserName: { $eq: 'administrator' } },
{ 'Company.CompanyName': { $eq: 'edt5' } }
]
}
},
{ unwind: '$Company' },
{
$lookup: {
from: 'companies',
localField: 'CompanyID',
foreignField: 'CompanyID',
as: 'Company'
}
},
])
With MongoDB 3.4, you can run an aggregation pipeline that uses the $addFields pipeline and a $filter operator to only return the Company array with elements that match the given condition. You can then wrap the $filter expression with the $arrayElemAt operator to return a single document which in essence incorporates the $unwind functionality by flattening the array.
Follow this example to understand the above concept:
db.users.aggregate([
{ "$match": { "UserName": "administrator" } },
{
"$lookup": {
"from": 'companies',
"localField": 'CompanyID',
"foreignField": 'CompanyID',
"as": 'Company'
}
},
{
"$addFields": {
"Company": {
"$arrayElemAt": [
{
"$filter": {
"input": "$Company",
"as": "comp",
"cond": {
"$eq": [ "$$comp.CompanyName", "edt5" ]
}
}
}, 0
]
}
}
}
])
Below answer is for mongoDB 3.6 or later.
Given that:
You have a collection users with a field CompanyID and a collection of companies with a field CompanyID
you want to lookup Companies on Users by matching CompanyID, where additionally:
each User must match condition: User.UserName equals administrator
each Company on User must match condition: CompanyName equals edt5
The following query will work for you:
db.users.aggregate([
{ $match: { UserName: 'administrator' } },
{
$lookup: {
from: 'companies',
as: 'Company',
let: { CompanyID: '$CompanyID' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$CompanyID', '$$CompanyID'] },
{ $eq: ['$CompanyName', 'edt5'] },
]
}
}
}
]
}
},
])
Explanation:
This is the way to perform left join queries with conditions more complex than simple foreign / local field equality match.
Instead of using localField and foreignField, you use:
let option where you can map local fields to variables,
pipeline option where you can specify aggregation Array.
In pipeline you can use $match filter, with $expr, where you can reuse variables defined earlier in let.
More info on $lookup
Nice tutorial
here is code for fitering array inside lookup.
const userId = req.userData.userId;
const limit = parseInt(req.params.limit);
const page = parseInt(req.params.page);
Collection.aggregate([
{ $match: {} },
{ $sort: { count: -1 } },
{ $skip: limit * page },
{ $limit: limit },
{
$lookup: {
from: Preference.collection.name,
let: { keywordId: "$_id" },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ["$keyword", "$$keywordId"] },
{
$eq: ["$user", mongoose.Types.ObjectId(userId)],
},
],
},
},
},
],
as: "keywordData",
},
},
{
$project: {
_id: 0,
id: "$_id",
count: 1,
for: 1,
against: 1,
created_at: 1,
updated_at: 1,
keyword: 1,
selected: {
$cond: {
if: {
$eq: [{ $size: "$keywordData" }, 0],
},
then: false,
else: true,
},
},
},
}])