$lookup and $match Mongodb golang - mongodb

I want to get document with foreign key by using $lookup and $match on MongoDB.
There is a "Jobs" collection which stores Job document. In Job document there are two field using as foreing key "creatorParent" and "Children".
CreatorParent is a foreign key for "Users" collection and Children array contains id for user's children.
When I list the whole jobs, I want to retrieve detail from "Users" collection for both CreatorParent ID and ChildrenID. I want to marshall "Job" document with ParentDetail and ChildDetail. I don't want to write a custom method for that. Is it possible to handle it with MongoDB query?
By the way I'm beginner on MongoDB so should store needed details on Children and CreatorParent instead of storing ObjectId?
Users document:
{
"_id" : ObjectId("58daf84877733645eaa9b44f"),
"email" : "meto93#gmail.com",
"password" : "vpGl+Fjnef616cRgNbCkwaFDpSI=",
"passwordsalt" : "99397F4A9D3A499D96694547667E74595CE994D2E83345D6953EF866303E8B65",
"children" : [
{
"_id" : ObjectId("58daf84977733645eaa9b450"),
"name" : "Mert",
"age" : 5,
"additionalinformation" : "ilk cocuk",
"creationtime" : ISODate("2017-03-28T23:56:56.952Z"),
"userid" : ObjectId("58daf84877733645eaa9b44f"),
"gender" : null
},
{
"_id" : ObjectId("58daf84977733645eaa9b451"),
"name" : "Sencer",
"age" : 7,
"additionalinformation" : "ikinci cocuk",
"creationtime" : ISODate("2017-03-28T23:56:56.952Z"),
"userid" : ObjectId("58daf84877733645eaa9b44f"),
"gender" : null
}
]
}
Job
{
"_id" : ObjectId("58db0a2d77733645eaa9b453"),
"creationtime" : ISODate("2017-03-29T01:13:17.509Z"),
"startingtime" : ISODate("2017-04-03T13:00:00.000Z"),
"endingtime" : ISODate("2017-04-03T17:00:00.000Z"),
"children" : [
ObjectId("58daf84977733645eaa9b450"),
ObjectId("58daf84977733645eaa9b451")
],
"creatorparent" : ObjectId("58daf84877733645eaa9b44f"),
"applicants" : []
}

If I understood it correctly. A similar solution is achievable using MongoDB 3.4's $addFields and $lookup aggregation steps.
Mongo aggregation:
[
{
$addFields: {
"job":"$$ROOT"
}
},
{
$unwind: {
path : "$children"
}
},
{
$lookup: {
"from" : "users",
"localField" : "creatorParent",
"foreignField" : "_id",
"as" : "creatorParent"
}
},
{
$lookup: {
"from" : "users",
"localField" : "children",
"foreignField" : "_id",
"as" : "children"
}
},
{
$group: {
"_id": "$_id",
"job": { "$first": "$job" },
"creatorParent" : { "$first" : "$creatorParent" },
"children": { "$addToSet": { $arrayElemAt: [ "$children", 0 ] } }
}
}
]
The output will look like the following:
{ "_id" : ObjectId("58da9cb6340c630315348114"),
"job" : {
"_id" : ObjectId("58da9cb6340c630315348114"),
"name" : "Developer",
"creatorParent" : ObjectId("58da9c79340c630315348113"),
"children" : [
ObjectId("58da9c6d340c630315348112"),
ObjectId("58da9c5f340c630315348111")
],
"hourly_rate" : 12.0,
"additional_information" : "other infos"
},
"creatorParent" : [
{
"_id" : ObjectId("58da9c79340c630315348113"),
"name" : "The Boss",
"age" : 40.0
}
],
"children" : [
{
"_id" : ObjectId("58da9c5f340c630315348111"),
"name" : "James",
"age" : 28.0
},
{
"_id" : ObjectId("58da9c6d340c630315348112"),
"name" : "Andrew",
"age" : 26.0
}
]}
UPDATE:
If you substitute the last $group stage with this:
{
"_id": "$_id",
"name": { "$first": "$name" },
"jobstatus": { "$first": "$jobstatus" },
"hourlyrate": { "$first":"$hourlyrate" },
"creatorparent" : { "$first" : "$creatorparent" },
"children": { "$addToSet": { $arrayElemAt: [ "$children", 0 ] } }
}
Then you can achieve what you would like to, but in this $group stage you have to specify every field of job one-by-one with the $first expression.

Related

Unable to aggregate two collections using lookup in MongoDB Atlas

I have an orders collection that looks like this:
{
"_id" : "wJNEiSYwBd5ozGtLX",
"orderId" : 52713,
"createdAt" : ISODate("2020-01-31T04:34:13.790Z"),
"status" : "closed",
"orders" : [
{
"_id" : "ziPzwLuZrz9MNkaRT",
"productId" : 10290,
"quantity" : 2
}
]
}
I have an products collection that looks like this
{
"_id" : "238cwwLkZa6gKNN86",
"productId" : 10290,
"title" : "Product Title",
"price" : 9.9
}
I am trying to merge the price information into the orders information.
Something like:
{
"_id" : "wJNEiSYwBd5ozGtLX",
"orderId" : 52713,
"createdAt" : ISODate("2020-01-31T04:34:13.790Z"),
"status" : "closed",
"orders" : [
{
"_id" : "ziPzwLuZrz9MNkaRT",
"productId" : 10290,
"quantity" : 2,
"price": 9.9
}
]
}
If I try a $lookup command on MongoDB Atlas Dashboard like this:
{
from: 'products',
localField: 'orders.productId',
foreignField: 'productId',
as: 'priceInfo'
}
The aggregated output is (not what I wanted):
{
"_id" : "wJNEiSYwBd5ozGtLX",
"orderId" : 52713,
"createdAt" : ISODate("2020-01-31T04:34:13.790Z"),
"status" : "closed",
"orders" : [
{
"_id" : "ziPzwLuZrz9MNkaRT",
"productId" : 10290,
}
],
"priceInfo": [
{
"_id" : "238cwwLkZa6gKNN86",
"productId" : 10290,
"title" : "Product Title",
"price" : 9.9
}
]
}
I do not need a separate priceInfo array. It will be best if I have the product details information merged into the "orders" array. What should be the aggregation lookup syntax to achieve the desired output?
Demo - https://mongoplayground.net/p/bLqcN7tauWU
Read - $lookup $unwind $first $set $push $group
db.orders.aggregate([
{ $unwind: "$orders" }, // break array of orders into individual documents
{
$lookup: { // join
"from": "products",
"localField": "orders.productId",
"foreignField": "productId",
"as": "products"
}
},
{
$set: {
"orders.price": { "$arrayElemAt": [ "$products.price", 0 ] } // set the price
}
},
{
$group: { // group records back
_id: "$_id",
createdAt: { $first: "$createdAt" },
status: { $first: "$status" },
orderId: { $first: "$orderId" },
orders: { $push: "$orders" }
}
}
])

Whats the alternative to $replaceRoot on mongoDB? $replaceRoot is incompatible with documentDB

The problem: I'm trying to make a query on MongoDB, but I'm using the DocumentDb from amazon, where some operations are no supported. I wanted to find an alternative to get the same result, if possible. Basically I want to change the root of the result, instead of being the first entity, I need it to be some merging of some values in different levels of the document.
So, I have the following structure in my collection:
{
"_id" : ObjectId("5e598bf4d98f7c70f9aa3b58"),
"status" : "active",
"invoices" : [
{
"_id" : ObjectId("5e598bf13b24713f50600375"),
"value" : 1157.52,
"receivables" : [
{
"situation" : {
"status" : "active",
"reason" : []
},
"rec_code" : "001",
"_id" : ObjectId("5e598bf13b24713f50600374"),
"expiration_date" : ISODate("2020-03-25T00:00:00.000Z"),
"value" : 1157.52
}
],
"invoice_code" : 9773,
"buyer" : {
"legal_name" : "test name",
"buyer_code" : "223132165498797"
}
},
],
"seller" : {
"code" : "321654897986",
"name" : "test name 2"
}
}
What I want to achieve is to list all "receivables" like this, where the _id is the _id of the receivable:
[{
"_id" : ObjectId("5e598bf13b24713f50600374"),
"situation" : {
"status" : "active",
"reason" : []
},
"rec_code" : "001",
"expiration_date" : ISODate("2020-03-25T00:00:00.000Z"),
"value" : 1157.52,
"status" : "active",
"seller" : {
"cnpj" : "321654897986",
"name" : "test name 2"
},
"invoice_code" : 9773.0,
"buyer" : {
"legal_name" : "test name",
"cnpj" : "223132165498797"
}
}]
This I can do with $replaceRoot in with the query below on MongoDB, but using documentDB I can't use $replaceRoot or $mergeObjects. Do you know how can I get the same result with other operators?:
db.testCollection.aggregate([
{ $unwind: "$invoices" },
{ $replaceRoot: {
newRoot: {
$mergeObjects: ["$$ROOT","$invoices"]}
}
},
{$project: {"_id": 0, "value": 0, "created_at": 0, "situation": 0}},
{ $unwind: "$receivables" },
{ $replaceRoot: {
newRoot: {
$mergeObjects: ["$receivables", "$$ROOT"]
}
}
},
{$project:{"created_at": 0, "receivables": 0, "invoices": 0}}
])
After going through mongodb operations, I could get a similar result fro what I wanted with the following query without $replaceRoot. It turns out it was a better query, I think:
db.testCollection.aggregate([
{$unwind: "$invoices"},
{$project : {
created_at: 1,
seller: "$seller",
buyer: "$invoices.buyer",
nnf: "$invoices.nnf",
receivable: '$invoices.receivables'
}
},
{$unwind: "$receivable"},
{$project : {
_id: '$receivable._id',
seller: 1,
buyer: 1,
invoice_code: 1,
receivable: 1,
created_at: 1,
}
},
{$sort: {"created_at": -1}},
])
This query resulted in the following structure list:
[{
"created_at" : ISODate("2020-03-06T09:47:26.161Z"),
"seller" : {
"name" : "Test name",
"cnpj" : "21231232131232"
},
"buyer" : {
"cnpj" : "21322132164654",
"legal_name" : "Test name 2"
},
"invoice_code" : 66119,
"receivable" : {
"rec_code" : "001",
"_id" : ObjectId("5e601bb5efff82b92935bad4"),
"expiration_date" : ISODate("2020-03-17T00:00:00.000Z"),
"value" : 6540.7,
"situation" : {
"status" : "active",
"reason" : []
}
},
"_id" : ObjectId("5e601bb5efff82b92935bad4")
}]
Support for $replaceRoot was added to Amazon DocumentDB in January 2021.

How to display name with the id but with datas in 2 collections MongoDb?

I just want change a Id by thea name corresponding. But id is in a collection and name in other collection. "$lookup" Mongo doesn't work in my case...
The first collection "parameter" contains items with the "category_id":
{"_id" : ObjectId("56cc8827b9e4ed0fd42a4569"),
"data" : {
"capacity" : NumberInt(60),
"categories" : [
{
"category_id" : "5964961294ff4a37988e8f9b",
"nbMax" : NumberInt(1),
"nbRes" : NumberInt(0)
},
{
"category_id" : "596495c994ff4a37988e8f99",
"nbMax" : NumberInt(1),
"nbRes" : NumberInt(0)
},
],
},
"type" : "launcher",
"name" : "launcherp01",
"description" : "",
"_class" : "parameter"
}
....
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4847"),
"data" : {
"capacity" : NumberInt(60),
"categories" : [
{
"category_id" : "596495c994ff4a37988e8f99",
"nbMax" : NumberInt(1),
"nbRes" : NumberInt(0)
},
{
"category_id" : "8864961294ff4a37988e8f3b",
"nbMax" : NumberInt(1),
"nbRes" : NumberInt(0)
},
],
},
"type" : "launcher",
"name" : "launcherp01",
"description" : "",
"_class" : "parameter"
}
.....
The second Collection "reference" contains the description of categories with the _id (same as category_id in first collection) and the name:
{
"_id" : ObjectId("596495c994ff4a37988e8f99"),
"taskType" : "qc",
"type" : "category",
"name" : "**qcSupportNormal01**",
"_class" : "reference",
}
{
"_id" : ObjectId("5964961294ff4a37988e8f9b"),
"taskType" : "transcode",
"type" : "category",
"name" : "tsSupportNormal01",
"_class" : "reference",
}
I want something like that (item with categories names not id):
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4569"),
"type" : "launcher",
"categories" : [
"qcSupportNormal01", //--->> name from reference collection
"tsSupportNormal01", //--->> name from reference collection
}
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4847"),
"type" : "launcher",
"categories" : [
"qcSupportNormal01", //--->> name from reference collection
"qptestNormal01", //--->> name from reference collection
...
My query:
db.parameters.aggregate([
///////////////////////item filter
{$match: {
type:{ $in: [ "launcher" ] } ,
}},
///////////////////// foreign field
{$lookup: {
from: "references",
localField: "categories",
foreignField: "_id",
as: "references"
}},
/////////////////// projection
{$project:
{_id:1,type:1,categories:"$data.categories.category_id"
}},
])
but the result is always with id not the name:
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4569"),
"type" : "launcher",
"categories" : [
"5964961294ff4a37988e8f9b", //--->> id not name from reference collection !
"596495c994ff4a37988e8f99", //--->> id not name from reference collection !
...
How to have the category name not the id.
Very simple in Sql (joint with foreign key and reference table) but complex in Mongo query ...
Thanks for your help
je déteste le langage Mongo !
There are a couple of issues with your approach:
You are trying to call $lookup between an ObjectId and a string value.
For your categories field you are mapping it with the category_id from the data.categories whereas it should have come from references.name.
The field category_id is nested inside of an array and when you are calling the $lookup you are matching it with the whole array instead of the key field.
So, now the best approach would be to convert the field category_id to an ObjectId and then call the $lookup for the field(instead of modifying the actual field i have appended a new field named as cat_id).
Here is how i would do it:
db.parameter.aggregate([
{
$match: {
type: { $in: ["launcher"] },
}
},
{ $unwind: "$data.categories" },
{ $addFields: { "data.categories.cat_id": { $convert: { input: "$data.categories.category_id", to: "objectId" } } } },
{
$lookup: {
from: "references",
localField: "data.categories.cat_id",
foreignField: "_id",
as: "ref"
}
},
{
$project:
{
_id: 1, type: 1, categories: "$ref.name"
}
}
]);
Which will give you an output as:
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4569"),
"type" : "launcher",
"categories" : [
"tsSupportNormal01"
]
},
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4569"),
"type" : "launcher",
"categories" : [
"**qcSupportNormal01**"
]
},
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4847"),
"type" : "launcher",
"categories" : [
"**qcSupportNormal01**"
]
},
{
"_id" : ObjectId("56cc8827b9e4ed0fd42a4847"),
"type" : "launcher",
"categories" : [ ]
}

MongoDB aggregate two collections, return additional field as count

(See edit below)
I am trying to aggregate data from two separate collections within the same MongoDB database.
The "accounts" collection contains user information (cleansed):
{
_id: ObjectId("5c0d64a4224a2900108c005f"),
"username" : "mike22",
"email" : "mike22#<domain>.com",
"country" : GB,
"created" : ISODate("2018-11-26T23:37:49.051Z")
},
{
_id: ObjectId("5a0d64a4527h2880108c0445"),
"username" : "mike23",
"email" : "mike23#<domain>.com",
"country" : DE,
"created" : ISODate("2018-11-26T23:37:49.051Z")
},
{
_id: ObjectId("5a3334a45zzz2884448c0445"),
"username" : "mike24",
"email" : "mike24#<domain>.com",
"country" : DE,
"created" : ISODate("2018-11-26T23:37:49.051Z")
}
The "devices" collection contains device definitions for all users. A user is likely to have many devices defined in this collection and many users devices are in this collection.
A single device within this collection is defined as follows:
{
"_id" : ObjectId("5c10138c73bbe0001018e415"),
"capabilities" : [
"BrightnessController",
"PowerController"
],
"displayCategories" : [
"LIGHT"
],
"friendlyName" : "Test1",
"description" : "Test device 1",
"reportState" : true,
"username" : "mike22",
"endpointId" : 11,
"__v" : 0
},
{
"_id" : ObjectId("5c10138c73bbe0001018e415"),
"capabilities" : [
"PowerController"
],
"displayCategories" : [
"SWITCH"
],
"friendlyName" : "Test2",
"description" : "Test device 2",
"reportState" : true,
"username" : "mike23",
"endpointId" : 12,
"__v" : 0
},
{
"_id" : ObjectId("5c10138c73bbe0001018e415"),
"capabilities" : [
"PowerController"
],
"displayCategories" : [
"SMARTPLUG"
],
"friendlyName" : "Test3",
"description" : "Test device 3",
"reportState" : true,
"username" : "mike22",
"endpointId" : 13,
"__v" : 0
}
I'm able to use the aggregate below to show me a count of device per-user:
db.accounts.aggregate([
{
$lookup: {
from : "devices",
localField : "username",
foreignField : "username",
as : "userdevs"
},
},
{ $unwind:"$userdevs" },
{ $group : { _id : "$username", count : { $sum : 1 } } }
])
Example output from the data/ aggregate above:
{ "_id" : "mike22", "count" : 2 },
{ "_id" : "mike23", "count" : 1 }
(Note user with no devices is now missing/ should be there with a zero count?!)
However, I want to return all fields for each user plus a new field which shows me the count of devices they have in the "devices" collection. The output I am looking for is as below:
{
"_id" : ObjectId("5c0d64a4224a2900108c005f"),
"username" : "mike22",
"email" : "mike22#<domain>.com",
"country" : GB,
"created" : ISODate("2018-11-26T23:37:49.051Z"),
"countDevices": 2
},
{
"_id" : ObjectId("5a0d64a4527h2880108c0445"),
"username" : "mike23",
"email" : "mike23#<domain>.com",
"country" : DE,
"created" : ISODate("2018-11-26T23:37:49.051Z"),
"countDevices": 1
},
{
"_id" : ObjectId("5a0d64a4527h2880108c0445"),
"username" : "mike24",
"email" : "mike24#<domain>.com",
"country" : DE,
"created" : ISODate("2018-11-26T23:37:49.051Z"),
"countDevices": 0
}
Edit 16/12: So I am nearly there with the aggregate below. Zero-count users are missing though.
use users
db.accounts.aggregate([
{
$lookup: {
from : "devices",
localField : "username",
foreignField : "username",
as : "userdevs"
},
},
{ $unwind: "$userdevs"},
{ $group : { _id : {
_id: "$_id",
username: "$username",
email: "$email",
country: "$country",
region: "$region",
},
countDevices : { $sum : 1 } } }
])
2nd Edit 16/12:
I have found the aggregate needed below:
db.accounts.aggregate([
{ "$lookup": {
"from": "devices",
"let": { "username": "$username" },
"pipeline": [
{ "$match": {
"$expr": { "$eq": [ "$$username", "$username" ] }
}},
{ "$count": "count" }
],
"as": "deviceCount"
}},
{ "$addFields": {
"countDevices": { "$sum": "$deviceCount.count" }
}}
])
First of All, you can flatten the answer you have got with a projection like below:
{ $project : {
_id : '$_id._id',
username : '$_id.username',
email : '$_id.email',
country : '$_id.country',
region : '$_id.region',
countDevices: 1
}
}
add this after the $group in your pipeline, you will get your result as you wanted in the question.
About zero-count users, there is a way to handle this in database using mongoDB, as explained in detail here but I do not recommend it, its better that you handle this kind of problem client side.
As-per second edit, the aggregate I used is as below:
db.accounts.aggregate([
{ "$lookup": {
"from": "devices",
"let": { "username": "$username" },
"pipeline": [
{ "$match": {
"$expr": { "$eq": [ "$$username", "$username" ] }
}},
{ "$count": "count" }
],
"as": "deviceCount"
}},
{ "$addFields": {
"countDevices": { "$sum": "$deviceCount.count" }
}}
])

Mongodb aggregation retrieving document from inside of document

I have problems with MongoDB's aggregation.
In my "Job" document, it has creatorParent(single value) and Children(array of mongodb object id). In the "User" document, user has children array with child details.
When user request for retrieving this document I want aggregate child details, if array contains id of child.
I wrote an aggregation with some help, It worked for creatorParent but whatever I've tried, it didn't worked for children.
db.getCollection('Jobs').aggregate([
{
$unwind: {
path : "$children"
}
},
{
$lookup: {
"from" : "Users",
"localField" : "creatorparent",
"foreignField" : "_id",
"as" : "creatorparent"
}
},
{
$lookup: {
"from" : "Users",
"localField" : "children",
"foreignField" : "children",
"as" : "children"
}
}
])
Users document:
{
"_id" : ObjectId("58daf84877733645eaa9b44f"),
"email" : "meto93#gmail.com",
"password" : "vpGl+Fjnef616cRgNbCkwaFDpSI=",
"passwordsalt" : "99397F4A9D3A499D96694547667E74595CE994D2E83345D6953EF866303E8B65",
"children" : [
{
"_id" : ObjectId("58daf84977733645eaa9b450"),
"name" : "Mert",
"age" : 5,
"additionalinformation" : "ilk cocuk",
"creationtime" : ISODate("2017-03-28T23:56:56.952Z"),
"userid" : ObjectId("58daf84877733645eaa9b44f"),
"gender" : null
},
{
"_id" : ObjectId("58daf84977733645eaa9b451"),
"name" : "Sencer",
"age" : 7,
"additionalinformation" : "ikinci cocuk",
"creationtime" : ISODate("2017-03-28T23:56:56.952Z"),
"userid" : ObjectId("58daf84877733645eaa9b44f"),
"gender" : null
}
]
}
Job
{
"_id" : ObjectId("58db0a2d77733645eaa9b453"),
"creationtime" : ISODate("2017-03-29T01:13:17.509Z"),
"startingtime" : ISODate("2017-04-03T13:00:00.000Z"),
"endingtime" : ISODate("2017-04-03T17:00:00.000Z"),
"children" : [
ObjectId("58daf84977733645eaa9b450"),
ObjectId("58daf84977733645eaa9b451")
],
"creatorparent" : ObjectId("58daf84877733645eaa9b44f"),
"applicants" : []
}
Try this:
db.jobs.aggregate(
[
{
$unwind: {
path : "$children",
}
},
{
$lookup: {
"from" : "users",
"localField" : "creatorparent",
"foreignField" : "_id",
"as" : "creatorparent"
}
},
{
$lookup: {
"from" : "users",
"localField" : "children",
"foreignField" : "children._id",
"as" : "children"
}
},
{
$addFields: {
children : {$arrayElemAt : ["$children",0]}
}
},
{
$addFields: {
"children":"$children.children"
}
},
{
$unwind: {
path : "$children",
}
},
{
$group: {
"_id": "$_id",
"name": { "$first": "$name" },
"jobstatus" : { "$first": "$jobstatus" },
"hourlyrate" : { "$first": "$hourlyrate" },
"creatorparent" : { "$first" : "$creatorparent" },
"children": { "$addToSet": "$children" }
}
},
]
);