mongoDB group documents after unwind - mongodb

In my mongodb books collection I have documents that look like:
{
_id: ObjectId(625efa44f1ba751c8275ea51),
contributors:[ObjectId(625efa44f1ba751c8275ea52), ObjectId(625efa44f1ba751c8275ea53)]
//other fields
}
And I want to do a query that returns me documents like:
{
_id: ObjectId(625efa44f1ba751c8275ea51),
contributors:[
{
_id: ObjectId(625efa44f1ba751c8275ea52),
first_name: 'Luigi'
//many other fields
},
{
_id: ObjectId(625efa44f1ba751c8275ea53),
first_name: 'Mario'
//many other fields
},
]
//other fields
}
I did an unwind on contributors and a lookup with my users collection and now I need to group them. I haven't used it before but I did something like:
{
$group:{
_id: '_id'
}
}
But I don't know what to do next in order to preserve all the fields from books and also from users.
Do you have any idea?

If the $lookup result is small enough (<16MB document size limit),
you can simply do a $lookup.
db.books.aggregate([
{
"$lookup": {
"from": "users",
"localField": "contributors",
"foreignField": "_id",
"as": "contributors"
}
}
])
Here is the Mongo Playground for your reference.
If the $lookup result will exceed 16 MB limit, you can still $unwind. Just use $firstto regroup the other fields after the$unwind`
db.books.aggregate([
{
"$lookup": {
"from": "users",
"localField": "contributors",
"foreignField": "_id",
"as": "contributors"
}
},
{
"$unwind": "$contributors"
},
{
"$group": {
"_id": "_id",
bookName: {
$first: "$bookName"
},
"contributors": {
"$push": "$contributors"
}
}
}
])
Here is the Mongo playground for your reference.

Related

Multiple joins trough array field

noSQL beginner here.
For a current use case we are syncing an external relational DB (dataverse) with a changing schema to an mongoDB instance. Because we have no control over the schema and each entity on his one can change we aren't able to normalize the data leading me to the need for nested lookups from an array.
i have created an sample playground to illustrate an rough example of the data structure: https://mongoplayground.net/p/ObWOC5choPq
I need to return the data in roughly the following format:
{
orderID,
products: [
{
productID,
pictureURL,
},
{
productID,
pictureURL,
}
]
}
I'm able to perform both lookups seperatly but aren't able to return the picture data as part of an product object in the products array. Could anyone point me in the right direction?
Kind regards,
Nomis
Maybe something like this:
db.orders.aggregate([
{
"$lookup": {
"from": "products",
"localField": "products.productID",
"foreignField": "_id",
"as": "products"
}
},
{
$unwind: {
path: "$products",
preserveNullAndEmptyArrays: true
}
},
{
"$lookup": {
"from": "pictures",
"localField": "products.pictureID",
"foreignField": "_id",
"as": "products.pictures"
}
},
{
$unwind: {
path:"$products.pictures",
preserveNullAndEmptyArrays: true
}
},
{
$project: {
_id: 1,
products: {
productID: "$products._id",
pictureURL: "$products.pictures.bloburl"
}
}
},
{
$group: {
_id: "$_id",
products: {
$push: "$products"
}
}
},
{
$project: {
orderID: "$_id",
products: 1,
_id: 0
}
}
])
$lookup orders with products
unwind to convert products to object
$lookup orders with pictures to nested in products
$unwind pictures to object
$project the fields as expected in the final result
$group by order_id.
$project one more time to rename order _id to orders
playground

Using a where on an array with id's inside aggregate mongodb

Hi im really struggling finding a solution to this.
Im trying to see if an array contains a value that I give
I want all tickets of a specific organization.
The organization is saved in the user (Creator) in an array named organizations.
Models
ticket {
creator: "ObjectId"
}
user {
organizations: ["ObjectId", "ObjectId"]
}
This is what I have now
Ticket.aggregate([
{
"$lookup": {
"from": User.collection.name,
"localField": "creator",
"foreignField": "_id",
"as": "creator"
}
},
{ "$unwind": "$creator" },
{ "$match": { "creator.organizations": {
"$elemMatch": {"$in": ["60a77d5b57d8c960829a0343"]}}}
},
{ "$set": {"creator": "$creator._id"}},
])
This doesn't work tho
I read that you can't use $elemMatch inside an aggregate because its a query.
How do I achieve this? I've seen people saying to use a $filter but I have no clue how to make that.
$elemMatch is not required here. This can be achieved only using $match as shown below.
const mongoose = require("mongoose");
Ticket.aggregate([{
"$lookup": {
"from": User.collection.name,
"localField": "creator",
"foreignField": "_id",
"as": "creator"
}
},
{
"$unwind": "$creator"
},
// Modified code
{
"$match": {
"creator.organizations": mongoose.Types.ObjectId("60a77d5b57d8c960829a0343")
}
},
{
"$set": {
"creator": "$creator._id"
}
},
])

Array is reordered when using $lookup

I have this aggregation:
db.getCollection("users").aggregate([
{
"$match": {
"_id": "5a708a38e6a4078bd49f01d5"
}
},
{
"$lookup": {
"from": "user-locations",
"localField": "locations",
"as": "locations",
"foreignField": "_id"
}
}
])
It works well, but there is one small thing that I don't understand and I can't fix.
In the query output, the locations array is reordered by ObjectId and I really need to keep the original order of data.
Here is how the locations array from the users collection looks like
'locations' : [
ObjectId("5b55e9820b720a1a7cd19633"),
ObjectId("5a708a38e6a4078bd49ef13f")
],
And here is the result after the aggregation:
'locations' : [
{
'_id' : ObjectId("5a708a38e6a4078bd49ef13f"),
'name': 'Location 2'
},
{
'_id' : ObjectId("5b55e9820b720a1a7cd19633"),
'name': 'Location 1'
}
],
What am I missing here? I really have no idea how to proceed with this issue.
Could you give me a push?
$lookup does not guarantee order of result documents, you can try a approach to manage natural order of document,
$unwind deconstruct locations array and add auto index number will start from 0,
$lookup with locations
$set to select first element from locations
$sort by index field in ascending order
$group by _id and reconstruct locations array
db.users.aggregate([
{ $match: { _id: "5a708a38e6a4078bd49f01d5" } },
{
$unwind: {
path: "$locations",
includeArrayIndex: "index"
}
},
{
$lookup: {
from: "user-locations",
localField: "locations",
foreignField: "_id",
as: "locations"
}
},
{ $set: { locations: { $arrayElemAt: ["$locations", 0] } } },
{ $sort: { index: 1 } },
{
$group: {
_id: "$_id",
locations: { $push: "$locations" }
}
}
])
Playground
From this closed bug report:
When using $lookup, the order of the documents returned is not guaranteed. The documents are returned in "natural order" - as they are encountered in the database. The only way to get a guaranteed consistent order is to add a $sort stage to the query.
Basically the way any Mongo query/pipeline works is that it returns documents in the order they were matched, meaning the "right" order is not guaranteed especially if there's indes usage involved.
What you should do is add a $sort stage as suggested, like so:
db.collection.aggregate([
{
"$match": {
"_id": "5a708a38e6a4078bd49f01d5"
}
},
{
"$lookup": {
"from": "user-locations",
"let": {
"locations": "$locations"
},
"pipeline": [
{
"$match": {
"$expr": {
"$setIsSubset": [
[
"$_id"
],
"$$locations"
]
}
}
},
{
$sort: {
_id: 1 // any other sort field you want.
}
}
],
"as": "locations",
}
}
])
You can also keep the original $lookup syntax you're using and just $unwind, $sort and then $group to restore the structure.

Mongodb 3.4 version can not Join on string to an ObjectId field [duplicate]

I have two collections
User
{
"_id" : ObjectId("584aac38686860d502929b8b"),
"name" : "John"
}
Role
{
"_id" : ObjectId("584aaca6686860d502929b8d"),
"role" : "Admin",
"userId" : "584aac38686860d502929b8b"
}
I want to join these collection based on the userId (in role collection) - _id ( in user collection).
I tried the below query:
db.role.aggregate({
"$lookup": {
"from": "user",
"localField": "userId",
"foreignField": "_id",
"as": "output"
}
})
This gives me expected results as long as i store userId as a ObjectId. When my userId is a string there are no results.
Ps: I tried
foreignField: '_id'.valueOf()
and
foreignField: '_id'.toString()
. But no luck to match/join based on a ObjectId-string fields.
Any help will be appreciated.
You can use $toObjectId aggregation from mongodb 4.0 which converts String id to ObjectId
db.role.aggregate([
{ "$lookup": {
"from": "user",
"let": { "userId": "$_id" },
"pipeline": [
{ "$addFields": { "userId": { "$toObjectId": "$userId" }}},
{ "$match": { "$expr": { "$eq": [ "$userId", "$$userId" ] } } }
],
"as": "output"
}}
])
Or you can use $toString aggregation from mongodb 4.0 which converts ObjectId to String
db.role.aggregate([
{ "$addFields": { "userId": { "$toString": "$_id" }}},
{ "$lookup": {
"from": "user",
"localField": "userId",
"foreignField": "userId",
"as": "output"
}}
])
This is not possible as of MongoDB 3.4. This feature has already been requested, but hasn't been implemented yet. Here are the corresponding tickets:
SERVER-22781: Allow $lookup between ObjectId (_id.str) and
string
SERVER-24947: Need a type conversion mechanism for booleans,
ISODates, ObjectID
For now you'll have to store userId as ObjectId
EDIT
The previous tickets were fixed in MongoDB 4.0. You can now achieve this with the folowing query:
db.user.aggregate([
{
"$project": {
"_id": {
"$toString": "$_id"
}
}
},
{
"$lookup": {
"from": "role",
"localField": "_id",
"foreignField": "userId",
"as": "role"
}
}
])
result:
[
{
"_id": "584aac38686860d502929b8b",
"role": [
{
"_id": ObjectId("584aaca6686860d502929b8d"),
"role": "Admin",
"userId": "584aac38686860d502929b8b"
}
]
}
]
try it online: mongoplayground.net/p/JoLPVIb1OLS
I think the previous answer has an error on the $toObjectId case. The let statement applies to the db collection on which the function aggregate is called (i.e 'role') and not on the collection pointed by "from" (i.e 'user').
db.role.aggregate([
{ "$lookup": {
"let": { "userObjId": { "$toObjectId": "$userId" } },
"from": "user",
"pipeline": [
{ "$match": { "$expr": { "$eq": [ "$_id", "$$userObjId" ] } } }
],
"as": "userDetails"
}}
])
Or
db.role.aggregate([
{ "$project": { "userObjId": { "$toObjectId": "$userId" } } },
{ "$lookup": {
"localField": "userObjId",
"from": "user",
"foreignField": "$_id",
"as": "userDetails"
}}
])
And
db.user.aggregate([
{ "$project": { "userStrId": { "$toString": "$_id" }}},
{ "$lookup": {
"localField": "userStrId",
"from": "role",
"foreignField": "userId",
"as": "roleDetails"
}}
])
For example we have two collections:
Authors like {'_id': ObjectId(), name: 'John Smith'}
Messages like {'_id': ObjectId(), text: 'Message Text', authorId: 'stringId'}
And we need to get messages with author names and other data
Then:
db.messages.aggregate([
{
$addFields: {
'$authorObjectId': {$toObjectId: $authorId}
}
},
{
$lookup: {
from: 'authors',
localField: '$authorObjectId',
foreignField: '_id',
as: 'author'
}
}
])
Explained:
Our aggregation pipeline has two steps:
First: We add additional field to messages which contains converted string authorId to ObjectId()
Second: We use this field as localField (in messages) to be compared with foreignField '_id' (in authors)

MongoDB: Lookup - Collections with Same Fields

I've two collections Users and Notes. Both collections contain an id property and the Notes collection has an userid that is the id of some user on the Users collection.
Now, I'm trying to aggregate (join) some user information into Notes:
db.getCollection("Notes").aggregate(
{
"$lookup": {
"from": "Users",
"let": {
"idForeignField": "$id"
},
"pipeline": [
{
"$match": {
"$expr": {
"$and": [{
"$eq": ["$userid", "$$idForeignField"]
}]
}
}
}
],
"as": "Users#joined"
}
}
);
What I get in a empty Users#joined array. Why? Shouldn't my query work? Is the problem caused by the fact that both collections have an id property? If yes how can I tell let and match what is the right collection?
Update: alternatively a simpler query works just fine:
db.getCollection("Notes").aggregate(
{
$lookup:
{
from: "Users",
localField: "userid",
foreignField: "id",
as: "Users#joined"
}
}
);
However I would like to do it with let and a pipeline in order to add more match conditions.
Thank you.
Your let variable must be userid
db.getCollection("Notes").aggregate([
{ "$lookup": {
"from": "Users",
"let": { "ifForeignField": "$userid" },
"pipeline": [
{ "$match": { "$expr": { "$and": [{ "$eq": ["$id", "$$ifForeignField"] }] }}}
],
"as": "Users#joined"
}}
])