Mongodb 3.2 aggregation $lookup for nested document - mongodb

I have two collections: memberships and categories.
In the categories I have stored array of objects called ageGroups.
memberships : [{
_id: ObjectId,
ageCategory: ObjectId,
ageGroup: ObjectId
}]
categories: {
_id: ObjectId,
ageGroups: [{
_id: ObjectId,
name: String
}, {
_id: ObjectId,
name: String
}, {
_id: ObjectId,
name: String
}]
}
There are ageCategory and ageGroup properties in memberships collection.
Now i need to aggregate these two and get result like this
[{
id: 'membership id here',
ageGroup: 'Age Group name here'
},{
id: '2 membership id here',
ageGroup: '2 membership age Group name here'
}]
So I tried
db.memberships.aggregate([
{
$lookup: {
from: 'categories', localField: 'ageCategory', foreignField: '_id', as: 'ageCategory'
}
},
{
$unwind: '$ageCategory'
},
{
$project: {
'group': {
$filter: {
input : '$ageCategory.ageGroups',
as : 'group',
cond : 'group._id' == '$_id'
}
}
}
},
{
$unwind: '$group'
},
{
$project: {
'group' : '$group.name'
}
}
]);
This query return empty result. As much as I understand, problem in filter condition
$filter: {
input : '$ageCategory.ageGroups',
as : 'group',
cond : 'group._id' == '$_id'
}
when I tried to set true in cond, it pushed new document to result, for each group, even if ageGroup not that, which is assigned to membership. Thanks for help!
UPD
I found solution,
{
$project: {
'group': {
$filter: {
input : '$ageCategory.ageGroups',
as : 'group',
cond : { $eq:['$$group._id', '$ageGroup']}
}
}
}
},
I have added double dollar sign with $eq operator and problem has been solved. Now I search better solution and I would like have a query which get similar result
{
"_id" : // member id,
"group" : "1-3 members"
}
{
"_id" : //member id,
"group" : "17-19 members"
}
without storing ageCategory in memberships table, only with ageGroup id.

Related

Mongoose GroupBy with Populate

i have User document and and Category document which is connected to shop document which look like
var ShopSchema = mongoose.Schema(
{
shopName: {
type: String,
required: [true, 'A Shop must have name'],
},
phone: {
type: String,
validate: [validator.isMobilePhone,"Phone number is not a valid number"],
required: [true, 'A Shop must have hepline number'],
},
shopStatus: {
type: Boolean,
default: false,
select: false
},
managerId:{
type: mongoose.Schema.ObjectId,
ref:'User',
require: [true,'Shop must have to a manager!']
},
createdAt:{
type:Date,
default: Date.now()
},
shopCategory:{
type: mongoose.Schema.ObjectId,
ref:'Category',
require: [true,'Shop must have a Category!']
},
},
{
toJSON: { virtuals: true },
toObject: { virtuals: true }
}
);
const Shop = mongoose.model("Shop", ShopSchema);
Now i am writing a query to get managers shop(groupBy managerId) with populate of Manager and Category data. but i am getting empty array everytime.
const doc = await Shop.aggregate([
{
$match: { shopStatus: {$ne: false} }
},
{
$group: {
_id:'$managerId',
numofShop: {$sum: 1},
shopCategory: {$first: "$shopCategory"}
}
},
{
$lookup: {
from: "User",
localField : "_id",
foreignField: "_id",
as: "Manager"
}
},
{
$lookup: {
from: "Category",
localField : "shopCategory",
foreignField: "_id",
as: "Category"
}
},
])
here is how my final result look like but this is an empty array.
{
"_id": "5f467660f630e804ec07fad8",
"numofShop": 2,
"Manager": [],
"Category": []
},
{
"_id": "5f44d2f4ff04993b40684bf9",
"numofShop": 1,
"Manager": [],
"Category": []
}
I want to find All shops groupBy managerId(Who own it). there is field of managerId referencing to User document who created this shop. i want data like this
{
"_id": "5f44d2f4ff04993b40684bf9",
"Manager": {},//this is populated through User Schema
"numofShop": 1,
"Shop": [
{
"shopName":"Grocery Shop",
"phone" : "111111",
"shopCategory" :{"_id": "","name":"Electronics"}
}
]
}
.....
.....
Find Example Data here
Shops- http://txt.do/1fwi0
3 shops created by two users
User - http://txt.do/1fwio
Categories- http://txt.do/1fwij
There are 2 fixes,
in second lookup you have no shopCategory field in localField because you have not defined shopCategory in $group,
{
$group: {
_id:'$managerId',
numofShop: {$sum: 1},
shopCategory: { $first: "$shopCategory" } // define here
}
},
in first lookup with user collection, there is no managerId field and you have assigned in _id field, then you can use _id in localField,
{
$lookup: {
from: "User",
localField : "_id", // change this from managerId
foreignField: "_id",
as: "Manager"
}
},
Updated things after updated question, Playground1, Playground2
for your second edit, you can't put condition for populate in find method, how to put condition in populate, you can follow the answer1 and answer2

Mongodb lookup with array

I have two collections first one is
user_profile collection
const userProfileSchema = mongoose.Schema({
phone_number: {
type: String,
required: false,
},
primary_skills: [
{
skill_id: {
type: mongoose.Schema.Types.ObjectId,
ref: 'Skill'
},
years: Number,
}
]
});
sample data
{
"phone_number":"222",
"primary_skills":[{skill_id:1,years:12},{skill_id:2,years:13}]
}
in the primary_skills the key skill_id is mapped with another collection named skills
skills collection
const skillSchema = mongoose.Schema({
name: {
type: String,
required: true,
unique:true,
},
});
sample data
[
{
id:1,
name:'php'
},
{
id:2,
name:'java'
}
]
I want to fetch all values in the user_profile collection along with the respective skills name
expected output:
{
"phone_number":"222",
"primary_skills":[{
name:"php",skill_id:1,years:12
},{
name:"java",skill_id:2,years:13}
]
}
I found a similar thread to my question MongoDB lookup when foreign field is an array of objects but it's doing the opposite of what I want
This is the query I tried
profile.aggregate([{
$lookup:{
from:'skills',
localField:'primary_skills.skill_id',
foreignField:'_id',
'as':'primary_skills'
}
}])
This works fine but it didn't contain the years key
You need to do it with $unwind and $group,
$unwind primary_skills because its an array and we need to lookup sub document wise
db.user_profile.aggregate([
{
$unwind: "$primary_skills"
},
$lookup to join primary_skills, that you have already did
{
$lookup: {
from: "skills",
localField: "primary_skills.skill_id",
foreignField: "id",
as: "primary_skills.name"
}
},
$unwind primary_skills.name that we have stored join result, its array and we are unwinding to do object
{
$unwind: {
path: "$primary_skills.name"
}
},
$addFields replace field name that we have object and we need only name
{
$addFields: {
"primary_skills.name": "$primary_skills.name.name"
}
},
$group by _id because we have unwind and we need to combine all documents
{
$group: {
_id: "$_id",
phone_number: {
$first: "$phone_number"
},
primary_skills: {
$push: "$primary_skills"
}
}
}
])
Playground: https://mongoplayground.net/p/bDmrOwmASn5

$lookup using multiple criteria mongodb java aggregation

Have 2 following collections:
user collection
{
userId:user1,
creationTimeStamp:2019-11-05T08:15:30
status:active
},
{
userId:user2,
creationTimeStamp:2019-10-05T08:15:30
status:active
}
document collection
{
userId:user1,
category:Development
published:true
},
{
userId:user2,
category:Development
published:false
}
I want to join these two collections and filter users such that documents which are of development category and are not published from active users between creationtimestamp
How can I write a mongodb java aggregation in order to get a result like this:
{
userId: user2,
status:active,
category:Development,
published:false
}
You could run below aggregation query on the document collection to get the expected result
[{$match: {
category:'development',
published: false
}}, {$lookup: {
from: 'user',
localField: 'userId',
foreignField: 'userId',
as: 'JoinedTable'
}}, {$unwind: {
path: '$JoinedTable'
}}, {$group: {
_id: '$_id',
userId: {
$first: '$userId'
},
status: {
$first: '$JoinedTable.status'
},
category: {
$first: '$category'
},
published: {
$first: '$published'
},
}}]
Explanation:
1. filter documents using match for criteria category: 'development' & published: false
2. join document collection with user collection with key userId
3. unwind the joined collection field to convert array to object
4. project the fields needed using groups.
Hope this helps!
You haven't mentioned about the duplicate of userId in User collection.
So the script is
[{
$match: {
category: "Development",
published: false
}
}, {
$lookup: {
from: 'user',
localField: 'userId',
foreignField: 'userId',
as: 'joinUser'
}
}, {
$unwind: {
path: "$joinUser",
preserveNullAndEmptyArrays: true
}
}, {
$match: {
"joinUser.status": "active"
}
}, {
$addFields: {
"status": "$joinUser.status"
}
}, {
$project: {
_id: 0,
userId: 1,
category: 1,
published: 1,
status: 1
}
}]
And the java code,
include these imports
import static org.springframework.data.mongodb.core.aggregation.Aggregation.match;
import static org.springframework.data.mongodb.core.aggregation.Aggregation.lookup;
import static org.springframework.data.mongodb.core.aggregation.Aggregation.unwind;
import static org.springframework.data.mongodb.core.aggregation.Aggregation.project;
method is,
public Object findAllwithVideos() {
Aggregation aggregation=Aggregation.newAggregation(
match(Criteria.where("category").is("Development").and("published").is(false)),
lookup("user","userId","userId","joinUser"),
unwind("joinUser",true),
new AggregationOperation(){
#Override
public Document toDocument(AggregationOperationContext aggregationOperationContext){
return new Document("$addFields",
new Document("status","$joinUser.status")
);
}
},
project("userId","category","published","status")
).withOptions(AggregationOptions.builder().allowDiskUse(Boolean.TRUE).build());
return mongoTemplate.aggregate(aggregation, mongoTemplate.getCollectionName(Document.class), Object.class);
}

MongoDB $lookup with nested object with nested array

I have 2 collections.
cases
_id: ObjectId.
name: string.
info: {
[here can be many different fields with diff types]
relatedEntities: [
{ role: string;
id: ObjectId;
} <--- here can be a lot of entities
]
}
entities
_id: ObjectId.
type: string,
name: string,
info: {
[here can be many different fields with diff types]
}
I need to retrieve all cases and for each case.info.entities object I need to have field data which will equal to entity document ( case.info.entities.id === entity_id)
Example what I need to have
_id: ObjectId.
name: string.
info: {
[here can be many different fields with diff types]
entities: [
{ role: string;
id: ObjectId;
data: {
_id: ObjectId.
type: string,
name: string,
info: {
[here can be many different fields with diff types]
}
}
} <--- here can be a lot of entities
]
}
How to do it in a proper way?
At the moment I implemented this is that way:
{ $unwind: "$info.relatedEntities" },
{ $lookup: {
"from": "entities",
"localField": "info.relatedEntities.entity",
"foreignField": "_id",
"as": "info.relatedEntities.entityObject"
}},
{ $group: {
"_id": "$_id",
"templateType":{$first: "$templateType"},
"info":{$first: "$info"},
"relatedEntities": {
$push: "$info.relatedEntities"
}
}}
It's working, but required additional parsing when data is retrieved, but I'd like to do it without workarounds..
You have done almost everything. Based on your query and model I have given below query (Field names might be different). Hope it helps.
db.cases.aggregate([
{ $unwind: '$info.relatedEntities' },
{ $lookup: {
from: 'entities',
localField: 'info.relatedEntities.entity',
foreignField: '_id',
as: 'info.relatedEntities.entityObject'
}
},
{ $group: {
_id: {
_id : '$_id',
templateType : '$templateType',
name : '$name',
info : {
address : "$info.address",
}
},
'relatedEntities': {
$push: {
role : '$info.relatedEntities.role',
entity : '$info.relatedEntities.entity',
data : { $arrayElemAt: [ '$info.relatedEntities.entityObject', 0 ] }
}
}
}
},
{
$project : {
_id : '$_id._id',
name : '$_id.name',
templateType : '$_id.templateType',
info : {
address : '$_id.info.address',
entities : '$relatedEntities'
}
}
}
]).pretty()

When I search on multiple field included lookup with more then one field with $regex MongoDB perform slow

I have Book collection which have Name, Description, Publisher ObjectID field, Array of Authors ID, Categories ID field etc. I need to search book with name, description, publisher name, author and category name with $regex.
To do that in aggregation pipe first I populate authors, publisher, categories with $lookup and then use $match with $or operator over the field.
My query works but it perform very slow ( approximately 11s ) where Book collection contain only 70 thousand documents.
What steps should I need in collection model, Indexing or Query good performance?
Book Model:
{
"_id" : ObjectId("5a2934934410bf8b0e547989"),
"publisher" : ObjectId("5a27e7b68021772210b125d4"),
"is_enabled" : true,
"published_at" : ISODate("2017-12-07T12:31:15.166Z"),
"author" : [
ObjectId("5a27c5754b0efc477f37a131"),
ObjectId("5a27c5754b0efc47737a1512"),
ObjectId("5a27c5754b0efc477f37a145"),
],
"category" : [
ObjectId("5a27e22ffb6110b11c326cd7"),
ObjectId("5a27e22ffb6110b11c326ced"),
ObjectId("5a27e22ffb6110b11c326d2d"),
ObjectId("5a27e22ffb6110b11c326e45")
]
"published_year" : "2017"
}
Query I executed:
Book.aggregate(
[
{
$match: {
"is_enabled": { $eq: true },
}
},
{
$lookup:
{
from: "authors",
localField: "author",
foreignField: "_id",
as: "author"
}
},
{
$lookup:
{
from: "categories",
localField: "category",
foreignField: "_id",
as: "category"
}
},
{
$lookup:
{
from: "publishers",
localField: "publisher",
foreignField: "_id",
as: "publisher"
}
},
{
$match: {
$or: [
{ "author.name": new RegExp(params.expression, 'i') },
{ "category.name": new RegExp(params.expression, 'i') },
{ "publisher.name": new RegExp(params.expression, 'i') },
{ "description": new RegExp(params.expression, 'i') },
{ "name": new RegExp(params.expression, 'i') },
{ "published_year": params.terms }
]
}
},
{
$project: {
previous_price: "$previous_price",
price: "$price",
name: "$name",
seo_url: "$seo_url",
click_url: "book",
author: "$author",
authorObj: {
name: { $arrayElemAt: ["$author.name", 0] },
}
}
},
{ $sort: { name: 1 } }
]
)
.skip(8 * (params.pagenum - 1))
.limit(8)
.exec((err, product) => {
if (err)
reject(err);
else
resolve(product);
})
You can create index for fields is_enabled, author, category and publisher like bellow.
db.coll.createIndex( { is_enabled: 1 } )
db.coll.createIndex( { author: 1 } )
db.coll.createIndex( { category: 1 } )
db.coll.createIndex( { publisher: 1 } )
that will increase the performance for first match stage and for lookup.
you can also create index for name, description and published_year but I am not sure how will affect of this index for last match stage because you used $or condition. As far I know still unable to optimize indexed queries that uses $or, $in (<=3.2). You can try that. It will be helpful if you use $and condition query. If you use $and query then you can also create multi key index for name, description and published_year. like
db.coll.createIndex( { name: 1, description: 1 published_year:1 } )
and then you should follow the same order in match condition
{$match: { name: 'xx', description:'yy', published_year: 2017}}