Mongoose querying a reference collection data using aggrigate/populate - mongodb

I am a newbie in MongoDB and Express JS
I have a Product model looks like
const productSchema = mongoose.Schema({
name: String,
title: String,
brand_id: { type: mongoose.Schema.Types.ObjectId, ref: 'brand' },
varient:[{
country_id: { type: mongoose.Schema.Types.ObjectId,ref: 'countries'},
max_retail_price : Number,
profit_margin : Number
}],
and Order model
const orderTransactionSchema = mongoose.Schema({
shop_id: { type: mongoose.Schema.Types.ObjectId, ref: 'shop' },
brand_id:{ type: mongoose.Schema.Types.ObjectId, ref: 'brand' },
product_id: { type: mongoose.Schema.Types.ObjectId, ref: 'product' },
product_varient_id:{ type: mongoose.Schema.Types.ObjectId, ref: 'product.varient._id' },
transaction_id:{ type: mongoose.Schema.Types.ObjectId, ref: 'transaction' }
})
module.exports = mongoose.model('order', orderTransactionSchema );
In my Product collection, each product can have multiple variants.
But in order collection, a user can order only one product variant under a product.
I am trying to display orders with particular product details and variant details, But the problem is when I try to display it using either populate/Aggregate I am getting all the variants in the response array. actually, I want only one product and variant details as in the order collection.
This is what I tried
order.aggregate([{ $match :{} },
{
$lookup: {
from: "products",
localField: "product_id",
foreignField: "_id",
as: "product_data"
}
},
]).exec(function(err,result){
console.log(result);
});
and I am getting the output as
{ _id: 5c8a010b8feeb875abc1b066,
shop_id: 5c7d194ca10ea45c0c03a0ee,
brand_id: 5c41a8c34272c61a176b7639,
product_varient_id: 5c41a9f3f8e1e71aa75b4f32,
transaction_id: 5c6670d5b6c63d0762c6cc77,
product_id: 5c41aac4d45a731af564c433,
product_data:
[ { _id: 5c41aac4d45a731af564c433,
brand_id: 5c41a8c34272c61a176b7639,
image: 'test.jpg',
varient: //getting all the varients here
[ { _id: 5c4ee531bc20b27948b3aa98,
sell_rate_local: 66,
country_id: 5c01149d3c20440a6b2e4928 },
{ _id: 5c4ee53bbc20b27948b3aa99,
sell_rate_local: 66,
country_id: 5c00e1697dd7a23f08bdae68 } ],
__v: 0 } ] } ]
In the Order table, there is porduct_id and product_varient_id is there
I want to populate only the product with product_varient_id.
I also tried something with Populate
order.find().
populate([
{ path: 'shop_id', model: 'shop',select : 'name accounts' }, //it works
{ path: 'transaction_id', model: 'transaction' }, //it wrks
{ path: 'product_varient_id', model: 'product', select:'product.varient.name'},
]).then(result =>
{
// console.log(result);
}).catch(err =>{
// console.log(err);
});
These are the sample product and order document
Order Document Sample :
{
"_id":"5c77a025d65a892f6acf1803",
"shop_id":"5c7d194ca10ea45c0c03a0ee",
"brand_id":"5c41a8b44272c61a176b7638",
"product_varient_id":"5c41a9f3f8e1e71aa75b4f32",
"buy_rate":10,
"buy_rate_after_discount":20,
"product_mrp":30,
"sell_rate":40,
"customer_mobile":123456789,
"status":true,
"transaction_id":"5c6670c9b6c63d0762c6cc76",
"product_id":"5c41a95ff8e1e71aa75b4f30",
"createdAt":"2019-02-28T08:47:33.097Z",
"updatedAt":"2019-02-28T08:47:33.097Z",
"__v":0
}
Product document Sample :
{
"_id":"5c41aac4d45a731af564c433",
"recharge_type":[
"5c00d9cf7dd7a23f08bdae5e"
],
"name":"25 OC - Product 1",
"title":"First installation recharge",
"description":"0.1 credit for first time installation",
"offer_message":"Hi.. You got 0.1 credits..!!",
"brand_id":"5c41a8c34272c61a176b7639",
"buy_rate":20,
"profit_margin":80,
"image":"test.jpg",
"varient":[
{
"_id":"5c4ee531bc20b27948b3aa98",
"display_name":"testlia",
"profit_margin":66,
"max_retail_price":66,
"sell_rate":66,
"sell_rate_local":66,
"country_id":"5c01149d3c20440a6b2e4928"
},
{
"_id":"5c4ee53bbc20b27948b3aa99",
"display_name":"testrinu",
"profit_margin":66,
"max_retail_price":66,
"sell_rate":66,
"sell_rate_local":66,
"country_id":"5c00e1697dd7a23f08bdae68"
}
],
"createdAt":"2019-01-18T10:30:28.991Z",
"updatedAt":"2019-01-28T11:19:23.662Z",
"__v":0
}

MongoDB 3.6 or above have new lookup syntax
db.orders.aggregate([{
$lookup: {
from: "products",
let: {
"productId": "$product_id",
"productVarientId": "$product_varient_id"
},
pipeline: [
{ $match: {
$expr: { $eq: [ "$_id", "$$productId" ]}
}
},
{ $addFields: {
varient: {
$filter: {
input: "$varient",
as: "varient",
cond: { $eq: [ "$$productVarientId", "$$varient._id" ] }
}
}
}
}
],
as: "product_data"
}
}])
You can check datasample Here

Related

MongoDB Realm: Query a Relationship

here is my problem:
export const PostSchema = {
name: "Post",
primaryKey: "_id",
properties: {
_id: "string",
comments: "Comment[]"
},
};
export const CommentSchema = {
name: "Comment",
primaryKey: "_id",
properties: {
_id: "string",
post: "Post"
},
};
// Original Query:
const posts = realm.objects("Post");
console.log(posts);
// Original Query query results in:
[
{
_id: "post_1",
comments: [],
},
{
_id: "post_2",
comments: [],
},
...
];
// I'm looking for a query with the following result:
[
{
_id: "post_1",
comments: [
{
_id: "Only the first (newest) comment post_1",
},
],
},
{
_id: "post_2",
comments: [
{
_id: "Only the first (newest) comment post_2",
},
],
},
...
];
//My current solution therefor is:
let data = realm.objects("Post").map((post) => {
return {
...post.toJSON(),
comments: m.linkingObjects("Comment", "post").slice(0, 1),
};
});
I'm not sure if this is the right/optimal solution.
If I understood correctly, then I make an extra request for the first comment for each post?
So if I have 20 posts, that's 20 extra requests? That would be very impractical?
Is there a better way?
I also tried Inverse Relationships.
But with this method I always get all the comments for a post which is a lot of unnecessary data?
THX for any help :-)
db.posts.aggregate([
{
$lookup: {
from: "comments",
localField: "comments",
foreignField: "_id",
as: "comments",
pipeline: [
{ $sort: { _id: 1 } },
{ $limit: 1 }
]
}
}
])
mongoplayground

Aggregate data and populate in one request

I am a bit puzzled by populate in MongoDB.
I've got a Schema:
import { Schema, Document, model } from "mongoose";
export interface ProductGroupType {
id: Schema.Types.ObjectId,
title: String,
name: String,
description: String,
}
const ProductGroupSchema: Schema<Document<ProductGroupType>> = new Schema({
title: { type: String, trim: true },
name: { type: String, trim: true },
description: { type: String, trim: true },
}, { collection: "productGroups", timestamps: true });
export const ProductGroupModel = model('ProductGroup', ProductGroupSchema);
and products
import { Schema, Document, model } from "mongoose";
import { plugin as autocomplete } from 'mongoose-auto-increment';
const ProductSchema: Schema<Document<IProduct>> = new Schema({
article: Number,
name: String,
category: { type: Schema.Types.ObjectId, ref: 'ProductCategory' },
group: { type: Schema.Types.ObjectId, ref: 'ProductGroup' },
price: { type: Number, default: 0 },
discount: { type: Number, default: 0 },
stock: {
available: { type: Number, default: 0 },
reserved: { type: Number, default: 0 },
},
images: [Object],
description: String,
productDetails: Object,
}, { collection: "products", timestamps: true });
ProductSchema.plugin(autocomplete, {
model: 'Product',
field: 'article',
startAt: 10000,
});
export const ProductModel = model('Product', ProductSchema);
I need to make a request and group on the MongoDB side data by the field 'group'.
I can make this like this:
await ProductModel.aggregate([
{ $match: { category: Types.ObjectId(queryCategory.id) } },
{
$group: {
_id: '$group',
products: {
$push: {
id: '$_id',
name: '$name',
article: '$article',
price: '$price',
discount: '$discount',
description: '$description',
group: '$groupName',
}
},
count: { $sum: 1 },
}
},
]);
but the output here is:
[
{ _id: 61969583ad32e113f87d0e99, products: [ [Object] ], count: 1 },
{
_id: 61993fff452631090bfff750,
products: [ [Object], [Object] ],
count: 2
}
]
almost what I need but I've been playing around with population and I cannot make it work with Aggregation framework.
I already tried to use the 'lookup' operator but it returns an empty array and doesn't want to work.
That's how I wanted to make it work:
const products: Array<IProduct> = await ProductModel.aggregate([
{ $match: { category: Types.ObjectId(queryCategory.id) } },
{
$group: {
_id: '$group',
products: {
$push: {
id: '$_id',
name: '$name',
article: '$article',
price: '$price',
discount: '$discount',
description: '$description',
group: '$groupName',
}
},
count: { $sum: 1 },
}
},
{
$lookup: {
"from": "productGroups",
"localField": "group",
"foreignField": "_id",
"as": "groupName"
},
},
]);
Is it possible to get the same result as I've got now but populate in the same query group field?
So far the only way I've managed to populate it like this as the second request:
await ProductGroupModel.populate( products.map( (product: any) => {
return {
_id: new ProductGroupModel(product),
products: product.products,
count: product.count,
}
} ), { "path": "_id" } )
In a MongoDB aggregation pipeline, the $group stage passes along only those field explicitly declared in the stage.
In the same pipeline you show, the documents passed along by the $group stage would contain the fields:
_id
products
count
When the exector arrives a the $lookup stage, none of the documents contain a field named group.
However, the value previously contained in the group field still exists, in the _id field.
In the $lookup stage, use
"localField": "_id",
to find documents based on that value.

Mongoose query on multiple populated fields

I have three collections i.e Events, News and FuneralNews.
I have another Notifications collection that is just a combination of all three and contains the reference Id of either/one of the three collections.
I want to fetch only those Notifcations whose eventId OR newsId OR funeralNewsId field isActive is true
EventsSchema:
var EventsSchema = new Schema({
...
isActive: Boolean
});
FuneralNewsSchema:
var FuneralNewsSchema = new Schema({
...
isActive: Boolean
});
NewsSchema:
var NewsSchema = new Schema({
...
isActive: Boolean
});
NotificationSchema:
var NotificationSchema = new Schema({
type: {
type: String,
required: true
},
creationDate: {
type: Date,
default: Date.now
},
eventId: {type: Schema.Types.ObjectId, ref: 'Events'},
newsId: {type: Schema.Types.ObjectId, ref: 'News'},
funeralNewsId: {type: Schema.Types.ObjectId, ref: 'FuneralNews'},
organisationId: {type: Schema.Types.ObjectId, ref: 'Organization'}
});
This was my query before I need a check on isActive property of referenced collection:
let totalItems;
const query = { organisationId: organisationId };
Notification.find(query)
.countDocuments()
.then((count) => {
totalItems = count;
return Notification.find(query, null, { lean: true })
.skip(page * limit)
.limit(limit)
.sort({ creationDate: -1 })
.populate("eventId")
.populate("newsId")
.populate("funeralNewsId")
.exec();
})
.then((notifications, err) => {
if (err) throw new Error(err);
res.status(200).json({ notifications, totalItems });
})
.catch((err) => {
next(err);
});
Now I don't know how to check on isActive field of three populated collections prior population.
I have seen other questions like this and this but being a newbie can't edit it according to my use-case. Any help would be highly appreciated
use $lookup for each objectId refrence
then group by _id of null to get data and add myCount as total number put original data to array
and use unwind to destruct array and use addField
model
.aggregate([
{
$lookup: {
from: "Events", // events collection name
localField: "eventId",
foreignField: "_id",
as: "events",
},
},
{
$lookup: {
from: "FuneralNews", //FuneralNews collection name
localField: "funeralNewsId",
foreignField: "_id",
as: "funeralnews",
},
},
{
$lookup: {
from: "News", // news collection name
localField: "newsId",
foreignField: "_id",
as: "news",
},
},
{
$match: {
$or: [
{ "news.isActive": true },
{ "events.isActive": true },
{ "funeralnews.isActive": true },
],
},
},
{
$group: {
_id: null,
myCount: {
$sum: 1,
},
root: {
$push: "$$ROOT",
},
},
},
{
$unwind: {
path: "$root",
},
},
{
$addFields: {
"root.total": "$myCount",
},
},
{
$replaceRoot: {
newRoot: "$root",
},
},
{
$sort: {
creationDate: -1,
},
},
])
.skip(page * limit)
.limit(limit);

Problem using aggregation in mongodb retrieving data from two collections

i am strugling with a query that i don't know how to perform... I have two collections,
Tarifas Collection
tarifaConfig = new Schema({
producto: { type: String },
titulo: { type: String },
bloqueo: { type: Boolean },
margen: { type: Number },
precioVenta: { type: Number },
precioVentaIva: { type: Number },
})
const tarifaSchema = new Schema({
codigo: { type: String },
titulo: { type: String },
margen: { type: Number },
estado: { type: Boolean },
bloqueo: { type: Boolean },
configs: [tarifaConfig]
})
Producto Collection
const productosSchema = new Schema({
ref: { type: String },
nombre: { type: String },
precioCompra: { type: Number },
precioCompraIva: { type: Number },
precioVenta: { type: Number },
precioVentaIva: { type: Number },
iva: { type: Number },
})
Now i am using an Aggregation method to retrieve both collection in a response
productosModel.aggregate([
{
$match: { _id: ObjectId(req.params.id) }
},
{
$lookup: {
from: "tarifas",
as: "tarifas",
pipeline: []
}
}
]).then((producto) => {
res.json(producto);
})
This is working and gives me both collections in the response... but..
In tarifa's collection i have a propertie called 'configs' that is an array with lot of sub collections... this sub collections are a config of each product that i have,
So what i need to do is, retrieve all tarifas that has a configs for the product, and if the configs does not contain retrieve the tarifa with a empty array.
Expected result
{
ref: 'rbe34',
nombre: 'bike',
precioCompra: 10,
precioCompraIva: 12.1,
precioVenta: "",
precioVentaIva: "",
iva: 21,
tarifas:[
{
codigo: 'NOR',
titulo: 'Normal tarifa',
margen: 33,
estado: true,
bloqueo: true,
configs: [], ///HERE I NEED A EMPTY ARRAY IF THERE IS NOT ANY CONFIG THAT MATCH WITH THE PRODUCT ID,
}
]
}
i tried to add $match in my aggregation pipeline.
productosModel.aggregate([
{
$match: { _id: ObjectId(req.params.id) }
},
{
$lookup: {
from: "tarifas",
as: "tarifas",
pipeline: [
{ $match: { 'configs.producto': req.params.id } }
]
}
}
])
But if there is not any config that match the product it doesn't retrieve the rest of Tarifa's collection
It seems you are trying to $filter the array after you retrieve it.
This pipeline will return only the configs for which the producto field from the config matches the ref field from the product.
[
{
$match: { _id: ObjectId(req.params.id) }
},
{
$lookup: {
from: "tarifas",
as: "tarifas",
pipeline: [
{
$addFields: {
"tarifas.configs":{ $filter:{
input: "$tarifas.configs",
cond: {$eq:["$$this.producto","$ref"]}
} }
}
}
]
}
},
]
Change the fields in the $eq array to the ones you need to match.

Mongoose populate and sort by length struggle

I have the following mongoose schema:
const userSchema = new mongoose.Schema({
email: { type: String, unique: true },
fragments: [{type: mongoose.Schema.Types.ObjectId, ref: 'Fragment'}]
}, { timestamps: true, collection: 'user' });
And
const fragmentSchema = new mongoose.Schema({
text: String,
owner: {type: mongoose.Schema.Types.ObjectId, ref: 'User'},
}, { timestamps: true, collection: 'fragment' });
In the data, I have a reference in the Fragment, but not in the User:
User:
{
"_id" : ObjectId("58373e571cbccb010012bfcd"),
"email" : "email#example.com",
// no "fragments": [ObjectId('58075ce37b7f2f01002b718f')] etc.
}
Fragment:
{
"_id" : ObjectId("58075ce37b7f2f01002b718f"),
"text" : "Donc, il faut changer de méthode",
"owner" : ObjectId("58075ce27b7f2f01002b717f")
}
I would like to query users sorted by the count of number of fragments, and I can't achieve this ...
First, I'd like to make this work:
User.find({_id: '58075ce27b7f2f01002b717f'})
.populate('fragments').exec(console.log)
returns
{
_id: 58075ce27b7f2f01002b717f,
email: 'bububg#hotmail.fr',
fragments: []
}
while I should have at least the above fragment included.
And regarding the sorted query, here's where I am now:
User.aggregate([
{ "$project": {
"email": 1,
"fragments": 1,
"nbFragments": { "$size": { "$ifNull": [ "$fragments", [] ] } }
}},
{ "$sort": { "nbFragments": -1 } }
], console.log)
At least it runs, but all the nbFragments fields are set to 0. This might be related to the fact that .populate('fragments') doesn't work but I can't be sure.
Thanks for the help, I did not expect so much trouble using Mongodb...
EDIT: thanks #Veeram, unfortunately your solution is not working:
User.find({}).find({_id: '58075ce27b7f2f01002b717f'}).populate('fragments').exec(console.log)
[ { _id: 58075ce27b7f2f01002b717f,
email: 'email#example.com',
// no fragments entry
} ]
while I updated my schema:
userSchema.virtual('fragments', {
ref: 'Fragment',
localField: '_id',
foreignField: 'owner',
options: { sort: { number: 1 }}, // Added sort just as an example
});
And regarding the aggregate, with:
User.aggregate([{
$lookup: {
from: 'Fragment',
localField: '_id',
foreignField: 'owner',
as: 'fragments'
}
}, { "$project": {
"email": 1,
"fragments": 1,
"nbFragments": {
"$size": { "$ifNull": [ "$fragments", [] ] } }
}}, { "$sort": { "nbFragments": -1 } }
]).exec(console.log)
I get:
{
_id: 58075ce27b7f2f01002b717f,
email: 'email#example.com',
fragments: [] // fragments are always empty while they shouldn't!
}
Tested with following data
User:
{
"_id" : ObjectId("58373e571cbccb010012bfcd"),
"email" : "email#example.com"
}
Fragment:
{
"_id" : ObjectId("58075ce37b7f2f01002b718f"),
"text" : "Donc, il faut changer de méthode",
"owner" : ObjectId("58373e571cbccb010012bfcd")
}
Response
[{"_id":"58373e571cbccb010012bfcd","email":"email#example.com","fragments":[{"_id":"58075ce37b7f2f01002b718f","text":"Donc, il faut changer de méthode","owner":"58373e571cbccb010012bfcd"}],"nbFragments":1}]
You define schema to use owner to populate the fragments also called virtual population. http://mongoosejs.com/docs/populate.html
const userSchema = new mongoose.Schema({
email: { type: String, unique: true }
}, { timestamps: true, collection: 'user' });
var User = mongoose.model("User", userSchema);
const fragmentSchema = new mongoose.Schema({
text: String,
owner: {type: mongoose.Schema.Types.ObjectId, ref: 'User'},
}, { timestamps: true, collection: 'fragment' });
var Fragment = mongoose.model("Fragment", fragmentSchema);
userSchema.virtual('fragments', {
ref: 'Fragment',
localField: '_id',
foreignField: 'owner',
options: { sort: { text: -1 }}, // Added sort just as an example
});
This will now work as expected, but I don't know a way to sort on some dynamic field like count of number of fragments in mongoose. I don't think it is possible
User.find({_id: '58373e571cbccb010012bfcd'})
.populate('fragments').exec(function (err, user) {
console.log(JSON.stringify(user));
});
Okay now for dynamic sorting, you have to use alternative raw mongo query with a $lookup (equivalent of populate).
const userSchema = new mongoose.Schema({
email: { type: String, unique: true }
}, { timestamps: true, collection: 'user' });
var User = mongoose.model("User", userSchema);
const fragmentSchema = new mongoose.Schema({
text: String,
owner: {type: mongoose.Schema.Types.ObjectId, ref: 'User'},
}, { timestamps: true, collection: 'fragment' });
User.aggregate([{
$lookup: {
from: 'fragment',
localField: '_id',
foreignField: 'owner',
as: 'fragments'
}
}, { "$project": {
"email": 1,
"fragments": 1,
"nbFragments": {
"$size": { "$ifNull": [ "$fragments", [] ] } }
}}, { "$sort": { "nbFragments": -1 } }
]).exec(function (err, user) {
console.log(JSON.stringify(user));
})