Problem using aggregation in mongodb retrieving data from two collections - mongodb

i am strugling with a query that i don't know how to perform... I have two collections,
Tarifas Collection
tarifaConfig = new Schema({
producto: { type: String },
titulo: { type: String },
bloqueo: { type: Boolean },
margen: { type: Number },
precioVenta: { type: Number },
precioVentaIva: { type: Number },
})
const tarifaSchema = new Schema({
codigo: { type: String },
titulo: { type: String },
margen: { type: Number },
estado: { type: Boolean },
bloqueo: { type: Boolean },
configs: [tarifaConfig]
})
Producto Collection
const productosSchema = new Schema({
ref: { type: String },
nombre: { type: String },
precioCompra: { type: Number },
precioCompraIva: { type: Number },
precioVenta: { type: Number },
precioVentaIva: { type: Number },
iva: { type: Number },
})
Now i am using an Aggregation method to retrieve both collection in a response
productosModel.aggregate([
{
$match: { _id: ObjectId(req.params.id) }
},
{
$lookup: {
from: "tarifas",
as: "tarifas",
pipeline: []
}
}
]).then((producto) => {
res.json(producto);
})
This is working and gives me both collections in the response... but..
In tarifa's collection i have a propertie called 'configs' that is an array with lot of sub collections... this sub collections are a config of each product that i have,
So what i need to do is, retrieve all tarifas that has a configs for the product, and if the configs does not contain retrieve the tarifa with a empty array.
Expected result
{
ref: 'rbe34',
nombre: 'bike',
precioCompra: 10,
precioCompraIva: 12.1,
precioVenta: "",
precioVentaIva: "",
iva: 21,
tarifas:[
{
codigo: 'NOR',
titulo: 'Normal tarifa',
margen: 33,
estado: true,
bloqueo: true,
configs: [], ///HERE I NEED A EMPTY ARRAY IF THERE IS NOT ANY CONFIG THAT MATCH WITH THE PRODUCT ID,
}
]
}
i tried to add $match in my aggregation pipeline.
productosModel.aggregate([
{
$match: { _id: ObjectId(req.params.id) }
},
{
$lookup: {
from: "tarifas",
as: "tarifas",
pipeline: [
{ $match: { 'configs.producto': req.params.id } }
]
}
}
])
But if there is not any config that match the product it doesn't retrieve the rest of Tarifa's collection

It seems you are trying to $filter the array after you retrieve it.
This pipeline will return only the configs for which the producto field from the config matches the ref field from the product.
[
{
$match: { _id: ObjectId(req.params.id) }
},
{
$lookup: {
from: "tarifas",
as: "tarifas",
pipeline: [
{
$addFields: {
"tarifas.configs":{ $filter:{
input: "$tarifas.configs",
cond: {$eq:["$$this.producto","$ref"]}
} }
}
}
]
}
},
]
Change the fields in the $eq array to the ones you need to match.

Related

Aggregate data and populate in one request

I am a bit puzzled by populate in MongoDB.
I've got a Schema:
import { Schema, Document, model } from "mongoose";
export interface ProductGroupType {
id: Schema.Types.ObjectId,
title: String,
name: String,
description: String,
}
const ProductGroupSchema: Schema<Document<ProductGroupType>> = new Schema({
title: { type: String, trim: true },
name: { type: String, trim: true },
description: { type: String, trim: true },
}, { collection: "productGroups", timestamps: true });
export const ProductGroupModel = model('ProductGroup', ProductGroupSchema);
and products
import { Schema, Document, model } from "mongoose";
import { plugin as autocomplete } from 'mongoose-auto-increment';
const ProductSchema: Schema<Document<IProduct>> = new Schema({
article: Number,
name: String,
category: { type: Schema.Types.ObjectId, ref: 'ProductCategory' },
group: { type: Schema.Types.ObjectId, ref: 'ProductGroup' },
price: { type: Number, default: 0 },
discount: { type: Number, default: 0 },
stock: {
available: { type: Number, default: 0 },
reserved: { type: Number, default: 0 },
},
images: [Object],
description: String,
productDetails: Object,
}, { collection: "products", timestamps: true });
ProductSchema.plugin(autocomplete, {
model: 'Product',
field: 'article',
startAt: 10000,
});
export const ProductModel = model('Product', ProductSchema);
I need to make a request and group on the MongoDB side data by the field 'group'.
I can make this like this:
await ProductModel.aggregate([
{ $match: { category: Types.ObjectId(queryCategory.id) } },
{
$group: {
_id: '$group',
products: {
$push: {
id: '$_id',
name: '$name',
article: '$article',
price: '$price',
discount: '$discount',
description: '$description',
group: '$groupName',
}
},
count: { $sum: 1 },
}
},
]);
but the output here is:
[
{ _id: 61969583ad32e113f87d0e99, products: [ [Object] ], count: 1 },
{
_id: 61993fff452631090bfff750,
products: [ [Object], [Object] ],
count: 2
}
]
almost what I need but I've been playing around with population and I cannot make it work with Aggregation framework.
I already tried to use the 'lookup' operator but it returns an empty array and doesn't want to work.
That's how I wanted to make it work:
const products: Array<IProduct> = await ProductModel.aggregate([
{ $match: { category: Types.ObjectId(queryCategory.id) } },
{
$group: {
_id: '$group',
products: {
$push: {
id: '$_id',
name: '$name',
article: '$article',
price: '$price',
discount: '$discount',
description: '$description',
group: '$groupName',
}
},
count: { $sum: 1 },
}
},
{
$lookup: {
"from": "productGroups",
"localField": "group",
"foreignField": "_id",
"as": "groupName"
},
},
]);
Is it possible to get the same result as I've got now but populate in the same query group field?
So far the only way I've managed to populate it like this as the second request:
await ProductGroupModel.populate( products.map( (product: any) => {
return {
_id: new ProductGroupModel(product),
products: product.products,
count: product.count,
}
} ), { "path": "_id" } )
In a MongoDB aggregation pipeline, the $group stage passes along only those field explicitly declared in the stage.
In the same pipeline you show, the documents passed along by the $group stage would contain the fields:
_id
products
count
When the exector arrives a the $lookup stage, none of the documents contain a field named group.
However, the value previously contained in the group field still exists, in the _id field.
In the $lookup stage, use
"localField": "_id",
to find documents based on that value.

How can i limit each ID i pass to $in operator inside the $match stage to only 4 elements

I have an aggregate like this :
const files = await File.aggregate([
{
$match: { facilityId: { $in: facilities } }
},
{
$sort: { createdAt: 1 }
},
{
$project: {
file: 0,
}
}
])
And i would like to have each "facility" return only 4 files, i used to do something like facilities.map(id => query(id)) but id like to speed things up in production env.
Using $limit will limit the whole query, that's not what i want, i tried using $slice in the projection stage but got en error :
MongoError: Bad projection specification, cannot include fields or add computed fields during an exclusion projection
how can i achieve that in a single query ?
Schema of the collections is :
const FileStorageSchema = new Schema({
name: { type: String, required: true },
userId: { type: String },
facilityId: { type: String },
patientId: { type: String },
type: { type: String },
accessed: { type: Boolean, default: false, required: true },
file: {
type: String, //
required: true,
set: AES.encrypt,
get: AES.decrypt
},
sent: { type: Boolean, default: false, required: true },
},
{
timestamps: true,
toObject: { getters: true },
toJSON: { getters: true }
})
And i would like to returns all fields except for the file fields that contains the encrypted blob encoded as base64.
Also: i have the feeling that my approach is not correct, what i really would like to get is being able to query all facilityId at once but limited to the 4 latest file created for each facility, i though using an aggregate would help me achieve this but im starting to think it's not how its done.
From the question the schema is not clear. So I have two answers based on two Schemas. Please use what works for you
#1
db.collection.aggregate([
{
$match: {
facilityId: {
$in: [
1,
2
]
}
}
},
{
$group: {
_id: "$facilityId",
files: {
$push: "$file"
}
}
},
{
$project: {
files: {
$slice: [
"$files",
0,
4
],
}
}
}
])
Test Here
#2
db.collection.aggregate([
{
$match: {
facilityId: {
$in: [
1,
2
]
}
}
},
{
$project: {
facilityId: 1,
file: {
$slice: [
"$file",
4
]
}
}
}
])
Test Here

$in operator not working in MongoDB Aggregation

I'm trying to make a discover page for a social media website. The discover page queries the database for all posts that satisfy four things:
User has not already liked post
Post tags do not violate user's filtered tag content
Post text content does not violate user's filtered post content
And finally the part of the aggregation giving me trouble:
Post tagIds contain a given tagId from user (a post using the same tag that the user already follows)
Here's the function:
const asyncFetchTagPosts = async (
query,
//here's a given tag that a user already follows
tagId,
likedPostIds,
Post,
User,
mongoose,
handleFilterTagRegex,
handleFilterPostContentRegex
) => {
var recastTagId = mongoose.Types.ObjectId(tagId)
var user = await User.findOne({ blogName: query })
var filteredTagRegex = handleFilterTagRegex(user)
var filteredPostContentRegex = handleFilterPostContentRegex(user)
var posts = await Post.aggregate([
{
$lookup: {
from: 'posts',
let: {
likedPostIds: likedPostIds,
tagId: recastTagId,
filteredTagRegex: filteredTagRegex,
filteredPostContentRegex: filteredPostContentRegex
},
pipeline: [
{
$match: {
$expr: {
$and: [
{ $not: { $in: ["$_id", "$$likedPostIds"] } },
{ $not: [
{
$regexMatch: {
input: "$tagTitles",
regex: "$$filteredTagRegex"
}
}
]
},
{ $not: [
{
$regexMatch: {
input: "$allText",
regex: "$$filteredPostContentRegex"
}
}
]
},
{ $or: [
//here's the bad expression, $tagIds won't resolve to an array
{ $in: [ "$$tagId", "$tagIds" ] },
]
}
]
}
}
}
],
as: 'posts'
}
},
{ $unwind: '$posts' },
{ $replaceRoot: { "newRoot": "$posts" } },
{ $sort: { "notesHeatLastTwoDays": -1 } },
{ $limit: 5 }
])
return posts
}
Here's the Post model:
import mongoose from 'mongoose';
const Schema = mongoose.Schema;
const options = { discriminatorKey: 'kind' }
const PostSchema = new Schema({
user: {
type: Schema.Types.ObjectId,
ref: 'User'
},
allText: {
type: String
},
descriptions: [
{
kind: String,
content: String,
displayIdx: Number
}
],
descriptionImages: [
{
type: Schema.Types.ObjectId,
ref: 'Image'
}
],
tagIds: [
{
type: Schema.Types.ObjectId,
ref: 'Tag'
}
],
tagTitles: {
type: String
},
mentions: [
{
type: Schema.Types.ObjectId,
ref: 'Mention'
}
],
notesCount: {
type: Number,
default: 0
},
notesHeatLastTwoDays: {
type: Number,
default: 0
},
createdAt: {
type: Date,
default: Date.now
},
updatedAt: {
type: Date,
default: Date.now
},
kind: {
type: String,
default: 'Post'
}
}, options)
const Post = mongoose.model('Post', PostSchema, 'posts')
export default Post;
I keep getting this error:
Error: $in requires an array as a second argument, found: missing
When I comment out the last part of the query the aggregation works. It returns data in this shape:
{
_id: 60c18ee43730198901cfae9b,
descriptionImages: [],
//here's the array I'm trying to get to resolve in the aggregation
tagIds: [],
mentions: [],
notesCount: 1,
notesHeatLastTwoDays: 0,
kind: 'VideoPost',
descriptions: [],
createdAt: 2021-06-10T04:02:44.744Z,
updatedAt: 2021-06-11T08:51:38.166Z,
user: 608f213bb4a094bd91e02936,
videoLink: 60c3241a6c9ed4d1fc908270,
allText: '',
__v: 1,
tagTitles: ''
},
I thought using the $ operator in the aggregation gave me access to each document, does it just not work if you try to use the variable as the first expression?
you need to handle missing "$tagIds" by setting it to empty array []
{
$ifNull: [
"$tagIds",
[]
]
}
https://docs.mongodb.com/manual/reference/operator/aggregation/ifNull/
so you pipeline stage would be
{ $or: [
//here's the bad expression, $tagIds won't resolve to an array
{ $in: [ "$$tagId", { $ifNull: [ "$tagIds", [] ] } ] },
]
}

Mongoose: Filter doc and manipulate nested array

I have an image schema that has a reference to a category schema and a nested array that contains an object with two fields (user, createdAt)
I am trying to query the schema by a category and add two custom fields to each image in my query.
Here is the solution with virtual fields:
totalLikes: Count of all nested attributes
schema.virtual("totalLikes").get(function() {
return this.likes.length;
});
canLike: Check if user with id "5c8f9e676ed4356b1de3eaa1" is included in the nested array. If user is included it should return false otherwise true
schema.virtual("canLike").get(function() {
return !this.likes.find(like => {
return like.user === "5c8f9e676ed4356b1de3eaa1";
});
});
In sql it would be a simple SUBQUERY but I can't get it working in Mongoose.
Schema:
import mongoose from "mongoose";
const model = new mongoose.Schema(
{
category: {
type: mongoose.Schema.Types.ObjectId,
ref: "Category"
},
likes: [{
user: {
type: String,
required: true
},
createdAt: {
type: Date,
required: true
}
}]
})
here is a sample document:
[{
category:5c90a0777952597cda9e9c8d,
likes: [
{
_id: "5c90a4c79906507dac54e764",
user: "5c8f9e676ed4356b1de3eaa1",
createdAt:"2019-03-19T08:13:59.250+00:00"
}
]
},
{
category:5c90a0777952597cda9e9c8d,
likes: [
{
_id: "5c90a4c79906507dac54e764",
user: "5c8f9e676ed4356b1dw223332",
createdAt:"2019-03-19T08:13:59.250+00:00"
},
{
_id: "5c90a4c79906507dac54e764",
user: "5c8f9e676ed4356b1d8498933",
createdAt:"2019-03-19T08:13:59.250+00:00"
}
]
}]
Here is how it should look like:
[{
category:5c90a0777952597cda9e9c8d,
likes: [
{
_id: "5c90a4c79906507dac54e764",
user: "5c8f9e676ed4356b1de3eaa1",
createdAt:"2019-03-19T08:13:59.250+00:00"
}
],
totalLikes: 1,
canLike: false
},
{
category:5c90a0777952597cda9e9c8d,
likes: [
{
_id: "5c90a4c79906507dac54e764",
user: "5c8f9e676ed4356b1dw223332",
createdAt:"2019-03-19T08:13:59.250+00:00"
},
{
_id: "5c90a4c79906507dac54e764",
user: "5c8f9e676ed4356b1d8498933",
createdAt:"2019-03-19T08:13:59.250+00:00"
}
],
totalLikes: 2,
canLike: true
}]
Here is what I tried:
Resolver:
1) Tried in Mongoose call - Fails
const resources = await model.aggregate([
{ $match: {category: "5c90a0777952597cda9e9c8d"},
$addFields: {
totalLikes: {
$size: {
$filter: {
input: "$likes",
as: "el",
cond: "$$el.user"
}
}
}
},
$addFields: {
canLike: {
$match: {
'likes.user':"5c8f9e676ed4356b1de3eaa1"
}
}
}
}
])
2) Tried to change it after db call - works but not preferred solution
model.where({ competition: "5c90a0777952597cda9e9c8d" }).exec(function (err, records) {
resources = records.map(resource => {
resource.likes = resource.likes ? resource.likes: []
const included = resource.likes.find(like => {
return like.user === "5c8f9e676ed4356b1de3eaa1";
});
resource.set('totalLikes', resource.likes.length, {strict: false});
resource.set('canLike', !included, {strict: false});
return resource
});
})
Does anyone know how I can do it at runtime? THX
you can achieve it using aggregate
Model.aggregate()
.addFields({ // map likes so that it can result to array of ids
likesMap: {
$map: {
input: "$likes",
as: "like",
in: "$$like.user"
}
}
})
.addFields({ // check if the id is present in likesMap
canLike: {
$cond: [
{
$in: ["5c8f9e676ed4356b1de3eaa1", "$likesMap"]
},
true,
false
]
},
totalLikes: {
$size: "$likes"
}
})
.project({ // remove likesMap
likesMap: 0,
})

Mongoose querying a reference collection data using aggrigate/populate

I am a newbie in MongoDB and Express JS
I have a Product model looks like
const productSchema = mongoose.Schema({
name: String,
title: String,
brand_id: { type: mongoose.Schema.Types.ObjectId, ref: 'brand' },
varient:[{
country_id: { type: mongoose.Schema.Types.ObjectId,ref: 'countries'},
max_retail_price : Number,
profit_margin : Number
}],
and Order model
const orderTransactionSchema = mongoose.Schema({
shop_id: { type: mongoose.Schema.Types.ObjectId, ref: 'shop' },
brand_id:{ type: mongoose.Schema.Types.ObjectId, ref: 'brand' },
product_id: { type: mongoose.Schema.Types.ObjectId, ref: 'product' },
product_varient_id:{ type: mongoose.Schema.Types.ObjectId, ref: 'product.varient._id' },
transaction_id:{ type: mongoose.Schema.Types.ObjectId, ref: 'transaction' }
})
module.exports = mongoose.model('order', orderTransactionSchema );
In my Product collection, each product can have multiple variants.
But in order collection, a user can order only one product variant under a product.
I am trying to display orders with particular product details and variant details, But the problem is when I try to display it using either populate/Aggregate I am getting all the variants in the response array. actually, I want only one product and variant details as in the order collection.
This is what I tried
order.aggregate([{ $match :{} },
{
$lookup: {
from: "products",
localField: "product_id",
foreignField: "_id",
as: "product_data"
}
},
]).exec(function(err,result){
console.log(result);
});
and I am getting the output as
{ _id: 5c8a010b8feeb875abc1b066,
shop_id: 5c7d194ca10ea45c0c03a0ee,
brand_id: 5c41a8c34272c61a176b7639,
product_varient_id: 5c41a9f3f8e1e71aa75b4f32,
transaction_id: 5c6670d5b6c63d0762c6cc77,
product_id: 5c41aac4d45a731af564c433,
product_data:
[ { _id: 5c41aac4d45a731af564c433,
brand_id: 5c41a8c34272c61a176b7639,
image: 'test.jpg',
varient: //getting all the varients here
[ { _id: 5c4ee531bc20b27948b3aa98,
sell_rate_local: 66,
country_id: 5c01149d3c20440a6b2e4928 },
{ _id: 5c4ee53bbc20b27948b3aa99,
sell_rate_local: 66,
country_id: 5c00e1697dd7a23f08bdae68 } ],
__v: 0 } ] } ]
In the Order table, there is porduct_id and product_varient_id is there
I want to populate only the product with product_varient_id.
I also tried something with Populate
order.find().
populate([
{ path: 'shop_id', model: 'shop',select : 'name accounts' }, //it works
{ path: 'transaction_id', model: 'transaction' }, //it wrks
{ path: 'product_varient_id', model: 'product', select:'product.varient.name'},
]).then(result =>
{
// console.log(result);
}).catch(err =>{
// console.log(err);
});
These are the sample product and order document
Order Document Sample :
{
"_id":"5c77a025d65a892f6acf1803",
"shop_id":"5c7d194ca10ea45c0c03a0ee",
"brand_id":"5c41a8b44272c61a176b7638",
"product_varient_id":"5c41a9f3f8e1e71aa75b4f32",
"buy_rate":10,
"buy_rate_after_discount":20,
"product_mrp":30,
"sell_rate":40,
"customer_mobile":123456789,
"status":true,
"transaction_id":"5c6670c9b6c63d0762c6cc76",
"product_id":"5c41a95ff8e1e71aa75b4f30",
"createdAt":"2019-02-28T08:47:33.097Z",
"updatedAt":"2019-02-28T08:47:33.097Z",
"__v":0
}
Product document Sample :
{
"_id":"5c41aac4d45a731af564c433",
"recharge_type":[
"5c00d9cf7dd7a23f08bdae5e"
],
"name":"25 OC - Product 1",
"title":"First installation recharge",
"description":"0.1 credit for first time installation",
"offer_message":"Hi.. You got 0.1 credits..!!",
"brand_id":"5c41a8c34272c61a176b7639",
"buy_rate":20,
"profit_margin":80,
"image":"test.jpg",
"varient":[
{
"_id":"5c4ee531bc20b27948b3aa98",
"display_name":"testlia",
"profit_margin":66,
"max_retail_price":66,
"sell_rate":66,
"sell_rate_local":66,
"country_id":"5c01149d3c20440a6b2e4928"
},
{
"_id":"5c4ee53bbc20b27948b3aa99",
"display_name":"testrinu",
"profit_margin":66,
"max_retail_price":66,
"sell_rate":66,
"sell_rate_local":66,
"country_id":"5c00e1697dd7a23f08bdae68"
}
],
"createdAt":"2019-01-18T10:30:28.991Z",
"updatedAt":"2019-01-28T11:19:23.662Z",
"__v":0
}
MongoDB 3.6 or above have new lookup syntax
db.orders.aggregate([{
$lookup: {
from: "products",
let: {
"productId": "$product_id",
"productVarientId": "$product_varient_id"
},
pipeline: [
{ $match: {
$expr: { $eq: [ "$_id", "$$productId" ]}
}
},
{ $addFields: {
varient: {
$filter: {
input: "$varient",
as: "varient",
cond: { $eq: [ "$$productVarientId", "$$varient._id" ] }
}
}
}
}
],
as: "product_data"
}
}])
You can check datasample Here