How create mongodb aggregate query for multiple collections - mongodb

I have two models
export const StorySchema = new Schema({
type: { type: String, required: true },
texts: { type: Array, require: true },
});
export const TextSchema = new Schema({
textKey: { type: String, required: true },
text: { type: String, required: true },
});
My collections
// stories
[
{
"type": "export",
"texts": ["export", "download"]
},
...
]
// Text
[
{
"textKey": "export",
"text": "Export ....."
},
{
"textKey": "download",
"text": "Download ....."
},
...
]
I want to combine a field textKey from collection text with array texts from collection story and write field text from collection text into result query. I have to get a array of object
[
{
"type": "export",
"texts": ["Export .....", "Download ....."]
},
...
]
I tried to create a aggregate multiple collections
public async getStoriesList(): Promise<Story[]> {
return await this.storyModel.aggregate([{
$lookup: {
from: 'texts',
localField: 'texts',
foreignField: 'textKey',
as: 'texts',
},
}]);
}
But i got an empty array. Where I made an error? How I can create a aggregate?

You can't lookup on an array, you could achieve what you want using this aggregation but it may be slow if you have big collections:
db.stories.aggregate([
{
"$unwind": "$texts"
},
{
"$lookup": {
"from": "texts",
"localField": "texts",
"foreignField": "textKey",
"as": "data"
}
},
{
"$unwind": "$data"
},
{
"$group": {
"_id": "type",
"texts": {
"$push": "$data.text"
}
}
},
{
"$project": {
"type": "$_id",
"_id": 0,
"texts": 1
}
}
])

Related

Unable to populate and structure document with aggregate

So I have this schema which have foreign keys to other collections in the database. The document has around 60k posts and each post can have multiple categories and there are around 200 categories. So I'm trying to fetch and structure data based on the category's foreign key and populate the category details and count.
Here's how the main schema and category schema looks like:
const postSchema = new mongoose.Schema( {
post: {
type: mongoose.Schema.Types.ObjectId,
ref: 'PostDetails'
},
categories: [ {
category: {
type: mongoose.Schema.Types.ObjectId,
ref: 'Category'
},
subCategories: [ {
subCategory: {
type: mongoose.Schema.Types.ObjectId,
ref: 'Subcategory'
}
} ]
} ]
} );
const categorySchema = new mongoose.Schema( {
category: {
type: String,
},
categorySlug: {
type: String,
}
} );
I was successful in making the count but the returned data is not what I expected. The returned data shows the id of the categories and the count but no name and slug. Here's how it looks like:
[
{
"_id": [
"617acfd232c766589c23a90c"
],
"count": 876,
"category": []
}
]
I got the above output with the following query:
const aggregateStages = [
{
$group: {
_id: '$categories.category',
count: { $sum: 1 }
}
},
{
$lookup: {
from: "Category",
localField: "categories.category",
foreignField: "_id",
as: "category"
}
}
];
const categories = await Post.aggregate( aggregateStages ).exec();
I'm hoping to get the data as follows:
[
{
"_id": "617acfd232c766589c23a90c",
"count": 876,
"category": 'SomeCategory',
"categorySlug": 'some-category'
}
]
Where am I going wrong and how can I fix it?
SAMPLE DATA FROM DB AS REQUESTED BY MATT OESTREICH
POST DATA
{
"_id": "617adad39054bae2c983c34f",
"post": "617ad1c80597c78ed4cc151e",
"author": "617acc689b309fdbbbdfdfe0",
"categories": [{
"category": "617acfd232c766589c23a8d1",
"subCategories":[]
}]
}
CATEGORY DATA
{
"_id": "617acfd232c766589c23a8d1",
"category": "Lorem Ipsum",
"categorySlug": "lorem-ipsum"
}
Ok so, it looks like you can resolve this by using the $size operator. The $size operator will give you the length (or count) of elements in an array.
Live demo here
Database
db={
"post": [
{
"_id": ObjectId("617adad39054bae2c983c34f"),
"post": ObjectId("617ad1c80597c78ed4cc151e"),
"author": ObjectId("617acc689b309fdbbbdfdfe0"),
"categories": [
{
"category": ObjectId("617acfd232c766589c23a8d1"),
"subCategories": []
}
]
}
],
"categories": [
{
"_id": ObjectId("617acfd232c766589c23a8d1"),
"category": "Lorem Ipsum",
"categorySlug": "lorem-ipsum"
}
]
}
Query
db.post.aggregate([
{
"$lookup": {
"from": "categories",
"localField": "categories.category",
"foreignField": "_id",
"as": "found_categories"
}
},
{
"$project": {
_id: "$_id",
count: {
"$size": "$found_categories"
},
"category": {
"$first": "$found_categories.category"
},
"categorySlug": {
"$first": "$found_categories.categorySlug"
}
}
}
])
Result
[
{
"_id": ObjectId("617adad39054bae2c983c34f"),
"category": "Lorem Ipsum",
"categorySlug": "lorem-ipsum",
"count": 1
}
]
Although, I do not believe this will give you what you are looking for if more than one category is found. Please let me know if it doesn't work and I will try to help fix it.

Mongoose, aggregate query with $lookup return null?

varyantalt Model:
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const varyantaltSchema = new Schema({
_id: {
type: Schema.ObjectId
},
varyantid: {
type: String
},
altvaryantname: {
type: String
},
createdAt: {
type: Date,
default: Date.now
}
});
module.exports = mongoose.model("varyantalt", varyantaltSchema, "varyantalt");
varyant model:
const varyantSchema = new Schema({
_id: {
type: Schema.ObjectId
},
stokid: {
type: Schema.ObjectId
},
varyantname: {
type: String
},
createdAt: {
type: Date,
default: Date.now
}
});
module.exports = mongoose.model("varyant", varyantSchema, "varyant");
varyant collection:
{
"_id": ObjectId("5e32286c34fb7322bdd566ed"),
"stokid": ObjectId("5e28b4a2a1d9692b29a65b24"),
"varyantname": "RENK"
}
altvaryant collection:
{
"_id": ObjectId("5e3228df34fb7322bdd566f5"),
"varyantid": "5e32286c34fb7322bdd566ed",
"altvaryantname": "KIRMIZI",
"vars": [
{
"_id": ObjectId("5e35b1e410fce83f3370cd0a"),
"images": {
"imageurl": "http://",
"_id": ObjectId("5e35b1e410fce83f3370cd0b"),
"filename": "5320_7d93",
"path": "https://res",
"publicid": "panel"
}
}
]
},
{
"_id": ObjectId("5e3359e6fa4c5e4bd9112fb6"),
"varyantid": "5e32286c34fb7322bdd566ed",
"altvaryantname": "SARI",
"vars": [
{
"_id": ObjectId("5e35b1f610fce83f3370cd0d"),
"images": {
"imageurl": "http://",
"_id": ObjectId("5e35b1f610fce83f3370cd0e"),
"filename": "veli-fidan-1LT-2-450x450",
"path": "https://",
"publicid": "panel"
}
}
]
},
{
"_id": ObjectId("5e335b64fa4c5e4bd9112fc9"),
"varyantid": "5e32286c34fb7322bdd566ed",
"altvaryantname": "YEŞİL",
"vars": [
{
"_id": ObjectId("5e35b20010fce83f3370cd10"),
"images": {
"imageurl": "http://",
"_id": ObjectId("5e35b20010fce83f3370cd11"),
"filename": "maxresdefault-29-450x450",
"path": "https://",
"publicid": ""
}
}
]
}
Query:
varyant
.aggregate([
{
$match: {
stokid: mongoose.Types.ObjectId("5e28b4a2a1d9692b29a65b24")
}
},
{
$lookup: {
from: "varyantalt",
localField: "_id",
foreignField: "varyantid",
as: "vars"
}
},
{ $unwind: "$vars" }
])
.exec((err, locations) => {
if (err) throw err;
console.log("res::", locations);
});
I am trying to get results with $aggregate and $lookup query with mongoose
Why is my query returning empty? I couldn't find the problem.
Under $lookup operator from: 'varyantalt' I do not understand exactly whether it takes a blank value. Thanks in advance for your help.
In your varyantaltSchema schema, varyantid is defined as String, but it must be Schema.ObjectId.
They must be same because $lookup performs an equality match on the foreignField to the localField from the input documents
If you save your varyandid as Object id aggregate will work.

$lookup when foreignField is array

I have 2 collections, the first storing the animes watched by each user, their status etc:
const listSchema = mongoose.Schema({
user: {
type: Schema.Types.ObjectId,
ref: 'user'
},
animes: [{
_id: false,
anime: {
type: Schema.Types.ObjectId,
ref: 'anime',
unique: true
},
status: String,
episodes: Number,
rating: Number
}]
});
and the second storing all animes and the relevant information.
I want to show that second collection but adding status and episodes fields that are filled from the list of the logged in user.
I tried the following:
Anime.aggregate([
{
$lookup: {
'from': 'lists',
localField: '_id',
foreignField: 'animes.anime',
'as': 'name'
},
},
{
$unwind: '$animes'
},
{
$project:{
status:'$lists.animes.status'
}
}
]
)
but it returns an empty array.
I also don't know how to filter by userId seeing how the _id is in the foreign collection.
You can use below aggregation
{ "$lookup": {
"from": "lists",
"let": { "id": "$_id" },
"pipeline": [
{ "$match": { "$expr": { "$in": ["$$id", "$animes.anime"] }}},
{ "$unwind": "$animes" },
{ "$match": { "$expr": { "$eq": ["$animes.anime", "$$id"] }}}
],
"as": "name"
}}

How to order mongodb $lookup query

Overview
A few devices are collecting data and sending it to a Node/MongoDb endpoint. Then, the user would use an endpoint to get all that data into a json.
Models
Device Model
const deviceSchema = new Schema({
group: { type: Schema.Types.ObjectId, ref: 'Group' },
deviceId: { type: String, unique: true },
name: String,
notes: String,
pac: String,
endCertificate: String,
lat: Number,
lng: Number
});
Message Model
const messageSchema = new Schema({
deviceId: { type: String, required: true },
raw: { type: String, required: true },
receivedAt: { type: Date, default: Date.now() }
});
One device can have N messages
Problem to solve
I want to get a json that has all the devices and have an array containing
all the messages that belongs to that device.
[
{
"id":"5b86c323e95759603ad7ea54",
"deviceId":"Device 01",
"name":"Device bla",
"notes":"...",
"pac":"pac",
"lat":-20.817396,
"endCertificate":"cert",
"lng":-27.031321,
"messages":[
{
"id":"5b869a42e0b94041b5f21eed",
"deviceId":"Device 01",
"raw":"1111111",
"receivedAt":"2018-08-29T13:04:43.641Z",
"__v":0
},
{
"id":"5b8c782fef4f8e98783f6f35",
"deviceId":"Device 01",
"raw":"2222222",
"receivedAt":"2018-09-01T09:04:43.641Z",
"__v":0
},
{
"id":"5b8c7840ef4f8e98783f6f3e",
"deviceId":"Device 01",
"raw":"3333333",
"receivedAt":"2018-09-02T09:04:43.641Z",
"__v":0
}
]
},
{
"id":"5b8c28ec38c51813cd159bac",
"deviceId":"Device 02",
"name":"Device ...",
"notes":"...",
"lat":-27.812296,
"lng":-27.073314,
"__v":0,
"messages":[
{
"id":"5b8c784cef4f8e98783f6f43",
"deviceId":"Device 02",
"raw":"1111111",
"receivedAt":"2018-09-01T09:04:43.641Z",
"__v":0
}
]
}
]
My solution
To get a json as the one above I have:
const [results, itemCount] = await Promise.all([
Device.aggregate([
{ $match: {} },
{
$lookup: {
from: 'messageschemas',
localField: 'deviceId',
foreignField: 'deviceId',
as: 'messages'
}
}
]).limit(req.query.limit).skip(req.skip)
.exec(),
Device.countDocuments(match)
]);
res.setHeader('X-Total-Count', itemCount);
res.send(results);
My question
How can I order the messages I get from the $lookup into messages[] by
'receivedAt'?
You need to $unwind the messages array and can apply $sort with receivedAt field and finally $group to rollback again into the array.
Device.aggregate([
{ "$match": { }},
{ "$lookup": {
"from": "messageschemas",
"localField": "deviceId",
"foreignField": "deviceId",
"as": "messages"
}},
{ "$unwind": "$messages" },
{ "$sort": { "messages.receivedAt": 1 }},
{ "$group": {
"_id": "$_id",
"messages": { "$push": "$messages" }
}}
])
Which can be simply done with the 3.6 $lookup syntax
Device.aggregate([
{ "$match": { }},
{ "$lookup": {
"from": "messageschemas",
"let": { "deviceId": "$deviceId" },
"pipeline": [
{ "$match": { "$expr": { "$eq": [ "$deviceId", "$$deviceId" ] } } },
{ "$sort": { "receivedAt": 1 }}
],
"as": "messages"
}}
])

mongodb aggregate group by category

I am new in using MongoDB's aggregation framework and here I have below a schema:
var bookSoldSchema = new Schema({
buyer:{
type: Number
}
book: {
type: Schema.Types.ObjectId, ref: 'Book'
}
});
var bookSchema = new Schema({
bookName: { type: String },
categories: [{ type: Schema.Types.ObjectId, ref: 'BookCategory'}],
})
Each book has multiple categories, I would like to show the top 5 best seller categories and each category I need show top 3 books which where sold most, and I also need show the category name, the results which I need is:
[
{
categoryId: xxx,
categoryName:xxx,
top3books:[
{bookId:xxx, bookName:xxx},
{bookId:xxx, bookName:xxx},
{bookId:xxx, bookName:xxx},
]
},
{
categoryId: xxx,
categoryName:xxx,
top3books:[
{bookId:xxx, bookName:xxx},
{bookId:xxx, bookName:xxx},
{bookId:xxx, bookName:xxx},
]
}
]
How can I go about this in MongoDB?
You can run the following aggregation pipeline which assumes you have a model BookSold that uses the bookSoldSchema above, books and bookcategories underlying collections:
BookSold.aggregate([
{
"$lookup": {
"from": "books",
"localField": "book",
"foreignField": "_id",
"as": "book_join"
}
},
{ "$unwind": "$book_join" },
{ "$unwind": "$book_join.categories" },
{
"$lookup": {
"from": "bookcategories",
"localField": "book_join.categories",
"foreignField": "_id",
"as": "categories"
}
},
{ "$unwind": "$categories" },
{ "$sort": { "categories.name": 1, "buyer": -1 } },
{
"$group": {
"_id": "$categories.name",
"categoryId": { "$first": "$categories._id" },
"buyer": { "$first": "$buyer" },
"books": {
"$push": {
"bookId": "$book_join._id",
"bookName": "$book_join.bookName"
}
}
}
},
{ "$sort": { "buyer": -1 } },
{ "$limit": 5 },
{
"$project": {
"_id": 0,
"categoryId": 1,
"categoryName": "$_id",
"top3books": { "$slice": [ "$books", 3 ] }
}
}
], function(err, result) {
if (err) handleError(err);
console.log(JSON.stringify(result, null, 4));
})