mongodb aggregate skip limit count for pagination - mongodb

Question

I want to apply the pagination on this aggregated data (all the documents which matched and project with both collection 2 and 3). I have tried multiple
queries, i am passing limit 25 but it will get only 20 document, which changes require in this query for pagination
var pipeline = [{
$match: query
}, {
$limit: limit
}, {
$skip: skip
}, {
$lookup: {
from: "collection2",
localField: "collection1Field",
foreignField: "collection2Field",
as: "combined1"
}
}, {
"$unwind": "$combined1"
}, {
$lookup: {
from: "collection3",
localField: "collection1Field",
foreignField: "collection3Field",
as: "combined2"
}
}, {
"$unwind": "$combined2"
}, {
$project: {
"collection1Field1": 1,
"collection1Field2": 1,
"collection1Field3": 1,
"collection2Field.field1": 1,
"collection2Field.field2": 1,
"collection3Field.field1": 1,
"collection3Field.field2": 1,
}
}
];

Answer 1

I just had a similar issue. I was getting no results back on "page 2" due to the fact that I ran limit, then skip.
The docs for $skip:
Skips over the specified number of documents that pass into the stage
and passes the remaining documents to the next stage in the pipeline.
The docs for $limit:
Limits the number of documents passed to the next stage in the
pipeline.
This means that if you run limit before skip then the results returned by the limit are then potentially truncated by the skip.
For example if limit is 50 and skip is 50 (for say page 2) then the match will find items, limit the results to 50, and then subsequently skip 50, thus feeding 0 results into any stages afterwards.
I WOULD NOT RECOMMEND running $skip and $limit at the end of the query as the DB will have done pipeline operations on a substantial amount of data that will be skipped/limited out at the end. Additionally there is a limit to the amount of memory aggregations can use which will end queries in a error if the limit is exceeded (100MB - see bottom). There is an option to turn on disk use if you exceed this limit, but a good way to optimize your code without relying on disk is to skip + limit your results before entering any $lookup or $unwind steps.
The one exception to the rule is $sort which is intelligent enough to use $limit if $limit immediately follows $sort (docs):
When a $sort immediately precedes a $limit in the pipeline, the $sort
operation only maintains the top n results as it progresses, where n
is the specified limit, and MongoDB only needs to store n items in
memory. This optimization still applies when allowDiskUse is true and
the n items exceed the aggregation memory limit.
From your code I would recommend the following (skip then limit):
var pipeline = [{
$match: query
}, {
$skip: skip
}, {
$limit: limit
}, {
$lookup: {
from: "collection2",
localField: "collection1Field",
foreignField: "collection2Field",
as: "combined1"
}
}, {
"$unwind": "$combined1"
}, {
$lookup: {
from: "collection3",
localField: "collection1Field",
foreignField: "collection3Field",
as: "combined2"
}
}, {
"$unwind": "$combined2"
}, {
$project: {
"collection1Field1": 1,
"collection1Field2": 1,
"collection1Field3": 1,
"collection2Field.field1": 1,
"collection2Field.field2": 1,
"collection3Field.field1": 1,
"collection3Field.field2": 1,
}
}];
Docs regarding aggregatation pipeline limits:
Memory Restrictions Changed in version 2.6.
Pipeline stages have a limit of 100 megabytes of RAM. If a stage
exceeds this limit, MongoDB will produce an error. To allow for the
handling of large datasets, use the allowDiskUse option to enable
aggregation pipeline stages to write data to temporary files.
Changed in version 3.4.
The $graphLookup stage must stay within the 100 megabyte memory limit.
If allowDiskUse: true is specified for the aggregate() operation, the
$graphLookup stage ignores the option. If there are other stages in
the aggregate() operation, allowDiskUse: true option is in effect for
these other stages.

Answer 2

You want to paginate after you get the results.
var pipeline = [{
$match: query
}, {
$lookup: {
from: "collection2",
localField: "collection1Field",
foreignField: "collection2Field",
as: "combined1"
}
}, {
"$unwind": "$combined1"
}, {
$lookup: {
from: "collection3",
localField: "collection1Field",
foreignField: "collection3Field",
as: "combined2"
}
}, {
"$unwind": "$combined2"
}, {
$project: {
"collection1Field1": 1,
"collection1Field2": 1,
"collection1Field3": 1,
"collection2Field.field1": 1,
"collection2Field.field2": 1,
"collection3Field.field1": 1,
"collection3Field.field2": 1,
}
}, {
$limit: limit
}, {
$skip: skip
}
];

Answer 3

If you use some npm modules for pagination you can Implementing pagination very easily. for exapmle if you use mongoose-aggregate-paginate then you just add it into the schema like...
const mongoose = require('mongoose');
const mongooseAggregatePaginate = require('mongoose-aggregate-
paginate');
mongoose.Promise = global.Promise;
const chatSchema = new mongoose.Schema({
text: {
type: String,
},
},
{
collection: 'chat',
timestamps: true
});
chatSchema.plugin(mongooseAggregatePaginate);
const chat = mongoose.model('chat', chatSchema);
module.exports = chat;
After this whenever you need pagination, Your query should be
UserCtr.get = (req, res) => {
const { limit } = 10;
const { page } = req.query.page;
const aggregateRules = [{
$match: {
_id: req.user.id
},
{
//Perform your query
}
];
Chat.aggregatePaginate(aggregateRules, {
page,
limit
}, (err, docs,
pages, total) => {
if (!err) {
const results = {
docs,
total,
limit,
page,
pages,
};
res.status(200).json(results);
} else {
res.status(500).json(err);
}
});
};
It gives response like
{
"docs": [
{
"_id": "5a7676c938c185142f99c4c3",
},
{
"_id": "5a7676c938c185142f99c4c4",
},
{
"_id": "5a7676cf38c185142f99c4c5",
}
],
"total": 3,
"limit": 50,
"page": "1",
"pages": 1
}