If I have a multiple documents in a mongodb collection that look like this:
// document 1
{
_id: '123',
date: '5/10/15',
charges: [{
amount: 500,
description: 'foo',
},{
amount: 400,
description: 'bar',
}],
}
// document 2
{
_id: '456',
date: '5/11/15',
charges: [{
amount: 500,
description: 'foo',
},{
amount: 300,
description: 'foo',
}],
}
I want to create and array of all charges that have an amount of 500. The result should look like this:
[{
amount: 500,
description: 'foo'
}, {
amount: 500,
description: 'foo'
}]
What is the most efficient way to accomplish this?
Try this:
db.collection.aggregate(
[
{
$unwind: "$charges"
},
{
$match: {
amount: 500
}
}
]
);
Across documents you use the aggregation framework with $unwind and $group:
db.collection.aggregate([
// Match documents with the required criteria
{ "$match": { "charges.amount": 500 } },
// Unwind to de-normalize the content
{ "$unwind": "$charges" },
// Filter the de-normalized documents
{ "$match": { "charges.amount": 500 } },
// Group back the result
{ "$group": {
"_id": null,
"charges": { "$push": "$charges" }
}}
])
Or a bit more efficient in modern versions is to filter the array first:
db.collection.aggregate([
// Match documents with the required criteria
{ "$match": { "charges.amount": 500 } },
// Pre filter the array
{ "$redact": {
"$cond": {
"if": { "$eq": [{ "$ifNull": [ "$amount", 500 ] }, 500 ]},
"then": "$$DESCEND",
"else": "$$PRUNE"
}
}},
// Unwind to de-normalize the content
{ "$unwind": "$charges" },
// Group back the result
{ "$group": {
"_id": null,
"charges": { "$push": "$charges" }
}}
])
Future versions ( working in current development releases ) will have a more helpful $filter method:
db.collection.aggregate([
// Match documents with the required criteria
{ "$match": { "charges.amount": 500 } },
// Filter the array
{ "$project": {
"charges": {
"$filter": {
"input": "$charges",
"as": "charge",
"cond": {
"$eq": [ "$$charge.amount", 500 ]
}
}
}
}},
// Unwind to de-normalize the content
{ "$unwind": "$charges" },
// Group back the result
{ "$group": {
"_id": null,
"charges": { "$push": "$charges" }
}}
])
All result in:
{
"_id": null,
"charges": [
{
amount: 500,
description: 'foo'
}, {
amount: 500,
description: 'foo'
}
]
}
Related
I have the following mongodb structure...
[
{
track: 'Newcastle',
time: '17:30',
date: '22/04/2022',
bookmakers: [
{
bookmaker: 'Coral',
runners: [
{
runner: 'John',
running: true,
odds: 3.2
},
...
]
},
...
]
},
...
]
I'm trying to find filter the bookmakers array for each document to only include the objects that match the specified bookmaker values, for example:
{ 'bookmakers.bookmaker': { $in: ['Coral', 'Bet365'] } }
At the moment, I'm using the following mongodb query to only select the bookmakers that are specified, however I need to put the documents back together after they've been seperated by the '$unwind', is there a way I can do this using $group?
await HorseRacingOdds.aggregate([
{ $unwind: "$bookmakers" },
{
$group: {
_id: "$_id",
bookmakers: "$bookmakers"
}
},
{
$project: {
"_id": 0,
"__v": 0,
"lastUpdate": 0
}
}
])
How about a plain $addFields with $filter?
db.collection.aggregate([
{
"$addFields": {
"bookmakers": {
"$filter": {
"input": "$bookmakers",
"as": "b",
"cond": {
"$in": [
"$$b.bookmaker",
[
"Coral",
"Bet365"
]
]
}
}
}
}
},
{
$project: {
"_id": 0,
"__v": 0,
"lastUpdate": 0
}
}
])
Here is the Mongo playground for your reference.
i did this Aggregate pipeline , and i want add a field contains the Global Total of all groups total.
{ "$match": query },
{ "$sort": cursor.sort },
{ "$group": {
_id: { key:"$paymentFromId"},
items: {
$push: {
_id:"$_id",
value:"$value",
transaction:"$transaction",
paymentMethod:"$paymentMethod",
createdAt:"$createdAt",
...
}
},
count:{$sum:1},
total:{$sum:"$value"}
}}
{
//i want to get
...project groups , goupsTotal , groupsCount
}
,{
"$skip":cursor.skip
},{
"$limit":cursor.limit
},
])
you need to use $facet (avaialble from MongoDB 3.4) to apply multiple pipelines on the same set of docs
first pipeline: skip and limit docs
second pipeline: calculate total of all groups
{ "$match": query },
{ "$sort": cursor.sort },
{ "$group": {
_id: { key:"$paymentFromId"},
items: {
$push: "$$CURRENT"
},
count:{$sum:1},
total:{$sum:"$value"}
}
},
{
$facet: {
docs: [
{ $skip:cursor.skip },
{ $limit:cursor.limit }
],
overall: [
{$group: {
_id: null,
groupsTotal: {$sum: '$total'},
groupsCount:{ $sum: '$count'}
}
}
]
}
the final output will be
{
docs: [ .... ], // array of {_id, items, count, total}
overall: { } // object with properties groupsTotal, groupsCount
}
PS: I've replaced the items in the third pipe stage with $$CURRENT which adds the whole document for the sake of simplicity, if you need custom properties then specify them.
i did it in this way , project the $group result in new field doc and $sum the sub totals.
{
$project: {
"doc": {
"_id": "$_id",
"total": "$total",
"items":"$items",
"count":"$count"
}
}
},{
$group: {
"_id": null,
"globalTotal": {
$sum: "$doc.total"
},
"result": {
$push: "$doc"
}
}
},
{
$project: {
"result": 1,
//paging "result": {$slice: [ "$result", cursor.skip,cursor.limit ] },
"_id": 0,
"globalTotal": 1
}
}
the output
[
{
globalTotal: 121500,
result: [ [group1], [group2], [group3], ... ]
}
]
I have an aggregation that looks like this:
userSchema.statics.getCounts = function (req, type) {
return this.aggregate([
{ $match: { organization: req.user.organization._id } },
{
$lookup: {
from: 'tickets', localField: `${type}Tickets`, foreignField: '_id', as: `${type}_tickets`,
},
},
{ $unwind: `$${type}_tickets` },
{ $match: { [`${type}_tickets.createdAt`]: { $gte: new Date(moment().subtract(4, 'd').startOf('day').utc()), $lt: new Date(moment().endOf('day').utc()) } } },
{
$group: {
_id: {
groupDate: {
$dateFromParts: {
year: { $year: `$${type}_tickets.createdAt` },
month: { $month: `$${type}_tickets.createdAt` },
day: { $dayOfMonth: `$${type}_tickets.createdAt` },
},
},
userId: `$${type}_tickets.assignee_id`,
},
ticketCount: {
$sum: 1,
},
},
},
{
$sort: { '_id.groupDate': -1 },
},
{ $group: { _id: '$_id.userId', data: { $push: { groupDate: '$_id.groupDate', ticketCount: '$ticketCount' } } } },
]);
};
Which outputs data like this:
[
{
_id: 5aeb6b71709f43359e0888bb,
data: [
{ "groupDate": 2018-05-07T00:00:000Z", ticketCount: 4 }
}
]
Ideally though, I would have data like this:
[
{
_id: 5aeb6b71709f43359e0888bb,
data: [
{ "groupDate": 2018-05-07T00:00:000Z", assignedCount: 4, resolvedCount: 8 }
}
]
The difference being that the object for the user would output both the total number of assigned tickets and the total number of resolved tickets for each date.
My userSchema is like this:
const userSchema = new Schema({
firstName: String,
lastName: String,
assignedTickets: [
{
type: mongoose.Schema.ObjectId,
ref: 'Ticket',
index: true,
},
],
resolvedTickets: [
{
type: mongoose.Schema.ObjectId,
ref: 'Ticket',
index: true,
},
],
}, {
timestamps: true,
});
An example user doc is like this:
{
"_id": "5aeb6b71709f43359e0888bb",
"assignedTickets": ["5aeb6ba7709f43359e0888bd", "5aeb6bf3709f43359e0888c2", "5aec7e0adcdd76b57af9e889"],
"resolvedTickets": ["5aeb6bc2709f43359e0888be", "5aeb6bc2709f43359e0888bf"],
"firstName": "Name",
"lastName": "Surname",
}
An example ticket doc is like this:
{
"_id": "5aeb6ba7709f43359e0888bd",
"ticket_id": 120292,
"type": "assigned",
"status": "Pending",
"assignee_email": "email#gmail.com",
"assignee_id": "5aeb6b71709f43359e0888bb",
"createdAt": "2018-05-02T20:05:59.147Z",
"updatedAt": "2018-05-03T20:05:59.147Z",
}
I've tried adding multiple lookups and group stages, but I keep getting an empty array. If I only do one lookup and one group, I get the correct counts for the searched on field, but I'd like to have both fields in one query. Is it possible to have the query group on two lookups?
In short you seem to be coming to terms with setting up your models in mongoose and have gone overboard with references. In reality you really should not keep the arrays within the "User" documents. This is actually an "anti-pattern" which was just something mongoose used initially as a convention for keeping "references" for population where it did not understand how to translate the references from being kept in the "child" to the "parent" instead.
You actually have that data in each "Ticket" and the natural form of $lookup is to use that "foreignField" in reference to the detail from the local collection. In this case the "assignee_id" on the tickets will suffice for looking at matching back to the "_id" of the "User". Though you don't state it, your "status" should be an indicator of whether the data is actually either "assigned" as when in "Pending" state or "resolved" when it is not.
For the sake of simplicity we are going to consider the state "resolved" if it is anything other than "Pending" in value, but extending on the logic from the example for actual needs is not the problem here.
Basically then we resolve to a single $lookup operation by actually using the natural "foreign key" as opposed to keeping separate arrays.
MongoDB 3.6 and greater
Ideally you would use features from MongoDB 3.6 with sub-pipeline processing here:
// Better date calculations
const oneDay = (1000 * 60 * 60 * 24);
var now = Date.now(),
end = new Date((now - (now % oneDay)) + oneDay),
start = new Date(end.valueOf() - (4 * oneDay));
User.aggregate([
{ "$match": { "organization": req.user.organization._id } },
{ "$lookup": {
"from": Ticket.collection.name,
"let": { "id": "$_id" },
"pipeline": [
{ "$match": {
"createdAt": { "$gte": start, "$lt": end },
"$expr": {
"$eq": [ "$$id", "$assignee_id" ]
}
}},
{ "$group": {
"_id": {
"status": "$status",
"date": {
"$dateFromParts": {
"year": { "$year": "$createdAt" },
"month": { "$month": "$createdAt" },
"day": { "$dayOfMonth": "$createdAt" }
}
}
},
"count": { "$sum": 1 }
}},
{ "$group": {
"_id": "$_id.date",
"data": {
"$push": {
"k": {
"$cond": [
{ "$eq": ["$_id.status", "Pending"] },
"assignedCount",
"resolvedCount"
]
},
"v": "$count"
}
}
}},
{ "$sort": { "_id": -1 } },
{ "$replaceRoot": {
"newRoot": {
"$mergeObjects": [
{ "groupDate": "$_id", "assignedCount": 0, "resolvedCount": 0 },
{ "$arrayToObject": "$data" }
]
}
}}
],
"as": "data"
}},
{ "$project": { "data": 1 } }
])
From MongoDB 3.0 and upwards
Or where you lack those features we use a different pipeline process and a little data transformation after the results are returned from the server:
User.aggregate([
{ "$match": { "organization": req.user.organization._id } },
{ "$lookup": {
"from": Ticket.collection.name,
"localField": "_id",
"foreignField": "assignee_id",
"as": "data"
}},
{ "$unwind": "$data" },
{ "$match": {
"data.createdAt": { "$gte": start, "$lt": end }
}},
{ "$group": {
"_id": {
"userId": "$_id",
"date": {
"$add": [
{ "$subtract": [
{ "$subtract": [ "$data.createdAt", new Date(0) ] },
{ "$mod": [
{ "$subtract": [ "$data.createdAt", new Date(0) ] },
oneDay
]}
]},
new Date(0)
]
},
"status": "$data.status"
},
"count": { "$sum": 1 }
}},
{ "$group": {
"_id": {
"userId": "$_id.userId",
"date": "$_id.date"
},
"data": {
"$push": {
"k": {
"$cond": [
{ "$eq": [ "$_id.status", "Pending" ] },
"assignedCount",
"resolvedCount"
]
},
"v": "$count"
}
}
}},
{ "$sort": { "_id.userId": 1, "_id.date": -1 } },
{ "$group": {
"_id": "$_id.userId",
"data": {
"$push": {
"groupDate": "$_id.date",
"data": "$data"
}
}
}}
])
.then( results =>
results.map( ({ data, ...d }) =>
({
...d,
data: data.map(di =>
({
groupDate: di.groupDate,
assignedCount: 0,
resolvedCount: 0,
...di.data.reduce((acc,curr) => ({ ...acc, [curr.k]: curr.v }),{})
})
)
})
)
)
Which just really goes to show that even with the fancy features in modern releases, you really don't need them because there pretty much has always been ways to work around this. Even the JavaScript parts just had slightly longer winded versions before the current "object spread" syntax was available.
So that is really the direction you need to go in. What you certainly don't want is using "multiple" $lookup stages or even applying $filter conditions on what could potentially be large arrays. Also both forms here do their best to "filter down" the number of items "joined" from the foreign collection so as not to cause a breach of the BSON limit.
Particularly the "pre 3.6" version actually has a trick where $lookup + $unwind + $match occur in succession which you can see in the explain output. All stages actually combine into "one" stage there which solely returns only the items which match the conditions in the $match from the foreign collection. Keeping things "unwound" until we reduce further avoids BSON limit problems, as does the new form with MongoDB 3.6 where the "sub-pipeline" does all the document reduction and grouping before any results are returned.
Your one document sample would return like this:
{
"_id" : ObjectId("5aeb6b71709f43359e0888bb"),
"data" : [
{
"groupDate" : ISODate("2018-05-02T00:00:00Z"),
"assignedCount" : 1,
"resolvedCount" : 0
}
]
}
Once I expand the date selection to include that date, which of course the date selection can also be improved and corrected from your original form.
So it seems to make sense that your relationships are actually defined that way but it's just that you recorded them "twice". You don't need to and even if that's not the definition then you should actually instead record on the "child" rather than an array in the parent. We can juggle and merge the parent arrays, but that's counterproductive to actually establishing the data relations correctly and using them correctly as well.
How about something like this?
db.users.aggregate([
{
$lookup:{ // lookup assigned tickets
from:'tickets',
localField:'assignedTickets',
foreignField:'_id',
as:'assigned',
}
},
{
$lookup:{ // lookup resolved tickets
from:'tickets',
localField:'resolvedTickets',
foreignField:'_id',
as:'resolved',
}
},
{
$project:{
"tickets":{ // merge all tickets into one single array
$concatArrays:[
"$assigned",
"$resolved"
]
}
}
},
{
$unwind:'$tickets' // flatten the 'tickets' array into separate documents
},
{
$group:{ // group by 'createdAt' and 'assignee_id'
_id:{
groupDate:{
$dateFromParts:{
year:{ $year:'$tickets.createdAt' },
month:{ $month:'$tickets.createdAt' },
day:{ $dayOfMonth:'$tickets.createdAt' },
},
},
userId:'$tickets.assignee_id',
},
assignedCount:{ // get the count of assigned tickets
$sum:{
$cond:[
{ // by checking the 'type' field for a value of 'assigned'
$eq:[
'$tickets.type',
'assigned'
]
},
1, // if matching count 1
0 // else 0
]
}
},
resolvedCount:{
$sum:{
$cond:[
{ // by checking the 'type' field for a value of 'resolved'
$eq:[
'$tickets.type',
'resolved'
]
},
1, // if matching count 1
0 // else 0
]
}
},
},
},
{
$sort:{ // sort by 'groupDate' descending
'_id.groupDate':-1
},
},
{
$group:{
_id:'$_id.userId', // group again but only by userId
data:{
$push:{ // create an array
groupDate:'$_id.groupDate',
assignedCount:{
$sum:'$assignedCount'
},
resolvedCount:{
$sum:'$resolvedCount'
}
}
}
}
}
])
This is locations collection data.
{
_id: "1",
location: "loc1",
sublocations: [
{
_id: 2,
sublocation: "subloc1",
},
{
_id: 3,
sublocation: "subloc2",
}
]
},
{
_id: "4",
location: "loc2",
sublocations: [
{
_id: 5,
sublocation: "subloc1",
},
{
_id: 6,
sublocation: "subloc2",
}
]
}
This is products collection data
{
_id: "1",
product: "product1",
prices: [
{
_id: 2,
sublocationid: 2, //ObjectId of object in sublocations array
price: 500
},
{
_id: 3,
sublocationid: 5, //ObjectId of object in sublocations array
price: 200
}
]
}
Now I need to get the sublocation in product schema in the prices array. Expected result is as below.
{
_id: "1",
product: "product1",
prices: [
{
_id: 2,
sublocationid: 3,
sublocation: "subloc2",
price: 500
},
{
_id: 3,
sublocationid: 5,
sublocation: "subloc1"
price: 200
}
]
}
To achieve it, I did it like in the following way.
First, performing aggregation on locations collection - $unwind the sublocations array and store the $out in the new collection.
Second, perform aggregation on 'products' collection - $unwind the prices, $lookup the sublocationid from the new collection and $group them.
Third, after getting data delete the data of new collection.
Is there any other simplified way? Please let me know if there is any.
If you want to stick with 3.4 version, you can try this query:
db.products.aggregate([
{
$unwind: {
"path": "$prices"
}
},
{
$lookup: {
"from": "locations",
"localField": "prices.sublocationid",
"foreignField": "sublocations._id",
"as": "locations"
}
},
{
$unwind: {
"path": "$locations"
}
},
{
$unwind: {
"path": "$locations.sublocations"
}
},
{
$addFields: {
"keep": {
"$eq": [
"$prices.sublocationid",
"$locations.sublocations._id"
]
}
}
},
{
$match: {
"keep": true
}
},
{
$addFields: {
"price": {
"_id": "$prices._id",
"sublocationid": "$prices.sublocationid",
"sublocation": "$locations.sublocations.sublocation",
"price": "$prices.price"
}
}
},
{
$group: {
"_id": "$_id",
"product": { "$first": "$product" },
"prices": { "$addToSet": "$price" }
}
}
]);
It's not as nice as 3.6 version though, because of a higher memory consumption.
You can try below aggregation query in 3.6 version.
Since both local field and foreign field are array you have to $unwind both to do equality comparison.
For this you will have to use new $lookup syntax.
$match with $expr provides comparsion between document fields to look up the location's sublocation document for each product's sublocation id.
$project to project the matching sublocation doc.
$addFields with $arrayElemAt to convert the looked up sublocation array into a document.
$group to push all prices with matching sublocation's document for each product.
db.products.aggregate[
{
"$unwind": "$prices"
},
{
"$lookup": {
"from": "locations",
"let": {
"prices": "$prices"
},
"pipeline": [
{
"$unwind": "$sublocations"
},
{
"$match": {
"$expr": [
"$$prices.sublocationid",
"$sublocations._id"
]
}
},
{
"$project": {
"sublocations": 1,
"_id": 0
}
}
],
"as": "prices.sublocations"
}
},
{
"$addFields": {
"prices.sublocations": {
"$arrayElemAt": [
"$prices.sublocations",
0
]
}
}
},
{
"$group": {
"_id": "$_id",
"product": {
"$first": "$product"
},
"prices": {
"$push": "$prices"
}
}
}
])
I'm wondering how to perform a kind of union in an aggregate in MongoDB. Let's imaging the following document in a collection (the structure is for the sake of the example) :
{
linkedIn: {
people : [
{
name : 'Fred'
},
{
name : 'Matilda'
}
]
},
twitter: {
people : [
{
name : 'Hanna'
},
{
name : 'Walter'
}
]
}
}
How to make an aggregate that returns the union of the people in twitter and linkedIn ?
{
{ name :'Fred', source : 'LinkedIn'},
{ name :'Matilda', source : 'LinkedIn'},
{ name :'Hanna', source : 'Twitter'},
{ name :'Walter', source : 'Twitter'},
}
There are a couple of approaches to this that you can use the aggregate method for
db.collection.aggregate([
// Assign an array of constants to each document
{ "$project": {
"linkedIn": 1,
"twitter": 1,
"source": { "$cond": [1, ["linkedIn", "twitter"],0 ] }
}},
// Unwind the array
{ "$unwind": "$source" },
// Conditionally push the fields based on the matching constant
{ "$group": {
"_id": "$_id",
"data": { "$push": {
"$cond": [
{ "$eq": [ "$source", "linkedIn" ] },
{ "source": "$source", "people": "$linkedIn.people" },
{ "source": "$source", "people": "$twitter.people" }
]
}}
}},
// Unwind that array
{ "$unwind": "$data" },
// Unwind the underlying people array
{ "$unwind": "$data.people" },
// Project the required fields
{ "$project": {
"_id": 0,
"name": "$data.people.name",
"source": "$data.source"
}}
])
Or with a different approach using some operators from MongoDB 2.6:
db.people.aggregate([
// Unwind the "linkedIn" people
{ "$unwind": "$linkedIn.people" },
// Tag their source and re-group the array
{ "$group": {
"_id": "$_id",
"linkedIn": { "$push": {
"name": "$linkedIn.people.name",
"source": { "$literal": "linkedIn" }
}},
"twitter": { "$first": "$twitter" }
}},
// Unwind the "twitter" people
{ "$unwind": "$twitter.people" },
// Tag their source and re-group the array
{ "$group": {
"_id": "$_id",
"linkedIn": { "$first": "$linkedIn" },
"twitter": { "$push": {
"name": "$twitter.people.name",
"source": { "$literal": "twitter" }
}}
}},
// Merge the sets with "$setUnion"
{ "$project": {
"data": { "$setUnion": [ "$twitter", "$linkedIn" ] }
}},
// Unwind the union array
{ "$unwind": "$data" },
// Project the fields
{ "$project": {
"_id": 0,
"name": "$data.name",
"source": "$data.source"
}}
])
And of course if you simply did not care what the source was:
db.collection.aggregate([
// Union the two arrays
{ "$project": {
"data": { "$setUnion": [
"$linkedIn.people",
"$twitter.people"
]}
}},
// Unwind the union array
{ "$unwind": "$data" },
// Project the fields
{ "$project": {
"_id": 0,
"name": "$data.name",
}}
])
Not sure if using aggregate is recommended over a map-reduce for that kind of operation but the following is doing what you're asking for (dunno if $const can be used with no issue at all in the .aggregate() function) :
aggregate([
{ $project: { linkedIn: '$linkedIn', twitter: '$twitter', idx: { $const: [0,1] }}},
{ $unwind: '$idx' },
{ $group: { _id : '$_id', data: { $push: { $cond:[ {$eq:['$idx', 0]}, { source: {$const: 'LinkedIn'}, people: '$linkedIn.people' } , { source: {$const: 'Twitter'}, people: '$twitter.people' } ] }}}},
{ $unwind: '$data'},
{ $unwind: '$data.people'},
{ $project: { _id: 0, name: '$data.people.name', source: '$data.source' }}
])