Count articles grouping by tags mongodb - mongodb

I had a lot of articles with a field called tags, and is an array of tags _ids, and for statistics purpose I want to count how many articles we had by each tag. If tags were a simple tag _id, it's easy because I could group by tag, but is an array of tags, and I can't group by that field.
First I try with this:
db.note.aggregate([{$match: {
publishedAt: {
$gte: ISODate('2018-01-01'),
$lte: ISODate('2019-01-01')
}
}}, {$group: {
_id: "$tags",
"total": {
"$sum": 1
}
}}, {$lookup: {
from: 'tags',
localField: '_id',
foreignField: '_id',
as: 'tag'
}}, {$unwind: {
path: "$tag"
}}, {$project: {
total: 1,
"tag.name": 1
}}, {$sort: {
total: -1
}}])
But that doesn't work, that query, group by tags group, so I try to do this:
{
'$match': {
'publishedAt': {
'$gte': new Date(req.body.gte),
'$lte': new Date(req.body.lte)
}
}
},
{
'$unwind': {
'path': '$tags'
}
}, {
'$group': {
'_id': '$tags',
'total': {
'$sum': 1
}
}
}, {
'$lookup': {
'from': 'tags',
'localField': '_id',
'foreignField': '_id',
'as': 'tag'
}
}, {
'$project': {
'total': 1,
'tag.name': 1
}
}, {
'$sort': {
'total': -1
}
},
{
'$unwind': {
'path': '$tag'
}
}
)
But the problem with this, that group for the first tag from the array and I miss all other tags in that array.
What do you think will be the solution?

I had a lot of articles with a field called tags, and is an array of
tags _ids, and for statistics purpose I want to count how many
articles we had by each tag.
You can try this (I am assuming the following input documents):
notes:
{ _id: 1, name: "art-1", author: "ab", tags: [ "t1", "t2" ] },
{ _id: 2, name: "art-2", author: "cd", tags: [ "t1", "t3" ] },
{ _id: 3, name: "art-3", author: "wx", tags: [ "t4", "t3" ] },
{ _id: 4, name: "art-4", author: "yx", tags: [ "t1" ] }
tags:
{ _id: 1, id: "t1", name: "t1's name" },
{ _id: 2, id: "t2", name: "t2's name" },
{ _id: 3, id: "t3", name: "t3's name" },
{ _id: 4, id: "t4", name: "t4's name" }
The Query:
db.tags.aggregate( [
{
$lookup: {
from: "notes",
localField: "id",
foreignField: "tags",
as: "tag_matches"
}
},
{ $project: { id: 1, name: 1, _id: 0, count: { $size: "$tag_matches" } } }
] )
The Output:
{ "id" : "t1", "name" : "t1's name", "count" : 3 }
{ "id" : "t2", "name" : "t2's name", "count" : 1 }
{ "id" : "t3", "name" : "t3's name", "count" : 2 }
{ "id" : "t4", "name" : "t4's name", "count" : 1 }

Related

mongodb - summations of array length with same ids

I am creating a platform where people can share their memes. On one page I want to show them who are the most popular members on the platform. so, there is a collection of 'meme' and 'user'
for example,
There is two content with same ids:
{
_id: 1,
username: "name",
bio: "bio",
image: "url",
};
memes
{
_id: 0,
user_id: 1,
image: "meme1.jpg",
likes: [
{
user_id: 4
}
]
},
{
_id: 1,
user_id: 1,
image: "meme2.jpg",
likes: [
{
user_id: 5
},
{
user_id: 6
}
]
}
and I want to output something like this way
{
user_id:1,
username:"name"
likes:3,
}
I wrote this query using aggregate functions but I am not understanding how to identify ids are the same or not?
meme
.aggregate([
{
$lookup: {
from: "users",
localField: "user_id",
foreignField: "_id",
as: "userDetails",
},
},
{
$project: {
user_id: "$user_id",
username: "$userDetails.username",
likes: {
$size: "$likes",
},
},
},
{
$sort: { likes: 1 },
},
])
.exec()
.then((result) => {
console.log(result);
});
It will be easier to start query with users.
You can use $sum, $map, $size aggregations to get the total likes, and add it using $addFields.
db.users.aggregate([
{
$lookup: {
from: "memes",
localField: "_id",
foreignField: "user_id",
as: "userDetails"
}
},
{
$addFields: {
"likes": {
"$sum": {
"$map": {
"input": "$userDetails",
"in": {
"$size": "$$this.likes"
}
}
}
}
}
},
{
$project: {
_id: 0,
user_id: "$_id",
username: 1,
likes: 1
}
}
])
Playground
Result:
[
{
"likes": 3,
"user_id": 1,
"username": "name"
}
]
You could project the length of the likes-array and group each projection by the user_id and cound the results. Something like this should work:
db.getCollection('memes').aggregate([{
$lookup: {
from: "users",
localField: "user_id",
foreignField: "_id",
as: "userDetails"
}
}, {
"$project": {
"user_id": 1,
"likesSize": {
"$size": "$likes"
}
}
}, {
$group: {
_id: "$user_id",
"count": {
"$sum": "$likesSize"
}
}
}
])
The above query should return:
{
"_id" : 1,
"count" : 3
}

MongoDB/Mongoose: append related records to each aggreation result

Given the following Mongo collection called "members"
{
{name: "Joe", hobby: "Food"}, {name: "Lyn", hobby: "Food"},
{name: "Rex", hobby: "Play"}, {name: "Rex", hobby: "Shop"},...
}
I have an aggregation query that returns a paged set of records along with metadata for the total records found:
db.members.aggregate([
{
$facet: {
pipe1: [{ $count: 'count' }],
pipe2: [{ $skip: 0 }, { $limit: 4 }],
},
},
{
$unwind: '$pipe1',
},
{
$project: {
count: '$pipe1.count',
results: '$pipe2',
},
},
])
This gives me:
{count: 454, results: [<First 4 records here>]}
I am now trying to add to each record, an array of all member names that have the same hobby. So for the collection above, something like:
{
count: 454,
results: [
{name: "Joe", hobby: "Food", fanClub: ["Joe", "Lyn", "Alfred"]},
{name: "Lyn", hobby: "Food", fanClub: ["Joe", "Lyn", "Alfred"]},
{name: "Rex", hobby: "Play", fanClub: ["Rex"]},
{name: "Rex", hobby: "Shop", fanClub: ["Rex", "Rita"]}
]
}
I can't figure out how to run the follow up query within the aggregate. I've tried:
db.members.aggregate([
{
$facet: {
pipe1: [{ $count: 'count' }],
pipe2: [
{ $skip: 0 },
{ $limit: 2 },
{
$lookup: {
from: 'members',
pipeline: [{ $match: { hobby: '$hobby' } }],
as: 'fanClub',
},
},
],
},
},
{
$unwind: '$pipe1',
},
{
$project: {
count: '$pipe1.count',
results: '$pipe2',
},
},
])
Alas, the fanClub array is always empty.
Update 1
If I hardcode the hobby, for instance replace
{ $match: { hobby: '$hobby' }
with
{ $match: { hobby: 'Food' }
Then I do get results and all the fanClub arrays contain the results for Joe, Lyn and Alfred. So I must not be referring to the value within the pipeline correctly
Please try this :
db.membersHobby.aggregate([
{
$facet: {
pipe1: [{ $count: 'count' }],
pipe2: [{
$lookup:
{
from: "membersHobby",
let: { hobby: "$hobby" },
pipeline: [
{
$match:
{ $expr: { $eq: ["$hobby", "$$hobby"] } }
},
{ $project: { name: 1, _id: 0 } }
],
as: "fanClub"
}
}, { $skip: 0 }, { $limit: 4 }]
}
},
{
$unwind: '$pipe1'
},
{
$project: {
count: '$pipe1.count',
results: '$pipe2'
}
}
])
Result :
/* 1 */
{
"count" : 4,
"results" : [
{
"_id" : ObjectId("5e20a63ed3c98f2a7100fd4a"),
"name" : "Joe",
"hobby" : "Food",
"fanClub" : [
{
"name" : "Joe"
},
{
"name" : "Lyn"
}
]
},
{
"_id" : ObjectId("5e20a63ed3c98f2a7100fd4b"),
"name" : "Lyn",
"hobby" : "Food",
"fanClub" : [
{
"name" : "Joe"
},
{
"name" : "Lyn"
}
]
},
{
"_id" : ObjectId("5e20a63ed3c98f2a7100fd4c"),
"name" : "Rex",
"hobby" : "Play",
"fanClub" : [
{
"name" : "Rex"
}
]
},
{
"_id" : ObjectId("5e20a63ed3c98f2a7100fd4d"),
"name" : "Rex",
"hobby" : "Shop",
"fanClub" : [
{
"name" : "Rex"
}
]
}
]
}
If #srinivasy's answer meets your requierements, please grant my points him :)
If you want to get such structure:
{
count: 454,
results: [
{name: "Joe", hobby: "Food", fanClub: ["Joe", "Lyn", "Alfred"]},
{name: "Lyn", hobby: "Food", fanClub: ["Joe", "Lyn", "Alfred"]},
{name: "Rex", hobby: "Play", fanClub: ["Rex"]},
{name: "Rex", hobby: "Shop", fanClub: ["Rex", "Rita"]}
]
}
Use this query ($reduce is used to return single value, in you case fanClub as array):
db.members.aggregate([
{
$facet: {
pipe1: [
{
$count: "count"
}
],
pipe2: [
{
$skip: 0
},
{
$limit: 4
},
{
$lookup: {
from: "members",
let: {
hobby: "$hobby"
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$hobby",
"$$hobby"
]
}
}
}
],
as: "fanClub"
}
}
]
}
},
{
$unwind: "$pipe1"
},
{
$project: {
count: "$pipe1.count",
results: {
$map: {
input: "$pipe2",
as: "pipe2",
in: {
_id: "$$pipe2._id",
hobby: "$$pipe2.hobby",
name: "$$pipe2.name",
fanClub: {
$reduce: {
input: "$$pipe2.fanClub",
initialValue: [],
in: {
$concatArrays: [
"$$value",
[
"$$this.name"
]
]
}
}
}
}
}
}
}
}
])
MongoPlayground

MongoDB lookup when foreign field is an array

I've searched the internet and StackOverflow, but I cannot find the answer or even the question.
I have two collections, reports and users. I want my query to return all reports and indicate if the specified user has that report as a favorite in their array.
Reports Collection
{ _id: 1, name:"Report One"}
{ _id: 2, name:"Report Two"}
{ _id: 3, name:"Report Three"}
Users Collection
{_id: 1, name:"Mike", favorites: [1,3]}
{_id: 2, name:"Tim", favorites: [2,3]}
Desired Result for users.name="Mike"
{ _id: 1, name:"Report One", favorite: true}
{ _id: 2, name:"Report Two", favorite: false}
{ _id: 3, name:"Report Three", favorite: true}
All of the answers I can find use $unwind on the local (reports) field, but in this case the local field isn't an array. The foreign field is the array.
How can I unwind the foreign field? Is there a better way to do this?
I saw online that someone suggested making another collection favorites that would contain:
{ _id: 1, userId: 1, reportId: 1 }
{ _id: 2, userId: 1, reportId: 3 }
{ _id: 3, userId: 2, reportId: 2 }
{ _id: 4, userId: 2, reportId: 3 }
This method seems like it should be unnessesary. It should be simple to join onto an ID in a foreign array, right?
You can use $lookup with custom pipeline which will give you 0 or 1 result and then use $size to convert an array to single boolean value:
db.reports.aggregate([
{
$lookup: {
from: "users",
let: { report_id: "$_id" },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: [ "$name", "Mike" ] },
{ $in: [ "$$report_id", "$favorites" ] }
]
}
}
}
],
as: "users"
}
},
{
$project: {
_id: 1,
name: 1,
favorite: { $eq: [ { $size: "$users" }, 1 ] }
}
}
])
Alternatively if you need to use MongoDB version lower than 3.6 you can use regular $lookup and then use $filter to get only those users where name is Mike:
db.reports.aggregate([
{
$lookup: {
from: "users",
localField: "_id",
foreignField: "favorites",
as: "users"
}
},
{
$project: {
_id: 1,
name: 1,
favorite: { $eq: [ { $size: { $filter: { input: "$users", as: "u", cond: { $eq: [ "$$u.name", "Mike" ] } } } }, 1 ] }
}
}
])
"_id" : ObjectId("611fc392cfadfbba65d4f4bd"),
"t_name" : "Bahadur",
"t_age" : "22",
"trch" : "java",
"StudentsDetails" : [
{
"_id" : ObjectId("611fc41ccfadfbba65d4f4be"),
"s_name" : "Asin",
"s_age" : "18",
"trch" : "java",
"tsid" : ObjectId("611fc392cfadfbba65d4f4bd")
},
{
"_id" : ObjectId("611fc8f1a815fb2c737ae31f"),
"s_name" : "sonu",
"s_age" : "18",
"tsid" : ObjectId("611fc392cfadfbba65d4f4bd")
},
{
"_id" : ObjectId("611fc915a815fb2c737ae320"),
"s_name" : "monu",
"s_age" : "19",
"tsid" : ObjectId("611fc392cfadfbba65d4f4bd")
}
]
}
Create Trainer Collection
Create Scholar Collection
//query
db.Trainer.aggregate(
[`enter code here`
{`enter code here`
$lookup:`enter code here`
{`enter code here`
from: "scholar",`enter code here`
localField: "_id",`enter code here`
foreignField: "tsid",`enter code here`
as: "StudentsDetails"`enter code here`
}`enter code here`
}`enter code here`
]`enter code here`
).pretty();

Mongodb - $group with $addToSet and then $lookup

I've got the following query
db.getCollection('transportations').aggregate(
{
$group: {
_id: null,
departure_city_id: { $addToSet: "$departure.city_id" },
departure_station_id: { $addToSet: "$departure.station_id" }
}
}
);
and the result is
{
"_id" : null,
"departure_city_id" : [
ObjectId("5a2f5378334c4442ab5a63ea"),
ObjectId("59dae1efe408157cc1585fea"),
ObjectId("5a5bbfdc35628410f9fdcde9")
],
"departure_station_id" : [
ObjectId("5a2f53d1334c4442ab5a63ee"),
ObjectId("5a2f53c5334c4442ab5a63ed"),
ObjectId("5a5bc13435628410f9fdcdea")
]
}
Now i want to lookup each departure_city_id with the collection "areas" to get the "name" of the area and each departure_station_id with the collection "stations" to get also the "name" of the station
The result could be something like this
{
"_id" : null,
"departure_city_id" : [
{
_id: ObjectId("5a2f5378334c4442ab5a63ea"),
name: "City 1
},
{
_id: ObjectId("59dae1efe408157cc1585fea"),
name: "City 2
},
{
_id: ObjectId("5a5bbfdc35628410f9fdcde9"),
name: "City 3
}
],
"departure_station_id" : [
{
_id: ObjectId("5a2f53d1334c4442ab5a63ee"),
name: "Station 1
},
{
_id: ObjectId("5a2f53c5334c4442ab5a63ed"),
name: "Station 2
},
{
_id: ObjectId("5a5bc13435628410f9fdcdea"),
name: "Station 3
}
]
}
The $lookup aggregation pipeline stage NOW works directly with an array (on 3.3.4 version).
See: lookup between local (multiple)array of values and foreign (single) value
The answer of the question is just:
db.getCollection('transportations').aggregate(
{
$group: {
_id: null,
departure_city_id: { $addToSet: "$departure.city_id" },
departure_station_id: { $addToSet: "$departure.station_id" }
}
},
{
$lookup: {
from: "areas",
localField: "departure_city_id",
foreignField: "_id",
as: "departure_city_id"
}
},
{
$lookup: {
from: "stations",
localField: "departure_station_id",
foreignField: "_id",
as: "departure_station_id"
}
}
)

MongoDB's aggregation from nested key returns nothing

(Edit : this question was edited to better reflect the issue, which might be a little more complicated than the proposed related question.)
Let's say I have these two collections
products
{
_id: 'AAAA',
components: [
{ type: 'foo', items: [
{ itemId: 'item1', qty: 2 },
{ itemId: 'item2', qty: 1 }
] },
{ type: 'bar', items: [
{ itemId: 'item3', qty: 8 }
] }
]
}
items
{
_id: 'item1',
name: 'Foo Item'
}
{
_id: 'item2',
name: 'Bar Item'
}
{
_id: 'item3',
name: 'Buz Item'
}
And that I perform this query
db['products'].aggregate([
{ $lookup: {
from: 'items',
localField: 'components.items.itemId',
foreignField: '_id',
as: 'componentItems'
} }
]);
I get this
{
_id: 'AAAA',
components: [
{ type: 'foo', items: [
{ itemId: 'item1', qty: 2 },
{ itemId: 'item2', qty: 1 }
] }
{ type: 'bar', items: [
{ itemId: 'item3', qty: 8 }
] }
],
componentItems: [ ]
}
Why doesn't the aggregation read the local field value? How can I retrieve the foreign document without losing my original document structure?
Edit
I have read the jira issue and seen the proposed answer, however I don't know how this applies. This is not merely an array, but values from an object, inside an array. I am not sure how I can unwind this, and how to put it back together without losing the document structure.
Edit 2
The problem that I have is that I'm not sure how to group the results back together. With this query :
db['products'].aggregate([
{ $unwind: '$components' },
{ $unwind: '$components.items' },
{ $lookup: {
from: 'items',
localField: 'components.items.itemId',
foreignField: '_id',
as: 'componentsItems'
} }
]);
I get the "correct" result of
{ "_id" : "AAAA", "components" : { "type" : "foo", "items" : { "itemId" : "item1", "qty" : 2 } }, "componentsItems" : [ { "_id" : "item1", "name" : "Foo Item" } ] }
{ "_id" : "AAAA", "components" : { "type" : "foo", "items" : { "itemId" : "item2", "qty" : 1 } }, "componentsItems" : [ { "_id" : "item2", "name" : "Bar Item" } ] }
{ "_id" : "AAAA", "components" : { "type" : "bar", "items" : { "itemId" : "item3", "qty" : 8 } }, "componentsItems" : [ { "_id" : "item3", "name" : "Buz Item" } ] }
But, while I can unwind components.items, I cannot seem to unto this, as $group complains that
"the group aggregate field name 'components.items' cannot be used because $group's field names cannot contain '.'"
db['products'].aggregate([
{ $unwind: '$components' },
{ $unwind: '$components.items' },
{ $lookup: {
from: 'items',
localField: 'components.items.itemId',
foreignField: '_id',
as: 'componentsItems'
} },
{ "$group": {
"components.type": "$components.type",
"components.items": { $push: "$components.items" },
"componentsItems": { $push: "$componentsItems" }
} },
{ "$group": {
"_id": "$_id",
"components": { $push: "$components" },
"componentsItems": { $push: "$componentsItems" }
} }
]);
Edit 3
This query is, thus far, the closest that I found, except that components are not grouped back by type.
db['products'].aggregate([
{ $unwind: '$components' },
{ $unwind: '$components.items' },
{ $lookup: {
from: 'items',
localField: 'components.items.itemId',
foreignField: '_id',
as: 'componentsItems'
} },
{ $unwind: '$componentsItems' },
{ $group: {
"_id": "$_id",
"components": {
$push: {
"type": "$components.type",
"items": "$components.items"
}
},
"componentsItems": { $addToSet: "$componentsItems" }
} }
]);
Also: I am concerned that using $unwind and $group may affect the order of the components, which should be preserved. AFAIK, MongoDB preserve array order when storing documents. I'd hate for this functionality to be broken by the awkwardness of $lookup.
Here is my long and awkward solution :
db['products'].aggregate([
// unwind all... because $lookup cannot work with multi-values
{ $unwind: '$components' },
{ $unwind: '$components.items' },
// lookup... This is a 1:1 relationship but who cares, right?
{ $lookup: {
from: 'items',
localField: 'components.items.itemId',
foreignField: '_id',
as: 'componentsItems'
} },
// our 1:1 relationship is now an array, so this is required
// before grouping, so we don't end up with array of arrays
{ $unwind: '$componentsItems' },
// Group 1: put "components.items" in a temporary array
// and filter duplicates from "componentsItems"
{ $group: {
"_id": {
"i": "$_id",
"t": "$components.type"
},
"items": {
$push: "$components.items"
},
"componentsItems": { $addToSet: "$componentsItems" }
} },
// undo $push...
{ $unwind: "$componentsItems" },
// Group 2: put everything back together
{ $group: {
"_id": "$_id.i",
"items": {
$push: {
"type": "$_id.t",
"items": "$items"
}
},
"componentsItems": { $push: "$componentsItems" }
} }
]);
Edit
A better solution :
db['products'].aggregate([
// Return document, added a collection of "itemId"
{ $project: {
"_id": 1,
"components": 1,
"componentItemId": "$components.items.itemId"
} },
// Since there was two arrays, the field is an array of arrays...
{ $unwind: "$componentItemId" },
{ $unwind: "$componentItemId" },
// make 1:1 lookup...
{ $lookup: {
from: 'items',
localField: 'componentItemId',
foreignField: '_id',
as: 'componentsItems'
} },
// ... extract the 1:1 reference...
{ $unwind: "$componentsItems" },
// group back, ignoring the "componentItemId" field
{ $group: {
"_id": "$_id",
"components": { $first: "$components" },
"componentItems": { $addToSet: "$componentsItems" }
}}
]);
I'm not sure if there is yet a better solution, and I am concerned about performance, but this seems to be the only solutions I can think of.
The downside is that documents cannot be dynamic, and this query will need to be modified whenever the schema changes.
Update
This seems to be resolved in MongoDB 3.3.4 (not release at the time of writing this answer).