MongdDB: Combining query results of two collections as one - mongodb

There are two collections (view and click) like following:
# View collection
_id publisher_id created_at
617f8ea98e0f54f05e10e796 1 2021-11-01T00:00:00.000Z
617f8eab8e0f54f05e10e798 1 2021-11-01T00:00:00.000Z
617f8eac8e0f54f05e10e79a 1 2021-11-01T00:00:00.000Z
617f90cea187d30ebbecdee9 2 2021-11-01T00:00:00.000Z
# Click collection
_id publisher_id created_at
617f8ea98e0f54f05e10e796 1 2021-11-01T00:00:00.000Z
617f8eab8e0f54f05e10e798 2 2021-11-01T00:00:00.000Z
How can I get the following expected results with one query?
(or)
What is the best way for the following expected results?
# Expected For Publisher ID(1)
_id view_count click_count
2021/11/1 3 1
# Expected For Publisher ID(2)
_id view_count click_count
2021/11/1 1 1
Currently, I am using 2 queries for both collections and combining results as one in code.
For View
db.view.aggregate([
/*FirstStage*/
{
$match:
{
"$and":
[
{
"publisher_id": 1
},
{
"created_at": {$gte: new ISODate("2021-11-01"), $lt: new ISODate("2021-11-28")}
}
]
}
},
/*SecondStage*/
{
$group:
{
_id: {$dateToString: {format: '%Y/%m/%d', date: "$created_at"}},
count: {
$sum: 1
}
}
}
])
For Click
db.click.aggregate([
/*FirstStage*/
{
$match:
{
"$and":
[
{
"publisher_id": 1
},
{
"created_at": {$gte: new ISODate("2021-11-01"), $lt: new ISODate("2021-11-28")}
}
]
}
},
/*SecondStage*/
{
$group:
{
_id: {$dateToString: {format: '%Y/%m/%d', date: "$created_at"}},
count: {
$sum: 1
}
}
}
])

Because you are querying two different collections there is no "good" way to merge this into one query, the only way I can think of is using $facet, where the first stage is the "normal" one, and the other stage starts with a $lookup from the other collection.
This approach does add overhead, which is why I recommend to just keep doing the merge in code, however for the sake of answering here is a sample:
db.view.aggregate([
{
$facet: {
views: [
{
$match: {
"$and": [
{
"publisher_id": 1
},
{
"created_at": {
$gte: ISODate("2021-11-01"),
$lt: ISODate("2021-11-28")
}
}
]
}
},
],
clicks: [
{
$limit: 1
},
{
$lookup: {
from: "click",
let: {},
pipeline: [
{
$match: {
"$and": [
{
"publisher_id": 1
},
{
"created_at": {
$gte: ISODate("2021-11-01"),
$lt: ISODate("2021-11-28")
}
}
]
}
},
],
as: "clicks"
}
},
{
$unwind: "$clicks"
},
{
$replaceRoot: {
newRoot: "$clicks"
}
}
]
}
},
{
$project: {
merged: {
"$concatArrays": [
"$views",
"$clicks"
]
}
}
},
{
$unwind: "$merged"
},
{
$group: {
_id: {
$dateToString: {
format: "%Y/%m/%d",
date: "$merged.created_at"
}
},
count: {
$sum: 1
}
}
}
])
Mongo Playground

Related

Sort Mongodb documents by seeing if the _id is in another array

I have two collections - "users" and "follows". "Follows" simply contains documents with a "follower" field and a "followee" field that represent when a user follows another user. What I want to do is to be able to query the users but display the users that I (or whatever user is making the request) follow first. For example if I follow users "5" and "14", when I search the list of users, I want users "5" and "14" to be at the top of the list, followed by the rest of the users in the database.
If I were to first query all the users that I follow from the "Follows" collection and get an array of those userIDs, is there a way that I can sort by using something like {$in: [userIDs]}? I don't want to filter out the users that I do not follow, I simply want to sort the list by showing the users that I do follow first.
I am using nodejs and mongoose for this.
Any help would be greatly appreciated. Thank you!
Answer
db.users.aggregate([
{
$addFields: {
sortBy: {
$cond: {
if: {
$in: [ "$_id", [ 5, 14 ] ]
},
then: 0,
else: 1
}
}
}
},
{
$sort: {
sortBy: 1
}
},
{
$unset: "sortBy"
}
])
Test Here
If you don't want you on the list, then
db.users.aggregate([
{
$addFields: {
sortBy: {
$cond: {
if: {
$in: [ "$_id", [ 5, 14 ] ]
},
then: 0,
else: 1
}
}
}
},
{
$sort: {
sortBy: 1
}
},
{
$unset: "sortBy"
},
{
$match: {
"_id": { $ne: 1 }
}
}
])
Test Here
If you want to sort users first
db.users.aggregate([
{
$sort: {
_id: 1
}
},
{
$addFields: {
sortBy: {
$cond: {
if: {
$in: [
"$_id",
[
5,
14
]
]
},
then: 0,
else: 1
}
}
}
},
{
$sort: {
sortBy: 1,
}
},
{
$unset: "sortBy"
},
{
$match: {
"_id": {
$ne: 1
}
}
}
])
Test Here

How to query an array and retrieve it from MongoDB

Updated:
I have a document on the database that looks like this:
My question is the following:
How can I retrieve the first 10 elements from the friendsArray from database and sort it descending or ascending based on the lastTimestamp value.
I don't want to download all values to my API and then sort them in Python because that is wasting my resources.
I have tried it using this code (Python):
listOfUsers = db.user_relations.find_one({'userId': '123'}, {'friendsArray' : {'$orderBy': {'lastTimestamp': 1}}}).limit(10)
but it just gives me this error pymongo.errors.OperationFailure: Unknown expression $orderBy
Any answer at this point would be really helpful! Thank You!
use aggregate
first unwind
then sort according timestap
group by _id to create sorted array
use addfields and filter for getting first 10 item of array
db.collection.aggregate([
{ $match:{userId:"123"}},
{
"$unwind": "$friendsArray"
},
{
$sort: {
"friendsArray.lastTimeStamp": 1
}
},
{
$group: {
_id: "$_id",
friendsArray: {
$push: "$friendsArray"
}
},
},
{
$addFields: {
friendsArray: {
$filter: {
input: "$friendsArray",
as: "z",
cond: {
$lt: [
{
$indexOfArray: [
"$friendsArray",
"$$z"
]
},
10
]
}// 10 is n first item
}
}
},
}
])
https://mongoplayground.net/p/2Usk5sRY2L2
and for pagination use this
db.collection.aggregate([
{ $match:{userId:"123"}},
{
"$unwind": "$friendsArray"
},
{
$sort: {
"friendsArray.lastTimeStamp": 1
}
},
{
$group: {
_id: "$_id",
friendsArray: {
$push: "$friendsArray"
}
},
},
{
$addFields: {
friendsArray: {
$filter: {
input: "$friendsArray",
as: "z",
cond: {
$and: [
{
$gt: [
{
$indexOfArray: [
"$friendsArray",
"$$z"
]
},
10
]
},
{
$lt: [
{
$indexOfArray: [
"$friendsArray",
"$$z"
]
},
20
]
},
]
}// 10 is n first item
}
}
},
}
])
The translation of your find to aggregation(we need unwind that why aggregation is used) would be like the bellow query.
Test code here
Query (for descending replace 1 with -1)
db.collection.aggregate([
{
"$match": {
"userId": "123"
}
},
{
"$unwind": {
"path": "$friendsArray"
}
},
{
"$sort": {
"friendsArray.lastTimeStamp": 1
}
},
{
"$limit": 10
},
{
"$replaceRoot": {
"newRoot": "$friendsArray"
}
}
])
If you want to skip some before limit add one stage also
{
"$skip" : 10
}
To take the 10-20 messages for example.

Mongodb aggregation , group by items for the last 5 days

I'm trying to get the result in some form using mongodb aggregation.
here is my sample document in the collection:
[{
"_id": "34243243243",
"workType": "TESTWORK1",
"assignedDate":ISODate("2021-02-22T00:00:00Z"),
"status":"Completed",
},
{
"_id": "34243243244",
"workType": "TESTWORK2",
"assignedDate":ISODate("2021-02-21T00:00:00Z"),
"status":"Completed",
},
{
"_id": "34243243245",
"workType": "TESTWORK3",
"assignedDate":ISODate("2021-02-20T00:00:00Z"),
"status":"InProgress",
}...]
I need to group last 5 days data in an array by workType count having staus completed.
Expected result:
{_id: "TESTWORK1" , value: [1,0,4,2,3] ,
_id: "TESTWORK2" , value: [3,9,,3,5],
_id : "TESTWORK3", value: [,,,3,5]}
Here is what I'm trying to do, but not sure how to get the expected result.
db.testcollection.aggregate([
{$match:{"status":"Completed"}},
{$project: {_id:0,
assignedSince:{$divide:[{$subtract:[new Date(),$assignedDate]},86400000]},
workType:1
}
},
{$match:{"assignedSince":{"lte":5}}},
{$group : { _id:"workType", test :{$push:{day:"$assignedSince"}}}}
])
result: {_id:"TESTWORK1": test:[{5},{3}]} - here I'm getting the day , but I need the count of the workTypes on that day.
Is there any easy way to do this? Any help would be really appreciated.
Try this:
db.testcollection.aggregate([
{
$match: { "status": "Completed" }
},
{
$project: {
_id: 0,
assignedDate: 1,
assignedSince: {
$toInt: {
$divide: [{ $subtract: [new Date(), "$assignedDate"] }, 86400000]
}
},
workType: 1
}
},
{
$match: { "assignedSince": { "$lte": 5 } }
},
{
$group: {
_id: {
workType: "$workType",
assignedDate: "$assignedDate"
},
count: { $sum: 1 }
}
},
{
$group: {
_id: "$_id.workType",
values: { $push: "$count" }
}
}
]);

Referencing root _id in aggregate lookup match expression not working

This is my first experience using aggregate pipeline. I'm not able to get a "$match" expression to work inside the pipeline. If I remove the "_id" match, I get every document in the collection past the start date, but once I add the $eq expression, it returns empty.
I read a lot of other examples and tried many different ways, and this seems like it is correct. But the result is empty.
Any suggestions?
let now = new Date()
let doc = await Team.aggregate([
{ $match: { created_by: mongoose.Types.ObjectId(req.params.user_oid)} },
{ $sort: { create_date: 1 } },
{ $lookup: {
from: 'events',
let: { "team_oid": "$team_oid" },
pipeline: [
{ $addFields: { "team_oid" : { "$toObjectId": "$team_oid" }}},
{ $match: {
$expr: {
$and: [
{ $gt: [ "$start", now ] },
{ $eq: [ "$_id", "$$team_oid" ] }
]
},
}
},
{
$sort: { start: 1 }
},
{
$limit: 1
}
],
as: 'events',
}},
{
$group: {
_id: "$_id",
team_name: { $first: "$team_name" },
status: { $first: "$status" },
invited: { $first: "$invited" },
uninvited: { $first: "$uninvited" },
events: { $first: "$events.action" },
dates: { $first: "$events.start" } ,
team_oid: { $first: "$events.team_oid" }
}
}])
Example Docs (added by request)
Events:
_id:ObjectId("60350837c57b3a15a414d265")
invitees:null
accepted:null
sequence:7
team_oid:ObjectId("60350837c57b3a15a414d263")
type:"Calendar Invite"
action:"Huddle"
status:"Questions Issued"
title:"Huddle"
body:"This is a Huddle; you should receive new questions 5 days befor..."
creator_oid:ObjectId("5ff9e50a206b1924dccd691e")
start:2021-02-26T07:00:59.999+00:00
end:2021-02-26T07:30:59.999+00:00
__v:0
Team:
_id:ObjectId("60350837c57b3a15a414d263")
weekly_schedule:1
status:"Live"
huddle_number:2
reminders:2
active:true
created_by:ObjectId("5ff9e50a206b1924dccd691e")
team_name:"tESTI"
create_date:2021-02-23T13:50:47.172+00:00
__v:0
This is just a guess since you don't have schema in your question. But it looks like your have some of your _ids mixed up. Where you are currently trying to $match events whose _id is equal to a team_oid. Rather than the event's team_oid field being equal to the current 'team' _id.
I'm pretty confident this will produce the correct output. If you post any schema or sample docs I will edit it.
https://mongoplayground.net/p/5i1w2Ii7KCR
let now = new Date()
let doc = await Team.aggregate([
{ $match: { created_by: mongoose.Types.ObjectId(req.params.user_oid)} },
{ $sort: { create_date: 1 } },
{ $lookup: {
from: 'events',
// Set tea_oid as the current team _id
let: { "team_oid": "$_id" },
pipeline: [
{ $match: {
$expr: {
$and: [
{ $gt: [ "$start", now ] },
// Match events whose 'team_oid' field matches the 'team' _id set above
{ $eq: [ "$team_oid", "$$team_oid" ] }
]
},
}
},
{
$sort: { start: 1 }
},
{
$limit: 1
}
],
as: 'events',
}},
{
$group: {
_id: "$_id",
team_name: { $first: "$team_name" },
status: { $first: "$status" },
invited: { $first: "$invited" },
uninvited: { $first: "$uninvited" },
events: { $first: "$events.action" },
dates: { $first: "$events.start" } ,
team_oid: { $first: "$events.team_oid" }
}
}])

total of all groups totals using mongodb

i did this Aggregate pipeline , and i want add a field contains the Global Total of all groups total.
{ "$match": query },
{ "$sort": cursor.sort },
{ "$group": {
_id: { key:"$paymentFromId"},
items: {
$push: {
_id:"$_id",
value:"$value",
transaction:"$transaction",
paymentMethod:"$paymentMethod",
createdAt:"$createdAt",
...
}
},
count:{$sum:1},
total:{$sum:"$value"}
}}
{
//i want to get
...project groups , goupsTotal , groupsCount
}
,{
"$skip":cursor.skip
},{
"$limit":cursor.limit
},
])
you need to use $facet (avaialble from MongoDB 3.4) to apply multiple pipelines on the same set of docs
first pipeline: skip and limit docs
second pipeline: calculate total of all groups
{ "$match": query },
{ "$sort": cursor.sort },
{ "$group": {
_id: { key:"$paymentFromId"},
items: {
$push: "$$CURRENT"
},
count:{$sum:1},
total:{$sum:"$value"}
}
},
{
$facet: {
docs: [
{ $skip:cursor.skip },
{ $limit:cursor.limit }
],
overall: [
{$group: {
_id: null,
groupsTotal: {$sum: '$total'},
groupsCount:{ $sum: '$count'}
}
}
]
}
the final output will be
{
docs: [ .... ], // array of {_id, items, count, total}
overall: { } // object with properties groupsTotal, groupsCount
}
PS: I've replaced the items in the third pipe stage with $$CURRENT which adds the whole document for the sake of simplicity, if you need custom properties then specify them.
i did it in this way , project the $group result in new field doc and $sum the sub totals.
{
$project: {
"doc": {
"_id": "$_id",
"total": "$total",
"items":"$items",
"count":"$count"
}
}
},{
$group: {
"_id": null,
"globalTotal": {
$sum: "$doc.total"
},
"result": {
$push: "$doc"
}
}
},
{
$project: {
"result": 1,
//paging "result": {$slice: [ "$result", cursor.skip,cursor.limit ] },
"_id": 0,
"globalTotal": 1
}
}
the output
[
{
globalTotal: 121500,
result: [ [group1], [group2], [group3], ... ]
}
]