I am trying to get an aggregate work in mongo
Here is a simple json structure, and how I would like to achieve the aggregate.
// this is the format of the collection
[
{
_id: 6154f64df41fa3628ac2062a,
type: 'email',
platform: 'google',
apiDataId: '17c33ac4735c80bf',
timestamp: '2021-09-01',
userId: '6132a04892559282c40fd29a',
groupId: '6132a0cb74af9d82df74e918',
__v: 0
},
{
_id: 6154f64df41fa3628ac2062b,
type: 'call',
platform: 'yahoo',
apiDataId: '17c2d25e2ccf770d',
timestamp: '2021-09-01',
userId: '6132a04892559282c40fd29a',
groupId: '6132a0cb74af9d82df74e918',
__v: 0
},
{
_id: 6154f64df41fa3628ac2062c,
type: 'email',
platform: 'google',
apiDataId: '2021-09-03',
timestamp: '1632958029720',
userId: '6132a04892559282c40ff2a9',
groupId: '6132a0cb74af9d82df74e918',
__v: 0
},
{
_id: 6154f64df41fa3628ac2062d,
type: 'email',
platform: 'google',
apiDataId: '17c273deffc51cc9',
timestamp: '2021-09-04',
userId: '6132a04892559282c40fd29a',
groupId: '6132a0cb74af9d82df74e918',
__v: 0
},
{
_id: 6154f64df41fa3628ac2062e,
type: 'call',
platform: 'yahoo',
apiDataId: '17c14c85f6a89088',
timestamp: '2021-09-04',
userId: '6132a04892559282c40ff2a9',
groupId: '6132a0cb74af9d82df74e918',
__v: 0
}
]
I've started out wrting the aggregate, but I am having hard time understanding the aggregate,
aggregate([
{ $match: { groupId: groupId } },
{ $group: { _id: "$userId" } },
{ $group: { timestamp: "$timestamp" } },
{ $project: { "groupId": 0, "userId": 0} },
{ $count: "num_data" }
]).exec()
what I am trying to accomplish edited
[
groupId: '6132a04892559282c40fd29a',
userId: {
calls: [
"2021-09-01": {platform: "yahoo"},
"2021-09-03": {platform: "yahoo"},
...
],
emails: [
"2021-09-01": {platform: "google"},
"2021-09-04": {platform: "google"},
...
],
},
userId: { ... }
]
of course that aggregation doesn't work, I am having hard time understanding and trying to figure out the orders and what $things to use.
Is this aggregate is what you looking for?
db.collection.aggregate([
{
$match: {
groupId: "1"
}
},
{
$group: {
_id: "$userId",
count: {
$sum: 1
},
apiDataId: {
"$first": "$$ROOT"
}
}
},
{
"$project": {
"count": "$count",
"apiDataId": {
"timestamp": "$apiDataId.timestamp",
"platform": "$apiDataId.platform",
"type": "$apiDataId.type",
"data": "$apiDataId.data"
}
}
}
])
data
[
{
"groupId": "1",
"timestamp": "1",
"userId": "1",
"platform": "1",
"apiDataId": "1",
"type": "1",
"data": "1"
},
{
"groupId": "1",
"timestamp": "1",
"userId": "2",
"platform": "3",
"apiDataId": "4",
"type": "5",
"data": "6"
},
{
"groupId": "1",
"timestamp": "3",
"userId": "2",
"platform": "7",
"apiDataId": "8",
"type": "9",
"data": "9"
},
{
"groupId": "2",
"timestamp": "1",
"userId": "1",
"platform": "1",
"apiDataId": "1",
"type": "1",
"data": "1"
}
]
result
[
{
"_id": "2",
"apiDataId": {
"data": "6",
"platform": "3",
"timestamp": "1",
"type": "5"
},
"count": 2
},
{
"_id": "1",
"apiDataId": {
"data": "1",
"platform": "1",
"timestamp": "1",
"type": "1"
},
"count": 1
}
]
mongoplayground
Update: 2021-10-03
aggregate by userid then by date
db.collection.aggregate([
{
$match: {
__v: 0
}
},
{
$group: {
_id: {
u: "$userId",
t: "$timestamp"
},
count: {
$sum: 1
},
"timestampList": {
"$push": "$$ROOT"
}
}
},
{
$group: {
_id: "$_id.u",
count: {
$sum: 1
},
"userList": {
"$push": "$$ROOT"
}
}
}
])
Related
I am trying to create top 10 product list based on postType = "buy". My logic is a count postType = "buy" and sort the top 10 products from the logs collection. Here are my sample log collections.
[
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "3",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "view",
"product": "4",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "2",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "view",
"product": "2",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "share",
"product": "3",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "2",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "2",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "share",
"product": "2",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "1",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "1",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "viewvideo",
"product": "1",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "viewvideo",
"product": "2",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "viewvideo",
"product": "3",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "4",
},
{
"_id": "633dc5b761ff04e7ae8e8c0f",
"postType": "buy",
"product": "5",
}
]
I am trying to get count for totalBuybutton, totalShareButton, totalView if 'buy' keyword matched.
My expected output is:
[
{"product":1, "totalBuycount":2, "shareButtonCount":4, viewCount":4},
{"product":2, "totalBuycount":3, shareButtonCount":4, viewCount":4},
{"product":3, "totalBuycount":1, shareButtonCount":4, viewCount":4},
{"product":4, "totalBuycount":1, shareButtonCount":4, viewCount":4},
{"product":5, "totalBuycount":1, shareButtonCount":2, viewCount":4}
]
My current implementation is as below
aggregate([
{
$match: {
postType: "buybutton"
},
},
{
$group: {
_id: "$product",
count: {
$count: {}
}
}
},
{
$project: {
product: "$_id",
count: 1,
_id: 0,
},
},)
No idea, if this is what you are looking for.
The main part is
{
$group: {
_id: { postType: "$postType", product: "$product" },
count: { $count: {} }
}
}
Do you really need exactly the output as given in the question. It would require quite a lot of cosmetic work
db.collection.aggregate([
{
$group: {
_id: { postType: "$postType", product: "$product" },
count: { $count: {} }
}
},
{
$group: {
_id: "$_id.product",
data: { $push: "$$ROOT" }
}
},
{
$project: {
_id: 0,
product: "$_id",
data: {
$arrayToObject: {
$map: {
input: "$data", in: {
k: "$$this._id.postType", v: "$$this.count",
}
}
}
}
}
},
{ $replaceWith: { $mergeObjects: ["$$ROOT", "$data"] } },
{ $unset: "data" }
])
]
{ product: '5', buy: 1 },
{ product: '3', buy: 1, share: 1, viewvideo: 1 },
{ product: '2', buy: 3, view: 1, viewvideo: 1, share: 1 },
{ product: '1', viewvideo: 1, buy: 2 },
{ product: '4', buy: 1, view: 1 }
]
Mongo Playground
I'm trying to aggreate a collection of transactions into a running total of owners by day.
The initial collection looks like this:
[
{ "to": "A", "from": "0", "ts": 1 },
{ "to": "A", "from": "0", "ts": 1 },
{ "to": "B", "from": "0", "ts": 1 },
{ "to": "B", "from": "0", "ts": 2 },
{ "to": "C", "from": "0", "ts": 3 },
{ "to": "A", "from": "B", "ts": 4 }
]
What I would like to get is something like this:
[
{
"ts": 1,
"holdings": [
{ "owner": "0", "holdings": -3 },
{ "owner": "A", "holdings": 2 },
{ "owner": "B", "holdings": 1 }
]
},
{
"ts": 2,
"holdings": [
{ "owner": "0", "holdings": -4 },
{ "owner": "A", "holdings": 2 },
{ "owner": "B", "holdings": 2 }
]
},
{
"ts": 4,
"holdings": [
{ "owner": "0", "holdings": -5 },
{ "owner": "A", "holdings": 3 },
{ "owner": "B", "holdings": 1 },
{ "owner": "C", "holdings": 1 }
]
}
]
I've already understood how to generate this for a single ts that I'm setting, but I don't know how to do it across all ts.
The aggregation pipeline for a single ts looks like this:
db.collection.aggregate([
// start with: { "to": "A", "from": "0", "ts": 1 }
{
// create a doc with an array with subset of fields:
// { "_id": ObjectId("5a934e000102030405000000"),
// "data": [ { "change": 1, "owner": "A", "ts": "1" },
// { "change": -1, "owner": "0", "ts": "1" } ] }
$project: {
data: [
{
owner: '$to',
ts: '$ts',
change: 1,
},
{
owner: '$from',
ts: '$ts',
change: -1,
},
],
},
},
{
// unwind the array into 2 docs:
// { "_id": ObjectId("5a934e000102030405000000"), "data": { "change": 1, "owner": "A", "ts": "1" } },
// { "_id": ObjectId("5a934e000102030405000000"), "data": { "change": -1, "owner": "0", "ts": "1" } },
$unwind: '$data',
},
{
// use data as root:
// { "data": { "change": 1, "owner": "A", "ts": "1" } },
// { "data": { "change": -1, "owner": "0", "ts": "1" } }
$replaceRoot: {
newRoot: '$data',
},
},
{
// select day to calc totals
$match: {
ts: {
$lt: 6,
},
},
},
{
// sum totals, grouped by owner
$group: {
_id: '$owner',
//_id: null,
holdings: {
$sum: '$change',
},
},
},
])
This gives the correct result for a particular day (selected in the match stage). I don't understand how I can now generalize that to all days.
One way to do it is using $setWindowFields, which has a built-in accumulation:
db.collection.aggregate([
{
$project: {
ts: "$ts",
data: [{owner: "$to", change: 1}, {owner: "$from", change: -1}]
}
},
{$unwind: "$data"},
{
$group: {
_id: {ts: "$ts", owner: "$data.owner"},
holdings: {$sum: "$data.change"}
}
},
{
$setWindowFields: {
partitionBy: "$_id.owner",
sortBy: {"_id.ts": 1},
output: {
cumulativeHoldings: {
$sum: "$holdings",
window: {documents: ["unbounded", "current"]}
}
}
}
},
{
$group: {
_id: "$_id.ts",
holdings: {$push: {owner: "$_id.owner", holdings: "$cumulativeHoldings"}}
}
}
])
Playground
Thank you in advance for any help!
I've a collection QR with schema similar to this:
var qrSchema = new Schema({
qrId: { type: String, index: true },
owner: { type: Schema.Types.ObjectId, ref: 'User' },
qrName: { type: String },
qrCategory: { type: String, index: true },
shortUrl: { type: String}}
})
And collection Datas similar to this:
var dataSchema = new Schema({
qrId: { type: String, index: true}
city: { type: String},
device: { type: String},
date: { type: Date, index:true},
})
The relation between QR and Datas is 1-to-many.
I've an aggregate like this:
Model.QR.aggregate([
{ $match: {
$and: [
{ owner: mongoose.Types.ObjectId(user._id) },
{
$expr: {
$cond: [
{ $in: [ category, [ null, "", "undefined" ]] },
true,
{ $eq: [ "$qrCategory", category ] }
]
}
}
]
}
},
{ $lookup:
{
"from": "datas",
"localField": "qrId",
"foreignField": "qrId",
"as": "data"
}
},
{
$project: {
_id: 0,
qrId: 1,
qrName: 1,
qrCategory: 1,
shortUrl: 1,
data: {
$filter: {
input: "$data",
as: "item",
cond: {
$and: [
{ $gte: [ "$$item.date", date.start ] },
{ $lte: [ "$$item.date", date.end ] }
] }
}
}
}
},
{
$group: {
_id: { "qrId": "$qrId", "qrName": "$qrName", "qrCategory": "$qrCategory", "shortUrl": "$shortUrl" },
data: {
$push: {
dataItems: "$data",
count: {
$size: { '$ifNull': ['$data', []] }
}
}
}
}
},
{
$sort: {
"data.count": -1
}
},
{
$limit: 10,
}]).exec((err, results) => { })
Which is returning something like:
[
{
"_id": {
"qrId": "0PRA",
"qrName": "Campaign 0PRA",
"qrCategory": "html",
"shortUrl": "http://someurl.com/0PRA"
},
"data": [
{
"dataItems": [
{
"_id": "6200f2a8c0cf7a1c49233c7f",
"qrId": "0PRA",
"device": "iOS",
"city": "Beijing",
},
{
"_id": "6200f2eac0cf7a1c49233c80",
"qrId": "0PRA",
"device": "AndroidOS",
"city": "Beijing",
},
{
"_id": "6200f3a4c0cf7a1c49233c81",
"qrId": "0PRA",
"device": "AndroidOS",
"city": "Beijing",
},
{
"_id": "6200f632c0cf7a1c49233c88",
"qrId": "0PRA",
"device": "AndroidOS",
"city": "Nanchang",
},
{
"_id": "6201b342c0cf7a1c49233caa",
"qrId": "0PRA",
"device": "iOS",
"city": "Taizhou",
}
],
"count": 5
}
]
},
{
"_id": {
"qrId": "NQ17",
"qrName": "Campaign NQ17",
"qrCategory": "menu",
"shortUrl": "http://someurl.com/NQ17"
},
"data": [
{
"dataItems": [
{
"_id": "6200f207c0cf7a1c49233c7a",
"qrId": "NQ17",
"device": "iOS",
"city": "Singapore"
},
{
"_id": "8200f207c1cf7a1c49233c7a",
"qrId": "NQ17",
"device": "iOS",
"city": "Singapore"
},
{
"_id": "6200ac5db44f23b9ec2b6040",
"qrId": "NQ17",
"device": "AndroidOS",
"city": "San Antonio"
}
],
"count": 3
}
]
}
]
I'm trying to include the most frequent device and city in the results after the count of dataItems, like this:
[
{
"_id": {
"qrId": "0PRA",
"qrName": "Campaign 0PRA",
"qrCategory": "html",
"shortUrl": "http://someurl.com/0PRA"
},
"data": [
{
"dataItems": [
{
"_id": "6200f2a8c0cf7a1c49233c7f",
"qrId": "0PRA",
"device": "iOS",
"city": "Beijing",
},
{
"_id": "6200f2eac0cf7a1c49233c80",
"qrId": "0PRA",
"device": "AndroidOS",
"city": "Beijing",
},
{
"_id": "6200f3a4c0cf7a1c49233c81",
"qrId": "0PRA",
"device": "AndroidOS",
"city": "Beijing",
},
{
"_id": "6200f632c0cf7a1c49233c88",
"qrId": "0PRA",
"device": "AndroidOS",
"city": "Nanchang",
},
{
"_id": "6201b342c0cf7a1c49233caa",
"qrId": "0PRA",
"device": "iOS",
"city": "Taizhou",
}
],
"count": 5,
"topDevice": "AndroidOS", // <---- trying to add this
"topLocation": "Beijing" // <---- trying to add this
}
]
},
{
"_id": {
"qrId": "NQ17",
"qrName": "Campaign NQ17",
"qrCategory": "menu",
"shortUrl": "http://someurl.com/NQ17"
},
"data": [
{
"dataItems": [
{
"_id": "6200f207c0cf7a1c49233c7a",
"qrId": "NQ17",
"device": "iOS",
"city": "Singapore"
},
{
"_id": "8200f207c1cf7a1c49233c7a",
"qrId": "NQ17",
"device": "iOS",
"city": "Singapore"
},
{
"_id": "6200ac5db44f23b9ec2b6040",
"qrId": "NQ17",
"device": "android",
"city": "San Antonio"
}
],
"count": 3,
"topDevice": "iOS", // <---- trying to add this
"topLocation": "Singapore" // <---- trying to add this
}
]
}
]
Is this possible?
Thank you very much in advance for any help or hints!
Method 1
Use $function will be way more easier. MongoDB version >= 4.4
Sort function in js
db.collection.aggregate([
{
"$set": {
"data": {
"$map": {
"input": "$data",
"as": "d",
"in": {
"count": "$$d.count",
"dataItems": "$$d.dataItems",
"topDevice": {
$function: {
body: "function(arr) {return arr.sort((a,b) =>arr.filter(v => v===a).length-arr.filter(v => v===b).length).pop() }",
args: [ "$$d.dataItems.device" ],
lang: "js"
}
},
"topLocation": {
$function: {
body: "function(arr) {return arr.sort((a,b) =>arr.filter(v => v===a).length-arr.filter(v => v===b).length).pop() }",
args: [ "$$d.dataItems.city" ],
lang: "js"
}
}
}
}
}
}
}
])
mongoplayground
Method 2
db.qr.aggregate([
{
"$match": {
owner: {
"$in": [
"1",
"2"
]
}
}
},
{
"$lookup": {
"from": "data",
"localField": "qrId",
"foreignField": "qrId",
"as": "data",
"pipeline": [
{
"$match": {
"$and": [
{
"date": {
"$gte": ISODate("2021-09-01T01:23:25.184Z")
}
},
{
"date": {
"$lte": ISODate("2021-09-02T11:23:25.184Z")
}
}
]
}
},
{
"$facet": {
"deviceGroup": [
{
"$group": {
"_id": "$device",
"sum": {
"$sum": 1
}
}
},
{
"$sort": {
sum: -1
}
},
{
"$limit": 1
}
],
"cityGroup": [
{
"$group": {
"_id": "$city",
"sum": {
"$sum": 1
}
}
},
{
"$sort": {
sum: -1
}
},
{
"$limit": 1
}
],
"all": []
}
}
]
}
},
{
"$set": {
"data": {
"$first": "$data.all"
},
"topDevice": {
"$first": {
"$first": "$data.deviceGroup._id"
}
},
"topLocation": {
"$first": {
"$first": "$data.cityGroup._id"
}
}
}
},
{
$group: {
_id: {
"qrId": "$qrId",
"qrName": "$qrName",
"qrCategory": "$qrCategory",
"shortUrl": "$shortUrl"
},
data: {
$push: {
dataItems: "$data",
topDevice: "$topDevice",
topLocation: "$topLocation",
count: {
$size: {
"$ifNull": [
"$data",
[]
]
}
}
}
}
}
}
])
mongoplayground
Query
add the match you need, i didn't understand what the match should do
lookup on qrId
filter to keep only the start<=dates<=end (replace the 1 and 100)
facet to group all-documents, the topDevice the topLocation
$set to bring those data out from the nested locations they are
count is added as the size of all-documents
*maybe i am missing something, but try it(first part i think its like YuTing answer)
Test code here
QR.aggregate(
[{"$lookup":
{"from":"Datas",
"localField":"qrId",
"foreignField":"qrId",
"pipeline":
[{"$match":{"$and":[{"date":{"$gte":1}}, {"date":{"$lte":100}}]}},
{"$facet":
{"dataItems":[],
"topDevice":
[{"$group":{"_id":"$device", "count":{"$sum":1}}},
{"$sort":{"count":-1}}, {"$limit":1}],
"topLocation":
[{"$group":{"_id":"$city", "count":{"$sum":1}}},
{"$sort":{"count":-1}}, {"$limit":1}]}}],
"as":"data"}},
{"$set":{"data":{"$arrayElemAt":["$data", 0]}}},
{"$set":
{"dataItems":"$data.dataItems",
"count":{"$size":"$data.dataItems"},
"topDevice":
{"$getField":
{"field":"_id", "input":{"$arrayElemAt":["$data.topDevice", 0]}}},
"topLocation":
{"$getField":
{"field":"_id",
"input":{"$arrayElemAt":["$data.topLocation", 0]}}},
"data":"$$REMOVE"}}])
I have a MongoDB schema that looks like this
const ProductModel = new Schema({
subcategory: {
type : mongoose.Schema.Types.ObjectId,
ref : "Subcategory",
},
product_name: {
type: String
},
description: {
type: String
},
price: {
type: Number
},
});
And a subcategory schema:
const SubcategoryModel = new Schema({
subcategoryName: {
type: String,
}
});
The input query before aggregation looks like this:
[
{
"_id": "111",
"subcategory": {
"_id": "456",
"categoryName": "Sneakers",
},
"product_name": "Modern sneaker",
"description": "Stylish",
"price": 4400
},
{
"_id": "222",
"subcategory": {
"_id": "456",
"categoryName": "Sneakers",
},
"product_name": "Blue shoes",
"description": "Vived colors",
"price": 7500
},
{
"_id": "333",
"subcategory": {
"_id": "123",
"categoryName": "Jackets",
"__v": 0
},
"product_name": "Modern jacket",
"description": "Stylish",
"price": 4400
},
}
]
The final result of the query should look like this:
{
"Sneakers":[
{
"product_name":"Modern sneaker",
"description":"Stylish",
"price":"4400"
},
{
"product_name":"Blue shoes",
"description":"Vived colors",
"price":"7500"
},
"Jackets":{
"...."
}
]
}
Subcategory before aggregation:
"subcategories": [
{
"_id": "123",
"categoryName": "Jackets",
},
{
"_id": "456",
"categoryName": "Sneakers",
}
]
I'm trying to populate the subcategory, And then group the products by their subcategoryName field.
You can use this aggregation query:
First $lookup to do the join between Product and Subcategory creating the array subcategories.
Then deconstructs the array using $unwind.
$group by the name of subproduct adding the entire object using $$ROOT.
The passes the fields you want using $project.
And replaceRoot to get key value into arrays as Sneakers and Jackets.
db.Product.aggregate([
{
"$lookup": {
"from": "Subcategory",
"localField": "subcategory.categoryName",
"foreignField": "categoryName",
"as": "subcategories"
}
},
{
"$unwind": "$subcategories"
},
{
"$group": {
"_id": "$subcategories.categoryName",
"data": {
"$push": "$$ROOT"
}
}
},
{
"$project": {
"data": {
"product_name": 1,
"description": 1,
"price": 1
}
}
},
{
"$replaceRoot": {
"newRoot": {
"$arrayToObject": [
[
{
"k": "$_id",
"v": "$data"
}
]
]
}
}
}
])
Example here
With your provided data, result is:
[
{
"Sneakers": [
{
"description": "Stylish",
"price": 4400,
"product_name": "Modern sneaker"
},
{
"description": "Vived colors",
"price": 7500,
"product_name": "Blue shoes"
}
]
},
{
"Jackets": [
{
"description": "Stylish",
"price": 4400,
"product_name": "Modern jacket"
}
]
}
]
I have Json which have values like state_city details this contains information like which city belongs to which state -
Need to query it for particular state name which will gives me all cities that belongs to that state.
db.collection.find({
"count": 10,
"state.name": "MP"
})
[
{
"collection": "collection1",
"count": 10,
"state": [
{
"name": "MH",
"city": "Mumbai"
},
{
"name": "MH",
"city": "Pune"
},
{
"name": "UP",
"city": "Kanpur"
},
{
"name": "CG",
"city": "Raipur"
}
]
},
{
"collection": "collection2",
"count": 20,
"state": [
{
"name": "MP",
"city": "Indore"
},
{
"name": "MH",
"city": "Bhopal"
},
{
"name": "UP",
"city": "Kanpur"
},
{
"name": "CG",
"city": "Raipur"
}
]
}
]
You have to use aggregate query to get only matching elements in array :
db.collection.aggregate([{
$unwind: "$content.state"
},
{
$match: {
"content.state.name": "MH",
"count": 10
}
},
{
$group: {
_id: "$content.state.city",
}
},
{
$addFields: {
key: 1
}
},
{
$group: {
_id: "$key",
cities: {
$push: "$_id"
}
}
},
{
$project: {
_id: 0,
cities: 1
}
}
])
This query will return :
{
"cities": [
"Pune",
"Mumbai"
]
}
The following query would be the solution.
db.collection.find({ "count": 10, "state":{"name": "MP"}})
For more complex queries, $elemMatch is also available.