Mongodb Aggregation get Data per user - mongodb

Report table sample data
{
"_id" : ObjectId("614415f4a6566a001623b622"),
"record" : [
{
"dateTime" : ISODate("2021-09-17T04:13:39.465Z"),
"status" : "time-in",
"month" : 9,
"day" : 17,
"year" : 2021,
"time" : 1631852019465.0,
"date" : ISODate("2021-09-17T00:00:00.000Z"),
},
{
"dateTime" : ISODate("2021-09-17T04:14:01.182Z"),
"status" : "time-out",
"month" : 9,
"day" : 17,
"year" : 2021,
"time" : 1631852041182.0,
"date" : ISODate("2021-09-17T00:00:00.000Z"),
}
],
"uid" : ObjectId("614415b0a6566a001623b80b"),
"date" : ISODate("2021-09-17T00:00:00.000Z"),
"status" : "time-out",
"createdAt" : ISODate("2021-09-17T04:13:40.102Z"),
"updatedAt" : ISODate("2021-09-17T04:14:01.831Z"),
"__v" : 0
}
Users table sample data
{
"_id" : ObjectId("615c0f6db30aff375cd05ac1"),
"displayName" : "test test",
"firstName" : "test",
"lastName" : "test",
"email" : "test#gmail.com",
"brand" : "Jollibee",
"phone" : "+632312312312312",
"role" : 1,
"isVerified" : true,
"isArchived" : false,
"createdAt" : ISODate("2021-10-05T08:40:13.208Z"),
"updatedAt" : ISODate("2021-10-05T08:40:13.208Z"),
"__v" : 0
}
I have a data like this
db.getCollection('users').aggregate([
{
"$match": { brand: "Jollibee" }
},
{
$lookup: {
from: "orders",
let: { id: 'id' },
pipeline: [
{
$match: {
date: { $gte: ISODate("2020-11-01"), $lt: ISODate("2021-11-31") },
}
}
],
as: "orders",
},
},
{
$project: {
"_id": 1,
"name": 1,
"orders": 1
}
}
])
when I'm using this aggregation I'm getting all the data inserted per user.
What I want to happen is that. I will only get the data that belong to the user and not all the data of all users.
Added the sample documents for each collection

You are not comparing the userIds of both collections. You should add that on your $match. Playground
db.users.aggregate([
{
"$match": {
brand: "Jollibee"
}
},
{
$lookup: {
from: "orders",
let: {
id: "$_id"
},
pipeline: [
{
$match: {
date: {
$gte: ISODate("2020-11-01"),
$lt: ISODate("2021-11-30")
},
$expr: {
$eq: [
"$uid",
"$$id"
]
}
}
}
],
as: "orders",
},
},
{
$project: {
"_id": 1,
"name": 1,
"orders": 1
}
}
])

Related

Unable to aggregate two collections using lookup in MongoDB Atlas

I have an orders collection that looks like this:
{
"_id" : "wJNEiSYwBd5ozGtLX",
"orderId" : 52713,
"createdAt" : ISODate("2020-01-31T04:34:13.790Z"),
"status" : "closed",
"orders" : [
{
"_id" : "ziPzwLuZrz9MNkaRT",
"productId" : 10290,
"quantity" : 2
}
]
}
I have an products collection that looks like this
{
"_id" : "238cwwLkZa6gKNN86",
"productId" : 10290,
"title" : "Product Title",
"price" : 9.9
}
I am trying to merge the price information into the orders information.
Something like:
{
"_id" : "wJNEiSYwBd5ozGtLX",
"orderId" : 52713,
"createdAt" : ISODate("2020-01-31T04:34:13.790Z"),
"status" : "closed",
"orders" : [
{
"_id" : "ziPzwLuZrz9MNkaRT",
"productId" : 10290,
"quantity" : 2,
"price": 9.9
}
]
}
If I try a $lookup command on MongoDB Atlas Dashboard like this:
{
from: 'products',
localField: 'orders.productId',
foreignField: 'productId',
as: 'priceInfo'
}
The aggregated output is (not what I wanted):
{
"_id" : "wJNEiSYwBd5ozGtLX",
"orderId" : 52713,
"createdAt" : ISODate("2020-01-31T04:34:13.790Z"),
"status" : "closed",
"orders" : [
{
"_id" : "ziPzwLuZrz9MNkaRT",
"productId" : 10290,
}
],
"priceInfo": [
{
"_id" : "238cwwLkZa6gKNN86",
"productId" : 10290,
"title" : "Product Title",
"price" : 9.9
}
]
}
I do not need a separate priceInfo array. It will be best if I have the product details information merged into the "orders" array. What should be the aggregation lookup syntax to achieve the desired output?
Demo - https://mongoplayground.net/p/bLqcN7tauWU
Read - $lookup $unwind $first $set $push $group
db.orders.aggregate([
{ $unwind: "$orders" }, // break array of orders into individual documents
{
$lookup: { // join
"from": "products",
"localField": "orders.productId",
"foreignField": "productId",
"as": "products"
}
},
{
$set: {
"orders.price": { "$arrayElemAt": [ "$products.price", 0 ] } // set the price
}
},
{
$group: { // group records back
_id: "$_id",
createdAt: { $first: "$createdAt" },
status: { $first: "$status" },
orderId: { $first: "$orderId" },
orders: { $push: "$orders" }
}
}
])

Facing a problem with the lookup in the second (student) table that matches all incoming output records mongodb aggregation

I'm facing a problem with the lookup in the second (student) table that matches all incoming output records of the first(test) table. I have two collections "tests" and "students". "Test" collection contains all school tests and the "student" table contains student's attended tests. Student table contains "pastTest"(test attended in past with status "pass" or "fail")array. I want to retrieve student who passed all incoming tests (we retrieve from the tests table)
test table: _id (primary ket)
student.pastTests.testId (need to match with test._id)
Test Document:
{
"_id" : ObjectId("5c9b5c1005729b2bf23f3290"),
"testDate" : {
"term" : 1,
"week" : 7
},
"retestDate" : {
"term" : 1,
"week" : 10
},
"testOrder" : "1.1",
"testDateScheduled" : true,
"retestDateScheduled" : true
}
Student Document:
{
"_id" : ObjectId("5c92dd994e8e6b2c1647d0d0"),
"completedYears" : [],
"firstName" : "Andrew",
"lastName" : "Jonhson",
"teacherId" : ObjectId("5bf36b1076696374e65feb4f"),
"yearGroup" : "0",
"schoolId" : 40001,
"currentTest" : ObjectId("5c9b5c1005729b2bf23f3290"),
"pastTests" : [
{
"_id" : ObjectId("5d3570645045863d373f6db1"),
"testId" : ObjectId("5c9b5c1005729b2bf23f3290"),
"status" : "pass"
},
{
"_id" : ObjectId("5d425af07708f5636c3bec1c"),
"testId" : ObjectId("5c9b5fc460e39c2c58e44109"),
"status" : "pass"
},
{
"_id" : ObjectId("5d5e54a875fab079f4d03570"),
"testId" : ObjectId("5c9b6492bb581c2ceb553fef"),
"status" : "fail"
},
],
"createdAt" : ISODate("2019-03-21T00:40:57.401Z"),
"updatedAt" : ISODate("2020-09-24T19:55:38.291Z"),
"__v" : 0,
"holdTests" : [],
"completedTests" : [],
"className" : "dd",
}
Query:
db.getCollection('tests').aggregate([
{
$match: {
yearGroup: '-1',
$or : [
{
$and: [
{'retestDateScheduled': true},
{ 'retestDate.term': { $lt: 4 } },
]
},
{
$and: [
{'testDateScheduled': true},
{ 'testDate.term': { $lt: 4 } },
]
}
]
}
},
{
$lookup: {
from: 'students',
let: {testId: '$_id', schoolId: 49014, yearGroup: '-1'},
pipeline: [
]
}
}
])
Note: Initial match query returns all tests of the term-1, now I have to retrieve students who passed in all tests of the term-1.
Lookup stage is pending - facing problem with lookup in second (student) table who match all incoming output records of first(test) collection
Thanks in advance !!
Try this:
db.tests.aggregate([
{
$match: {
// Your match condition
}
},
{
$group: {
_id: null,
term_1_testIds: { $push: "$_id" },
test_count: { $sum: 1 }
}
},
{
$lookup: {
from: "students",
let: { term_1_testIds: '$term_1_testIds', schoolId: 40001, totalTestCount: "$test_count" },
pipeline: [
{
$match: {
$expr: { $eq: ["$schoolId", "$$schoolId"] }
}
},
{ $unwind: "$pastTests" },
{
$match: {
"pastTests.status": "pass",
$expr: { $in: ["$pastTests.testId", "$$term_1_testIds"] }
}
},
{
$group: {
_id: "$_id",
firstName: { $first: "$firstName" },
yearGroup: { $first: "$yearGroup" },
schoolId: { $first: "$schoolId" },
currentTest: { $first: "$currentTest" },
passedTestCount: { $sum: 1 },
pastTests: { $push: "$pastTests" }
}
},
{
$match: {
$expr: { $eq: ["$passedTestCount", "$$totalTestCount"] }
}
}
],
as: "students"
}
}
]);
Output:
{
"_id" : null,
"term_1_testIds" : [
ObjectId("5c9b5c1005729b2bf23f3290"),
ObjectId("5c9b5fc460e39c2c58e44109"),
ObjectId("5c9b6492bb581c2ceb553fef")
],
"test_count" : 3,
"students" : [
{
"_id" : ObjectId("5c92dd994e8e6b2c1647d0d1"),
"firstName" : "Dheemanth",
"yearGroup" : "0",
"schoolId" : 40001,
"currentTest" : ObjectId("5c9b5c1005729b2bf23f3290"),
"passedTestCount" : 3,
"pastTests" : [
{
"_id" : ObjectId("5d3570645045863d373f6db1"),
"testId" : ObjectId("5c9b5c1005729b2bf23f3290"),
"status" : "pass"
},
{
"_id" : ObjectId("5d425af07708f5636c3bec1c"),
"testId" : ObjectId("5c9b5fc460e39c2c58e44109"),
"status" : "pass"
},
{
"_id" : ObjectId("5d5e54a875fab079f4d03570"),
"testId" : ObjectId("5c9b6492bb581c2ceb553fef"),
"status" : "pass"
}
]
}
]
}
This how my tests collection looks like
/* 1 createdAt:3/27/2019, 5:24:58 PM*/
{
"_id" : ObjectId("5c9b6492bb581c2ceb553fef"),
"name" : "Test 3"
},
/* 2 createdAt:3/27/2019, 5:04:28 PM*/
{
"_id" : ObjectId("5c9b5fc460e39c2c58e44109"),
"name" : "Test 2"
},
/* 3 createdAt:3/27/2019, 4:48:40 PM*/
{
"_id" : ObjectId("5c9b5c1005729b2bf23f3290"),
"name" : "Test 1"
}
This is how my students collection looks like:
/* 1 createdAt:3/21/2019, 6:10:57 AM*/
{
"_id" : ObjectId("5c92dd994e8e6b2c1647d0d1"),
"firstName" : "Dheemanth",
"yearGroup" : "0",
"schoolId" : 40001,
"currentTest" : ObjectId("5c9b5c1005729b2bf23f3290"),
"pastTests" : [
{
"_id" : ObjectId("5d3570645045863d373f6db1"),
"testId" : ObjectId("5c9b5c1005729b2bf23f3290"),
"status" : "pass"
},
{
"_id" : ObjectId("5d425af07708f5636c3bec1c"),
"testId" : ObjectId("5c9b5fc460e39c2c58e44109"),
"status" : "pass"
},
{
"_id" : ObjectId("5d5e54a875fab079f4d03570"),
"testId" : ObjectId("5c9b6492bb581c2ceb553fef"),
"status" : "pass"
}
]
},
/* 2 createdAt:3/21/2019, 6:10:57 AM*/
{
"_id" : ObjectId("5c92dd994e8e6b2c1647d0d0"),
"firstName" : "Andrew",
"yearGroup" : "0",
"schoolId" : 40001,
"currentTest" : ObjectId("5c9b5c1005729b2bf23f3290"),
"pastTests" : [
{
"_id" : ObjectId("5d3570645045863d373f6db1"),
"testId" : ObjectId("5c9b5c1005729b2bf23f3290"),
"status" : "pass"
},
{
"_id" : ObjectId("5d425af07708f5636c3bec1c"),
"testId" : ObjectId("5c9b5fc460e39c2c58e44109"),
"status" : "pass"
},
{
"_id" : ObjectId("5d5e54a875fab079f4d03570"),
"testId" : ObjectId("5c9b6492bb581c2ceb553fef"),
"status" : "fail"
}
]
}
Also:
In your first $match stage, $and operator is redundant inside $or array it should be like this:
{
$match: {
yearGroup: '-1',
$or: [
{
'retestDateScheduled': true,
'retestDate.term': { $lt: 4 }
},
{
'testDateScheduled': true,
'testDate.term': { $lt: 4 }
}
]
}
}

MongoDb: find deeply nested object with $lookup

I have collection like this:
collection name is - account.
and it has sub-documents like account > buildings > gateways > devices.
{
"_id" : ObjectId("5e1fe45cd05bfb0cc549297d"),
"apiCallCount" : 0,
"email" : "info#data.com",
"password" : "dummy",
"userName" : "AAAAA",
"companyName" : "The AAAAAA",
"apiKey" : "5e1fe45cd05bfb0cc549297c",
"solutionType" : "VVVVVV",
"parentCompany" : "",
"buildings" : [
{
"_id" : ObjectId("5e1fe5e3d05bfb0cc5494146"),
"buildingName" : "xxxxxx",
"address" : "xxx",
"suite" : "101",
"floor" : "22",
"timeZone" : "us/eastern",
"gateways" : [
{
"_id" : ObjectId("5e1fe624d05bfb0cc549453a"),
"gatewayName" : "CC-GW-THF-001",
"gatewayKey" : "gk_5e1fe624d05bfb0cc549453a",
"suite" : "area1",
"devices" : [
{
"_id" : ObjectId("5e1fe751d05bfb0cc549578d"),
"serialNumber" : "129300000013",
"area" : "area1",
"connectionStatus" : 1,
"gatewayKey" : "gk_5e1fe624d05bfb0cc549453a",
"applicationNumber" : 30,
"firmwareVersion" : "1.0",
"needsAttention" : false,
"verificationCode" : "GAAS",
"createdAt" : ISODate("2020-01-16T04:32:17.899Z"),
"updatedAt" : ISODate("2020-01-16T08:53:54.460Z")
}
],
"createdAt" : ISODate("2020-01-16T04:27:16.678Z"),
"updatedAt" : ISODate("2020-01-16T08:53:54.460Z")
},
{
"_id" : ObjectId("5e1fe651d05bfb0cc54947f0"),
"gatewayName" : "AA-GW-THF-002",
"gatewayKey" : "gk_5e1fe651d05bfb0cc54947f0",
"suite" : "area2",
"devices" : [
{
"_id" : ObjectId("5e1fe7a9d05bfb0cc5495cdf"),
"serialNumber" : "129300000012",
"area" : "area2",
"connectionStatus" : 0,
"gatewayKey" : "gk_5e1fe651d05bfb0cc54947f0",
"applicationNumber" : 30,
"firmwareVersion" : "1.0",
"needsAttention" : false,
"verificationCode" : "VG3K",
"createdAt" : ISODate("2020-01-16T04:33:45.698Z"),
"updatedAt" : ISODate("2020-01-16T08:54:17.604Z")
}
],
"createdAt" : ISODate("2020-01-16T04:28:01.532Z"),
"updatedAt" : ISODate("2020-01-16T08:54:17.604Z")
},
],
"createdAt" : ISODate("2020-01-16T04:26:11.941Z"),
"updatedAt" : ISODate("2020-01-16T08:56:32.657Z")
}
],
"createdAt" : ISODate("2020-01-16T04:19:40.310Z"),
"updatedAt" : ISODate("2020-04-06T18:18:39.628Z"),
"__v" : 1,
}
I have accountId, buildingId, gatewayId, deviceId.
I am trying to find matched device object using $lookup operator.
I think I have to fist find the building object by using buildingId and then filter gateway under that building using gatewayId and then find device object using deviceId that I have.
I basically need to have access to device object fields to project in final output.
Having difficulty in coming up with correct pipleline for usingg lookup operator.
I have this so far:
db.getCollection('test').aggregate([
{
$lookup: {
from: 'account',
let: {
accountId: "$accountId"
},
pipeline: [
{
"$match": {
"$expr": {
"$eq": ["$_id", "$$accountId"]
}
}
},
],
as: "accountDetails"
}
}, {
$unwind: "$accountDetails"
}, {
$lookup: {
from: 'account',
let: {
accountId: "$accountId",
buildingId: "$buildingId",
buildings: "$accountDetails"
},
pipeline: [
{
"$match": {
"$expr": {
"$eq": ["$buildings._id", "$$buildingId"] // how to dig through nested document to get to devices ?
}
}
},
],
as: "buildingDetails"
}
}
{
$project: { ... ...
}
])
if I do this:
{
$lookup: {
from: 'account',
localField: "accountId",
foreignField: "_id",
as: "accountDetails"
}
},
accountDetails gives me access the account document based on accountId. but I need to get to buildings > gateways > devices and find the matching device.
UPDATE:
I forgot to mention, I am working with 2 collections here.
sensingresults and accounts.
Main purpose is to aggregate data from sensingresults, but also find deviceId from account collection and return with the result.
That's why lookup is needed to join 2 collections?
UPDATE2:
Current output:
{
"accountId": ObjectId("5e1fe45cd05bfb0cc549297d"),
"avgZoneCountNumber": 0,
"avgZoneCountNumberInstant": 0,
"buildingId": ObjectId("5e1fe5e3d05bfb0cc5494146"),
"companyName": "The AAAAAA",
"createdAt": ISODate("1970-01-01T00:00:00Z"),
"dateHour": "2020-03-19T18",
"deviceId": ObjectId("5e1fe81ed05bfb0cc5496406"),
"gatewayId": ObjectId("5e1fe6a6d05bfb0cc5494d25"),
"minuteBucket": 1
}
Expected result:
{
"accountId": ObjectId("5e1fe45cd05bfb0cc549297d"),
"avgZoneCountNumber": 0,
"avgZoneCountNumberInstant": 0,
"buildingId": ObjectId("5e1fe5e3d05bfb0cc5494146"),
"createdAt": ISODate("1970-01-01T00:00:00Z"),
"dateHour": "2020-03-19T18",
"deviceId": ObjectId("5e1fe81ed05bfb0cc5496406"),
"gatewayId": ObjectId("5e1fe6a6d05bfb0cc5494d25"),
"minuteBucket": 1,
"serialNumber: 1, // this value should come from device object
"area": 1 // this value should come from device object
}
You can find nested device using $filter, $arrayElemAt and $let:
device: {
$let: {
vars: {
building: {
$arrayElemAt: [ { $filter: { input: "$company_name.buildings", cond: { $eq: [ "$$this._id", "$buildingId" ] }} }, 0 ]
}
},
in: {
$let: {
vars: {
gateway: {
$arrayElemAt: [ { $filter: { input: "$$building.gateways", cond: { $eq: [ "$$this._id", "$gatewayId" ] }} }, 0 ]
}
},
in: { $arrayElemAt: [ { $filter: { input: "$$gateway.devices", cond: { $eq: [ "$$this._id", "$deviceId" ] }} }, 0 ] }
}
}
}
}
Full Solution

mongodb find the document by id and then group the result based on name field

I have a collection with multiple documents like
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 12.41
},
{
"date" : "2015-05-19",
"value" : 12.45
},
],
"Name" : "ABC Banking",
"scheme":"ABC1",
"createdDate" : "21-01-2018"
}
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 13.41
},
{
"date" : "2015-05-19",
"value" : 13.45
},
],
"Name" : "ABC Banking",
"scheme":"ABC2",
"createdDate" : "21-01-2018"
}
I am Querying collection based on Number field like
db.getCollection('mfhistories').find({'Number':53})
to get all the documents with this Number.
Now I want to group all the collection with Name 'ABC Banking' into an array. so that I will get result based on Name.
so the result should be like
{
"Name":"ABC Banking",
[
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 13.41
},
{
"date" : "2015-05-19",
"value" : 13.45
},
],
"scheme":"ABC1",
"createdDate" : "21-01-2018"
},
{
"_id" : ObjectId("5a64d076bfd103df081967ae"),
"status" : "",
"Number" : 53,
"values" : [
{
"date" : "2015-05-18",
"value" : 13.41
},
{
"date" : "2015-05-19",
"value" : 13.45
}
],
"scheme":"ABC2",
"createdDate" : "21-01-2018"
}
]
}
Please help..
Thanks,
J
You can use Aggregation Framework for that:
db.col.aggregate([
{
$match: { Number: 53, Name: "ABC Banking" }
},
{
$group: {
_id: "$Name",
docs: { $push: "$$ROOT" }
}
},
{
$project: {
Name: "$_id",
_id: 0,
docs: 1
}
}
])
$$ROOT is a special variable which captures entire document. More here.
db.mfhistories.aggregate(
// Pipeline
[
// Stage 1
{
$match: {
Number: 53
}
},
// Stage 2
{
$group: {
_id: {
Name: '$Name'
},
docObj: {
$addToSet: '$$CURRENT'
}
}
},
// Stage 3
{
$project: {
Name: '$_id.Name',
docObj: 1,
_id: 0
}
}
]
);

Get average of values over a period of time

I have the following collection:
{
"_id" : ObjectId("58503934034b512b419a6eab"),
"website" : "https://www.google.com",
"name" : "Google",
"keywords" : [
"Search",
"Websites",
],
"tracking" : [
{
"_id" : ObjectId("5874aa1df63258286528598d"),
"position" : 0,
"created_at" : ISODate("2017-01-1T09:32:13.831Z"),
"real_url" : "https://www.google.com",
"keyword" : "Search"
},
{
"_id" : ObjectId("5874aa1ff63258286528598e"),
"keyword" : "Page",
"real_url" : "https://www.google.com",
"created_at" : ISODate("2017-01-1T09:32:15.832Z"),
"found_url" : "https://google.com/",
"position" : 3
},
{
"_id" : ObjectId("5874aa21f63258286528598f"),
"keyword" : "Search",
"real_url" : "https://www.foamymedia.com",
"created_at" : ISODate("2017-01-2T09:32:17.017Z"),
"found_url" : "https://google.com/",
"position" : 2
},
{
"_id" : ObjectId("5874aa21f63258286528532f"),
"keyword" : "Search",
"real_url" : "https://www.google.com",
"created_at" : ISODate("2017-01-2T09:32:17.017Z"),
"found_url" : "https://google.com/",
"position" : 1
},
]
}
What I want to do is group all of the keywords together and calculate the average for that particular day, over a certain period.
So let's say for example:
Between: 2017-01-01 to 2017-01-31 the following keywords was tracked:
2017-01-01:
'Seach' => 1,
'Page' => 3,
Average = 2
2017-01-02:
'Search' => 4,
'Page' => 6,
Average = 5
....
So in the end result, I would be finished with (in this case):
{
"_id" : ObjectId("5874dccb9cd90425e41b7c54"),
"website" : "www.google.com",
"averages" : [
"2",
"5"
]
}
You can try something like this.
$unwind the tracking array followed by $sort on tracking.keyword and tracking.created_at.$group by day to get average for day across all categories. Final $group to push all the day's average values into array for a website.
db.website.aggregate([{
$match: {
"_id": ObjectId("58503934034b512b419a6eab")
}
}, {
$lookup: {
from: "seo_tracking",
localField: "website",
foreignField: "real_url",
as: "tracking"
}
}, {
$unwind: "$tracking"
}, {
$sort: {
"tracking.keyword": 1,
"tracking.created_at": -1
}
}, {
$group: {
_id: {
$dayOfMonth: "$tracking.created_at"
},
"website": {
$first: "$website"
},
"website_id": {
$first: "$_id"
},
"averageByDay": {
$avg: "$tracking.position"
}
}
}, {
$group: {
"_id": "$website_id",
"website": {
$first: "$website"
},
"average": {
$push: "$averageByDay"
}
}
}]);