I have a collection like this:
{
"_id" : ObjectId("5f4e81f1da5ea3cb7c248a8f"),
"type" : "TYPE_1",
"updateTime" : ISODate("2020-08-24T11:10:43.219+0000")
}
{
"_id" : ObjectId("5f4e8206da5ea3cb7c248a90"),
"type" : "TYPE_1",
"updateTime" : ISODate("2020-09-24T11:10:43.219+0000")
}
{
"_id" : ObjectId("5f4e821fda5ea3cb7c248a91"),
"type" : "TYPE_2",
"updateTime" : ISODate("2020-09-25T11:10:43.219+0000")
}
I want to know how many documents there are of each type and also obtain the date of the last global modification. For now I can get these results like this:
db.getCollection("test").aggregate(
// Pipeline
[
// Stage 1
{
$group: {
_id : "$type",
count: { $sum: 1 },
lastUpdate: { "$max": "$updateTime" }
}
},
// Stage 2
{
$sort: {
lastUpdate : -1
}
},
]
);
With which I get the results this way:
{
"_id" : "TYPE_2",
"count" : 1.0,
"lastUpdate" : ISODate("2020-09-25T11:10:43.219+0000")
}
{
"_id" : "TYPE_1",
"count" : 2.0,
"lastUpdate" : ISODate("2020-09-24T11:10:43.219+0000")
}
So I have both the sum of each document and the last modification (thanks to the sort).
But I would like to project and get the results like this, in a single result document:
{
"type1" : 2.0,
"type2" : 1.0,
"lastUpdate" : ISODate("2020-09-25T11:10:43.219+0000")
}
#varman's answer is good, this is just in different way,
$group you have already done by your self
$group create types array to combine all documents
$replaceWith to replace root with field types to convert $arrayToObject
db.collection.aggregate([
{
$group: {
_id: "$type",
count: { $sum: 1 },
lastUpdate: { $max: "$updateTime" }
}
},
{
$group: {
_id: null,
types: {
$push: {
k: "$_id",
v: "$count"
}
},
lastUpdate: { $max: "$lastUpdate" }
}
},
{
$replaceWith: {
$mergeObjects: [
{ lastUpdate: "$lastUpdate" },
{ $arrayToObject: "$types" }
]
}
}
])
Playground
You can use following stages after your stage.
{
$group: {
_id: null,
data: {
$push: {
type: "$_id",
count: "$count"
}
},
lastUpdate: {
$first: "$lastUpdate"
}
}
},
{
$project: {
data: {
$arrayToObject: {
$map: {
input: "$data",
in: {
k: "$$this.type",
v: "$$this.count"
}
}
}
},
lastUpdate: 1
}
},
{
$addFields: {
"data.lastUpdate": "$lastUpdate"
}
},
{
"$replaceRoot": {
"newRoot": "$data"
}
}
Working Mongo playground
Related
I have a collection of 1000 documents like this:
{
"_id" : ObjectId("628b63d66a5951db6bb79905"),
"index" : 0,
"name" : "Aurelia Gonzales",
"isActive" : false,
"registered" : ISODate("2015-02-11T04:22:39.000+0000"),
"age" : 41,
"gender" : "female",
"eyeColor" : "green",
"favoriteFruit" : "banana",
"company" : {
"title" : "YURTURE",
"email" : "aureliagonzales#yurture.com",
"phone" : "+1 (940) 501-3963",
"location" : {
"country" : "USA",
"address" : "694 Hewes Street"
}
},
"tags" : [
"enim",
"id",
"velit",
"ad",
"consequat"
]
}
I want to group those by year and gender. Like In 2014 male registration 105 and female registration 131. And finally return documents like this:
{
_id:2014,
male:105,
female:131,
total:236
},
{
_id:2015,
male:136,
female:128,
total:264
}
I have tried till group by registered and gender like this:
db.persons.aggregate([
{ $group: { _id: { year: { $year: "$registered" }, gender: "$gender" }, total: { $sum: NumberInt(1) } } },
{ $sort: { "_id.year": 1,"_id.gender":1 } }
])
which is return document like this:
{
"_id" : {
"year" : 2014,
"gender" : "female"
},
"total" : 131
}
{
"_id" : {
"year" : 2014,
"gender" : "male"
},
"total" : 105
}
Please guide to figure out from this whole.
db.collection.aggregate([
{
"$group": { //Group things
"_id": "$_id.year",
"gender": {
"$addToSet": {
k: "$_id.gender",
v: "$total"
}
},
sum: { //Sum it
$sum: "$total"
}
}
},
{
"$project": {//Reshape it
g: {
"$arrayToObject": "$gender"
},
_id: 1,
sum: 1
}
},
{
"$project": { //Reshape it
_id: 1,
"g.female": 1,
"g.male": 1,
sum: 1
}
}
])
Play
Just add one more group stage to your aggregation pipeline, like this:
db.persons.aggregate([
{ $group: { _id: { year: { $year: "$registered" }, gender: "$gender" }, total: { $sum: NumberInt(1) } } },
{ $sort: { "_id.year": 1,"_id.gender":1 } },
{
$group: {
_id: "$_id.year",
male: {
$sum: {
$cond: {
if: {
$eq: [
"$_id.gender",
"male"
]
},
then: "$total",
else: 0
}
}
},
female: {
$sum: {
$cond: {
if: {
$eq: [
"$_id.gender",
"female"
]
},
then: "$total",
else: 0
}
}
},
total: {
$sum: "$total"
}
},
}
]);
Here's the working link. We are grouping by year in this last step, and calculating the counts for gender conditionally and the total is just the total of the counts irrespective of the gender.
Besides #Gibbs mentioned in the comment which proposes the solution with 2 $group stages,
You can achieve the result as below:
$group - Group by year of registered. Add gender value into genders array.
$sort - Order by _id.
$project - Decorate output documents.
3.1. male - Get the size of array from $filter the value of "male" in "genders" array.
3.2. female - Get the size of array from $filter the value of "female" in "genders" array.
3.3. total - Get the size of "genders" array.
Propose this method if you are expected to count and return the "male" and "female" gender fields.
db.collection.aggregate([
{
$group: {
_id: {
$year: "$registered"
},
genders: {
$push: "$gender"
}
}
},
{
$sort: {
"_id": 1
}
},
{
$project: {
_id: 1,
male: {
$size: {
$filter: {
input: "$genders",
cond: {
$eq: [
"$$this",
"male"
]
}
}
}
},
female: {
$size: {
$filter: {
input: "$genders",
cond: {
$eq: [
"$$this",
"female"
]
}
}
}
},
total: {
$size: "$genders"
}
}
}
])
Sample Mongo Playground
I'm trying to out a collection, after some stages in the MongoDB aggregation pipeline.
the out collection needs to contain an id with sequence numeric id (1,2,3,4 etc...).
db.getCollection('MY_COLLECTION').aggregate([{
$addFields: {
"dataEntries.targetCol1": "$dataEntries.col1",
"dataEntries.targetCol2": "$dataEntries.col2"
}
},
{
$project: {
"_id": 0,
"dataEntries.col1": 0,
"dataEntries.col2": 0,
"dataEntries.col3": 0,
"dataEntries.col4": 0
}
},
{
$unionWith: {
coll: "MY_COLLECTION",
pipeline: [{
$addFields: {
"dataEntries.targetCol1": "$dataEntries.col3",
"dataEntries.targetCol2": "$dataEntries.col4"
}
},
{
$project: {
"_id": 0,
"dataEntries.col1": 0,
"dataEntries.col2": 0,
"dataEntries.col3": 0,
"dataEntries.col4": 0
}
}
]
}
},
{
$out: "test_out"
}
])
The result collection contains an objectId as the id.
/* 1 */
{
"_id" : ObjectId("6239cb749482cf6b13248a80"),
"dataEntries" : {
"targetCol1" : "101-A",
"targetCol2" : "101"
}
}
/* 2 */
{
"_id" : ObjectId("6239cb749482cf6b13248a81"),
"dataEntries" : {
"targetCol1" : "101-B",
"targetCol2" : "101"
}
}
There is a way to set the id to sequence numeric counter for every document?
The expected results:
/* 1 */
{
"_id" : 1,
"dataEntries" : {
"targetCol1" : "101-A",
"targetCol2" : "101"
}
}
/* 2 */
{
"_id" : 2,
"dataEntries" : {
"targetCol1" : "101-B",
"targetCol2" : "101"
}
}
In MongoDB 5.0 it is very simple:
db.collection.aggregate([
{
$setWindowFields: {
sortBy: { _id: 1, },
output: { _id: { $documentNumber: {} } }
}
}
])
In earlier versions, it is a little more to do:
db.collection.aggregate([
{
$group: {
_id: null, data: { $push: "$$ROOT" }
}
},
{
$set: {
data: {
$map: {
input: "$data",
in: {
$mergeObjects: ["$$this",
{ _id: {$add: [{ $indexOfArray: ["$data._id", "$$this._id"] }, 1]} }
]
}
}
}
}
},
{ $unwind: "$data" },
{ $replaceWith: "$data" }
])
MongoDB version:4.2.17.
Trying out aggregation on data in a collection.
Example data:
{
"_id" : "244",
"pubName" : "p1",
"serviceIdRef" : "36e9c779-7865-4b74-a30b-e4d6a0cc5295",
"serviceName" : "my-service",
"subName" : "c1",
"pubState" : "INVITED"
}
I would like to:
Do a match by something (let’s say subName) and group by serviceIdRef and then limit to return X entries
Also return for each of the serviceIdRefs, the count of the documents in each of ACTIVE or INVITED states. And Y (for this example, say Y=3) documents that are in this state.
For example, the output would appear as (in brief):
[
{
serviceIdRef: "36e9c779-7865-4b74-a30b-e4d6a0cc5295",
serviceName:
state:[
{
pubState: "INVITED"
count: 200
sample: [ // Get those Y entries (here Y=3)
{
// sample1 like:
"_id" : "244",
"pubName" : "p1",
"serviceIdRef" : "36e9c779-7865-4b74-a30b-e4d6a0cc5295",
"serviceName" : "my-service",
"subName" : "c1",
"pubState" : "INVITED"
},
{
sample2
},
{
sample3
}
]
},
{
pubState: "ACTIVE", // For this state, repeat as we did for "INVITED" state above.
......
}
]
}
{
repeat for another service
}
]
So far I have written this but am not able to get those Y entries. Is there a (better) way?
This is what I have so far (not complete and not exactly outputs in the format above):
db.sub.aggregate(
[{
$match:
{
"subName": {
$in: ["c1", "c2"]
},
"$or": [
{
"pubState": "INVITED",
},
{
"pubState": "ACTIVE",
}
]
}
},
{
$group: {
_id: "$serviceIdRef",
subs: {
$push: "$$ROOT",
}
}
},
{
$sort: {
_id: -1,
}
},
{
$limit: 22
},
{
$facet:
{
facet1: [
{
$unwind: "$subs",
},
{
$group:
{
_id: {
"serviceName" : "$_id",
"pubState": "$subs.pubState",
"subState": "$subs.subsState"
},
count: {
$sum: 1
}
}
}
]
}
}
])
You have to do the second $group stage to manage nested structure,
$match your conditions
$sort by _id in descending order
$group by serviceIdRef and pubState, get first required fields and prepare the array for sample, and get count of documents
$group by only serviceIdRef and construct the state array
$slice for limit the document in sample
db.collection.aggregate([
{
$match: {
subName: { $in: ["c1", "c2"] },
pubState: { $in: ["INVITED", "ACTIVE"] }
}
},
{ $sort: { _id: -1 } },
{
$group: {
_id: {
serviceIdRef: "$serviceIdRef",
pubState: "$pubState"
},
serviceName: { $first: "$serviceName" },
sample: { $push: "$$ROOT" },
count: { $sum: 1 }
}
},
{
$group: {
_id: "$_id.serviceIdRef",
serviceName: { $first: "$serviceName" },
state: {
$push: {
pubState: "$_id.pubState",
count: "$count",
sample: { $slice: ["$sample", 22] }
}
}
}
}
])
Playground
I have documents storing IoT data.
Following MongoDB schema design best practices for IoT, I came to documents with the following structure:
"_id" : "AQ106_2020-09-12T09",
"date" : "2020-09-12T09:00:00.000Z",
"station" : {
"name" : "AQ106",
"loc" : {
"type" : "Point",
"coordinates" : [
14.339263,
40.814224
]
},
"properties" : {
}
},
"samples" : [
{
"t" : ISODate("2020-09-12T11:02:00.000+02:00"),
"data" : {
"pm1_mg_m3" : 2.7,
"pm2_5_mg_m3" : 4.6,
"pm10_mg_m3" : 12,
"P0" : 152,
"P1" : 16,
"P2" : 4.7,
"P3" : 0.8,
"P4" : 0.86,
"P5" : 0.6,
"P6" : 0.28,
"P7" : 0.152,
"P8" : 0.094,
"P9" : 0.092,
"P10" : 0.019,
"P11" : 0,
"P12" : 0,
"P13" : 0.0188,
"P14" : 0,
"P15" : 0,
"P16" : 0,
"P17" : 0,
"P18" : 0,
"P19" : 0,
"P20" : 0,
"P21" : 0,
"P22" : 0,
"P23" : 0,
"temp_celsius" : 32.59,
"humRelPercent" : 34,
"press_mBar" : 1010.79,
"CO2mA" : 4,
"NO2_WE_mV" : 226.419,
"NO2_AE_mV" : 229.553,
"OX_WE_mV" : 252.287,
"OX_AE_mV" : 220.419,
"CO_WE_mV" : 509.077,
"AE_WE_mV" : 348.51,
"batt_V" : 13.5,
"source_V" : 17.6
}
},
.... additional arrays
}
Now I want to compute hourly or daily averages (or another metric) to populate a new collection with only summarised data.
I coded the following solution for hourly means:
db.collection.aggregate([{$match: {
'station.name':'AQ104'
}}, {$unwind: {
path: "$samples"
}}, {$group: {
_id: "$date",
P0: {
$avg : "$samples.data.P0"
},
temp:{
$avg:"$samples.data.temp_celsius"
}
}}])
This works but I need to manually create a field for each property in samples.data in the original document and that's tedious.
Moreover, how to group both by date and station.name?
You can find a working example here.
Thanks.
Let's start with the easy question, how to group on multiple fields? With a simple syntax change:
{
$group: {
_id: {
date: "$date",
station: "$station.name"
}
}
Now for the second question this will be a bit more tedious. Mongo does not support "merging" objects by their keys with custom logic (in this case $avg). So we will have to convert the object to an array. unwind it, calculate the average per field and eventually group to restore the required structure like so:
db.collection.aggregate([
{
$match: {
"station.name": "AQ106"
}
},
{
$unwind: {
path: "$samples"
}
},
{
$addFields: {
objArr: {
"$objectToArray": "$samples.data"
}
}
},
{
$unwind: "$objArr"
},
{
$group: {
_id: {
date: "$date",
station: "$station.name",
objKey: "$objArr.k"
},
value: {
$avg: "$objArr.v"
}
}
},
{
$addFields: {
data: {
"$arrayToObject": [
[
{
k: "$_id.objKey",
v: "$value"
}
]
]
}
}
},
{
$group: {
_id: {
date: "$_id.date",
station: "$_id.station"
},
data: {
"$mergeObjects": "$data"
}
}
},
{
$replaceRoot: {
newRoot: {
"$mergeObjects": [
"$data",
"$_id"
]
}
}
}
])
MongoPlayground
------- EDIT ---------
For Mongo v4.4+ you can use $accumulator which allows you to execute custom javascript code in your pipeline. I am unsure how this will fare against the native Mongo pipeline in terms of performance in scale.
One thing to note is that I added the initial $addFields stage under the assumption that different samples may have different keys. if this is not the case it is not needed.
db.collection.aggregate([
{
$addFields: {
sampleKeys: {
$reduce: {
input: {
$map: {
input: "$samples",
as: "sample",
in: {
$map: {
input: {
"$objectToArray": "$$sample.data"
},
as: "sampleArrItem",
in: "$$sampleArrItem.k"
}
}
}
},
initialValue: [],
in: {
"$setUnion": [
"$$this",
"$$value"
]
}
}
}
}
},
{
$addFields: {
samples: {
$accumulator: {
init: function(keys){
return keys.map(k => {return {k: {v: 0, c: 0}}});
},
initArgs: ["$sampleKeys"],
accumulateArgs: ["$samples"],
accumulate: function(state, sample){
Object.keys(state).forEach((key) => {
if (key in sample.data) {
state[key].v += sample.data[key];
state[key].c++;
};
});
return state;
},
merge: function(state1, state2){
Object.keys(state1).forEach((key) => {
state1[key].v += state2[key].v;
state1[key].c += state2[key].c;
});
return state1;
},
lang: "js"
}
}
}
},
{
$replaceRoot: {
newRoot: {
$mergeObject: [
"$samples",
{station: "$station.name", date: "$date"},
]
}
}
}
])
I partially resolved my question in terms of grouping by multiple fields (MongoDB documentation was not so clear at this regard, in my opinion)
db.collection.aggregate([
{
$unwind: {
path: "$samples"
}
},
{
$group: {
_id: {
date: "$date",
station: "$station.name"
},
P0: {
$avg: "$samples.data.P0"
},
temp: {
$avg: "$samples.data.temp_celsius"
}
}
}
])
Here the updated working example.
Thanks to Tom Slabbaert, I solved my question with the following query:
db.collection.aggregate([
{
$unwind: {
path: "$samples"
}
},
{
$addFields: {
objArr: {
"$objectToArray": "$samples.data"
}
}
},
{
$unwind: "$objArr"
},
{
$group: {
_id: {
date: "$date",
station: "$station",
objKey: "$objArr.k"
},
value: {
$avg: "$objArr.v"
}
}
},
{
$addFields: {
data: {
"$arrayToObject": [
[
{
k: "$_id.objKey",
v: "$value"
}
]
]
}
}
},
{
$group: {
_id: {
date: "$_id.date",
station: "$_id.station"
},
data: {
"$mergeObjects": "$data"
}
}
},
{
"$project": {
_id: "$_id.date",
station: "$_id.station",
data: 1
}
}
])
Here
I wonder if it is possible to simplify the above solution using the new $function operator.
Thanks.
I am trying the mongo rank calculation based on count and mentioned in below db schema. I am not getting the expecting results. Anyone help to resolve this?
Mongo Query:
db.company.aggregate([
{
"$group": {
"_id": {
"name1": "$name1",
"name2": "$name2",
},
"expanded": {
"$push": {
"name1": "$name1",
"name2": "$name2",
}
},
"count": { "$sum": 1 }
}
},
{ "$sort": { "count": -1 } },
{
$unwind: {
path: '$expanded',
includeArrayIndex: 'count'
}
}
]);
Expecting results like
Name|Count|Rank
Google|3|1
FB|2|2
Yahoo|1| 3
DB Schema :
{
"_id" : 1.0,
"name1" : "Yahoo",
"name2" : "Google",
"salary" : 1000.0
}
/* 2 */
{
"_id" : 2.0,
"name1" : "FB",
"name2" : "Google",
"salary" : 2000.0
}
/* 3 */
{
"_id" : 3.0,
"name1" : "Google",
"name2" : "FB",
"salary" : 1500.0
}
It seems like you should count name1 and name2 separately so you can create a temporary 2-element array and then run $unwind on that array. Additionally to get rank you have to $group by null to get single array of all groups, try:
db.collection.aggregate([
{
$project: {
key: [ "$name1", "$name2" ]
}
},
{
$unwind: "$key"
},
{
$group: {
_id: "$key",
count: { $sum: 1 }
}
},
{
$sort: {
count: -1
}
},
{
$group: {
_id: null,
groups: { $push: "$$ROOT" }
}
},
{
$unwind: {
path: '$groups',
includeArrayIndex: 'rank'
}
},
{
$project: {
_id: 0,
name: "$groups._id",
rank: { $add: [ "$rank", 1 ] },
count: "$groups.count"
}
}
])
Mongo Playground
try this
db.company.aggregate([
{
$group: {
_id:null,
names1: {$push: "$name1"},
names2: {$push:"$name2"},
}
},
{
$project: {
_id: 0,
names:{$concatArrays: ["$names1", "$names2"]}
}
},
{$unwind: "$names"},
{$sortByCount: "$names"},
{$addFields:{name: "$_id"}},
{
$group : {
_id: null,
records : { $push : {count : "$count", name : "$name"}}
}
},
{
$project: {
total_docs: {$size: "$records"},
records: 1
}
},
{$unwind: "$records"},
{
$project: {
_id: 0,
name: "$records.name",
count:"$records.count",
rank: {
$add:[
{
$subtract:["$total_docs", "$records.count"]
}, 1]
}
}
}])