RESTHeart filtering and sorting by sub documents property - mongodb

I m working with mongodb and restheart.
In my nosql db i have a unique document with this structure:
{
"_id": "docID",
"users": [
{
"userID": "12",
"elements": [
{
"elementID": "1492446877599",
"events": [
{
"id": 1,
"date": 356
},
{
"id": 2,
"date": 123
}
]
}
]
},
{
"userID": "11",
"elements": [
{
"elementID": "14924",
"events": [
{
"id": 1,
"date": 123
},
{
"id": 2,
"date": 356
}
]
},
{
"elementID": "14925",
"events": [
{
"id": 1,
"date": 12
},
{
"id": 2,
"date": 36
}
]
}
]
}
i need to filter the user with userID = 11 and i need to order his events by ascending date.
i was trying with:
http://myhost:port/myCollection?keys={"users":{"$elemMatch":{"userID":"11"}}}&sort_by={"users.elements.events.date":-1}
but it doesn t work.

db.v.aggregate([
{ $unwind : '$users'},
{ $match : { 'users.userID' : '11' }} ,
{ $unwind : '$users.elements'},
{ $unwind : '$users.elements.events'},
{ $sort : {'users.elements.events.date': 1}},
{ $group : {
_id : '$_id',
elementID : { $first : '$users.elements.elementID' },
userID : { $first : '$users.userID' },
events : { $push : '$users.elements.events'}
}
},
{ $project : {
_id : 1,
userID : 1,
'elements.elementID' : '$elementID',
'elements.events' : '$events'
}
}
]);
This will give you following :
{
"_id" : ObjectId("5911ba55f0d9c285c561ea33"),
"userID" : "11",
"elements" : {
"elementID" : "14924",
"events" : [
{
"id" : 1,
"date" : 123
},
{
"id" : 2,
"date" : 356
}
]
}
}

Related

MongoDB Aggregation function

I have the following JSON Documents in Mongo collection named "Movies"
{
"_id": "5ed0c9700b9e8b0e2c542054",
"movie_name": "Jake 123",
"score": 20,
"director": "Jake"
},
{
"_id": "5ed0a9840b9e8b0e2c542053",
"movie_name": "Avatar",
"director": "James Cameroon",
"score": 50,
"boxoffice": [
{
"territory": "US",
"gross": 2000
},
{
"territory": "UK",
"gross": 1000
}
]
},
{
"_id": "5ed0a9630b9e8b0e2c542052",
"movie_name": "Titanic",
"score": 100,
"director": "James Cameroon",
"boxoffice": [
{
"territory": "US",
"gross": 1000
},
{
"territory": "UK",
"gross": 500
}
],
"actors": [
"Kate Winselet",
"Leonardo De Caprio",
"Rajinikanth",
"Kamalhaasan"
]
}
I run the below query which finds the maximum collection of a country of various movies. My intention is to find the maximum collection and the corresponding territory.
db.movies.aggregate([
{$match: {"boxoffice" : { $exists: true, $ne : []}}},
{$project: {
"title":"$movie_name", "max_boxoffice": {$max : "$boxoffice.gross"},
"territory" : "$boxoffice.territory" } }
])
I get the result as follows. How do I get the correct territory that corresponds to the collection?
{
"_id" : ObjectId("5ed0a9630b9e8b0e2c542052"),
"title" : "Titanic",
"max_boxoffice" : 1000,
"territory" : [
"US",
"UK"
]
},
{
"_id" : ObjectId("5ed0a9840b9e8b0e2c542053"),
"title" : "Avatar",
"max_boxoffice" : 2000,
"territory" : [
"US",
"UK"
]
}
Expected output:
Avatar and Titanic has collected more money in US. I wanted territories to display the values of them
{
"_id" : ObjectId("5ed0a9630b9e8b0e2c542052"),
"title" : "Titanic",
"max_boxoffice" : 1000,
"territory" : "US"
},
{
"_id" : ObjectId("5ed0a9840b9e8b0e2c542053"),
"title" : "Avatar",
"max_boxoffice" : 2000,
"territory" : "US"
}
For this specific requirement, you can use $set (aggregation). $set appends new fields to existing documents. and we can include one or more $set stages in an aggregation operation to achieve this like:
db.movies.aggregate([
{
$match: { "boxoffice": { $exists: true, $ne: [] } }
},
{
$set: {
boxoffice: {
$filter: {
input: "$boxoffice",
cond: { $eq: ["$$this.gross", { $max: "$boxoffice.gross" }]}
}
}
}
},
{
$set: {
boxoffice: { $arrayElemAt: ["$boxoffice", 0] }
}
},
{
$project: {
"title": "$movie_name",
"max_boxoffice": "$boxoffice.gross",
"territory": "$boxoffice.territory"
}
}
])
Mongo Playground

Forming an array with aggregation in MongoDB

I have a document in MongoDB 3.4 with the following structure:
{
"_id" : ObjectId("5e3419e468d01013eadb83dc"),
"id_station" : "62",
"fiware_service" : null,
"fiware_servicepath" : null,
"id_fiware_name" : "CE_del_medio",
"attrName" : "15",
"attrType" : "float",
"attrValue" : 0.33,
"id_sensor_station_absolute" : "15_62",
"recvTimeTs" : 1580387045,
"recvTime" : "2020-01-30T12:24:05.00Z",
"id_fiware" : "15",
"sensor_type" : [
{
"name" : "id",
"type" : "String",
"value" : "15"
},
{
"name" : "img",
"type" : "String",
"value" : "assets/img/contrast.png"
},
{
"name" : "manufacturer",
"type" : "String",
"value" : "Hortisis"
},
{
"name" : "medida",
"type" : "String",
"value" : "mS/cm"
},
{
"name" : "name_comun",
"type" : "String",
"value" : "CE del medio"
},
{
"name" : "place",
"type" : "String",
"value" : "interior"
},
{
"name" : "timestamp",
"type" : "DateTime",
"value" : "2020-01-30T12:24:05.00Z"
},
{
"name" : "type",
"type" : "String",
"value" : "fertigation"
}
]
}
I need to convert the sensor_type field to an array with only one object, as follows:
{
"_id":"15_62",
"medidas":[
{
"_id":"5e3419e468d01013eadb83dc",
"marca":"Hortisis",
"modelo":"Estacion",
"fabricante":"Hortisis",
"id_station":"15",
"sensor_type":[
{
"name":"15",
"type":"fertigation",
"place":"interior",
"img":"assets/img/contrast.png",
"name_comun":"Temp. Suelo",
"medida":"mS/cm"
}
],
"attrName":"15",
"attrValue":0.33,
"recvTimeTs":1580387045,
"recvTime":"2020-01-30T12:24:05.00Z",
"id_sensor_station_absolute":"15_62"
}
]
}
As you can really see it is formatting the sensor_type field = name : value.
I'm working with NODEJS and mongoose.
This is my query: (first I search, sort, only show the first value and then with the project I give format, the problem is that I don't know how to tell the project to put that format if I put "sensor_type": "$latest.attributes.name") it only shows the names and I don't know how to put it in the mentioned format.
Datagreenhouse.aggregate([
{ "$match": { "id_sensor_station_absolute": { "$in": array3 } } }, // "id_station": { "$in": id_station },
{ "$sort": { "recvTime": -1 } },
{
"$group": {
"_id": "$id_sensor_station_absolute",
"latest": { "$first": "$$ROOT" },
}
},
{
"$project": {
"_id": 1,
"id_station": "$latest.id_station",
//"id_sensor_station_absolute": "$id_sensor_station_absolute",
"attrName": "$latest.attrName",
"attrValue": "$latest.attrValue",
"recvTimeTs": "$latest.recvTimeTs",
"recvTime": "$latest.recvTime",
"id_sensor_station_absolute": "$latest.id_sensor_station_absolute",
"sensor_type": "$latest.attributes",
"name": { $arrayElemAt: ["$latest.attributes", 0] },
"type": { $arrayElemAt: ["$latest.attributes", 1] },
"place": { $arrayElemAt: ["$latest.attributes", 2] },
"img": { $arrayElemAt: ["$latest.attributes", 1] },
"name_comun": { $arrayElemAt: ["$latest.attributes", 4] },
"medida": { $arrayElemAt: ["$latest.attributes", 3] },
"interfaz": { $arrayElemAt: ["$latest.attributes", 6] },
}
}
], (err, DatagreenhouseRecuperado) => {
if (err) return res.status(500).send({ message: 'Error al realizar la peticion' + err })
if (!DatagreenhouseRecuperado) return res.status(404).send({ message: 'Error el usuario no existe' })
res.status(200).send({ DatagreenhouseRecuperado })
})
Thank you for your help. Best regards.
Since version 3.4.4, MongoDB introduced a magnific operator: $arrayToObject
This operator allows us transmute array key:value pair into object.
Syntax
RAW DATA $map $arrayToObject
sensor_type : [ sensor_type : [ sensor_type : {
{ \ { \
"name" : "manufacturer", ----> k: "manufacturer", --->
"type" : "String", / v: "Hortisis" / "manufacturer" : "Hortisis"
"value" : "Hortisis"
} }
] ] }
db.datagreenhouses.aggregate([
{
"$match": {} // setup your match criteria
},
{
"$sort": {
"recvTime": -1
}
},
{
$group: {
_id: "$id_sensor_station_absolute",
medidas: {
$push: {
_id: "$_id",
"marca": "Hortisis", // don't know where you get this value
"modelo": "Estacion", // don't know where you get this value
"id_station": "$id_station",
"attrName": "$attrName",
"attrValue": "$attrValue",
"recvTimeTs": "$recvTimeTs",
"recvTime": "$recvTime",
"id_sensor_station_absolute": "$id_sensor_station_absolute",
"sensor_type": {
$arrayToObject: {
$map: {
input: "$sensor_type",
in: {
k: "$$this.name",
v: "$$this.value"
}
}
}
}
}
}
}
}
])
MongoPlayground
[
{
"_id": "15_62",
"medidas": [
{
"_id": ObjectId("5e3419e468d01013eadb83dc"),
"attrName": "15",
"attrValue": 0.33,
"id_sensor_station_absolute": "15_62",
"id_station": "62",
"marca": "Hortisis",
"modelo": "Estacion",
"recvTime": "2020-01-30T12:24:05.00Z",
"recvTimeTs": 1.580387045e+09,
"sensor_type": {
"id": "15",
"img": "assets/img/contrast.png",
"manufacturer": "Hortisis",
"medida": "mS/cm",
"name_comun": "CE del medio",
"place": "interior",
"timestamp": "2020-01-30T12:24:05.00Z",
"type": "fertigation"
}
}
]
}
]
All you need to do is transform data to the desired result with an easy to handle object ($unwind medidas field, transform and then $group again)
Note: If your MongoDB is earlier 3.4.4 version, follow update procedure:
Install MongoDB 3.4.4 or newer
Make mongodump with new version MongoBD
Stop old MongoBD
Remove /data directory (make backup)
Start new MongoDB and run mongorestore

Join Same Collection in Mongo

Below is the sample collection document record that i want to join the same collection with different child array elements.
Sample Collection Record :
{
"_id": "052dc2aa-043b-4cd7-a3f2-f3fe6540ae52",
"Details": [
{
"Id": "104b0bb1-d4a5-469b-b1fd-b4822e96dcb0",
"Number": "12345",
"Percentages": [
{
"Code": "55555",
"Percentage": "45"
},
{
"Code": "55333",
"Percentage": "50"
}
]
},
{
"Id": "104b0bb1-d4a5-469b-b1fd-b4822e96dcb0",
"Number": "55555",
"Percentages": [
{
"Code": "55555",
"Percentage": "45"
}
]
}
],
"Payments": [
{
"Id": "61ee1a6f-3334-4f33-ab6c-51c646b75c41",
"Number": "12345"
}
]
}
The mongo Pipeline query which i would like to fetch the Percentages Array with matched conditions whose Details.Number and Payment.Number should be same
Result:
"Percentages": [
{
"Code": "55555",
"Percentage": "45"
},
{
"Code": "55333",
"Percentage": "50"
}]
How to bring the result by joining the same collections child elements using aggregate ?
Following query does what you want:
db.collection.aggregate([
{$unwind : "$Details"},
{$unwind : "$Details.Percentages"},
{$unwind : "$Payments"}, // $unwind all your arrays
{
$addFields : { //This include new `isMatch` field, which is gonna be true, only if Details.Number = Payment.Number
"isMatch" : {$cond: { if: { $eq: [ "$Details.Number", "$Payments.Number" ] }, then: true, else: false }}
}
},
{
$match : { // This ignores all others, for which Details.Number != Payment.Number
"isMatch" : true
}
},
{
$group : { // This will return only the Percentage objects
_id : null,
"Percentages" : {$push : "$Details.Percentages"}
}
},
{
$project : { // To ignore "_id" field
_id : 0,
"Percentages" : 1
}
}
])
Result:
{
"Percentages" : [
{
"Code" : "55555",
"Percentage" : "45"
},
{
"Code" : "55333",
"Percentage" : "50"
}
]
}
Hope this helps!

mongo aggregation framework group by quarter/half year/year

I have a database with this schema structure :
{
"name" : "Carl",
"city" : "paris",
"time" : "1-2018",
"notes" : [
"A",
"A",
"B",
"C",
"D"
]
}
And this query using the aggregation framework :
db.getCollection('collection').aggregate(
[{
"$match": {
"$and": [{
"$or": [ {
"time": "1-2018"
}, {
"time": "2-2018"
} ]
}, {
"name": "Carl"
}, {
"city": "paris"
}]
}
}, {
"$unwind": "$notes"
}, {
"$group": {
"_id": {
"notes": "$notes",
"time": "$time"
},
"count": {
"$sum": 1
}
}
}
, {
"$group": {
"_id": "$_id.time",
"count": {
"$sum": 1
}
}
}, {
"$project": {
"_id": 0,
"time": "$_id",
"count": 1
}
}])
It working correcly and i'm getting these results these results :
{
"count" : 4.0,
"time" : "2-2018"
}
{
"count" : 4.0,
"time" : "1-2018"
}
My issue is that i'd like to keep the same match stage and i'd like to group by quarter.
Here the result i'd like to have :
{
"count" : 8.0,
"time" : "1-2018" // here quarter 1
}
Thanks

Aggregate analytics data by day/next day logic

I wrote a server for mobile app analytics in which I have a sharded(!Upd) collection with events as follows:
{
"event": "install",
"userId": "a",
"time": 2014-02-09,
"data" : ...
},
{
"event": "login",
"userId": "a",
"time": 2014-02-12,
"data" : ...
},
{
"event": "install",
"userId": "b",
"time": 2014-4-29,
"data" : ...
},
{
"event": "login",
"userId": "b",
"time": 2014-4-30,
"data" : ...
}
...
I need to select users, who have the event install and not login in next day after the install event (in other words, I want to select Users who install the app, but do not login in next day). So the output for above data should be:
{
"userId": "a",
"data" : ...
}
How do carry out this task with aggregation framework or mapreduce? Or maybe another solution?
This is a bit tricky one :-)
You can do it with aggregation, if time is just a date field (without time data),
then
having collection
{
"_id" : ObjectId("57694365ef9176ec54960a66"),
"event" : "install",
"userId" : "a",
"time" : ISODate("2014-09-02T00:00:00.000Z")
},{
"_id" : ObjectId("57694365ef9176ec54960a67"),
"event" : "login",
"userId" : "a",
"time" : ISODate("2014-12-02T00:00:00.000Z")
},{
"_id" : ObjectId("57694365ef9176ec54960a68"),
"event" : "install",
"userId" : "b",
"time" : ISODate("2014-04-29T00:00:00.000Z")
},{
"_id" : ObjectId("57694365ef9176ec54960a69"),
"event" : "login",
"userId" : "b",
"time" : ISODate("2014-04-30T00:00:00.000Z")
}
we can use aggregate query:
var match = {
$match : {
"event" : "install"
}
};
var projectNextDayDate = {
$project : {
_id : 1,
event : 1,
userId : 1,
time : 1,
nextDay : {
$add : ["$time", 24 * 60 * 60 * 1000]
}
}
}
var lookup = {
$lookup : {
from : "zella",
localField : "nextDay",
foreignField : "time",
as : "mergedDocs"
}
}
var nowMatchUsers = {
$project : {
_id : 1,
event : 1,
userId : 1,
time : 1,
nextDay : 1,
mergedDocs : {
$filter : {
input : "mergedDocs",
as : "m",
cond : {
$eq : ["$$m.userId", "$userId"]
}
}
}
}
}
var findEmptyArrays = {
$match : {
mergedDocs : []
}
}
db.zella.aggregate([match, projectNextDayDate, lookup, findEmptyArrays])
with this output:
{
"_id" : ObjectId("57694365ef9176ec54960a66"),
"event" : "install",
"userId" : "a",
"time" : ISODate("2014-09-02T00:00:00.000Z"),
"nextDay" : ISODate("2014-09-03T00:00:00.000Z"),
"mergedDocs" : []
}
The assumption here is that time is a date 2014-09-02T00:00:00.000
the other way to merge collection is to use user ID, as a $lookup point, but then there will be more logic to filter result set and performance could be dropped.
You could try running the following aggregation pipeline:
db.test.aggregate([
{
"$project": {
"event": 1,
"userId": 1,
"time": 1,
"data": 1,
"dayAfter": {
"$add": [ "$time", 24 * 60 * 60 * 1000 ]
}
}
},
{ "$match": { "event": { "$in": ["install", "login"] } } },
{
"$group": {
"_id": "$userId",
"eventsTimeLine": {
"$push": {
"event": "$event",
"time": "$time",
"dayAfter": "$dayAfter"
}
},
"data": { "$push": "$data" }
}
},
{ "$unwind": "$eventsTimeLine" },
{ "$sort": { "eventsTimeLine.event": 1 } },
{
"$group": {
"_id": "$_id",
"dayAfterInstall": { "$first": "$eventsTimeLine.dayAfter" },
"loginTime": { "$last": "$eventsTimeLine.time" },
"data": { "$first": "$data" }
}
},
{
"$project": {
"isChurn": { "$ne": [ "$loginTime", "$dayAfterInstall" ] },
"userId": "$_id", "data": 1, "_id": 0
}
},
{ "$match" : { "isChurn" : true } }
])
Here another solution with mapreduce and aggregation:
var mapFunction = function() {
if (this.event != 'install' && this.event != 'login'){
return;
}
var value = {data: this.data, count: 1};
if (this.event == 'install'){
var nextDay = new Date(this.date.getTime() + 24 * 60 * 60 * 1000)
emit({userId:this.userId, nextDayAfterInstall:nextDay}, value );
} else
if (this.event == 'login'){
emit({userId:this.userId, nextDayAfterInstall:this.date}, value );
}
};
var reduceFunction = function(event, values) {
var value = { data: null, count: 1 };
for (var index = 0; index < values.length; ++index) {
value.count += values[index].count;
value.data = values[index].data;
}
return value ;
};
db.events.mapReduce(
mapFunction,
reduceFunction,
{ out: "case1_mr_out" }
)
var groupByUserId = {
$group :
{
_id : { userId: "$_id.userId" },
data : { $last: '$value.data' },
count : { $max: '$value.count' }
}
}
var filterWhereOnlyOne = {
$match : {
"count" : 1
}
};
db.case1_mr_out.aggregate([groupByUserId,filterWhereOnlyOne])