Remove redundant data from sensors by using date and value - mongodb

Question

I'm developing an application that collects data from sensors and I need to reduce the amount of data that is stored in a mongodb database by using a value (temperature) and a date (timestamp).
The document have the following format:
{
temperature: 10,
timestamp: ISODate("2016-04-29T14:37:50.370Z")
sensorCode:"SENSOR_A1"
}
The problem is that sensors sent data too much frequently so there are too many documents with redudant data in a short period of time (let's say 10 minutes). I meant it is not useful to have multiple equal values in a very short period of time.
Example: here there are data from a sensor that is reporting temperature is 10
// collection: datasensors
[
{
temperature: 10,
timestamp: ISODate("2016-04-29T14:37:50.370Z")
sensorCode:"SENSOR_A1"
},
{
temperature: 10,
timestamp: ISODate("2016-04-29T14:38:50.555Z")
sensorCode:"SENSOR_A1"
},
{
temperature: 10,
timestamp: ISODate("2016-04-29T14:38:51.654Z")
sensorCode:"SENSOR_A1"
}
,
{
temperature: 10,
timestamp: ISODate("2016-04-29T14:50:20.335Z")
sensorCode:"SENSOR_A1"
}
]
Because a minute precission is not required, I would like to remove all documents from 2016-04-29T14:37:50.370Z to 2016-04-29T14:38:51.32Z except one. So the result should be this:
[
{
temperature: 10,
timestamp: ISODate("2016-04-29T14:38:51.654Z")
sensorCode:"SENSOR_A1"
},
{
temperature: 10,
timestamp: ISODate("2016-04-29T14:50:20.335Z")
sensorCode:"SENSOR_A1"
}
]
The remove operation I want to perform should "reduce" equal temperatures in time ranges less than 10 minutes to one value.
Is there any technique to achieve this?

Answer 1

I simplified my solution and decided to keep every unique measurement received in 10 minutes time window.
Mongo 3.2 is required for that
adding a time mark will separate measurements in 10 minutes time groups
Then we are preserving first record in group and storing all ids for futher process
Then removing id of document we want to keep from an array of all ids (let say documents to delete)
Finally as forEach loop we are deleting not needed ids - this line is commented :-)
Copy code below to mongo console, execute and verify ids to delete, then un-comment and GO!
var addTimeMark = {
$project : {
_id : 1,
temperature : 1,
timestamp : 1,
sensorCode : 1,
yearMonthDay : {
$substr : [{
$dateToString : {
format : "%Y%m%d%H%M",
date : "$timestamp"
}
}, 0, 11]
}
}
}
var getFirstRecordInGroup = {
// take only first record froum group
$group : {
_id : {
timeMark : "$yearMonthDay",
sensorCode : "$sensorCode",
temperature : "$temperature"
},
id : {
$first : "$_id"
},
allIds : {
$push : "$_id"
},
timestamp : {
$first : "$timestamp"
},
totalEntries : {
$sum : 1
}
}
}
var removeFirstIdFromAllIds = {
$project : {
_id : 1,
id : 1,
timestamp : 1,
totalEntries : 1,
allIds : {
$filter : {
input : "$allIds",
as : "item",
cond : {
$ne : ["$$item", "$id"]
}
}
}
}
}
db.sensor.aggregate([
addTimeMark,
getFirstRecordInGroup,
removeFirstIdFromAllIds,
]).forEach(function (entry) {
printjson(entry.allIds);
// db.sensor.deleteMany({_id:{$in:entry.allIds}})
})
below document outlook after each step:
{
"_id" : ObjectId("574b5d8e0ac96f88db507209"),
"temperature" : 10,
"timestamp" : ISODate("2016-04-29T14:37:50.370Z"),
"sensorCode" : "SENSOR_A1",
"yearMonthDay" : "20160429143"
}
2:
{
"_id" : {
"timeMark" : "20160429143",
"sensorCode" : "SENSOR_A1",
"temperature" : 10
},
"id" : ObjectId("574b5d8e0ac96f88db507209"),
"allIds" : [
ObjectId("574b5d8e0ac96f88db507209"),
ObjectId("574b5d8e0ac96f88db50720a"),
ObjectId("574b5d8e0ac96f88db50720b")
],
"timestamp" : ISODate("2016-04-29T14:37:50.370Z"),
"totalEntries" : 3
}
and last;
{
"_id" : {
"timeMark" : "20160429143",
"sensorCode" : "SENSOR_A1",
"temperature" : 10
},
"id" : ObjectId("574b5d8e0ac96f88db507209"),
"allIds" : [
ObjectId("574b5d8e0ac96f88db50720a"),
ObjectId("574b5d8e0ac96f88db50720b")
],
"timestamp" : ISODate("2016-04-29T14:37:50.370Z"),
"totalEntries" : 3
}