MongoDB count number of new documents per minute based on _id - mongodb

Question

I want to create a statistic on how many new documents are stored each minute.
Since the _id field with standard ObjectID contains already the timestamp of the document creation I think it should be possible to somehow use it.
On Stackoverflow i found the following map reduce code to get it done when there is a dedicated field for the creation data
Map-Reduce count number of documents in each minute MongoDB
map = function() {
var created_at_minute = new Date(this.created_at.getFullYear(),
this.created_at.getMonth(),
this.created_at.getDate(),
this.created_at.getHours(),
this.created_at.getMinutes());
emit(created_at_minute, {count: 1});
}
reduce = function(key, values) {
var total = 0;
for(var i = 0; i < values.length; i++) { total += values[i].count; }
return {count: total};
}
According to the Mongo DB Documentation (http://docs.mongodb.org/manual/reference/object-id/) it should be possible to get the timestamp from the _id by calling ObjectId("507f191e810c19729de860ea").getTimestamp().
Right now I have no idea if it is possible at all to use this getTimestamp() inside of the map function.
Has anybody an idea how to do it or is there a better way ?
I need it to be implementable in python or php

Answer 1

You can do this with M/R indeed. getTimestamp() works in M/R as it runs in JavaScript on the server, it doesn't matter whether your client language is PHP or Python:
map = function() {
var datetime = this._id.getTimestamp();
var created_at_minute = new Date(datetime.getFullYear(),
datetime.getMonth(),
datetime.getDate(),
datetime.getHours(),
datetime.getMinutes());
emit(created_at_minute, {count: 1});
}
reduce = function(key, values) {
var total = 0;
for(var i = 0; i < values.length; i++) { total += values[i].count; }
return {count: total};
}
db.so.mapReduce( map, reduce, { out: 'inline' } );
db.inline.find();
Which outputs something like:
{ "_id" : ISODate("2013-08-05T15:24:00Z"), "value" : { "count" : 9 } }
{ "_id" : ISODate("2013-08-05T15:26:00Z"), "value" : { "count" : 2 } }
However, I would suggest you don't use M/R but instead turn to the aggregation framework as it's much faster because can use indexes and run concurrently. Right now, the A/F does not have an operator to get the timestamp out of an ObjectID field yet though so you will have to store the time at the moment of insertion as well. F.e. with documents like this:
db.so.drop();
db.so.insert( { date: new ISODate( "2013-08-05T15:24:15" ) } );
db.so.insert( { date: new ISODate( "2013-08-05T15:24:19" ) } );
db.so.insert( { date: new ISODate( "2013-08-05T15:24:25" ) } );
db.so.insert( { date: new ISODate( "2013-08-05T15:24:32" ) } );
db.so.insert( { date: new ISODate( "2013-08-05T15:24:45" ) } );
db.so.insert( { date: new ISODate( "2013-08-05T15:25:15" ) } );
db.so.insert( { date: new ISODate( "2013-08-05T15:25:15" ) } );
db.so.aggregate( [
{ $group: {
_id: {
y: { '$year': '$date' },
m: { '$month': '$date' },
d: { '$dayOfMonth': '$date' },
h: { '$hour': '$date' },
i: { '$minute': '$date' },
},
count: { $sum : 1 }
} }
] );
Which outputs:
{
"result" : [
{
"_id" : {
"y" : 2013,
"m" : 8,
"d" : 5,
"h" : 15,
"i" : 25
},
"count" : 2
},
{
"_id" : {
"y" : 2013,
"m" : 8,
"d" : 5,
"h" : 15,
"i" : 24
},
"count" : 5
}
],
"ok" : 1
}