Mongodb distinct aggregation of 3 billion documents - mongodb

Question

I have a huge collection with 3 billion documents. Each document looks like the following:
"_id" : ObjectId("54c1a013715faf2cc0047c77"),
"service_type" : "JE",
"receiver_id" : NumberLong("865438083645"),
"time" : ISODate("2012-12-05T23:07:36Z"),
"duration" : 24,
"service_description" : "NQ",
"receiver_cell_id" : null,
"location_id" : "658_55525",
"caller_id" : NumberLong("475035504705")
I would like to get the list of distinct users (they should at least appear once as a caller 'caller_id'), their counts (how many times each user appeared in the collection as either caller or receiver) and the count of locations if they are callers (i.e., the count for each location_id per user).
I want to end up with the following:
"number_of_records" : 20,
"locations" : [{location_id: 658_55525, count:5}, {location_id: 840_5425, count:15}],
"user" : NumberLong("475035504705")
I tried the solution described here and here but they are not efficient enough (extremely slow). What would be an efficient way to achieve this?

Answer 1

Use aggregation for your result:
db.<collection>.aggregate([
{ $group : { _id : { user: "$caller_id", localtion: '$location_id'} , count : { $sum : 1} } },
{ $project : { _id : 0, _id : '$_id.user', location : '$_id.localtion', count : '$count' } },
{ $group : { _id : '$_id', 'locations' : { $push : { location_id : '$location', count : '$count' } }, number_of_records : {$sum : '$count'} } },
{ $project : { _id : 0, user : '$_id', locations : '$locations', number_of_records : '$number_of_records'} },
{ $out : 'outputCollection'},
])
The output will be:
{
"0" : {
"locations" : [
{
"location_id" : "840_5425",
"count" : 8
},
{
"location_id" : "658_55525",
"count" : 5
}
],
"number_of_records" : 13,
"user" : NumberLong(475035504705)
}
}
Update using allowDiskUse:
var pipe = [
{ $group : { _id : { user: "$caller_id", localtion: '$location_id'} , count : { $sum : 1} } },
{ $project : { _id : 0, _id : '$_id.user', location : '$_id.localtion', count : '$count' } },
{ $group : { _id : '$_id', 'locations' : { $push : { location_id : '$location', count : '$count' } }, number_of_records : {$sum : '$count'} } },
{ $project : { _id : 0, user : '$_id', locations : '$locations', number_of_records : '$number_of_records'} },
{ $out : 'outputCollection'},
];
db.runCommand(
{ aggregate: "collection",
pipeline: pipe,
allowDiskUse: true
}
)

Answer 2

A map-reduce solution would be more suitable here rather than an aggregation pipeline, simply because it avoids two unwinds. If you could bring out an aggregation solution with a single unwind, that would be it. But the below map-reduce solution is one way to do it, though you would need to measure its running time against large data and see if it works for you.
The map function:
var map = function(){
emit(this.caller_id,
{locs:[{"location_id":this.location_id,"count":1}]});
}
The reduce function:
var reduce = function(key,values){
var result = {locs:[]};
var locations = {};
values.forEach(function(value){
value.locs.forEach(function(loc){
if(!locations[loc.location_id]){
locations[loc.location_id] = loc.count;
}
else{
locations[loc.location_id]++;
}
})
})
Object.keys(locations).forEach(function(k){
result.locs.push({"location_id":k,"count":locations[k]});
})
return result;
}
The finalize function:
var finalize = function(key,value){
var total = 0;
value.locs.forEach(function(loc){
total += loc.count;
})
return {"total":total,"locs":value.locs};
}
Invoking map-reduce:
db.collection.mapReduce(map,reduce,{"out":"t1","finalize":finalize});
Aggregating the result once the map-reduce produces its output.
db.t1.aggregate([
{$project:{"_id":0,
"number_of_records":"$value.total",
"locations":"$value.locs","user":"$_id"}}
])
Sample o/p:
{
"number_of_records" : 3,
"locations" : [
{
"location_id" : "658_55525",
"count" : 1
},
{
"location_id" : "658_55525213",
"count" : 2
}
],
"user" : 2
}
{
"number_of_records" : 1,
"locations" : [
{
"location_id" : "658_55525",
"count" : 1
}
],
"user" : NumberLong("475035504705")
}
The map-reduce java script code should be self explanatory.