Twitter data - Finding the most mentioned user in MongoDB - mongodb

Question

Lets say I have stream data from the Twitter API, and I have the data stored as documents in the MongoDB. What I'm trying to find is the count of screen_name under entities.user_mentions.
{
"_id" : ObjectId("50657d5844956d06fb5b36c7"),
"contributors" : null,
"text" : "",
"entities" : {
"urls" : [ ],
"hashtags" : [
{
"text" : "",
"indices" : [
26,
30
]
},
{
"text" : "",
"indices" : []
}
],
"user_mentions" : [
{
"name":"Twitter API",
"indices":[4,15],
"screen_name":"twitterapi",
"id":6253282, "id_str":"6253282"
}]
},
...
I have attempted to use map reduce:
map = function() {
if (!this.entities.user_mentions.screen_name) {
return;
}
for (index in this.entities.user_mentions.screen_name) {
emit(this.entities.user_mentions.screen_name[index], 1);
}
}
reduce = function(previous, current) {
var count = 0;
for (index in current) {
count += current[index];
}
return count;
}
result = db.runCommand({
"mapreduce" : "twitter_sample",
"map" : map,
"reduce" : reduce,
"out" : "user_mentions"
});
But its not quite working...

Answer 1

Since entities.user_mentions is an array, you want to emit a value for each screen_name in the map():
var map = function() {
this.entities.user_mentions.forEach(function(mention) {
emit(mention.screen_name, { count: 1 });
})
};
Then count the values by unique screen_name in the reduce():
var reduce = function(key, values) {
// NB: reduce() uses same format as results emitted by map()
var result = { count: 0 };
values.forEach(function(value) {
result.count += value.count;
});
return result;
};
Note: to debug your map/reduce JavaScript functions, you can use print() and printjson() commands. The output will appear in your mongod log.
EDIT: For comparison, here is an example using the new Aggregation Framework in MongoDB 2.2:
db.twitter_sample.aggregate(
// Project to limit the document fields included
{ $project: {
_id: 0,
"entities.user_mentions" : 1
}},
// Split user_mentions array into a stream of documents
{ $unwind: "$entities.user_mentions" },
// Group and count the unique mentions by screen_name
{ $group : {
_id: "$entities.user_mentions.screen_name",
count: { $sum : 1 }
}},
// Optional: sort by count, descending
{ $sort : {
"count" : -1
}}
)
The original Map/Reduce approach is best suited for a large data set, as is implied with Twitter data. For a comparison of Map/Reduce vs Aggregation Framework limitations see the related discussion on the StackOverflow question MongoDB group(), $group and MapReduce.