mongodb mapreduce exclude nested field - mongodb

Question

I am mongodb newbie! I am trying to process some tweeter data. my goal is to group users on each time interval (for simplicity, daily interval) and count his unique hashtags on that day. My idea to build new DB which is only contains user, date and hashtags. Here is data format:
> db.sampledDB.findOne()
{
"_id" : NumberLong("2334234"),
"replyid" : NumberLong(-1),
"userid" : NumberLong(21313),
"replyuserid" : NumberLong(-1),
"createdAt" : ISODate("2013-07-02T22:35:06Z"),
"tweettext" : "RT #BBCBreaking: Plane carrying Bolivia President Morales is diverted to Austria on suspicion US fugitive #Snowden is on board - Bolivian m…",
"screenName" : "x83",
"name" : "david x",
"retweetCount" : NumberLong(0),
"retweet_id" : NumberLong("12313223"),
"retweet_userid" : NumberLong(123123123),
"source" : "Twitter for Windows Phone",
"hashtags" : [
{
"start" : 106,
"end" : 114,
"text" : "Snowden"
}
],
"mentions" : [
{
"start" : 3,
"end" : 15,
"id" : NumberLong(876678),
"screenName" : "BBCBreaking",
"name" : "BBC Breaking News"
}
],
"media" : [ ]
}
I use mapReduce like this:
MAP:
map = function(){
//format date to year/month/day
var format = this.createdAt.getFullYear() + '/' + (this.createdAt.getMonth()+1) + '/' + this.createdAt.getDate();
var key = {userid:this.userid, date:format};
emit(key,{hashtags:this.hashtags}); }
REDUCE:
reduce = function(key,values){
var result = {a:[]};
for (var idx=0;idx<values.length;idx++){
result.a.push(values[idx].hashtag);
}
return result};
it results to:
{
"_id" : {
"userid" : NumberLong(7686787),
"date" : "2013/7/5"
},
"value" : {
"hashtag" : [
{
"start" : 24,
"end" : 44,
"text" : "SıkSöylenenYalanlar"
},
{
"start" : 45,
"end" : 60,
"text" : "ZimmermanTrial"
},
{
"start" : 61,
"end" : 84,
"text" : "ZaynMalikYouArePerfect"
},
{
"start" : 85,
"end" : 99,
"text" : "TrayvonMartin"
},
{
"start" : 100,
"end" : 110,
"text" : "Wimbledon"
},
{
"start" : 111,
"end" : 118,
"text" : "Футбол"
},
{
"start" : 119,
"end" : 127,
"text" : "Snowden"
},
{
"start" : 128,
"end" : 138,
"text" : "TFFistifa"
}
]
}
},
{
"_id" : {
"userid" : NumberLong(45666),
"date" : "2013/7/5"
},
"value" : {
"hashtag" : [
{
"start" : 24,
"end" : 44,
"text" : "SıkSöylenenYalanlar"
},
{
"start" : 45,
"end" : 60,
"text" : "ZimmermanTrial"
},
{
"start" : 61,
"end" : 84,
"text" : "ZaynMalikYouArePerfect"
},
{
"start" : 85,
"end" : 99,
"text" : "TrayvonMartin"
},
{
"start" : 100,
"end" : 110,
"text" : "Wimbledon"
},
{
"start" : 111,
"end" : 118,
"text" : "Футбол"
},
{
"start" : 119,
"end" : 127,
"text" : "Snowden"
},
{
"start" : 128,
"end" : 138,
"text" : "TFFistifa"
}
]
}
},
But I just want to keep the text element of hashtags. I tried to change the reducer to values[idx].hashtag.text or values[idx].hashtag["text"] which did not help.
UPDATE:
I suspect my problem is similar to MapReduce problem, but I dont know to fix mine

Answer 1

You might also consider using the aggregation framework which can produce the results shown below. The pipeline would look similar to this:
{$project: {
userid: "$userid",
"hashtags": "$hashtags.text",
date: {
year: { $year: "$createdAt" },
month: { $month: "$createdAt"},
day: {$dayOfMonth: "$createdAt"} }}},
{$unwind: "$hashtags" },
{ $group: { _id : {
date: "$date",
userid: "$userid"},
hashtags: { $addToSet:"$hashtags" }
}} )
Might produce a result like:
[
{
"_id" : {
"date" : {
"year" : 2013,
"month" : 8,
"day" : 4
},
"userid" : NumberLong(362337301)
},
"hashtags" : [
"tagger",
"stackoverflow",
"twitter"
]
}, /* more */
A brief explanation of the aggregation framework pipeline:
Using $project, grab only the fields that will matter through the rest of the pipeline. Before doing this, if there was a specific date or range that would have been desired, using $match would have been a great step to filter some of the results efficiently). Note that the createdAt field has been split into the respective pieces so that the time of day will later be ignored when grouping. After the projection has occurred, the new field will be called date in the example. Here, the hash tags have been simplified to be only the text property, and the name reused as "hashtags".
Next, as "hashtags" is an array at this point (would look like: ['tagger', 'stackoverflow', 'twitter'] for example, the pipeline creates a new document for each element in the "hashtag" array.
Finally, the grouping pipeline operator uses the combination of userid and date as a grouper, and adds all unique hash tags to a field called "hashtags".
As an alternative to splitting the date, you can also just treat the createdAt field as a string, and remove the time by using this in the pipeline:
date: {$substr: ["$createdAt",0, 10] }
It would produce something like:
2013-07-02
Edit
As you've pointed out, there is currently a 16MB limit in the document that is output from an Aggregation. While this is scheduled to be changed in the 2.6 version of MongoDB, you may be able to get a MapReduce as well that work. It's a bit messier given a MapReduce wasn't necessarily intended for this type of work, so the results may not be necessarily what you want.
map = function() {
var format = this.createdAt.getFullYear() + '/'
+ (this.createdAt.getMonth()+1) + '/' + this.createdAt.getDate();
var key = {userid:this.userid, date:format};
var hashtags = this.hashtags || [];
for(var i=0, l=hashtags.length; i < l; i++) {
emit(key, hashtags[i].text);
}
};
reduce = function(key, values){
values = values || [];
var tag;
var tags = {};
for(var i=0, l=values.length; i<l ; i++) {
tag = values[i] || "";
if (tag.length > 0) {
tags[tag] = "";
}
};
values = [];
for(var t in tags) {
values.push(t);
}
return values.join(',');
};
Instead of emitting the array, it emits each hash tag in the map. The reduce eliminates duplicates using a simple associative array and then returns a joined string with all of the hash tags. MongoDB does not support returning an array of results via the reduce function (the idea is that a reduce should be providing one result, not an array of results).
Results:
{
"_id" : {
"userid" : NumberLong(262317302),
"date" : "2013/7/2"
},
"value" : "Wisconsin,Space,Cheese"
}
If you don't need to do this work frequently, you could also just write a shell script in the MongoDB console that extracts the hash tags into a new collection. Then, just run it when you need to.

Answer 2

here is how I managed to produce the same result as the answer above. just for presenting another solution.
map = function(){
var day = this.createdAt.getFullYear() + '/' + (this.createdAt.getMonth()+1) + '/' + this.createdAt.getDate();
var key = {userid:this.userid, date:day};
var values = {hashtags:[]};
for (var idx=0;idx<this.hashtags.length;idx++){
values.hashtags.push(this.hashtags[idx].text);
}
emit(key,values);
};
reduce = function(key,values){
hashtag_list = {hashtags: []} ;
for(var i in values) {
hashtag_list.hashtags= values[i].hashtags.concat(hashtag_list.hashtags);
}
return hashtag_list;
}

Answer 3

Try:
values[idx].text
hashtag is not a property of the object, but text is.