Related
I have produced the below output using mongodb aggregation (including $group pipeline inside levelsCount field) :
{
"_id" : "1",
"name" : "First",
"levelsCount" : [
{ "_id" : "level_One", "levelNum" : 1, "count" : 1 },
{ "_id" : "level_Three", "levelNum" : 3, "count" : 1 },
{ "_id" : "level_Four", "levelNum" : 4, "count" : 8 }
]
}
{
"_id" : "2",
"name" : "Second",
"levelsCount" : [
{ "_id" : "level_One", "levelNum" : 1, "count" : 5 },
{ "_id" : "level_Two", "levelNum" : 2, "count" : 2 },
{ "_id" : "level_Three", "levelNum" : 3, "count" : 1 },
{ "_id" : "level_Four", "levelNum" : 4, "count" : 3 }
]
}
{
"_id" : "3",
"name" : "Third",
"levelsCount" : [
{ "_id" : "level_One", "levelNum" : 1, "count" : 1 },
{ "_id" : "level_Two", "levelNum" : 2, "count" : 3 },
{ "_id" : "level_Three", "levelNum" : 3, "count" : 2 },
{ "_id" : "level_Four", "levelNum" : 4, "count" : 3 }
]
}
Now, I need to sort these documents based on the levelNum and count fields of levelsCount array elements. I.e. If two documents both had the count 5 forlevelNum: 1 (level_One), then the sort goes to compare the count of levelNum: 2 (level_Two) field and so on.
I see how $sort pipeline would work on multiple fields (Something like { $sort : { level_One : 1, level_Two: 1 } }), But the problem is how to access those values of levelNum of each array element and set that value as a field name to do sorting on that. (I couldn't handle it even after $unwinding the levelsCount array).
P.s: The initial order of levelsCount array's elements may differ on each document and is not important.
Edit:
The expected output of the above structure would be:
// Sorted result:
{
"_id" : "2",
"name" : "Second",
"levelsCount" : [
{ "_id" : "level_One", "levelNum" : 1, "count" : 5 }, // "level_One's count: 5" is greater than "level_One's count: 1" in two other documents, regardless of other level_* fields. Therefore this whole document with "name: Second" is ordered first.
{ "_id" : "level_Two", "levelNum" : 2, "count" : 2 },
{ "_id" : "level_Three", "levelNum" : 3, "count" : 1 },
{ "_id" : "level_Four", "levelNum" : 4, "count" : 3 }
]
}
{
"_id" : "3",
"name" : "Third",
"levelsCount" : [
{ "_id" : "level_One", "levelNum" : 1, "count" : 1 },
{ "_id" : "level_Two", "levelNum" : 2, "count" : 3 }, // "level_Two's count" in this document exists with value (3) while the "level_Two" doesn't exist in the below document which mean (0) value for count. So this document with "name: Third" is ordered higher than the below document.
{ "_id" : "level_Three", "levelNum" : 3, "count" : 2 },
{ "_id" : "level_Four", "levelNum" : 4, "count" : 3 }
]
}
{
"_id" : "1",
"name" : "First",
"levelsCount" : [
{ "_id" : "level_One", "levelNum" : 1, "count" : 1 },
{ "_id" : "level_Three", "levelNum" : 3, "count" : 1 },
{ "_id" : "level_Four", "levelNum" : 4, "count" : 8 }
]
}
Of course, I'd prefer to have an output document in the below format, But the first problem is to sort all docs:
{
"_id" : "1",
"name" : "First",
"levelsCount" : [
{ "level_One" : 1 },
{ "level_Three" : 1 },
{ "level_Four" : 8 }
]
}
You can sort by levelNum as descending order and count as ascending order,
db.collection.aggregate([
{
$sort: {
"levelsCount.levelNum": -1,
"levelsCount.count": 1
}
}
])
Playground
For key-value format result of levelsCount array,
$map to iterate loop of levelsCount array
prepare key-value pair array and convert to object using $arrayToObject
{
$addFields: {
levelsCount: {
$map: {
input: "$levelsCount",
in: {
$arrayToObject: [
[{ k: "$$this._id", v: "$$this.levelNum" }]
]
}
}
}
}
}
Playground
I have a mongodb collection as follow :
[
{
"_id" : ObjectId("5ba0e5a99e7537012371855a"),
"user_id" : 2,
"action" : 0,
"source" : 1,
"service" : "FootPlus",
"free" : false,
"source_detail" : {
"source_type" : "landing",
"source_id" : 2,
"promoter_id" : 1
},
"created_at" : ISODate("2018-09-18T11:46:49.000Z")
}
{
"_id" : ObjectId("5ba0e5cc9e7537013d57e37a"),
"user_id" : 2,
"action" : 1,
"service" : "FootPlus",
"source" : 0,
"created_at" : ISODate("2018-09-18T11:47:24.000Z"),
"source_detail" : {
"source_type" : "landing",
"source_id" : 2,
"promoter_id" : 1
}
}
]
I want to group by source_detail.promoter_id and count action = 0 and action = 1
and also divide count of action = 0 to count of action = 1 per promoter
so my result should be :
[
{
"promoter_id": 2
"action_0": 27,
"action_1": 9,
"devide": 3
},
{
"promoter_id": 3
"action_0": 18,
"action_1": 3,
"devide": 6
}
]
how can I achieve this result ?
thanks.
You can use below aggregation.
Something like
db.colname.aggregate([
{"$group":{
"_id":"$source_detail.promoter_id",
"action_0":{"$sum":{"$cond":[{"$eq":["$action",0]},1,0]}},
"action_1":{"$sum":{"$cond":[{"$eq":["$action",1]},1,0]}}
}},
{"$addFields":{"divide":{"$divide":["$action_0","$action_1"]}}}
])
I am getting confused with a simple MongoDB query that I can't figure it out where the problem is. I have a collection like this:
{
"_id" : NumberLong(1939026454),
"username" : "5144269288",
"_type" : 1,
"group_id" : 416,
"user_id" : NumberLong(426661),
"credit_used" : 0.0,
"retry_count" : 1,
"successful" : true,
"type_details" : {
"in_bytes" : 0,
"sub_service_qos" : "",
"sub_service_name" : "rating-group-103",
"out_bytes" : 0,
"sub_service_charging" : "FreeIPTV",
"remote_ip" : ""
},
"logout_time" : ISODate("2017-11-06T07:16:09.000Z"),
"before_credit" : 4560.2962,
"ras_id" : 18,
"caller_id" : "",
"isp_id" : 0,
"duration" : NumberLong(14500),
"details" : {
"connect_info" : "rate-group=103",
"sub_service" : "rating-group-103",
"diameter_request_type" : "initial"
},
"unique_id_value" : "918098048;falcon;Dehghan01000000001415716f9a113697;529;falcon-02b4e8a7__103",
"charge_rule_details" : [
{
"start_in_bytes" : 0,
"charge_rule_id" : 3682,
"start_out_bytes" : 0,
"stop_time" : ISODate("2017-11-06T07:16:09.000Z"),
"stop_out_bytes" : 0,
"start_time" : ISODate("2017-11-06T03:14:29.000Z"),
"charge_rule_desc" : "rating-group-103",
"stop_in_bytes" : 0
}
],
"unique_id" : "acct_session_id",
"login_time" : ISODate("2017-11-06T03:14:29.000Z")
}
I need to filter documents that login_time is between two given dates and type_details.sub_service_name is some string.
I tried this:
db.getCollection('connection_log_partial_data').find({
"type_details" : {"sub_service_name": "rating-group-103"},
"login_time": {
"$gt": ISODate("2016-11-06T03:14:29.000Z"),
"$lt": ISODate("2017-11-06T03:14:29.000Z")
}
});
but it fetches 0 records. Any suggestions?
your query should be something like:
db.getCollection('connection_log_partial_data').find({
"type_details.sub_service_name" : "rating-group-103",
"login_time": {
"$gte": ISODate("2016-11-06T03:14:29.000Z"),
"$lte": ISODate("2017-11-06T03:14:29.000Z")
}
});
I have a MovieRatings database with columns userId, movieId, movie-categoryId, reviewId, movieRating and reviewDate.
In my mapper I want to extract userId -> (movieId, movieRating)
And then in the reducer I want to group all movieId, movieRating pair by user.
Here is my attempt:
Map function:
var map = function() {
var values={movieId : this.movieId, movieRating : this.movieRating};
emit(this.userId, values);}
Reduce function:
var reduce = function(key,values) {
var ratings = [];
values.forEach(function(V){
var temp = {movieId : V.movieId, movieRating : V.movieRating};
Array.prototype.push.apply(ratings, temp);
});
return {userId : key, ratings : ratings };
}
Run MapReduce:
db.ratings.mapReduce(map, reduce, { out: "map_reduce_step1" })
Output: db.map_reduce_step1.find()
{ "_id" : 1, "value" : { "userId" : 1, "ratings" : [ ] } }
{ "_id" : 2, "value" : { "userId" : 2, "ratings" : [ ] } }
{ "_id" : 3, "value" : { "userId" : 3, "ratings" : [ ] } }
{ "_id" : 4, "value" : { "userId" : 4, "ratings" : [ ] } }
{ "_id" : 5, "value" : { "userId" : 5, "ratings" : [ ] } }
{ "_id" : 6, "value" : { "userId" : 6, "ratings" : [ ] } }
{ "_id" : 7, "value" : { "userId" : 7, "ratings" : [ ] } }
{ "_id" : 8, "value" : { "userId" : 8, "ratings" : [ ] } }
{ "_id" : 9, "value" : { "userId" : 9, "ratings" : [ ] } }
{ "_id" : 10, "value" : { "userId" : 10, "ratings" : [ ] } }
{ "_id" : 11, "value" : { "userId" : 11, "ratings" : [ ] } }
{ "_id" : 12, "value" : { "userId" : 12, "ratings" : [ ] } }
{ "_id" : 13, "value" : { "userId" : 13, "ratings" : [ ] } }
{ "_id" : 14, "value" : { "userId" : 14, "ratings" : [ ] } }
{ "_id" : 15, "value" : { "movieId" : 1, "movieRating" : 3 } }
{ "_id" : 16, "value" : { "userId" : 16, "ratings" : [ ] } }
I am not getting the expected output. In fact, this output makes no sense to me!
Here is the python equivalent of what I am trying to do in the reducer (just in case the purpose of reducer wasn't clear above) :
def reducer_ratings_by_user(self, user_id, itemRatings):
#Group (item, rating) pairs by userID
ratings = []
for movieID, rating in itemRatings:
ratings.append((movieID, rating))
yield user_id, ratings
Edit 1 #chridam
Here is an outline of what I really want to do here :
Movies.csv file looks like :
userId,movieId,movie-categoryId,reviewId,movieRating,reviewDate
1,1,1,1,5,7/12/2000
2,1,1,2,5,7/12/2000
3,1,1,3,5,7/12/2000
4,1,1,4,4,7/12/2000
5,1,1,5,4,7/12/2000
6,1,1,6,5,7/15/2000
1,2,1,7,4,7/25/2000
8,1,1,8,4,7/28/2000
9,1,1,9,3,8/3/2000
...
...
I import this into mongoDB :
mongoimport --db SomeName --collection ratings --type csv --headerline --file Movies.csv
Then I am trying to apply the map-reduce function as define above. After that I will export it back to a csv by doing somethig like :
mongoexport --db SomeName --collection map_reduce_step1 --csv --out movie_ratings_out.csv --fields ...
This movie_ratings_out.csv file should be like :
userId, movieId1, rating1, movieId2, rating2 ,...
1,1,5,2,4
...
...
So each row contains all the (movie,rating) pair for every user.
Edit 2
Sample :
db.ratings.find().pretty()
{
"_id" : ObjectId("57f4a0dd9cb74fc4d344a40f"),
"userId" : 4,
"movieId" : 1,
"movie-categoryId" : 1,
"reviewId" : 4,
"movieRating" : 4,
"reviewDate" : "7/12/2000"
}
{
"_id" : ObjectId("57f4a0dd9cb74fc4d344a410"),
"userId" : 5,
"movieId" : 1,
"movie-categoryId" : 1,
"reviewId" : 5,
"movieRating" : 4,
"reviewDate" : "7/12/2000"
}
{
"_id" : ObjectId("57f4a0dd9cb74fc4d344a411"),
"userId" : 4,
"movieId" : 2,
"movie-categoryId" : 1,
"reviewId" : 6,
"movieRating" : 5,
"reviewDate" : "7/15/2000"
}
{
"_id" : ObjectId("57f4a0dd9cb74fc4d344a412"),
"userId" : 4,
"movieId" : 3,
"movie-categoryId" : 1,
"reviewId" : 2,
"movieRating" : 5,
"reviewDate" : "7/12/2000"
}
...
Then after MapReduce expected output json is :
{
"_id" : ....,
"userId" : 4,
"movieList" : [ {
"movieId" : 2
"movieRating" : 5
},
{
"movieId" : 1
"movieRating" : 4
}
...
]
}
{
"_id" : ....,
"userId" : 5,
"movieList" : ...
}
...
You just need to run an aggregation pipeline which consists of a $group stage that summarize documents. This groups input documents by a specified identifier expression and applies the accumulator expression(s). The $group pipeline operator is similar to the SQL's GROUP BY clause. In SQL, you can't use GROUP BY unless you use any of the aggregation functions. The same way, you have to use an aggregation function in MongoDB as well. You can read more about the aggregation functions here.
The accumulator operator you would need to create the movieList array is $push.
Another pipeline which follows after the $group stage is the $project operator which is used to select or reshape each document in the stream, include, exclude or rename fields, inject computed fields, create sub-document fields, using mathematical expressions, dates, strings and/or logical (comparison, boolean, control) expressions - similar to what you would do with the SQL SELECT clause.
The last step is the $out pipeline which writes the resulting documents of the aggregation pipeline to a collection. It must be the last stage in the pipeline.
So as a result, you can run the following aggregate operation:
db.ratings.aggregate([
{
"$group": {
"_id": "$userId",
"movieList": {
"$push": {
"movieId": "$movieId",
"movieRating": "$movieRating",
}
}
}
},
{
"$project": {
"_id": 0, "userId": "$_id", "movieList": 1
}
},
{ "$out": "movie_ratings_out" }
])
Using the sample 5 documents above, the sample output if you query db.getCollection('movie_ratings_out').find({}) would yield:
/* 1 */
{
"_id" : ObjectId("57f52636b9c3ea346ab1d399"),
"movieList" : [
{
"movieId" : 1.0,
"movieRating" : 4.0
}
],
"userId" : 5.0
}
/* 2 */
{
"_id" : ObjectId("57f52636b9c3ea346ab1d39a"),
"movieList" : [
{
"movieId" : 1.0,
"movieRating" : 4.0
},
{
"movieId" : 2.0,
"movieRating" : 5.0
},
{
"movieId" : 3.0,
"movieRating" : 5.0
}
],
"userId" : 4.0
}
Im having a hard time trying to run some nested queries with a conditional statement of an item inside an array.
this is how my documents looks like.
I would like to get a summary such as sum and average and alarmedCount (count every time Channels.AlarmStatus == "alarmed") of each "Channel" based on Channels.Id. I got sum and average to work but cant get the right query for alarmedCount
{
"_id" : "55df8e4cd8afa4ccer1915ee"
"location" : "1",
"Channels" : [{
"_id" : "55df8e4cdsafa4cc0d1915r1",
"ChannelId" : 1,
"Value" : 14,
"AlarmStatus" : "normal"
},
{
"_id" : "55df8e4cdsafa4cc0d1915r9",
"ChannelId" : 2,
"Value" : 20,
"AlarmStatus" : "alarmed"
},
{
"_id" : "55df8e4cdsafa4cc0d1915re",
"ChannelId" : 3,
"Value" : 10,
"AlarmStatus" : "alarmed"},
]
}
{
"_id" : "55df8e4cd8afa4ccer1915e0"
"location" : "1",
"Channels" : [{
"_id" : "55df8e4cdsafa4cc0d19159",
"ChannelId" : 1,
"Value" : 50,
"AlarmStatus" : "normal"
},
{
"_id" : "55df8e4cdsafa4cc0d1915re",
"ChannelId" : 2,
"Value" : 16,
"AlarmStatus" : "normal"
},
{
"_id" : "55df8e4cdsafa4cc0d1915g7",
"ChannelId" : 3,
"Value" : 9,
"AlarmStatus" : "alarmed"},
]
}
I got it to work to group them and show some calculations
using this aggregate
db.records.aggregate( [
{
"$unwind" : "$Channels"
},
{
"$group" : {
"_id" : "$Channels.Id",
"documentSum" : { "$sum" : "$Channels.Value" },
"documentAvg" : { "$avg" : "$Channels.Value" }
}
}
] )
the result looks like this:
{
"result" : [
{
"_id" : 1,
"documentSum" : 64,
"documentAvg" : 32
},
{
"_id" : 2,
"documentSum" : 36,
"documentAvg" : 18
},
{
"_id" : 3,
"documentSum" : 19,
"documentAvg" : 9.5
},
],
"ok" : 1.0000000000000000
}
I would like to get this type of result
{
"result" : [
{
"_id" : 1,
"documentSum" : 64,
"documentAvg" : 32,
"AlarmedCount" : 0
},
{
"_id" : 2,
"documentSum" : 36,
"documentAvg" : 18,
"AlarmedCount" : 1
},
{
"_id" : 3,
"documentSum" : 19,
"documentAvg" : 9.5,
"AlarmedCount" : 2
}
],
"ok" : 1.0000000000000000
}
Use a project-step before your group-step to convert the field AlarmedStatus to 1 or 0 depending on its value:
$project: {
"Channels.value":"$Channels.Value",
"Channels.AlarmCount":{ $cond: {
if: { $eq: ["$Channels.AlarmedStatus", "alarmed"] },
then: 1,
else: 0 }
}
}
Then sum the newly created field to get the aggregated count:
$group : {
"_id" : "$Channels.Id",
"documentSum" : { "$sum" : "$Channels.Value" },
"documentAvg" : { "$avg" : "$Channels.Value" },
"AlarmCount" : { "$sum" : "$Channels.AlarmCount" }
}