I am new to Mongo aggregation.I want to calculate the difference betwen Two values (The last collection for each day -The first collection for each day).the data base record data every 5 mn for many ressource name.The structucture of the document is :
{
_id : ObjectId("5820511a95d447ed648b45d6"),
DeviceName : "OLT01FTV",
ResourceName : "CM MAC:00-07-11-11-39-20",
CollectionTime : ISODate("2016-11-07T09:30:00.000+01:00"),
GranularityPeriod : 5,
A : 0,
B: 17,
C: 4,
D: 21,
E: 3,
F: 0
}
A,B...F are the differrent counters.
Below, the illustration of that I'm trying to have :
result
([
{ "$match": {
"CollectionTime": {
$gte: ISODate("2016-09-05T00:00:00.000Z"),
$lt: ISODate("2016-10-07T00:00:00.000Z")
}
}},
{ "$unwind": "$u2000" },
{ "$group": {
"_id": null,
"firstUC": { "$first": "$UC" },
"lastUC": { "$last": "$UC" },
"firstSM-MISS": { "$first": "$SM-MISS" },
"lastSM-MISS": { "$last": "$SM-MISS" }
}},
{ "$project": {
"diff": {
"$divide": [
{ "$subtract": [ "$firstUC", "$lastUC" ] },
{ "$subtract": [ "$firstSM-MISS", "$lastSM-MISS" ] }
]
}
}}
])
This will get you the difference between the 'A' values for your above scenario. You can add the other fields if you want to get the difference for them also.
db.collection.aggregate([
{ "$match": {
"CollectionTime": {
$gte: ISODate("2016-11-01T00:00:00.000Z"),
$lt: ISODate("2016-11-30T00:00:00.000Z")
}
}},
{ "$sort": { "CollectionTime": 1 } },
{ "$group": {
"_id": null,
"firstA": { "$first": "$A" },
"lastA": { "$last": "$A" }
}},
{ "$project": {
_id: 0,
diffA: {
$subtract: [ "$lastA", "$firstA"]
}
}}
])
* EDIT *
So I'm using the following sample documents I created with the following to match your schema:
// Create 3 Documents 1 second apart
for (var i = 1; i < 4; i++) {
db.foo.insert({
DeviceName : "OLT01FTV",
ResourceName : "CM MAC:00-07-11-11-39-20",
CollectionTime : new Date(),
GranularityPeriod : 5,
A : 1*i,
B: 2*i,
C: 3*i,
D: 4*i,
E: 5*i,
F: 6*i
})
sleep(1000); // To add a delay between insertions so we can visibly see the date difference
}
This results in the following 3 documents being created:
> db.foo.find().pretty()
{
"_id" : ObjectId("582b1a6ced19a7334a5dee31"),
"DeviceName" : "OLT01FTV",
"ResourceName" : "CM MAC:00-07-11-11-39-20",
"CollectionTime" : ISODate("2016-11-15T14:23:40.934Z"),
"GranularityPeriod" : 5,
"A" : 1,
"B" : 2,
"C" : 3,
"D" : 4,
"E" : 5,
"F" : 6
}
{
"_id" : ObjectId("582b1a6ded19a7334a5dee32"),
"DeviceName" : "OLT01FTV",
"ResourceName" : "CM MAC:00-07-11-11-39-20",
"CollectionTime" : ISODate("2016-11-15T14:23:41.936Z"),
"GranularityPeriod" : 5,
"A" : 2,
"B" : 4,
"C" : 6,
"D" : 8,
"E" : 10,
"F" : 12
}
{
"_id" : ObjectId("582b1a6eed19a7334a5dee33"),
"DeviceName" : "OLT01FTV",
"ResourceName" : "CM MAC:00-07-11-11-39-20",
"CollectionTime" : ISODate("2016-11-15T14:23:42.939Z"),
"GranularityPeriod" : 5,
"A" : 3,
"B" : 6,
"C" : 9,
"D" : 12,
"E" : 15,
"F" : 18
}
The first step of the aggregation pipeline will match on all documents between the date range - which I set to beginning of November... so no worry there, then the sorting will sort by collection time:
After the grouping we have one document with the firstA and lastA value:
{ "_id" : null, "firstA" : 1, "lastA" : 3 }
And finally - perform the subtract in the projection and hide the ID field:
{ "diffA" : 2 }
Related
I'm trying to figure out what I'm doing wrong, I have collected the following, "Subset of data", "Desired output"
This is how my data objects look
[{
"survey_answers": [
{
"id": "9ca01568e8dbb247", // As they are, this is the key to groupBy
"option_answer": 5, // Represent the index of the choosen option
"type": "OPINION_SCALE" // Opinion scales are 0-10 (meaning elleven options)
},
{
"id": "ba37125ec32b2a99",
"option_answer": 3,
"type": "LABELED_QUESTIONS" // Labeled questions are 0-x (they can change it from survey to survey)
}
],
"survey_id": "test"
},
{
"survey_answers": [
{
"id": "9ca01568e8dbb247",
"option_answer": 0,
"type": "OPINION_SCALE"
},
{
"id": "ba37125ec32b2a99",
"option_answer": 3,
"type": "LABELED_QUESTIONS"
}
],
"survey_id": "test"
}]
My desired output is:
[
{
id: '9ca01568e8dbb247'
results: [
{ _id: 5, count: 1 },
{ _id: 0, count: 1 }
]
},
{
id: 'ba37125ec32b2a99'
results: [
{ _id: 3, count: 2 }
]
}
]
Active query
Model.aggregate([
{
$match: {
'survey_id': survey_id
}
},
{
$unwind: "$survey_answers"
},
{
$group: {
_id: "$survey_answers.option_answer",
count: {
$sum: 1
}
}
}
])
Current output
[
{
"_id": 0,
"count": 1
},
{
"_id": 3,
"count": 2
},
{
"_id": 5,
"count": 1
}
]
I added your records to my db. Post that I tried your commands one by one.
$unwind results you similar to -
> db.survey.aggregate({$unwind: "$survey_answers"})
{ "_id" : ObjectId("5c3859e459875873b5e6ee3c"), "survey_answers" : { "id" : "9ca01568e8dbb247", "option_answer" : 5, "type" : "OPINION_SCALE" }, "survey_id" : "test" }
{ "_id" : ObjectId("5c3859e459875873b5e6ee3c"), "survey_answers" : { "id" : "ba37125ec32b2a99", "option_answer" : 3, "type" : "LABELED_QUESTIONS" }, "survey_id" : "test" }
{ "_id" : ObjectId("5c3859e459875873b5e6ee3d"), "survey_answers" : { "id" : "9ca01568e8dbb247", "option_answer" : 0, "type" : "OPINION_SCALE" }, "survey_id" : "test" }
{ "_id" : ObjectId("5c3859e459875873b5e6ee3d"), "survey_answers" : { "id" : "ba37125ec32b2a99", "option_answer" : 3, "type" : "LABELED_QUESTIONS" }, "survey_id" : "test" }
I am not adding code for match since that is okay in your query as well
The grouping would be -
> db.survey.aggregate({$unwind: "$survey_answers"},{$group: { _id: { 'optionAnswer': "$survey_answers.option_answer", 'id':"$survey_answers.id"}, count: { $sum: 1}}})
{ "_id" : { "optionAnswer" : 0, "id" : "9ca01568e8dbb247" }, "count" : 1 }
{ "_id" : { "optionAnswer" : 3, "id" : "ba37125ec32b2a99" }, "count" : 2 }
{ "_id" : { "optionAnswer" : 5, "id" : "9ca01568e8dbb247" }, "count" : 1 }
You can group on $survey_answers.id to bring it into projection.
The projection is what you're missing in your query -
> db.survey.aggregate({$unwind: "$survey_answers"},{$group: { _id: { 'optionAnswer': "$survey_answers.option_answer", 'id':'$survey_answers.id'}, count: { $sum: 1}}}, {$project : {answer: '$_id.optionAnswer', id: '$_id.id', count: '$count', _id:0}})
{ "answer" : 0, "id" : "9ca01568e8dbb247", "count" : 1 }
{ "answer" : 3, "id" : "ba37125ec32b2a99", "count" : 2 }
{ "answer" : 5, "id" : "9ca01568e8dbb247", "count" : 1 }
Further you can add a group on id and add results to a set. And your final query would be -
db.survey.aggregate(
{$unwind: "$survey_answers"},
{$group: {
_id: { 'optionAnswer': "$survey_answers.option_answer", 'id':'$survey_answers.id'},
count: { $sum: 1}
}},
{$project : {
answer: '$_id.optionAnswer',
id: '$_id.id',
count: '$count',
_id:0
}},
{$group: {
_id:{id:"$id"},
results: { $addToSet: {answer: "$answer", count: '$count'} }
}},
{$project : {
id: '$_id.id',
answer: '$results',
_id:0
}})
Hope this helps.
I have a document to Mongodb update array of objects like this:
{
"_id" : 5,
"quizzes" : [
{ "wk" : 1, "score" : 10 },
{ "wk" : 2, "score" : 8 },
{ "wk" : 5, "score" : 8 }
]
}
And I want to add new field in each object of quizzes array.
The expected result should be
{
"_id" : 5,
"quizzes" : [
{ "wk" : 1, "score" : 10, "day": 1 },
{ "wk" : 2, "score" : 8, "day": 2 },
{ "wk" : 5, "score" : 8, "day": 3 }
]
}
Any solution for this.
You can use Aggregation Framework:
db.col.aggregate([
{
$unwind: {
path: "$quizzes",
includeArrayIndex: "quizzes.day"
}
},
{
$group: {
_id: "$_id",
quizzes: {
$push: {
"score" : "$quizzes.score",
"wk" : "$quizzes.wk",
"day" : { $add: [ "$quizzes.day", 1 ] }
}
}
}
},
{
$out: "col"
}
])
To assign indexes to each element you can use $unwind with includeArrayIndex option. Those indexes start with 0 so we have to use $add to start with 1. Then you can group back by your _id property and use $out to save aggregation results to your collection.
I have the following records:
{ "_id" : 1, "c" : 120, "b" : [ { "f1" : 10 }, { "f1" : 10 } ] }
{ "_id" :2, "c" : 5, "b" : [ { "f1" : 10 }, { "f1" : 10 } ] }
I need the output this way:
{ "_id" : 1, 'total':140}
{ "_id" :2, 'total':25 }
where total = sum of value in 'c' with sum of values in f1 for same record.
When i unwind the field 'b' it creates two documents with same id and hence data is duplicated and when i sum it up, i get:
db.test2.aggregate([
{'$unwind':'$b'},
{'$project':{'total':{'$add':['$c','$b.f1']}}},
{'$group':{'_id':'$_id', 'total':{'$sum':'$total'}}}
])
outputs:
{ "_id" : 1, 'total':260}
{ "_id" :2, 'total':30 }
(not what i wanted, as it has added 120 and 5 again to total due to duplication during unwinding)
So i tried:
db.test2.aggregate([
{'$unwind':'$b'},
{'$group':{'_id':'$_id', 'c':{'$push': '$c'},'f1':{'$sum':'$b.f1'}}},
{'$project':{'total':{'$add':[{'$arrayElemAt':['$c',0]},'$f1']}}}
])
outputs:
{ "_id" : 1, 'total':140}
{ "_id" :2, 'total':25 }
( what i wanted)
Is there any other way to achieve this?
You can try below query. Sum operator to first calculate sum in array followed by add to calculate total with other field.
db.test2.aggregate([{
$project: {
total: {"$add":["$c", {"$sum":"$b.f1"}]}
}
}]
An alternative:
db.test2.aggregate([{
$project: {
_id: 0,
c: "$c",
b: {
$reduce: {
input: "$b.f1",
initialValue: 0,
in: {
$add: ["$$value", "$$this"]
}
}
}
}
},
{
$project: {
_id: 0,
total: {
$sum: ["$c", "$b"]
}
}
}
])
That would create result:
{
"total" : 140
}
{
"total" : 25
}
If you need the field _id then replace the _id: 0 in both $project to _id: 1
That would create this result:
{
"_id" : 1,
"total" : 140
}
{
"_id" : 2,
"total" : 25
}
This question already has answers here:
MongoDB - The argument to $size must be an Array, but was of type: EOO / missing
(3 answers)
Closed 5 years ago.
Are there computed fields in MongoDB?
In SQL I can do:
SELECT A+B AS C FROM MYTABLE WHERE C>10
Can I do something similar in MongoDB?
UPDATE
I did with projection:
db.segments.aggregate(
[
{
$project: {
"_id": 1,
numberOfRestrictions: { $size: "$Speed Restrictions" }
}
}
]
)
and it works.
Unfortunately, further pipelining does not:
db.segments.aggregate(
[
{
$project: {
"_id": 1,
numberOfRestrictions: { $size: "$Speed Restrictions" }
}
},
{
$match: {
"numberOfRestrictions": {
"$gt": 1
}
}
}
]
)
Latter causes error
The argument to $size must be an Array, but was of type: EOO
Yes. It is called aggregation pipelines. Specifically, you need to use a $project stage to create the C field, and then use a $match stage to find all documents which match the criterion.
Example
Let's create some documents first:
for( var i = 1; i <=10; i++){
db.agg.insert({a:i,b:i})
}
Which results in a collection looking like this:
> db.agg.find()
{ "_id" : ObjectId("56c1b5561a3b578f37a99d4d"), "a" : 1, "b" : 1 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d4e"), "a" : 2, "b" : 2 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d4f"), "a" : 3, "b" : 3 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d50"), "a" : 4, "b" : 4 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d51"), "a" : 5, "b" : 5 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d52"), "a" : 6, "b" : 6 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d53"), "a" : 7, "b" : 7 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d54"), "a" : 8, "b" : 8 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d55"), "a" : 9, "b" : 9 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d56"), "a" : 10, "b" : 10 }
Finding all documents for which C > 10
db.agg.aggregate([
// You need to include all fields you want to have
// in the resulting document within the $project stage
{ "$project":{ a:1, b:1, c:{ "$add": ["$a","$b"] }}},
{ "$match":{ c:{ "$gt":10 }}}
])
Returns the following result:
{ "_id" : ObjectId("56c1b5561a3b578f37a99d52"), "a" : 6, "b" : 6, "c" : 12 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d53"), "a" : 7, "b" : 7, "c" : 14 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d54"), "a" : 8, "b" : 8, "c" : 16 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d55"), "a" : 9, "b" : 9, "c" : 18 }
{ "_id" : ObjectId("56c1b5561a3b578f37a99d56"), "a" : 10, "b" : 10, "c" : 20 }
There is an operator called $expr that enables you to use aggregation framework operator inside the find() query.
For instance, the SQL query
SELECT A+B AS C FROM MYTABLE WHERE C>10
can be translated to a mongo query as
db.segments.find({
"$expr": {
"$gt": [
{ "$add": [ "$A", "$B" ] },
10
]
}
})
And for checking an array length it's similar
db.segments.find({
"$expr": {
"$gt": [
{ "$size": "$SpeedRestrictions" },
10
]
}
})
With the aggregation framework it's also possible to use $expr within a $match pipeline step:
db.segments.aggregate([
{ "$match": {
"$expr": {
{ "$gt": [
{ "$size": "$SpeedRestrictions" },
10
] }
}
} }
])
And if the $expr operator is not available, for backwards compatibility one can use $redact as
db.segments.aggregate([
{ "$redact": {
"$cond": [
{ "$gt": [
{ "$size": "$SpeedRestrictions" },
10
] },
"$$KEEP",
"$$PRUNE"
]
} }
])
The other approach is to use the $addFields pipeline operator for creating the computed fields and the $match operator for filtering documents based on that computed field:
db.collection.aggregate([
{ "$addFields": { "C": { "$add": [ "$A", "$B" ] } } },
{ "$match": { "C": { "$gt": 10 } } }
])
I have the following collection for messages:
{
"_id" : ObjectId("56214d5632001bae07a6e6b3"),
"sender_id" : 8,
"receiver_id" : 2,
"content" : "fdgfd",
"state" : 1,
"timestamp" : 1445023062899.0000000000000000
},
{
"_id" : ObjectId("56214d5c32001bae07a6e6b4"),
"sender_id" : 2,
"receiver_id" : 8,
"content" : "fasfa",
"state" : 1,
"timestamp" : 1445023068443.0000000000000000
},
{
"_id" : ObjectId("56214d8032001bae07a6e6b5"),
"sender_id" : 2,
"receiver_id" : 8,
"content" : "dfdsfds",
"state" : 1,
"timestamp" : 1445023104363.0000000000000000
},
{
"_id" : ObjectId("56214d8032001bae07a6e6b6"),
"sender_id" : 2,
"receiver_id" : 8,
"content" : "fdsf",
"state" : 1,
"timestamp" : 1445023104825.0000000000000000
},
{
"_id" : ObjectId("56214d8132001bae07a6e6b7"),
"sender_id" : 2,
"receiver_id" : 8,
"content" : "sfsdfs",
"state" : 1,
"timestamp" : 1445023105436.0000000000000000
},
{
"_id" : ObjectId("56214d8132001bae07a6e6b8"),
"sender_id" : 2,
"receiver_id" : 8,
"content" : "f",
"state" : 1,
"timestamp" : 1445023105963.0000000000000000
},
{
"_id" : ObjectId("56214d8432001bae07a6e6b9"),
"sender_id" : 2,
"receiver_id" : 8,
"content" : "qwqwqwq",
"state" : 1,
"timestamp" : 1445023108202.0000000000000000
},
{
"_id" : ObjectId("56214db032001bae07a6e6ba"),
"sender_id" : 9902,
"receiver_id" : 2,
"content" : "fsafa",
"state" : 1,
"timestamp" : 1445023152297.0000000000000000
}
I'm trying to get all unique users ids that had been messaging with user 2, along with the last content message. So the result should be:
[ { user: 8, lastContent: "qwqwqwq" }, { user: 9902, lastContent: "fsafa" } ]
By now, I have the following code:
db.getCollection('messenger').group({
keyf: function(doc) {
return { user: doc.user };
},
cond: {
$or : [
{ sender_id : 2 },
{ receiver_id : 2 }
]
},
reduce: function( curr, result ) {
result.user = (curr.sender_id == 2 ? curr.receiver_id : curr.sender_id);
result.content = curr.content;
},
initial: { } })
But I only get the last id. The result:
{
"0" : {
"user" : 9902.0000000000000000,
"content" : "fsafa"
} }
Can anyone help me with this?
You need to use the .aggregate() method. You need to reduce the size of documents in the pipeline using the $match operator which filter out all documents where the receiver_id is not equal to 2. After that you need to $sort your document by timestamp in descending order this will help us get the content of last message sent. Now comes the $group stage where you group your documents and use the $addToSet operator which returns array of distinct sender_id and distinct receiver_id and the $last operator to get the last message content. Now to get the user_ids we need union of distinct sender_id and receiver_id which we can get after $projection using the $setUnion operator.
db.messenger.aggregate([
{ "$match": {
"$or": [
{ "sender_id": 2 },
{ "receiver_id": 2 }
]
}},
{ "$sort": { "timestamp": 1 } },
{ "$group": {
"_id": null,
"receiver_id": {
"$addToSet": { "$receiver_id" }
},
"sender_id": {
"$addToSet": { "$sender_id" }
},
"lastContent": { "$last": "$content" }
}},
{ "$project": {
"_id": 0,
"lastContent": 1,
"user_ids": {
"$setUnion": [
"$sender_id",
"$receiver_id"
]
}
}}
])
Which returns:
{ "lastContent" : "fsafa", "user_ids" : [ 9902, 2, 8 ] }
Now if what you want is distinct user alongside their last content message with user 2 then here it is:
db.messenger.aggregate([
{ "$match": {
"$or": [
{ "sender_id": 2 },
{ "receiver_id": 2 }
]
}},
{ "$sort": { "timestamp": 1 } },
{ "$group": {
"_id": {
"sender": "$sender_id",
"receiver": "$receiver_id"
},
"lastContent": {
"$last": "$content"
},
"timestamp": { "$last": "$timestamp" },
"sender": { "$addToSet": "$sender_id" },
"receiver": { "$addToSet": "$receiver_id" }
}},
{ "$project": {
"_id": 0,
"user": {
"$setDifference": [
{ "$setUnion": [ "$sender", "$receiver" ] },
[ 2 ]
]
},
"lastContent": 1,
"timestamp": 1
}},
{ "$unwind": "$user" },
{ "$sort": { "timestamp": 1 } },
{ "$group": {
"_id": "$user",
"lastContent": { "$last": "$lastContent" }
} }
])
Which yields:
{ "_id" : 9902, "lastContent" : "fsafa" }
{ "_id" : 8, "lastContent" : "qwqwqwq" }