Related
Hello I heard that mongoDB was very good at aggregating data compared to, for example SQL server.
I tried to translate an SQL query to a mongoDB query but it's a complete failure :
For SQL server we had like 4 minutes which is manageable and now we top at 34 minutes for 128 days using this request in mongoDB :
function(thresholdObs, days)
{
var name = "period-" + days + "j"
var startDate = new Date('2020', '00', '01');
var endDate = new Date(startDate.getTime() + 1000 * 60 * 60 * 24 *[days]);
db.getCollection('measures').aggregate([
{
$match: {
$and: [
{
measureDate: {
$gte: startDate
}
},
{
measureDate: {
$lte: endDate
}
}
]
}
},
{
$addFields: {
multiplyIahobs: {
$cond: [
{$or: [
{$eq: ["$averageIAH", null]},
{$eq: ["$obs", null]}
]},
0,
{ $multiply: ["$averageIAH", "$obs"] }
]
},
multiplyPressionobs: {
$cond: [
{$or: [
{$eq: ["$averagePressure", null]},
{$eq: ["$obs", null]}
]},
0,
{ $multiply: ["$averagePressure", "$obs"] }
]
},
multiplyFuitesobs: {
$cond: [
{$or: [
{$eq: ["$averageLeakage", null]},
{$eq: ["$obs", null]}
]},
0,
{ $multiply: ["$averageLeakage", "$obs"] }
]
},
multiplyinspiratoryPressureobs: {
$cond: [
{$or: [
{$eq: ["$inspiratoryPressure", null]},
{$eq: ["$obs", null]}
]},
0,
{ $multiply: ["$inspiratoryPressure", "$obs"] }
]
},
multiplyexpiratoryPressureobs: {
$cond: [
{$or: [
{$eq: ["$expiratoryPressure", null]},
{$eq: ["$obs", null]}
]},
0,
{ $multiply: ["$expiratoryPressure", "$obs"] }
]
},
enoughDays: {
$cond: [ { $gte: ["$obs", thresholdObs ] }, 1, 0]
},
missingDays: {
$cond: [ { $lt: ["$obs", thresholdObs ] }, 1, 0]
},
daysWithoutUsage: {
$cond: [ { $eq: ["$obs", 0 ] }, 1, 0]
},
daysWithData: { $sum: 1 }
}
},
{
$group: {
_id: "$deviceID",
obsUsage: { $avg: "$obs" },
enoughDays: { $sum: "$enoughDays" },
missingDays: { $sum: "$missingDays" },
daysWithoutUsage: { $sum: "$daysWithoutUsage" },
daysWithData: { $sum: "$daysWithData" },
sumobs: { $sum: "$obs" },
sumMultiplyIahobs: { $sum: "$multiplyIahobs" },
sumMultiplyPressureObs: { $sum: "$multiplyPressionobs" },
sumMultiplyLeakageObs: { $sum: "$multiplyFuitesobs" },
sumMultiplyinspiratoryPressureobs: { $sum: "$multiplyinspiratoryPressureobs" },
sumMultiplyexpiratoryPressureobs: { $sum: "$multiplyexpiratoryPressureobs" },
}
},
{
$addFields: {
fullObs: { $divide: [ "$sumobs", days ] },
daysWithoutData: { $subtract: [ days, "$daysWithData" ]},
averageIAH: { $cond: [ { $eq: [ "$sumobs", 0 ] }, 0, { $divide: ["$sumMultiplyIahobs", "$sumobs"] } ] },
averagePressure: { $cond: [ { $eq: [ "$sumobs", 0 ] }, 0, { $divide: ["$sumMultiplyPressureObs", "$sumobs"] } ] },
averageLeakage: { $cond: [ { $eq: [ "$sumobs", 0 ] }, 0, { $divide: ["$sumMultiplyLeakageObs", "$sumobs"] } ] },
inspiratoryPressure: { $cond: [ { $eq: [ "$sumobs", 0 ] }, 0, { $divide: ["$sumMultiplyinspiratoryPressureobs", "$sumobs"] } ] },
expiratoryPressure: { $cond: [ { $eq: [ "$sumobs", 0 ] }, 0, { $divide: ["$sumMultiplyexpiratoryPressureobs", "$sumobs"] } ] },
}
},
{
$project: {
_id: 1,
[name] : {
obsUsage: "$obsUsage",
enoughDays: "$enoughDays",
missingDays: "$missingDays",
daysWithoutUsage: "$daysWithoutUsage",
daysWithData: "$daysWithData",
sumobs: "$sumobs",
fullObs: "$fullObs",
daysWithoutData: "$daysWithoutData",
averageIAH: "$averageIAH",
averagePressure: "$averagePressure",
averageLeakage: "$averageLeakage",
inspiratoryPressure: "$inspiratoryPressure",
expiratoryPressure: "$expiratoryPressure"
}
}
},
{
$merge: {
into: "periods",
on: "_id",
whenMatched: "merge",
whenNotMatched: "insert"
}
}
],{allowDiskUse: true})
}
Before throwing out the baby (mongoDB) with the bathwater (the query). I came here asking if my understanding on how to write a good query is off. Should I try to alter this query to make it work under 4 minutes ? Is it possible ? How ?
NB : measures collection contains 374.670.449 documents.
Sample documents :
{
_id: ObjectId('6127a15fef44a9ed52a5bf62'),
deviceId: 5,
measureDateAdded: ISODate('2013-03-15T10:30:35.753Z'),
measureDate: ISODate('2012-03-20T06:00:00.000Z'),
obs: 20,
averageIAH: NumberDecimal('5.50'),
averagePressure: NumberDecimal('6.30'),
averageLeakage: NumberDecimal('13.00'),
inspiratoryPressure: null,
expiratoryPressure: null
},
{
_id: ObjectId('6127a15fef44a9ed52a5bfc6'),
deviceId: 5,
measureDateAdded: ISODate('2013-03-15T10:30:40.063Z'),
measureDate: ISODate('2012-06-28T05:00:00.000Z'),
obs: 197,
averageIAH: NumberDecimal('5.30'),
averagePressure: NumberDecimal('6.90'),
averageLeakage: NumberDecimal('15.00'),
inspiratoryPressure: null,
expiratoryPressure: null
},
{
_id: ObjectId('6127aa2bef44a9ed52a0922a'),
deviceId: 367959,
measureDateAdded: ISODate('2019-01-19T14:13:19.620Z'),
measureDate: ISODate('2019-01-16T11:00:00.000Z'),
obs: 375,
averageIAH: NumberDecimal('2.00'),
averagePressure: NumberDecimal('9.60'),
averageLeakage: NumberDecimal('6.00'),
inspiratoryPressure: null,
expiratoryPressure: null
}
What cost the most seems to be the group with deviceId 4+minutes
I tried to run a query on huge data set but after lookup stage(stage 3) it took too much amount of time (approximately 12 hour) . How do I improve query performance . I am using single node with 16 cpu, 64 gb RAM with mongodb 4.0 version .
// Pipeline
[
// Stage 1
{
$match: {
"ID" : "XXXXX"
}
},
// Stage 2
{
$redact: {
$cond: [ { $or: [ { $eq: [ "$Masterid", "" ] }, { $or: [ { $eq: [ "$VKORG", "" ] }, { $or: [ { $eq: [ "$VTWEG", "" ] },{ $eq: [ "$UDATE", "" ] } ] } ] } ] } ] } ] } ] } ] }, "$$PRUNE", "$$KEEP" ]
}
},
// Stage 3
{
$lookup: {
from: "BU07-02", let: { raw_Masterid: "$Masterid", raw_VKORG: "$VKORG", raw_VTWEG: "$VTWEG", raw_SPART: "$SPART", raw_KUNNR: "$KUNNR",
raw_VBELN: "$VBELN", raw_POSNR: "$POSNR", raw_UDATE: "$UDATE", raw_KNUMH: "$KNUMH", raw_AUART: "$AUART", raw_FKART: "$FKART",
}, pipeline: [ { $match: { $expr: { $and: [ { $eq: [ "$$raw_Masterid", "$Masterid" ] },
{ $and: [ { $eq: [ "$$raw_VKORG", "$VKORG" ] }, { $and: [ { $eq: [ "$$raw_VTWEG", "$VTWEG" ] }, { $and: [ { $eq: [ "$$raw_SPART", "$SPART" ] },
{ $and: [ { $eq: [ "$$raw_KUNNR", "$KUNNR" ] }, { $and: [ { $eq: [ "$$raw_VBELN", "$AUBEL" ] }, { $and: [ { $eq: [ "$$raw_POSNR", "$AUPOS" ] },
{ $lte: [ "$$raw_UDATE", "$UDATE" ] } ] } ] } ] } ] } ] } ] } ] } } } ], as: "BU07-02"
}
},
// Stage 4
{
$unwind: {
path : "$BU07-02",
}
},
// Stage 5
{
$redact: {
$cond: [ { $and: [ { $eq: [ "$Masterid", "$BU07-02.Masterid" ] }, { $and: [ { $eq: [ "$VKORG", "$BU07-02.VKORG" ] }, { $and: [ { $eq:
[ "$VTWEG", "$BU07-02.VTWEG" ] }, { $and: [ { $eq: [ "$SPART", "$BU07-02.SPART" ] }, { $and: [ { $eq: [ "$KUNNR", "$BU07-02.KUNNR" ] },
{ $and: [ { $ne: [ "$BU07-02.FKART", "ZBS1" ] },
{ $and: [ { $ne: [ "$BU07-02.FKART", "ZBS2" ] }, { $and: [ { $ne: [ "$VBTYP", "U" ] }, { $and: [ { $ne: [ "$BU07-02.VBTYP", "U" ] },
{ $ne: [ "$BASE_ENTITY", true ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] }, "$$KEEP",
"$$PRUNE" ] }
},
],
);
Neither the pipeline form of $lookup nor the $expr operator are able to make efficient use of an index. This means that the $lookup stage is essentially reading every document in the BU07-02 collection from disk in order to evaluate the all of the criteria.
You might try selecting the criteria that you expect will usually return the fewest documents and use it in a foreignField form of $lookup, making sure there is an index on that field in the BU07-02, and then apply the remaining criteria use a $match or $filter.
I have ListPrice field in collection on that price Ii have to calculate min, max, median, avg of all data, active standardStatus , sold standardStatus.
I have tried to calculate using aggregation and for loop but it won't work
db.collection('selected_properties').aggregate([
{ presentation_id : ObjectId(req.body.presentation_id),
checked_status : true}
},
{
$lookup : { from :'properties', localField : 'property_id', foreignField : '_id', as : 'property_info'}
},
{
$unwind : {path : '$property_info', preserveNullAndEmptyArrays : true}
},
{
$sort : {'property_info.ListPrice' : 1}
},
{
$group:{
_id: "$user_id",
minActiveListPrice: { $min: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice','' ] } },
maxActiveListPrice: { $max: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice',0 ] } },
avgActiveListPrice: { $avg: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice','' ] } },
medianActiveListprice: { $push: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice','' ] } },
minsoldListPrice: { $min: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "S" ]},
'$property_info.ListPrice','' ] } },
maxsoldListPrice: { $max: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "S" ]},
'$property_info.ListPrice',0 ] } },
avgsoldListPrice: { $avg: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "S" ]},
'$property_info.ListPrice','' ] } },
avgPrice: { $avg: "$property_info.ListPrice" },
maxPrice: { $max: "$property_info.ListPrice" },
minPrice: { $min: "$property_info.ListPrice" },
}
median: { $push: "$property_info.ListPrice"}
}
},
db.collection('selected_properties').aggregate([
{
$match : { presentation_id : ObjectId(req.body.presentation_id),
checked_status : true}
},
{
$lookup : { from :'properties', localField : 'property_id',
foreignField : '_id', as : 'property_info'}
},
{
$unwind : {path : '$property_info', preserveNullAndEmptyArrays : true}
},
{
$sort : {'property_info.ListPrice' : 1}
},
{
$group:
{
_id: "$user_id",
minActiveListPrice: { $min: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice','' ] } },
maxActiveListPrice: { $max: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice',0 ] } },
avgActiveListPrice: { $avg: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice','' ] } },
medianActiveListprice: { $push: { $cond: [ {
$eq: [ "$property_info.StandardStatus", "A" ]},
'$property_info.ListPrice',null ] } },
avgPrice: { $avg: "$property_info.ListPrice" },
maxPrice: { $max: "$property_info.ListPrice" },
minPrice: { $min: "$property_info.ListPrice" },
median: { $push: "$property_info.ListPrice"}
}
},
{ "$project": {
"minActiveListPrice":1,
"maxActiveListPrice":1,
"avgActiveListPrice":1,
"avgPrice": 1,
"maxPrice": 1,
"minPrice": 1,
"medianActiveListpricevalue": {
$let: {
vars: {
arr: { $filter: {
input: "$medianActiveListprice",
as: "aa",
cond: {$ne:["$$aa",null]}
}},
},
in: { "$cond": {
"if": {
"$eq": [{$mod: [ {$size:"$$arr"}, 2 ]}, 0]
},
"then": {
$avg:[
{ $arrayElemAt: [ "$$arr", {$subtract:[{$divide: [ {$size:"$$arr"}, 2 ]},1]}]},
{ $arrayElemAt: [ "$$arr", {$divide: [ {$size:"$$arr"}, 2 ]}]}
]
},
"else": {
$arrayElemAt: [ "$$arr",{$floor : {$divide: [ {$size:"$$arr"}, 2 ]}}]
}
}}
}
},
"medianvalue":{ "$cond": {
"if": {
"$eq": [{$mod: [ {$size:"$median"}, 2 ]}, 0]
}
"then": {
$avg:[
{ $arrayElemAt: [ "$median", {$subtract:[{$divide: [ {$size:"$median"}, 2 ]},1]}]},
{ $arrayElemAt: [ "$median", {$divide: [ {$size:"$median"}, 2 ]}]}
]
},
"else": {
$arrayElemAt: [ "$median",{$floor : {$divide: [ {$size:"$median"}, 2 ]}}]
}
}}
} }
])
Consider this pipeline.
let Pipeline = [
{ $match: {
}},
{ $group: {
workHours: { $sum: { $divide: [ { $subtract: ['$workTime.end', '$workTime.start'] } , { $multiply: [3600, 1000] }] }},
breakHours: { $sum: { $divide: [ { $subtract: ['$breakTime.end', '$breakTime.start'] } , { $multiply: [3600, 1000] }] }},
weekEndHours: {
$cond: [
{ $or : [ { $eq : [{ $dayOfWeek : '$workTime.start' }, 1 ] }, {$eq : [{ $dayOfWeek : '$workTime.start' }, 7 ] }] },
{ $sum: { $divide: [ { $subtract: ['$workTime.end', '$workTime.start'] } , { $multiply: [3600, 1000] }] }},
0
]
}
}},
];
I'm trying to get sum of work hours, break hours and weekend hours. Work hours and break hours works fine. Now I want to use condition inside weekend hours which is if the day of the date is either 1 or 7 only then count the total hours in the key.
Then it is showing me the error 'The $cond accumulator is a unary operator'. I don't know what is wrong with my group query.
Got Solution and also extended pipeline conditions.
let Pipeline = [
{ $match: {
}},
{ $group: {
_id: '$memberId',
workHours: { $sum: { $divide: [ { $subtract: ['$workTime.end', '$workTime.start'] } , { $multiply: [3600, 1000] }] }},
breakHours: { $sum: { $divide: [ { $subtract: ['$breakTime.end', '$breakTime.start'] } , { $multiply: [3600, 1000] }] }},
weekEndHours: {
$sum: {
$cond: [
{ $or : [ { $eq : [{ $dayOfWeek : { date: '$workTime.start', timezone: timezoneOffset}}, 1 ] }, {$eq : [{ $dayOfWeek : { date: '$workTime.start', timezone: timezoneOffset}}, 7 ] }] },
{ $divide: [ { $subtract: ['$workTime.end', '$workTime.start'] } , { $multiply: [3600, 1000] }] },
0
]
}
},
weekDayHours: {
$sum: {
$cond: [
{ $or : [
{ $eq : [{ $dayOfWeek : { date: '$workTime.start', timezone: timezoneOffset}}, 2 ] },
{ $eq : [{ $dayOfWeek : { date: '$workTime.start', timezone: timezoneOffset}}, 3 ] },
{ $eq : [{ $dayOfWeek : { date: '$workTime.start', timezone: timezoneOffset}}, 4 ] },
{ $eq : [{ $dayOfWeek : { date: '$workTime.start', timezone: timezoneOffset}}, 5 ] },
{ $eq : [{ $dayOfWeek : { date: '$workTime.start', timezone: timezoneOffset}}, 6 ] }
]
},
{ $divide: [ { $subtract: ['$workTime.end', '$workTime.start'] } , { $multiply: [3600, 1000] }] },
0
]
}
},
}},
{ $addFields:{
totalWorkedHours: { $subtract: [ '$workHours', '$breakHours'] }
}}
];
My collection will look like this,
{
"_id" : ObjectId("591c5971240033283736860a"),
"status" : "Done",
"createdDate" : ISODate("2017-05-17T14:09:20.653Z")
"communications" : [
{
"communicationUUID" : "df07948e-4a14-468e-beb1-db55ff72b215",
"communicationType" : "CALL",
"recipientId" : 12345,
"createdDate" : ISODate("2017-05-18T14:09:20.653Z")
"callResponse" : {
"Status" : "completed",
"id" : "dsd45554545ds92a9bd2c12e0e6436d",
}
}
]}
{
"_id" : ObjectId("45sdsd59124003345121450a"),
"status" : "ToDo",
"createdDate" : ISODate("2017-05-17T14:09:20.653Z")
"communications" : [
{
"communicationUUID" : "45sds55-4a14-468e-beb1-db55ff72b215",
"communicationType" : "CALL",
"recipientId" : 1234,
"createdDate" : ISODate("2017-05-18T14:09:20.653Z")
"callResponse" : {
"Status" : "completed",
"id" : "84fe862f1924455dsds5556436d",
}
}
]}
Currently I am writing two aggregate query to achieve my requirement and my query will be below
db.collection.aggregate(
{ $project: {
dayMonthYear: { $dateToString: { format: "%d/%m/%Y", date: "$createdDate" } },
status: 1,
}},
{ $group: {
_id: "$dayMonthYear",
Pending: { $sum: { $cond : [{ $eq : ["$status", "ToDo"]}, 1, 0]} },
InProgress: { $sum: { $cond : [{ $eq : ["$status", "InProgress"]}, 1, 0]} },
Done: { $sum: { $cond : [{ $eq : ["$status", "Done"]}, 1, 0]} },
Total: { $sum: 1 }
}}
My output will be,
{"_id" : "17/05/2017", "Pending" : 1.0, "InProgress" : 0.0, "Done" : 1.0, "Total" : 2.0 }
Using above query I can able to get count but I need to find the count based on communication Status too so I am writing one more query to achieve,
db.collection.aggregate(
{"$unwind":"$communications"},
{ $project: {
dayMonthYear: { $dateToString: { format: "%d/%m/%Y", date: "$createdDate" } },
communications: 1
}},
{ "$group": {
_id: "$dayMonthYear",
"total_call": { $sum: { $cond : [{ $or : [ { $eq: [ "$communications.callResponse.Status", "failed"] },
{ $eq: [ "$communications.callResponse.Status", "busy"] },
{ $eq: [ "$communications.callResponse.Status", "completed"] },
{ $eq: [ "$communications.callResponse.Status", "no-answer"] }
]}, 1, 0 ] }},
"engaged": { $addToSet: { $cond : [{ $eq : ["$communications.callResponse.Status", "completed"]},
"$communications.recipientId", "null" ]} },
"not_engaged": { $addToSet: { $cond: [{ $or : [ { $eq: [ "$communications.callResponse.Status", "failed"] },
{ $eq: [ "$communications.callResponse.Status", "busy"] },
{ $eq: [ "$communications.callResponse.Status", "no-answer"] } ]},
"$communications.recipientId", "null" ] }}
}},
{ "$project": {
"_id": 1,
"total_call": 1,
"engaged": { "$setDifference": [ "$ngaged", ["null"] ] },
"not_engaged": { "$setDifference": [ "$not_engaged", ["null"] ] },
}},
{ "$project": {
"total_call": 1,
"engaged": { "$size": "$engaged" },
"not_engaged": { "$size": { "$setDifference": [ "$not_engaged", "$engaged" ] }},
}})
My output will be,
{"_id" : "18/05/2017", "total_call" : 2.0, "engaged" : 2, "not_engaged" : 0}
Using above query I can able to get count but I want to achieve it in single query
I am looking for output like
{"_id":"17/05/2017", "Pending" : 1.0, "InProgress" : 0.0, "Done" : 1.0, "total_call" : 0, "engaged" : 0, "not_engaged" : 0}
{"_id":"18/05/2017", "Pending" : 0.0, "InProgress" : 0.0, "Done" : 0.0, "total_call" : 2, "engaged" : 2, "not_engaged" : 0}
Can anyone suggest or provide me good way to get above result.
You can use $concatArrays to merge the status& createdDate documents followed by $group to count the occurrences.
db.collection.aggregate([
{
"$project": {
"statusandcreateddate": {
"$concatArrays": [
[
{
"status": "$status",
"createdDate": "$createdDate"
}
],
{
"$map": {
"input": "$communications",
"as": "l",
"in": {
"status": "$$l.callResponse.Status",
"createdDate": "$$l.createdDate"
}
}
}
]
}
}
},
{
"$unwind": "$statusandcreateddate"
},
{
"$group": {
"_id": {
"$dateToString": {
"format": "%d/%m/%Y",
"date": "$statusandcreateddate.createdDate"
}
},
"total_call": {
"$sum": {
"$cond": [
{
"$or": [
{
"$eq": [
"$statusandcreateddate.status",
"failed"
]
},
{
"$eq": [
"$statusandcreateddate.status",
"busy"
]
},
{
"$eq": [
"$statusandcreateddate.status",
"completed"
]
},
{
"$eq": [
"$statusandcreateddate.status",
"no-answer"
]
}
]
},
1,
0
]
}
},
"engaged": {
"$sum": {
"$cond": [
{
"$eq": [
"$statusandcreateddate.status",
"completed"
]
},
1,
0
]
}
},
"not_engaged": {
"$sum": {
"$cond": [
{
"$or": [
{
"$eq": [
"$statusandcreateddate.status",
"failed"
]
},
{
"$eq": [
"$statusandcreateddate.status",
"busy"
]
},
{
"$eq": [
"$statusandcreateddate.status",
"no-answer"
]
}
]
},
1,
0
]
}
},
"Pending": {
"$sum": {
"$cond": [
{
"$eq": [
"$statusandcreateddate.status",
"ToDo"
]
},
1,
0
]
}
},
"InProgress": {
"$sum": {
"$cond": [
{
"$eq": [
"$statusandcreateddate.status",
"InProgress"
]
},
1,
0
]
}
},
"Done": {
"$sum": {
"$cond": [
{
"$eq": [
"$statusandcreateddate.status",
"Done"
]
},
1,
0
]
}
}
}
}
])