can't convert from BSON type missing to Date - mongodb

I have a mongoDB aggregation which was working great when I tried it on MongoDB 4.0 but now I need to use it on MongoDB 3.4 and it's not working, I can't find why. All I suppose it that the bug occurs in the $project stage.
Here's the aggregation query :
{
"aggregate": true,
"pipeline": [
{
"$match": {
"field": {
"$exists": true
},
"field.objects": {
"$exists": true,
"$ne": []
},
"created_at": {
"$gte": {
"sec": 1551398400,
"usec": 0
},
"$lte": {
"sec": 1554076799,
"usec": 0
}
}
}
},
{
"$unwind": "$field.objects"
},
{
"$lookup": {
"from": "Object",
"localField": "field.objects.id",
"foreignField": "_id",
"as": "objects"
}
},
{
"$match": { // Some match clauses here
}
},
{
"$group": {
"_id": "$_id"
}
},
{
"$project": {
"year": {
"$year": "$created_at"
},
"month": {
"$month": "$created_at"
}
}
},
{
"$group": {
"_id": {
"date": {
"$concat": [
{
"$substr": [
"$year",
0,
4
]
},
"-",
{
"$cond": [
{
"$lte": [
"$month",
9
]
},
{
"$concat": [
"0",
{
"$substr": [
"$month",
0,
2
]
}
]
},
{
"$substr": [
"$month",
0,
2
]
}
]
},
"-01"
]
}
},
"total": {
"$sum": 1
}
}
}
],
"options": {
"cursor": true
},
"db": "db",
"collection": "Collection"
}
So, with MongoDB 4.0, I get the right result, but MongoDB 3.4 throws the following : can't convert from BSON type missing to Date. I looked a bit at changelogs but I didn't find anything.

Related

(MongoDB) Combining Union and Intersection in the same pipeline

I have the following aggregation pipeline running in the latest version of mongoDB and pymongo:
[
{
"$project": {
"union": {
"$setUnion": [
"$query_a",
"$query_b"
]
}
}
},
{
"$unwind": "$union"
},
{
"$group": {
"_id": "$union.ID",
"date_a": {
"$addToSet": "$union.date_a"
},
"date_b": {
"$addToSet": "$union.date_b"
}
}
},
{
"$unwind": "$date_a"
},
{
"$unwind": "$date_b"
},
{
"$project": {
"_id": 1,
"date_a": "$date_a",
"date_b": "date_b",
"diff": {
"$subtract": [
{
"$toInt": "$date_b"
},
{
"$toInt": "$date_a"
}
]
}
}
},
{
"$match": {
"diff": {
"$gt": 0,
"$lte": 20
}
}
},
]
This gives the union of the 2 pipelines query_a and query_b. After this union I want to get an intersection on ID with the pipeline query_c: (query_a UNION query_b) INTERSECTION query_c.
For this playground example the desired output would be:
[
{
"ID": "c80ea2cb-3272-77ae-8f46-d95de600c5bf",
},
{
"ID": "cdbcc129-548a-9d51-895a-1538200664e6",
}
]
You could change and augment your pipeline a little to get your desired output.
db.collection.aggregate([
{
"$project": {
"union": {
// do the intersection here
"$filter": {
"input": {
"$setUnion": [
"$query_a",
"$query_b"
]
},
"as": "elem",
"cond": {
// only take IDs in query_c
"$in": ["$$elem.ID", "$query_c.ID"]
}
}
}
}
},
{
"$unwind": "$union"
},
{
"$group": {
"_id": "$union.ID",
"date_a": {
"$addToSet": "$union.date_a"
},
"date_b": {
"$addToSet": "$union.date_b"
}
}
},
{
"$unwind": "$date_a"
},
{
"$unwind": "$date_b"
},
{
"$project": {
"diff": {
"$subtract": [
{
"$toInt": "$date_b"
},
{
"$toInt": "$date_a"
}
]
}
}
},
{
"$match": {
"diff": {
"$gt": 0,
"$lte": 20
}
}
},
{ // get unique _id's
"$group": {
"_id": "$_id"
}
},
{ // rename _id to ID
"$project": {
"_id": 0,
"ID": "$_id"
}
}
])
Try it on mongoplayground.net.
You can do it with:
Updating first $project stage to also project an array of IDs from query_c.
Using $set as a second stage where you would filter out all items from the union of query_a and query_b, that does not have ID that's in query_c.
You can do it like this:
{
"$project": {
"union": {
"$setUnion": [
"$query_a",
"$query_b"
]
},
"query_c": {
"$map": {
"input": "$query_c",
"in": "$$this.ID"
}
}
}
},
{
"$set": {
"union": {
"$filter": {
"input": "$union",
"cond": {
"$in": [
"$$this.ID",
"$query_c"
]
}
}
}
}
},
The rest of your Aggregation pipeline can remain the same.
Working example

Use $addToSet condition vise in mongodb

I have below mongodb query, in which i am using $addToSet, Now i want to use it condition vise.
Worksheet.aggregate([
{
"$group": {
"_id": null,
"todayBilling": {
"$sum": {
"$cond": [{ "$and" : [ { "$eq": [ "$isBilling", true] }, { $eq: [ "$date",new Date(moment().format('l'))]}] },"$hours",0 ]
}
},
"todayProjects": { "$addToSet": "$projectId" }
},
},
{ "$addFields": { "todayProjects": { "$size": "$todayProjects" }}},
{
"$lookup":{
"from": "projects",
"let": {},
"pipeline": [
{
"$group": { "_id": null, "count": { "$sum": 1 } }
}
],
"as": "totalProjects"
}
},
{'$unwind':'$totalProjects'}
])
Now, I want to get the count of todayProjects field if got result today date vise. means where "todayProjects": { "$addToSet": "$projectId" } exists, i want to use $cond with below condition:
{ $eq: [ "$date",new Date(moment().format('l'))]}

Aggregate Unexpected token :

Below aggregate mongodb query gives
Unexpected token : error
db.getCollection("products_data").aggregate(
{
"$unwind": {
"path": "$color",
"preserveNullAndEmptyArrays": true
}
},
{
"$match":{
"country":"UK",
"$or":[{
"$and":[
"$or":
[{
"$and":[
{"status":"drafted"},
{"color":{$in:["blue"]}}
]},
{"$and":[
{"status1":"complete"},
{"status2":{$nin:["n/a","drafted","complete"]}},
{"color":{$in:["green"]}}
]}
]
]
},{
"$and":[
"$or":
[
{ "$and":[
{"status":"drafted"},
{"color":{$in:["blue"]}}
]},
{"$and":[
{"status1":"complete"},
{"status2":{$nin:["n/a","drafted","complete"]}},
{"color":{$in:["green"]}}
]}
]
]
}
]
}
},
{
"$group":{
"_id":"$field",
"products":{$sum: 1},
"bid":{"$push":"$product_id"}
}
},
{
"$project":{
"field":"$_id",
"products":"$products",
"bid":1,
"_id":0
}
}
);
To fetch the aggregate count for the given specified condition.
Correct syntax to use aggregate and it's stages in pipeline
db.getCollection("products_data").aggregate([
{ "$unwind": { "path": "$color", "preserveNullAndEmptyArrays": true }},
{ "$match": {
"country": "UK",
"$or": [
{
"$and": [
{
"$or": [
{ "$and": [{ "status": "drafted" }, { "color": { "$in": ["blue"] }}] },
{ "$and": [{ "status1": "complete" }, { "status2": { "$nin": ["n/a", "drafted", "complete"] }}, { "color": { "$in": ["green"] }}]}
]
}
]
},
{
"$and": [
{
"$or": [
{ "$and": [{ "status": "drafted" }, { "color": { "$in": ["blue"] }}] },
{ "$and": [{ "status1": "complete" }, { "status2": { "$nin": ["n/a", "drafted", "complete"] }}, { "color": { "$in": ["green"] }}] }
]
}
]
}
]
}},
{ "$group": {
"_id": "$field",
"products": { "$sum": 1 },
"bid": { "$push": "$product_id" }
}},
{ "$project": { "field": "$_id", "products": "$products", "bid": 1, "_id": 0 }}
])

Mongo request slow

I'm trying Mongo with a million enties. My request is really slow.
Do you have any ideas to optimize it ?
db.financial_transaction.runCommand({
"aggregate": "financial_transaction",
"pipeline": [ {
"$match": {
"transaction_type": { "$in": [ 1, 2 ] },
"created_at": { "$gte": new ISODate("2016-03-13T00:00:00+01:00"), "$lte": new ISODate("2017-12-13T23:59:00+01:00") },
"type": { "$in": [ "A", "C", "E" ] },
"sid": { "$in": [ 1, 3, 7, 9, 11, 13 ] },
"context": { "$in": [ "CL", "RE" ] } }
}, {
"$group": { "_id": { "paymentType": "$payment_type",
"paymentMethod": "$payment_method",
"responseCode": "$response_code",
"reasonCode": "$reason_code"
},
"count": { "$sum": 1 }, "total_amount": { "$sum": "$requested_amount" } } }, { "$sort": { "count": -1 } } ]
});
Indexes:
_id_
idx_context
idx_payment_method
idx_response_code
idx_reason_code
idx_created_at
idx_transaction_type
idc_payment_method_created_at_transaction_type_origin_auth_system
idx_created_at_context_transaction_type
idx_updated_at
I made a gist with the explain result: https://gist.github.com/sanchobouillant/bd59403242ebb1ec45582dff74b457a2

How to get count of multiple fields based on value in mongodb?

Collection exists as below:
[
{"currentLocation": "Chennai", "baseLocation": "Bengaluru"},
{"currentLocation": "Chennai", "baseLocation": "Bengaluru"},
{"currentLocation": "Delhi", "baseLocation": "Bengaluru"},
{"currentLocation": "Chennai", "baseLocation": "Chennai"}
]
Expected Output:
[
{"city": "Chennai", "currentLocationCount": 3, "baseLocationCount": 1},
{"city": "Bengaluru", "currentLocationCount": 0, "baseLocationCount": 3},
{"city": "Delhi", "currentLocationCount": 1, "baseLocationCount": 0}
]
What I have tried is:
db.getCollection('users').aggregate([{
$group: {
"_id": "$baselocation",
baseLocationCount: {
$sum: 1
}
},
}, {
$project: {
"_id": 0,
"city": "$_id",
"baseLocationCount": 1
}
}])
Got result as:
[
{"city": "Chennai", "baseLocationCount": 1},
{"city": "Bengaluru", "baseLocationCount": "3"}
]
I'm not familiar with mongo, so any help?
MongoDB Version - 3.4
Neil Lunn and myself had a lovely argument over this topic the other day which you can read all about here: Group by day with Multiple Date Fields.
Here are two solutions to your precise problem.
The first one uses the $facet stage. Bear in mind, though, that it may not be suitable for large collections because $facet produces a single (potentially huge) document that might be bigger than the current MongoDB document size limit of 16MB (which only applies to the result document and wouldn't be a problem during pipeline processing anyway):
collection.aggregate(
{
$facet:
{
"current":
[
{
$group:
{
"_id": "$currentLocation",
"currentLocationCount": { $sum: 1 }
}
}
],
"base":
[
{
$group:
{
"_id": "$baseLocation",
"baseLocationCount": { $sum: 1 }
}
}
]
}
},
{ $project: { "result": { $setUnion: [ "$current", "$base" ] } } }, // merge results into new array
{ $unwind: "$result" }, // unwind array into individual documents
{ $replaceRoot: { newRoot: "$result" } }, // get rid of the additional field level
{ $group: { "_id": "$_id", "currentLocationCount": { $sum: "$currentLocationCount" }, "baseLocationCount": { $sum: "$baseLocationCount" } } }, // group into final result)
{ $project: { "_id": 0, "city": "$_id", "currentLocationCount": 1, "baseLocationCount": 1 } } // group into final result
)
The second one works based on the $map stage instead:
collection.aggregate(
{
"$project": {
"city": {
"$map": {
"input": [ "current", "base" ],
"as": "type",
"in": {
"type": "$$type",
"name": {
"$cond": {
"if": { "$eq": [ "$$type", "current" ] },
"then": "$currentLocation",
"else": "$baseLocation"
}
}
}
}
}
}
},
{ "$unwind": "$city" },
{
"$group": {
"_id": "$city.name",
"currentLocationCount": {
"$sum": {
"$cond": {
"if": { "$eq": [ "$city.type", "current" ] },
"then": 1,
"else": 0
}
}
},
"baseLocationCount": {
"$sum": {
"$cond": {
"if": { "$eq": [ "$city.type", "base" ] },
"then": 1,
"else": 0
}
}
}
}
}
)