I'm trying to create an aggregation pipeline wherein I need to aggregate the difference between 2 fields, and hence I try to apply a convert operation on string fields in the pipeline.
the document structure is as follows :
{
page_start : (String),
page_end : (String),
references : (list of strings)
}
match_1_stage = { "$match":{
"references":{"$exists":True},
"page_start": {"$exists": True},
"page_end": {"$exists": True}
}
}
pageConversionStage = {
"$addFields": {
"convertedPageStart": { "$convert": { "input": "page_start", "to": "decimal", "onError": "-1", "onNull": "-1" } },
"convertedPageEnd": { "$convert": { "input": "page_end", "to": "decimal", "onError": "-1", "onNull": "-1" } }
}
}
match_2_stage = { "$match":{
"convertedPageStart": {"$ne": "-1"},
"convertedPageEnd": {"$ne": "-1"}
}
}
project_stage = { "$project":{
"convertedPageStart":1,
"convertedPageEnd":1,
"no_of_pages":{ "$subtract": ["convertedPageEnd","convertedPageStart"] }
}
}
match_3_stage = { "$match":{
"no_of_pages":{ "$lt":100000 },
"no_of_pages":{ "$gte":0 },
}
}
group_stage = { "$group":{
"_id": "no_of_pages",
"totalreferences": { "$sum": { "$size": "$references" } }
}
}
res_set = articles.aggregate([match_1_stage, pageConversionStage, match_2_stage, project_stage, match_3_stage, group_stage])
The error I get is Failed to optimize pipeline :: caused by :: cant $subtract astring from a string, whereas I have added a conversion stage in the pipeline before the other stages, and I checked that mongodb pipelines are run sequentially, so the conversion should be taking place before the subtraction, which does not seem to be happening here. What could be going wrong?
You have to use
"convertedPageStart": { "$convert": { "input": "$page_start", ... } },
"convertedPageEnd": { "$convert": { "input": "$page_end", ... } }
"no_of_pages":{ "$subtract": ["$convertedPageEnd","$convertedPageStart"] }
i.e. you missed the $
Related
I have a Mongo collection of states, where each state contains an array of cities:
{
"_id":"636d1137cf1e57408486f795",
"state":"new york",
"cities":[
{
"cityid":"62bd8fa5396ba8aef4ad1041",
"name":"Yonkers"
},
{
"cityid":"62bd8fa5396ba8aef4ad1043",
"name":"Syracuse"
}
]
}
I need an update query that will lowercase every cities.name in the collection.
I can do an update with a literal value e.g.
db.states.updateMany(
{},
{ $set: { "cities.$[].name" : "some_value" } }
)
... , but I need the value to be based on the existing value. The closest I can get is something like this (but that doesn't work -- FieldPath field names may not start with '$')
db.states.updateMany(
{},
{ $set: { "cities.$[].name" : { $toLower: "cities.$[].name"} } }
)
You can chain up $map and $mergeObjects to perform the update. Put it in an aggregation pipeline in update.
db.collection.update({},
[
{
$set: {
cities: {
"$map": {
"input": "$cities",
"as": "c",
"in": {
"$mergeObjects": [
"$$c",
{
"name": {
"$toLower": "$$c.name"
}
}
]
}
}
}
}
}
])
Mongo Playground
I have a collection with many documents containing shipping prices:
{
"_id": {
"$oid": "5f7439c3bc3395dd31ca4f19"
},
"adapterKey": "transport1",
"pricegrid": {
"10000": 23.66,
"20000": 23.75,
"30000": 23.83,
"31000": 43.5,
"40000": 44.16,
"50000": 49.63,
"60000": 50.25,
"70000": 52,
"80000": 56.62,
"90000": 59,
"100000": 62.5,
"119000": 68.85,
"149000": 80,
"159000": 87,
"179000": 94,
"199000": 100.13,
"249000": 118.5,
"299000": 138.62,
"999000": 208.63
},
"zones": [
"25"
],
"franco": null,
"tax": 20,
"doc_created": {
"$date": "2020-09-30T07:54:43.966Z"
},
"idConfig": "0000745",
"doc_modified": {
"$date": "2020-09-30T07:54:43.966Z"
}
}
In pricegrid, all the properties can be different from one grid to another.
I'd like to update all the prices in the field "pricegrid" (price * 1.03 + 1).
I tried this :
db.shipping_settings.updateMany(
{ 'adapterKey': 'transport1' },
{
$mul: { 'pricegrid.$': 1.03 },
$inc: { 'pricegrid.$': 1}
}
)
Resulting in this error :
MongoServerError: Updating the path 'pricegrid.$' would create a conflict at 'grille.$'
So I tried with only $mul (planning on doing $inc in another query) :
db.livraison_config.updateMany(
{ 'adapterKey': 'transport1' },
{
$mul: { 'pricegrid.$': 1.03 }
}
)
But in that case, I get this error :
MongoServerError: The positional operator did not find the match needed from the query.
Could you please direct me on the correct way to write the request ?
You can use an aggregation pipeline in an update. $objectToArray pricegrid to convert it into an array of k-v tuple first. Then, do a $map to perform the computation. Finally, $arrayToObject to convert it back.
db.collection.update({
"adapterKey": "transport1"
},
[
{
$set: {
pricegrid: {
"$objectToArray": "$pricegrid"
}
}
},
{
"$set": {
"pricegrid": {
"$map": {
"input": "$pricegrid",
"as": "p",
"in": {
"k": "$$p.k",
"v": {
"$add": [
{
"$multiply": [
"$$p.v",
1.03
]
},
1
]
}
}
}
}
}
},
{
$set: {
pricegrid: {
"$arrayToObject": "$pricegrid"
}
}
}
])
Here is the Mongo playground for your reference.
You can do it with Aggregation framework:
$objectToArray - to transform pricegrid object to array so you can iterate of its items
$map to iterate over array generated in previous step
$sum and multiply to perform mathematical operations
$arrayToObject to transform updated array back to object
db.collection.update({
"adapterKey": "transport1"
},
[
{
"$set": {
"pricegrid": {
"$arrayToObject": {
"$map": {
"input": {
"$objectToArray": "$pricegrid"
},
"in": {
k: "$$this.k",
v: {
"$sum": [
1,
{
"$multiply": [
"$$this.v",
1.02
]
}
]
}
}
}
}
}
}
}
],
{
"multi": true
})
Working example
I might be wrong, but it looks like there's currently no support for this feature - there's actually an open jira-issue that addresses this topic. Doesn't look like this is going to be implemented though.
Can you help me with a situation...
I Have this json but I would like to return only the nodes.
{
"_id":{
"userArea":NumberInt(4927)
},
"pages":{
"12":{
"page":NumberInt(2635),
"progress":"COMPLETED",
"progressType":"USER_PROGRESS",
"end":11
},
"13":{
"page":NumberInt(2627),
"progress":"COMPLETED",
"progressType":"USER_PROGRESS",
"end":ISODate("2018-04-19T15:04:29.000+0000")
}
"14":{
"page":NumberInt(2627),
"progress":"CANCELLED",
"progressType":"USER_PROGRESS",
"end":ISODate("2018-04-19T15:04:29.000+0000")
}
}
}
This way.... without header
"12":{
"page":NumberInt(2635),
"progress":"COMPLETED",
"progressType":"USER_PROGRESS",
"end":11
},
"13":{
"page":NumberInt(2627),
"progress":"COMPLETED",
"progressType":"USER_PROGRESS",
"end":ISODate("2018-04-19T15:04:29.000+0000")
}
Can you help me? I need to filter only the completed!!!
You can use this aggregation query:
First use $objectToArray to generate an array and can filter the values using the value v. This is done into the input.
Then with that input values you can filter the elements whose progress value is not CANCELLED. And convert again to an object with $arrayToObject
db.collection.aggregate([
{
"$project": {
"pages": {
"$arrayToObject": {
"$filter": {
"input": {
"$objectToArray": "$pages"
},
"cond": {
"$ne": [
"$$this.v.progress",
"CANCELLED"
]
}
}
}
}
}
}
])
Example here
Sorry, on my first attemp I read "without header" and I thought you wanted using $replaceRoot but I think this new query is what you want (also avoiding $unwind).
But I think you say "without hedaer" to show the output simplified in the question. Also if you can't output the header simply add a new stage using $replaceRoot. Example here
You need to do many pipelines as in here
db.collection.aggregate([
{
"$project": {
"p": {
"$objectToArray": "$pages"
}
}
},
{
"$unwind": "$p"
},
{
"$match": {
"p.v.progress": "COMPLETED"
}
},
{
"$group": {
"_id": "$_id",
"p": {
"$addToSet": "$p"
}
}
},
{
$project: {
"a": {
"$arrayToObject": "$p"
}
}
},
{
"$replaceRoot": {
"newRoot": "$a"
}
}
])
Reshape so that you can apply filter.
Group again to get back to your original schema. Else, you can stop at $match pipeline.
If possible, change your schema, Do not use dynamic keys.
I have the following mongo db schema and I am trying to build an aggregate query that searches under github_open_issues under the repo key and can return me a match for all the values with repoA as the value. I have tried the following as my query however its not returning any result. Im a bit confused why this is not working as I have another db with a schema similar to this and this type of query works there but here something seems to be different and is not working. I have also put together this interactive example mongoplayground
query
db.collection.aggregate([
{
"$unwind": "$github_open_issues"
},
{
"$match": {
"github_open_issues.repo": {
"$in": [
"repoA"
]
}
}
},
])
schema
[
{
"github_open_issues": {
"0": {
"git_url": "https://github.com/",
"git_assignees": "None",
"git_open_date": "2019-09-26",
"git_id": 253113,
"repo": "repoA",
"git_user": "userA",
"state": "open"
},
"1": {
"git_url": "https://github.com/",
"git_assignees": "None",
"git_open_date": "2019-11-15",
"git_id": 294398,
"repo": "repoB",
"git_user": "userB",
"state": "open"
},
"2": {
"git_url": "https://github.com/",
"git_assignees": "None",
"git_open_date": "2021-04-12",
"git_id": 661208,
"repo": "repoA",
"state": "open"
}
},
"unique_label_seen": {
"568": {
"label_name": "some label",
"times_seen": 12,
"535": {
"label_name": "another label",
"times_seen": 1
}
}
}
}
]
$objectToArray convert github_open_issues object to array in key-value format
$filter to iterate loop of above converted array and filter your search condition
$match to filter github_open_issues not empty
$arrayToObject convert github_open_issues array to object
db.collection.aggregate([
{
$addFields: {
github_open_issues: {
$filter: {
input: { $objectToArray: "$github_open_issues" },
cond: { $in: ["$$this.v.repo", ["repoA"]] }
}
}
}
},
{ $match: { github_open_issues: { $ne: [] } } },
{ $addFields: { github_open_issues: { $arrayToObject: "$github_open_issues" } } }
])
Playground
You query is correct but you data in schema placed wrong inside github_open_issues.repo your objects are place by numbers like {"0": {values... }, "1":{values... }} which cannot get your desired value. You can check the playground now playground
I have a mongodb full of records with mixed types and need to sanitize data to make some sense in back end application.
To change type(NumberLong to String) of a normal field is easy, just cast it to string in Javascript like:
db.foo.find({ field: { $type: 18 } }).forEach(function (x) {
x.field = new String(x.field); // convert field to string
db.foo.save(x);
});
But how do I change array elements from NumberLong to String?
For example I have field:
"elements" : { "top" : {"random" : [ NumberLong(12) , NumberLong(20), NumberLong(13)] } }
and I need all the elements of elements.top.random as strings.
Do I need to do "foreach" element of the array or is there any better way?
Any Javascript guru can help me out?
Before posting I just could not find an answer, and now I did find something similar and adopted...
so here it is:
db.foo.find( {"elements.top.random": {$type:18}} ).forEach( function (x) {
var arr = [];
x.elements.top.random.forEach( function (e) { arr.push("" + e); } );
x.elements.top.random = arr;
db.foo.save(x);
});
We came across this issue today with a more recent version of MongoDB (v4.0.0). Since the save() API is deprecated, we had to use updateOne(), resulting in the following code:
db.foo.find( {"elements.top.random": {$type:18}} ).forEach(function (x) {
var stringValues = [];
x.elements.top.random.forEach(function(e) {
stringValues.push("" + e);
});
db.foo.updateOne(
{ _id: x._id },
{ $set: { "elements.top.random" : stringValues } }
);
});
With MongoDB v4.2+, you can do an update with aggregation pipeline. Use $reduce and $toString to do the string conversion and string concat.
db.collection.update({
"elements.top.random": {
$type: 18
}
},
[
{
$set: {
"elements.top.random": {
"$reduce": {
"input": "$elements.top.random",
"initialValue": "",
"in": {
"$concat": [
"$$value",
{
$toString: "$$this"
}
]
}
}
}
}
}
],
{
multi: true
})
Here is the Mongo playground for your reference.
With a slightly older version of MongoDB (v4.0+), you can still have $toString available. You can use the old "aggregate-then-update" approach.
db.collection.aggregate([
{
$match: {
"elements.top.random": {
$type: 18
}
}
},
{
$set: {
"elements.top.random": {
"$reduce": {
"input": "$elements.top.random",
"initialValue": "",
"in": {
"$concat": [
"$$value",
{
$toString: "$$this"
}
]
}
}
}
}
}
]).forEach(agg => {
db.collection.update(
{ _id: agg._id },
{ $set: { "elements.top.random" : agg.elements.top.random } }
)
})