Complex mongodb document search - mongodb

I'm attempting to write a find query where one of the keys is unknown at the time the query is run, for example on the following document I'm interested in returning the document if "setup" is true:
{
"a": {
"randomstringhere": {
"setup": true
}
}
}
However I can't work how to wildcard the "randomstringhere" field as it changes for each document in the collection.
Can somebody help?

There is not much you can do with that. But you can modify your collection schema like
{
"a": [
{
"keyName": "randomstringhere",
"setup": true
},
//...
]
}
you can than write query to look
{
'a' : { $elemMatch: { setup: true } ,
}

You can't do this with a single query, as with the current design you would need a mechanism to get all the random keys that you need and then assemble the query document that uses the $or operator in the event that you get a list of variable key name.
The first part of your operation is possible using Map-Reduce. The following mapreduce operation will populate a separate collection called collectionKeys with all the random keys as the _id values:
mr = db.runCommand({
"mapreduce": "collection",
"map" : function() {
for (var key in this.a) { emit(key, null); }
},
"reduce" : function() { },
"out": "collectionKeys"
})
To get a list of all the random keys, run distinct on the resulting collection:
db[mr.result].distinct("_id")
Example Ouput
["randomstring_1", "randomstring_2", "randomstring_3", "randomstring_4", ...]
Now given the list above, you can assemble your query by creating an object that will have its properties set within a loop. Normally your query document will have this structure:
var query = {
"$or": [
{ "a.randomstring_1.setup": true },
{ "a.randomstring_2.setup": true },
{ "a.randomstring_3.setup": true }
]
};
which you can then use in your query:
db.collection.find(query)
So using the above list of subdocument keys, you can dynamically construct the above using JavaScript's map() method:
mr = db.runCommand({
"mapreduce": "collection", // your collection name
"map" : function() { // map function
for (var key in this.a) { emit(key, null); }
},
"reduce" : function() { }, // empty reducer that doesn't do anything
"out": "collectionKeys" // output collection with results
})
var randomstringKeysList = db[mr.result].distinct("_id"),
orOperator = randomstringKeysList.map(function (key){
var o = {};
o["a."+ key +".setup"] = true;
return o;
}),
query = { "$or": orOperator };
db.collection.find(query);

Related

MongoDB Shell Script Update all field Names where there is space in field name

Using MongoDB shell script 3.2, how can I update all fields where field names have a space replace those with underscore?
{
"Some Field": "value",
"OtherField" :"Value",
"Another Field" : "Value"
}
update the above document as below
{
"Some_Field": "value",
"OtherField" :"Value",
"Another_Field" : "Value"
}
rename field can be done with something like this
db.CollectionName.update( { _id: 1 }, { $rename: { 'nickname': 'alias', 'cell': 'mobile' } } )
Challenging part here is filter, how to come up with a filter where there is a space in field name
This needs a two-step approach. First, you need a mechanism to get a list of all the keys with a space in your collection. Once you get the list, construct an object that maps those keys to their renamed values. You can then use that object as your $rename operator document. Consider using mapReduce to get the list of keys with spaces.
The following mapReduce operation will populate a separate collection with all the filtered keys as the _id values:
mr = db.runCommand({
"mapreduce": "CollectionName",
"map": function() {
var regxp = /\s/;
for (var key in this) {
if (key.match(regxp)) {
emit(key, null);
}
}
},
"reduce": function() {},
"out": "filtered_keys"
})
To get a list of all the spaced keys, run distinct on the resulting collection:
db[mr.result].distinct("_id")
["Some Field", "Another Field"]
Now given the list above, you can assemble your update document by creating an object that will have its properties set within a loop. Normally your update document will have this structure:
var update = {
"$rename": {
"Some Field": "Some_Field",
"Another Field": "Another_Field"
}
}
Thus
var update = { "$rename": {} };
db[mr.result].distinct("_id").forEach(function (key){
update["$rename"][key] = key.replace(/ /g,"_");
});
which you can then use in your update as
db.CollectionName.update({ }, update, false, true );
Thanks to #chridam that was a excellent query.
Had to make small changes to run query, Full working query.
mr = db.runCommand({
"mapreduce": "MyCollectionName",
"map": function() {
var regxp = /\s/;
for (var key in this) {
if (key.match(regxp)) {
emit(key, null);
}
}
},
"reduce": function() {},
"out": "filtered_keys"
})
db[mr.result].distinct("_id")
var update = { "$rename": {} };
db[mr.result].distinct("_id").forEach(function (key){
update["$rename"][key] = key.replace(/\s+/g, "_");
});
//print(update)
db.MyCollectionName.update({ }, update, false, true );

MongoDB - Remove Folder

I'm trying to delete all folders on MongoDB whose descriptions contain a number higher than 10. Can you tell me how to do that?
I've been trying desperately since hours...
Thanks very much!
Robomongo
You need a mechanism to get a list of the keys in the collection first, filter the list for the ones that have a number greater than 10 and then generate a query that you will use with the $unset operator in your update. Your update document should have this structure:
var update = {
"$unset": {
"p11": "",
"p12": "",
...
}
}
which you will use in your update as
db.collection.update({}, update, {multi: true});
You need the mapReduce() command to generate that update document. The following mapreduce operation will populate a separate collection with the document as the value:
db.collection.mapReduce(
function() {
var map = this;
for (var key in map) {
if (map.hasOwnProperty(key)){
num = parseInt(key.replace(/[^\d.]/g, '' ));
if (num > 10) emit(null, key);
}
}
},
function(key, values) {
return values.reduce(function(o, v) {
o[v] = "";
return o;
}, {});
},
{ "out": "filtered_keys" }
);
You can then run a query on the resultant collection to get the update document and do the actual update:
var update = {
"$unset": db.filtered_keys.findOne({"_id": null}).value
},
options = { "multi": true };
db.collection.update({}, update, options);

How can i remove empty string from a mongodb collection?

I have a "mongodb colllenctions" and I'd like to remove the "empty strings"with keys from it.
From this:
{
"_id" : ObjectId("56323d975134a77adac312c5"),
"year" : "15",
"year_comment" : "",
}
{
"_id" : ObjectId("56323d975134a77adac312c5"),
"year" : "",
"year_comment" : "asd",
}
I'd like to gain this result:
{
"_id" : ObjectId("56323d975134a77adac312c5"),
"year" : "15",
}
{
"_id" : ObjectId("56323d975134a77adac312c5"),
"year_comment" : "asd",
}
How could I solve it?
Please try executing following code snippet in Mongo shell which strips fields with empty or null values
var result=new Array();
db.getCollection('test').find({}).forEach(function(data)
{
for(var i in data)
{
if(data[i]==null || data[i]=='')
{
delete data[i]
}
}
result.push(data)
})
print(tojson(result))
Would start with getting a distinct list of all the keys in the collection, use those keys as your query basis and do an ordered bulk update using the Bulk API operations. The update statement uses the $unset operator to remove the fields.
The mechanism to get distinct keys list that you need to assemble the query is possible through Map-Reduce. The following mapreduce operation will populate a separate collection with all the keys as the _id values:
mr = db.runCommand({
"mapreduce": "my_collection",
"map" : function() {
for (var key in this) { emit(key, null); }
},
"reduce" : function(key, stuff) { return null; },
"out": "my_collection" + "_keys"
})
To get a list of all the dynamic keys, run distinct on the resulting collection:
db[mr.result].distinct("_id")
// prints ["_id", "year", "year_comment", ...]
Now given the list above, you can assemble your query by creating an object that will have its properties set within a loop. Normally your query will have this structure:
var keysList = ["_id", "year", "year_comment"];
var query = keysList.reduce(function(obj, k) {
var q = {};
q[k] = "";
obj["$or"].push(q);
return obj;
}, { "$or": [] });
printjson(query); // prints {"$or":[{"_id":""},{"year":""},{"year_comment":""}]}
You can then use the Bulk API (available with MongoDB 2.6 and above) as a way of streamlining your updates for better performance with the query above. Overall, you should be able to have something working as:
var bulk = db.collection.initializeOrderedBulkOp(),
counter = 0,
query = {"$or":[{"_id":""},{"year":""},{"year_comment":""}]},
keysList = ["_id", "year", "year_comment"];
db.collection.find(query).forEach(function(doc){
var emptyKeys = keysList.filter(function(k) { // use filter to return an array of keys which have empty strings
return doc[k]==="";
}),
update = emptyKeys.reduce(function(obj, k) { // set the update object
obj[k] = "";
return obj;
}, { });
bulk.find({ "_id": doc._id }).updateOne({
"$unset": update // use the $unset operator to remove the fields
});
counter++;
if (counter % 1000 == 0) {
// Execute per 1000 operations and re-initialize every 1000 update statements
bulk.execute();
bulk = db.collection.initializeOrderedBulkOp();
}
})
If you need to update a single blank parameter or you prefer to do parameter by parameter, you can use the mongo updateMany functionality:
db.comments.updateMany({year: ""}, { $unset : { year : 1 }})

How to get average value from a hashmap in MongoDB?

I have a time data in my Mongo database. Each document equal a minute and contain 60 seconds as objects with value for each. How to get average value of all seconds in one minute?
A document looking like that:
{
"_id" : ObjectId("55575e4062771c26ec5f2287"),
"timestamp" : "2015-05-16T18:12:00.000Z",
"values" : {
"0" : "26.17",
"1" : "26.17",
"2" : "26.17",
...
"58" : "24.71",
"59" : "25.20"
}
}
You could take two approaches here:
Changing the schema and use the aggregation framework to get the average by using the $avg operator OR
Apply Map-Reduce.
Let's look at the first option. Currently as it is, the schema will not make it possible to use the aggregation framework because of the dynamic keys in the values subdocument. The ideal schema that would favour the aggregation framework would have the values field be an array which contains embedded key/value documents like this:
/* 0 */
{
"_id" : ObjectId("5559d66c9bbec0dd0344e4b0"),
"timestamp" : "2015-05-16T18:12:00.000Z",
"values" : [
{
"k" : "0",
"v" : 26.17
},
{
"k" : "1",
"v" : 26.17
},
{
"k" : "2",
"v" : 26.17
},
...
{
"k" : "58",
"v" : 24.71
},
{
"k" : "59",
"v" : 25.20
}
]
}
With MongoDB 3.6 and newer, use the aggregation framework to tranform the hashmaps to an array by using the $objectToArray operator then use $avg to calculate the average.
Consider running the following aggregate pipeline:
db.test.aggregate([
{
"$addFields": {
"values": { "$objectToArray": "$values" }
}
}
])
Armed with this new schema, you would then need to update your collection to change the string values to int by iterating the cursor returned from the aggregate method and using bulkWrite as follows:
var bulkUpdateOps = [],
cursor = db.test.aggregate([
{
"$addFields": {
"values": { "$objectToArray": "$values" }
}
}
]);
cursor.forEach(doc => {
const { _id, values } = doc;
let temp = values.map(item => {
item.key = item.k;
item.value = parseFloat(item.v) || 0;
delete item.k;
delete item.v;
return item;
});
bulkUpdateOps.push({
"updateOne": {
"filter": { _id },
"update": { "$set": { values: temp } },
"upsert": true
}
});
if (bulkUpdateOps.length === 1000) {
db.test.bulkWrite(bulkUpdateOps);
bulkUpdateOps = [];
}
});
if (bulkUpdateOps.length > 0) {
db.test.bulkWrite(bulkUpdateOps);
}
If your MongoDB version does not support the $objectToArray operator in the aggregation framework, then to convert the current schema into the one above takes a bit of native JavaScript functions with the MongoDB find() cursor's forEach() function as follows (assuming you have a test collection):
var bulkUpdateOps = [],
cursor = db.test.find();
cursor.forEach(doc => {
const { _id, values } = doc;
let temp = Object.keys(values).map(k => {
let obj = {};
obj.key = k;
obj.value = parseFloat(doc.values[k]) || 0;
return obj;
});
bulkUpdateOps.push({
"updateOne": {
"filter": { _id },
"update": { "$set": { values: temp } },
"upsert": true
}
});
if (bulkUpdateOps.length === 1000) {
db.test.bulkWrite(bulkUpdateOps);
bulkUpdateOps = [];
}
});
if (bulkUpdateOps.length > 0) {
db.test.bulkWrite(bulkUpdateOps);
}
or
db.test.find().forEach(function (doc){
var keys = Object.keys(doc.values),
values = keys.map(function(k){
var obj = {};
obj.key = k;
obj.value = parseFloat(doc.values[k]) || 0;
return obj;
});
doc.values = values;
db.test.save(doc);
});
The collection will now have the above schema and thus follows the aggregation pipeline that will give you the average time in one minute:
db.test.aggregate([
{
"$fields": {
"average": { "$avg": "$values.value" }
}
}
])
Or for MongoDB 3.0 and lower
db.test.aggregate([
{ "$unwind": "$values" },
{
"$group": {
"_id": "$timestamp",
"average": {
"$avg": "$values.value"
}
}
}
])
For the above document, the output would be:
/* 0 */
{
"result" : [
{
"_id" : "2015-05-16T18:12:00.000Z",
"average" : 25.684
}
],
"ok" : 1
}
As for the other Map-Reduce option, the intuition behind the operation is you would use JavaScript to make the necessary transformations and calculate the final average. You would need to define three functions:
Map
When you tell Mongo to MapReduce, the function you provide as the map function will receive each document as the this parameter. The purpose of the map is to exercise whatever logic you need in JavaScript and then call emit 0 or more times to produce a reducible value.
var map = function(){
var obj = this.values;
var keys = Object.keys(obj);
var values = [];
keys.forEach(function(key){
var val = parseFloat(obj[key]);
var value = { count: 1, qty: val };
emit(this.timestamp, value);
});
};
For each document you need to emit a key and a value. The key is the first parameter to the emit function and represents how you want to group the values (in this case you will be grouping by the timestamp). The second parameter to emit is the value, which in this case is a little object containing the count of documents (always 1) and total value of each individual value object key i.e. for each second within the minute.
Reduce
Next you need to define the reduce function where Mongo will group the items you emit and pass them as an array to this reduce function It's inside the reduce function where you want to do the aggregation calculations and reduce all the objects to a single object.
var reduce = function(key, values) {
var result = {count: 0, total: 0 };
values.forEach(function(value){
result.count += value.count;
result.total += value.qty;
});
return result;
};
This reduce function returns a single result. It's important for the return value to have the same shape as the emitted values. It's also possible for MongoDB to call the reduce function multiple times for a given key and ask you to process a partial set of values, so if you need to perform some final calculation, you can also give MapReduce a finalize function.
Finalize
The finalize function is optional, but if you need to calculate something based on a fully reduced set of data, you'll want to use a finalize function. Mongo will call the finalize function after all the reduce calls for a set are complete. This would be the place to calculate the average of all the second values in a document/timestamp:
var finalize = function (key, value) {
value.average = value.total / value.count;
return value;
};
Putting It Together
With the JavaScript in place, all that is left is to tell MongoDB to execute a MapReduce:
var map = function(){
var obj = this.values;
var keys = Object.keys(obj);
var values = [];
keys.forEach(function(key){
var val = parseFloat(obj[key]);
var value = { count: 1, qty: val };
emit(this.timestamp, value);
});
};
var reduce = function(key, values) {
var result = {count: 0, total: 0 };
values.forEach(function(value){
result.count += value.count;
result.total += value.qty;
});
return result;
};
var finalize = function (key, value) {
value.average = value.total / value.count;
return value;
};
db.collection.mapReduce(
map,
reduce,
{
out: { merge: "map_reduce_example" },
finalize: finalize
}
)
And when you query the output collection map_reduce_example, db.map_reduce_example.find(), you get the result:
/* 0 */
{
"_id" : null,
"value" : {
"count" : 5,
"total" : 128.42,
"average" : 25.684
}
}
References:
A Simple MapReduce with MongoDB and C#
MongoDB docuumentation on mapReduce
This kind of data structure creates lots of conflicts and difficult to handled mongo operations. This case either you changed your schema design. But, if you not able to changed this schema then follow this :
In your schema having two major problem 1> keys dynamic and 2> values of given keys in string so you should use some programming code to calculating avg check below scripts
From ref this first calculated size of values
Object.size = function(obj) {
var size = 0,
key;
for (key in obj) {
if (obj.hasOwnProperty(key)) size++;
}
return size;
};
db.collectionName.find().forEach(function(myDoc) {
var objects = myDoc.values;
var value = 0;
// Get the size of an object
var size = Object.size(objects);
for (var key in objects) {
value = value + parseFloat(objects[key]); // parse string values to float
}
var avg = value / size
print(value);
print(size);
print(avg);
});

Remove multiple documents from array in MongoDB

My document contains an array like:
{
"differentialDiagnosis" : "IART/Flutter",
"explanation" : "The rhythm.",
"fileName" : "A115a JPEG.jpg",
"history" : "1 year old with fussiness",
"interpretationList" : [
{
"interpretations" : [
ObjectId("54efe7c8d6d5ca3d5c580a22"),
ObjectId("54efe80bd6d5ca3d5c580a26")
]
},
{
"interpretations" : [
ObjectId("54efe80bd6d5ca3d5c580a26"),
ObjectId("54efe82ad6d5ca3d5c580a28")
]
}
],
}
and I want to remove all occurrences of ObjectId("54efe80bd6d5ca3d5c580a26"),
but I write a query:
db.ekgs.update({'interpretationList.interpretations':ObjectId("54c09fb3581c4c8c218d1a40")}, {$pull:{ 'interpretationList.$.interpretations':{ ObjectId("54c09fb3581c4c8c218d1a40")}})
This removes only first occurrence of ObjectId("54efe80bd6d5ca3d5c580a26").
The reason your query is only removing the first occurrence is because, as explained in this page in the documentation, "the positional $ operator acts as a placeholder for the first element that matches the query document".
The problem is that it is really tricky to deal with these types of updates with schema having embedded arrays in embedded objects in embedded arrays. In order to get around this problem, if you are able to flatten the schema, then your update becomes much easier. So if instead, your document looked like this:
{
"differentialDiagnosis" : "IART/Flutter",
"explanation" : "The rhythm.",
"fileName" : "A115a JPEG.jpg",
"history" : "1 year old with fussiness",
"interpretations" : [
ObjectId("54efe7c8d6d5ca3d5c580a22"),
ObjectId("54efe80bd6d5ca3d5c580a26"),
ObjectId("54efe82ad6d5ca3d5c580a28")
]
}
Then your query would be as simple as the one below. (Remember to add { "multi": true } as an option if you want to update multiple documents).
db.ekgs.update(
{ "interpretations": ObjectId("54efe80bd6d5ca3d5c580a26")},
{ "$pull": { "interpretations": ObjectId("54efe80bd6d5ca3d5c580a26") }}
);
But I understand that you might not be able to change the schema. In that case, you can try a solution that requires a small script. In the mongo shell, you can use the following bit of JavaScript to do the operation.
// Get cursor with documents requiring updating.
var oid = ObjectId("54efe80bd6d5ca3d5c580a26");
var c = db.ekgs.find({ "interpretationList.interpretations": oid });
// Iterate through cursor, removing oid from each subdocument in interpretationList.
while (c.hasNext()) {
var isModified = false;
var doc = c.next();
var il = doc.interpretationList;
for (var i in il) {
var j = il[i].interpretations.length;
while (j--) {
// If oid to remove is present, remove it from array
// and set flag that the document has been modified.
if (il[i].interpretations[j].str === oid.str) {
il[i].interpretations.splice(j, 1);
isModified = true;
}
}
}
// If modified, update interpretationList for document.
if (isModified) {
db.ekgs.update({ "_id": doc._id }, { "$set": { "interpretationList": il }});
}
}
UPDATE: Example of how it might work using the Node.js driver.
// Get cursor with documents requiring updating.
var oid = new ObjectID("54efe80bd6d5ca3d5c580a26");
var ekgs = db.collection("ekgs");
ekgs.find({ "interpretationList.interpretations": oid },
function(err, c) {
if(err) throw err;
// Iterate through cursor, removing oid from each subdocument in interpretationList.
c.each(function(err, doc) {
if (err) throw err;
// If doc is null then the cursor is exhausted/empty and closed.
if (doc != null) {
var isModified = false;
var il = doc.interpretationList;
for (var i in il) {
var j = il[i].interpretations.length;
while (j--) {
// If oid to remove is present, remove it from array
// and set flag that the document has been modified.
if (il[i].interpretations[j].equals(oid)) {
il[i].interpretations.splice(j, 1);
isModified = true;
}
}
}
// If modified, update interpretationList for document.
if (isModified) {
ekgs.update({ "_id": doc._id },
{ "$set": { "interpretationList": il }},
function(err, res) {
if (err) throw err;
// Callback.
console.log(res);
});
}
}
});
});