how to query for exact mach in unknown number of subfields in mongodb - mongodb

I have a collection where documents can have an unknown number of sub documents:
"agent_id": {
"0":"1234",
"1":"2234",...etc
How do I search for an exact match in all the agent_id sub-fields?

You need to dynamically create an object with properties that are a concatenation of the embedded document name agent_id with the dot (.) and the field name, enclosed in quotes, something like this:
var query = {
"agent_id.0": "78343",
"agent_id.1": "78343",
"agent_id.2": "78343",
"agent_id.3": "78343",
...
"agent_id.n": "78343"
}
One way to create the object is generate the sub-documents keys with mapReduce. The following demonstrates this approach. In the Map-Reduce operation, an array of keys in the agent_id subdocument is generated to an output collection "collection_keys" and then used to produce the find() query expression:
Suppose you populate a sample collection
db.collection.insert([
{
"agent_id": {
"0":"1234",
"1":"2234",
"56":"8451",
"74":"1475",
"10":"1234"
}
},
{
"agent_id": {
"5":"5874",
"18":"2351"
}
}
])
Running the following mapReduce operation
var mr = db.runCommand({
"mapreduce" : "collection",
"map" : function() {
for (var key in this.agent_id) { emit(key, null); }
},
"reduce" : function(key, stuff) {
return null
},
"out": "collection" + "_keys"
});
var query = { "$or": [] },
value = "1234";
db[mr.result].distinct("_id").forEach(function (key){
var obj = {};
obj["agent_id." + key] = value;
query["$or"].push(obj)
});
printjson(query);
will produce:
{
"$or" : [
{
"agent_id.0" : "1234"
},
{
"agent_id.1" : "1234"
},
{
"agent_id.10" : "1234"
},
{
"agent_id.18" : "1234"
},
{
"agent_id.5" : "1234"
},
{
"agent_id.56" : "1234"
},
{
"agent_id.74" : "1234"
}
]
})
You can then use the query document in your find() query:
db.collection.find(query)
which will produce the result:
/* 0 */
{
"_id" : ObjectId("561d5312cd05efc95a1ea1f4"),
"agent_id" : {
"0" : "1234",
"1" : "2234",
"56" : "8451",
"74" : "1475",
"10" : "1234"
}
}

Related

How to use aggregation function mongo db-query

I am new in MongoDB and I would like to use the aggregation function where I want to check type == topic and get the following output
Expected output
[
{
conceptName : 59d98cfd1c5edc24e4024d00
totalCount : 2
},
{
conceptName : 59d98cfd1c5edc24e4024d03
totalCount : 1
}
]
Sample input db.GroupContents
{
"_id" : "5a0948bb1c5edc7a5000521a",
"type" : "topic",
"groupID" : "5a0948bb1c5edc7a5000521a",
"pedagogyID" : "59d98cfa1c5edc24e40249a3",
}
Sample input db.PedagogyNodes
{
"_id" : "59d98cfa1c5edc24e40249a3",
"latestVersion" : "59d98cfa1c5edc24e402497f_1",
"createdAt" : "2017-10-08 04:27:06",
"updatedAt" : "2017-10-08 04:27:06"
}
Sample input db.PedagogyVersions
{
"_id" : "59d98cfa1c5edc24e402497f_1",
"type" : "topic",
"contentNodes" : {
"LearningNodes" : [
"59d98cfd1c5edc24e4024d00",
"59d98cfd1c5edc24e4024d03",
"59d98cfd1c5edc24e4024d00",
]
},
"createdAt" : "2017-10-08 04:27:06",
"updatedAt" : "2017-10-08 04:27:06"
}
What I have tried so far
var groupID = "5a0948bb1c5edc7a5000521a"; // Step 1
var records;
var pnDoc;
var pvDoc;
db.GroupContents.find({groupID : groupID}).forEach(function (doc){ // Step 2
var pedagogyID = doc.pedagogyID;
var records = db.getSiblingDB('PedagogyService');
records.PedagogyNodes.find({_id : pedagogyID}).forEach(function (pnDoc) { // Step 3
var latestVersion = pnDoc.latestVersion;
// addded aggregate function here
records.PedagogyVersions.aggregate([
{
$match:{_id:latestVersion} // Step 4
},
{
$unwind:"$contentNodes.LearningNodes"
},
{
$group:
{
_id:"$contentNodes.LearningNodes",
count:{$sum:1}
}
}
])
})
});
I am unable to write db query based on my expected answer, please help.
Understand my requirement
Step : 1 => I am passing `groupID = 5a0948bb1c5edc7a5000521a`
Step : 2 => we have to check from GroupContents where groupID = groupID then we have to take `pedagogyID`
Step : 3 => we have to check from PedagogyNodes where _id = pedagogyID then we have to take `latestVersion`
Step : 4 => we have to check from PedagogyVersions where _id = latestVersion then we have to take `contentNodes->LearningNodes`
Step : 5 => Finally we have to do the aggregation then we have display the result
Try to unwind the LearningNodes array and then count them by grouping them together
db.PedagogyNodes.aggregate([
{
$unwind:"$contentNodes.LearningNodes"
},
{
$group:
{
_id:"$contentNodes.LearningNodes",
count:{$sum:1}
}
}
])
In case you need to do any matches you can use the $match stage
db.PedagogyNodes.aggregate([
{
$match:{type:"topic"}
},
{
$unwind:"$contentNodes.LearningNodes"
},
{
$group:
{
_id:"$contentNodes.LearningNodes",
count:{$sum:1}
}
}
])
Answering the edited question =>
You were not able to view the output on the console since mongoshell does not print script output on the screen. To do this, do the following:
var result = records.PedagogyVersions.aggregate([......]);
result.forEach(function(resultDoc){
print(tojson(resultDoc))
})
To see the result of your aggregation you have to pass the callback to be executed as parameter.
records.PedagogyVersions.aggregate([
{
$match:{_id:latestVersion} // Step 4
},
{
$unwind:"$contentNodes.LearningNodes"
},
{
$group:
{
_id:"$contentNodes.LearningNodes",
count:{$sum:1}
}
}
], function(err, results) {
console.log(results);
});

How to query MongoDb documents using the indices of embedded arrays

I am trying to learn how to use mongo queries to reach deep into a data tree. Specifically, I'm trying to remove the object below {"object": 'to remove'}
{
"_id" : ObjectId("7840f22736341b09154f7ebf"),
"username" : "nmay",
"fname" : "Nate",
"lname" : "May",
"data" : [
{
"monthNum" : 1,
"year" : 2016,
"days" : [
{
"date" : "2016-01-01T06:00:00.000Z",
"type1" : [],
"type2" : []
},
{
"date" : "2016-01-02T06:00:00.000Z",
"type1" : [
{"object": 'to remove'}
],
"type2" : []
}
]
}
]
}
so far I know how to query for the user _id, but I'm not sure how to remove the desired object using the indices in each array. In this example I want to remove data[0].days[1].type1[0]
Here is the query that I have so far:
app.delete('/user/:id/data/:monthIndex/days/:dayIndex/type1/:type1Index', function (req, res, next) {
var monthIndex = parseInt(req.params.monthIndex); // these console the value properly
var dayIndex = parseInt(req.params.dayIndex); // -1 is applied to the parameter to translate to array position
var type1Index = parseInt(req.params.type1Index);
db.users.update(
{ _id: mongojs.ObjectId(req.params.id) },
{ $pull: data.monthIndex.days.dayIndex.type1.type1Index }
);
}
It gives me the error
ReferenceError: data is not defined
Can someone demonstrate how I can pass this query my index parameters to remove the desired object?
Unfortunately, there is no way to remove an array element by its numerical index with a single operation in MongoDB. In order to do this, you need to unset desired element(s) first, and remove the resulting null-valued fields afterwards.
Your code should look something like this:
db.users.update(
{ _id : mongojs.ObjectId(req.params.id) },
{ $unset : { 'data.0.days.1.type1.0' : 1 } }
);
db.users.update(
{ _id : mongojs.ObjectId(req.params.id) },
{ $pull : { 'data.0.days.1.type1' : null } }
);
Edit by #bob: to pass in the parameters you have to build the query string, which is ugly:
var unset = {};
unset['$unset'] = {};
unset.$unset['data.' + req.params.monthIndex + '.days.' + req.params.dayIndex + '.foods.' + req.params.foodIndex] = 1;
db.users.update( { _id : mongojs.ObjectId(req.params.id) }, unset );
var pull = {};
pull['$pull'] = {};
pull.$pull['data.' + req.params.monthIndex + '.days.' + req.params.dayIndex + '.foods'] = null;
db.users.update( { _id : mongojs.ObjectId(req.params.id) }, pull );

MongoDB query to return documents that only have keys amongst a predefined set

The MongoDB query language allows filtering documents based on the existence or absence of a given field with the $exists operator.
Is there a way, with the MongoDB syntax, and given a set K of allowed fields, to exclude documents that have fields not in K from the results, but:
not knowing in advance which extra fields (outside K) can be encountered
not using JavaScript, that is, the $where operator?
Example:
{
"Some field" : "foo"
}
{
"Some field" : "bar",
"Some other field" : "foobar"
}
With the set K = [ "Some field" ], only the first document is to be returned.
Note how this is not to be confused with a projection, which would return both documents but removing the extra field.
I'm not sure if MongoDB do support such kind of operations out of box but you can achieve so with help of mapReduce.
Assuming your sample data set;
// Variable for map
var map = function () {
var isAcceptable = true;
Object.keys(this).forEach(function (key) {
if (key != "_id" && white_list.indexOf(key) == -1) {
isAcceptable = false;
}
});
if (isAcceptable == true) {
emit(1, this);
}
};
// Variable for reduce
var reduce = function (key, values) {
return values;
};
db.collection.mapReduce(
map,
reduce,
{
scope: {"white_list": ["Some field"]},
out: {"inline": 1}
}
);
Will return:
{
"results" : [
{
"_id" : 1,
"value" : {
"_id" : ObjectId("57cd7503e55de957c62fb9c8"),
"Some field" : "foo"
}
}
],
"timeMillis" : 13,
"counts" : {
"input" : 2,
"emit" : 1,
"reduce" : 0,
"output" : 1
},
"ok" : 1
}
Desired result will be in results.values of returned document. However, keep in mind limitation of MongoDB mapReduce and maximum size of BSON document.
Given a set of known fields K, you can construct a query that takes the set as input and gives a query with the $exists operator along with the corresponding fields projection. Using an example, suppose you have the following documents in a test collection
db.test.insert({ "fieldX": "foo", "fieldY": "bar", "fieldZ": 1 })
db.test.insert({ "fieldX": "123", "fieldY": "bar", "fieldZ": 2 })
db.test.insert({ "fieldY": "abc", "fieldZ": 3 })
db.test.insert({ "fieldX": "xyz", "fieldZ": 4 })
db.test.insert({ "fieldZ": 5 })
Then you can construct a query Q and a projection P from an input set K as follows:
var K = [ "fieldX", "fieldZ" ];
var or = K.map(function(field) {
var obj = {};
obj[field] = { "$exists": true };
return obj;
});
var P = K.reduce(function(doc, field) {
doc[field] = 1;
return doc;
}, {} );
var Q = { "$or": or };
db.test.find(Q, P);
Sample Output:
/* 1 */
{
"_id" : ObjectId("57cd78322c241f5870c82b7d"),
"fieldX" : "foo",
"fieldZ" : 1
}
/* 2 */
{
"_id" : ObjectId("57cd78332c241f5870c82b7e"),
"fieldX" : "123",
"fieldZ" : 2
}
/* 3 */
{
"_id" : ObjectId("57cd78332c241f5870c82b7f"),
"fieldZ" : 3
}
/* 4 */
{
"_id" : ObjectId("57cd78332c241f5870c82b80"),
"fieldX" : "xyz",
"fieldZ" : 4
}
/* 5 */
{
"_id" : ObjectId("57cd78332c241f5870c82b81"),
"fieldZ" : 5
}

Better Alternative to MongoDB db.collection.group()

I'm looking to group a collection of documents, then format them into something I can query dynamically, then query on the resulting collection. I've been able to group the collection and format it successfully using db.collection.group() but there are limitations when it comes to sharding that would prevent me from using it in production.
Here's that code:
var reduceFunction = function(current, result){
var provider = current.provider;
var collection = current.collection || provider;
result[provider] = result[provider] || {};
result[provider][collection] = result[provider][collection] || {};
result[provider][collection] = current.attributes;
}
var result = db.identity.group( {
key: { cookie: 1 },
cond: { organizationId: organization },
reduce: reduceFunction,
initial: {}
} );
Here's an example of the input documents:
{
"_id" : ObjectId("566c92712ae892aeb5278467"),
"provider" : "myProvider",
"collection": "myCollection",
"ip" : 111.111.111.111,
"cookie" : "12381902348asdf",
"updated" : ISODate("2015-12-12T21:32:33.824Z"),
"attributes" : {
company: "My Company"
}
}
Here's what my desired output would be (multiple input documents grouped by cookie)
{
"cookie" : "12381902348asdf",
"myProvider": {
"myCollection": {
"company": "myCompany"
}
},
"myOtherProvider": {
"myOtherCollection": {
"company": "myCompany"
}
}
}
so that way I could query using something like
collection.identity.find({"myProvider.myCollection.company": {$regex: /my/}})
This allows me to have conditions for multiple provider/collection/field groupings without having conflicts in the field names.

Distinct/Aggregation query Mongodb array, trim trailing space

I have a MongoDB collection which contains a colours array like :
myCollection :
{
_id : ...,
"colours" : [
{
"colourpercentage" : "42",
"colourname" : "Blue"
},
{
"colourpercentage" : "32",
"colourname" : "Red"
},
{
"colourpercentage" : "10",
"colourname" : "Green "
}
]
}
I would like to retrieve every distinct colourname of every entry of this collection, and be able to filter it with a search.
I tried with distinct but without success. I searched further and found that an aggregation could help me. For the moment I have :
db.getCollection('myCollection').aggregate([
{ "$match": { "colours.colourname": /Gre/ } }, # Gre is my search
{ "$unwind": "$colours" },
{ "$match": { "colours.colourname": /search/ } },
{ "$group": {
"_id": "$colours.colourname"
}}
])
It is working, but I get an array like :
{
"result" : [
{
"_id" : "Grey"
},
{
"_id" : "Light Green "
},
{
"_id" : "Light Green"
},
{
"_id" : "Green "
},
{
"_id" : "Green"
}
],
"ok" : 1.0000000000000000
}
And I would like to remove duplicate entries which have a space in the end and displays them like :
["Grey","Light Green","Green"]
One approach you could take is the Map-Reduce way even though the JavaScript interpreter driven mapReduce takes a bit longer than the aggregation framework but will work since you will be using some very useful native JavaScript functions that are lacking in the aggregation framework. For instance, in the map function you could use the trim() function to remove any trailing spaces in your colourname fields so that you can emit the "cleansed" keys.
The Map-Reduce operation would typically have the following map and reduce functions:
var map = function() {
if (!this.colours) return;
this.colours.forEach(function (c){
emit(c.colourname.trim(), 1)
});
};
var reduce = function(key, values) {
var count = 0;
for (index in values) {
count += values[index];
}
return count;
};
db.runCommand( { mapreduce : "myCollection", map : map , reduce : reduce , out : "map_reduce_result" } );
You can then query map_reduce_result collection with the regex to have the result:
var getDistinctKeys = function (doc) { return doc._id };
var result = db.map_reduce_result.find({ "_id": /Gre/ }).map(getDistinctKeys);
print(result); // prints ["Green", "Grey", "Light Green"]
-- UPDATE --
To implement this in Python, PyMongo's API supports all of the features of MongoDB’s map/reduce engine thus you could try the following:
import pymongo
import re
from bson.code import Code
client = pymongo.MongoClient("localhost", 27017)
db = client.test
map = Code("function () {"
" if (!this.colours) return;"
" this.colours.forEach(function (c){"
" emit(c.colourname.trim(), 1)"
" });"
"};")
reduce = Code("function (key, values) {"
" var count = 0;"
" for (index in values) {"
" count += values[index];"
" }"
" return count;"
" };")
result = db.myCollection.map_reduce(map, reduce, "map_reduce_result")
regx = re.compile("Gre", re.IGNORECASE)
for doc in result.find({"_id": regx}):
print(doc)