Upserting on embedded document - mongodb

I have the following document strucutre
{
"_id" : "NmBYYasdsa",
"objectId" : "asdsd"
"text" : "test",
....
"publishedAt" : ISODate("2015-05-28T15:31:51Z"),
"updatedAt" : ISODate("2015-05-28T15:31:51Z"),
"data" : {
.....
"likeCount" : 0,
"replyCount" : 0
}
}
That is use to synchronise my database with an external API. To do this, I poll the API once every minute and do a bulk upsert, matching on the object id to keep my database up to date.
Problem is that the data subdocument doesn't get updated when upserting, any ideas as to why?
My bulkwrite method
Mongo.Collection.prototype.upsertBulk = function(matcher, documents, options) {
if (_.isEmpty(documents)) { throw Error('Empty list of documents provided'); }
options = options || {};
var operations = documents.map(function(_document) {
_document._id = Random.id();
var operation = {
updateOne: {
filter: {},
update: { $set: _document },
upsert: true
},
};
operation['updateOne']['filter'][matcher] = _document[matcher];
return operation;
});
this.__mongoCollection(function(collection) {
collection.bulkWrite(operations, options, function(error, result) {
if (error) { throw error; }
return result;
});
});
};

Related

MongoDB update only a few records, create a new attribute, assign a value

I wasn't lucky finding any help on this anywhere else.
Basically I want to update only the few documents that do not have a given attribute already.
And the value for the update comes from a field that is already on the document.
This is what I tried but it didn't like the "from a field already on the document" part. Saying Cn doesn't exist.
db.getCollection('test').update(
// query
{ "id2" : { $exists: false } },
// update
{ id2: Cn },
// options
{
"multi" : true, // update all documents
"upsert" : false // don't insert new documents
}
);
Here is my test data
/* 1 */
{
"_id" : ObjectId("5912132c4a58677726d37168"),
"Cn" : "CA",
"id2" : "CAAB",
"Prov" : "AB"
}
/* 2 */
{
"_id" : ObjectId("591213404a58677726d37172"),
"Cn" : "CA",
"id2" : "CANZ",
"Prov" : "NZ"
}
/* 3 */
{
"_id" : ObjectId("591213534a58677726d37180"),
"Cn" : "CA",
"id2" : "CAMB",
"Prov" : "MB"
}
/* 4 */
{
"_id" : ObjectId("591213674a58677726d3718c"),
"Cn" : "US"
}
/* 5 */
{
"_id" : ObjectId("591213894a58677726d371a3"),
"Cn" : "MX"
}
All this should do is create a id2 on US & MX and give those new id2 attributes the corresponding values 'US' & 'MX'.
This would not be a big deal but I have able 144 countries & 10,000+ documents to add id2 to.
You can't update multiple items with an operator that only contains expressions, and self referencing doesn't work either.
Check out this question for work around solutions to get what you need done: Update MongoDB field using value of another field
You can try this :
db.your_collection.aggregate( [
{ $match : { id2 : { $exists: false }}},
{ $addFields: {
id2 : "$Cn"
}
},
{ $out : "your_collection" }
]);
This will remove all documents in collection which have id2 field.
You can do this in mongoose in a longer way :
db.getCollection('test').find({ "id2" : { $exists: false } }, function(err, docs){
if(err){
//handle errror
}
else if(!docs){
// no docs found
}
else{
for(var i=0; i< docs.length; i++){
db.getCollection('test')
.findByIdAndUpdate(
docs[i]._id
, { $set : { id2 : docs[i].Cn}}
, { new : true}
, function(err, doc){
if(err){
//handle error
}
else{
//doc was updated
}
});
}
}
});
I would not have gotten to this without Mihir Bhende pushing. Thanks for that.
var c = db.getCollection('test').find(
{ "id2" : { $exists: false }}
);
c.forEach(function(myDoc) {
print("doc", myDoc.Cn);
db.getCollection('test').update( {_id: myDoc._id}, {$set: { "id2": myDoc.Cn }}, function (err) {
if (err) { print("err"); }
});
});

Meteor Mongodb Bulk Update by _id not working

I'm trying to perform following bulk Update Operation by _id:
let bulk = Transactions.rawCollection().initializeOrderedBulkOp();
transactions.forEach((t) => {
let query = { '_id' : new Meteor.Collection.ObjectID(t._id._str) };
bulk.find( query ).update( { $set: { balance: t.balance, updated_at: new Date() } } );
});
bulk.execute(Meteor.bindEnvironment((err, res) => {
if (!err) {
callback(null, true);
} else {
console.log("Error Ocurred");
callback(true, null);
}
}));
But This find query by _id is not working. If I do query by any other field It works just fine. But I can't do that because Collection has only one Unique field as '_id'.
EDITED:
Following is Transaction MongoDB Document
{
"_id" : ObjectId("5899f8a15d79b02f6075100d"),
"transaction" : "Charge",
"description" : "01/01/2017 - 01/28/2017",
"created_by" : "SYSTEM",
"residency_id" : "6pFs3sBMZPtp3e5N9",
"value" : 25.82,
"balance" : 25.82,
"created_at" : ISODate("2017-02-07T16:41:05.718+0000"),
"updated_at" : ISODate("2017-02-07T18:08:05.378+0000")
}
Help Needed. TIA

Better Alternative to MongoDB db.collection.group()

I'm looking to group a collection of documents, then format them into something I can query dynamically, then query on the resulting collection. I've been able to group the collection and format it successfully using db.collection.group() but there are limitations when it comes to sharding that would prevent me from using it in production.
Here's that code:
var reduceFunction = function(current, result){
var provider = current.provider;
var collection = current.collection || provider;
result[provider] = result[provider] || {};
result[provider][collection] = result[provider][collection] || {};
result[provider][collection] = current.attributes;
}
var result = db.identity.group( {
key: { cookie: 1 },
cond: { organizationId: organization },
reduce: reduceFunction,
initial: {}
} );
Here's an example of the input documents:
{
"_id" : ObjectId("566c92712ae892aeb5278467"),
"provider" : "myProvider",
"collection": "myCollection",
"ip" : 111.111.111.111,
"cookie" : "12381902348asdf",
"updated" : ISODate("2015-12-12T21:32:33.824Z"),
"attributes" : {
company: "My Company"
}
}
Here's what my desired output would be (multiple input documents grouped by cookie)
{
"cookie" : "12381902348asdf",
"myProvider": {
"myCollection": {
"company": "myCompany"
}
},
"myOtherProvider": {
"myOtherCollection": {
"company": "myCompany"
}
}
}
so that way I could query using something like
collection.identity.find({"myProvider.myCollection.company": {$regex: /my/}})
This allows me to have conditions for multiple provider/collection/field groupings without having conflicts in the field names.

how to query for exact mach in unknown number of subfields in mongodb

I have a collection where documents can have an unknown number of sub documents:
"agent_id": {
"0":"1234",
"1":"2234",...etc
How do I search for an exact match in all the agent_id sub-fields?
You need to dynamically create an object with properties that are a concatenation of the embedded document name agent_id with the dot (.) and the field name, enclosed in quotes, something like this:
var query = {
"agent_id.0": "78343",
"agent_id.1": "78343",
"agent_id.2": "78343",
"agent_id.3": "78343",
...
"agent_id.n": "78343"
}
One way to create the object is generate the sub-documents keys with mapReduce. The following demonstrates this approach. In the Map-Reduce operation, an array of keys in the agent_id subdocument is generated to an output collection "collection_keys" and then used to produce the find() query expression:
Suppose you populate a sample collection
db.collection.insert([
{
"agent_id": {
"0":"1234",
"1":"2234",
"56":"8451",
"74":"1475",
"10":"1234"
}
},
{
"agent_id": {
"5":"5874",
"18":"2351"
}
}
])
Running the following mapReduce operation
var mr = db.runCommand({
"mapreduce" : "collection",
"map" : function() {
for (var key in this.agent_id) { emit(key, null); }
},
"reduce" : function(key, stuff) {
return null
},
"out": "collection" + "_keys"
});
var query = { "$or": [] },
value = "1234";
db[mr.result].distinct("_id").forEach(function (key){
var obj = {};
obj["agent_id." + key] = value;
query["$or"].push(obj)
});
printjson(query);
will produce:
{
"$or" : [
{
"agent_id.0" : "1234"
},
{
"agent_id.1" : "1234"
},
{
"agent_id.10" : "1234"
},
{
"agent_id.18" : "1234"
},
{
"agent_id.5" : "1234"
},
{
"agent_id.56" : "1234"
},
{
"agent_id.74" : "1234"
}
]
})
You can then use the query document in your find() query:
db.collection.find(query)
which will produce the result:
/* 0 */
{
"_id" : ObjectId("561d5312cd05efc95a1ea1f4"),
"agent_id" : {
"0" : "1234",
"1" : "2234",
"56" : "8451",
"74" : "1475",
"10" : "1234"
}
}

How to do a find iterating in a array field

Hi i have a expressjs app using mongodb.
At first i find a tv by id on my "tvs" collection, i get it but now i want to find all user info from other collection "users".
This is my JSON for each collection:
tvs
{
"_id" : ObjectId("5203af83396d285ea2ecff8f"),
"brand" : "LG",
"comments" : [{
"user" : ObjectId("521dc636eda03d0f9cab3568"),
"text" : "Sold!"
}, {
"user" : ObjectId("521b2785eda03d0f9cab3566"),
"text" : "Nice TV"
}],
"model" : "47LS5600",
"price" : 499.0
}
users
{
"_id" : ObjectId("521b2785eda03d0f9cab3566"),
"name" : {
"first" : "Ruben",
"last" : "Montes"
}
}
And this is my code
var tvs = db.collection("tvs");
var users = db.collection("users");
exports.findById = function (req, res) {
var id = req.params.id;
tvs.findOne({'_id': new BSON.ObjectID(id)}, function (err, tv) {
users.find( { _id : tv.comments.user_id }).toArray(function (err, items) {
res.send( { tv: tv, users: items } );
});
})
}
I need to know how to iterate the comments array from tvs collection to get the the info user that post a comment
users.find( { _id : tv.comments.user_id })
You can do a bit more logic to efficiently grab the users as a batch using the $in operator.
var mongodb = require('mongodb')
, MongoClient = require('mongodb').MongoClient
, Server = require('mongodb').Server;
MongoClient.connect('mongodb://127.0.0.1:27017/test', function (err, db) {
if (err) throw err;
var tvs = db.collection('tvs');
var users = db.collection('users');
var userNames = {};
var tvId = new mongodb.ObjectID("5203af83396d285ea2ecff8f"); // hard-code
// find a TV
tvs.findOne({ _id : tvId }, function (err, tv) {
var allUserIds = [];
if (tv && tv.comments) {
// build a list of all user IDs used in comments
// this doesn't filter duplicates right now
allUserIds = tv.comments.map(function (comment) {
return comment.user;
});
}
// using the list of UserIds, grab all of them ...,
// and just return the name
users.find({_id: { $in: allUserIds }}, { name: 1 })
.toArray(function (err, users_list) {
// if we got some
if (users_list && users_list.length > 0) {
for(var i= 0, len = users_list.length; i < len ; i++ ) {
userNames[users_list[i]._id] = users_list[i].name;
}
console.log("All comments ========");
// now all the usernames are indexed in userNames by Id
for(var i= 0, len = tv.comments.length; i < len ; i++ ) {
// swap id for name
tv.comments[i].user = userNames[tv.comments[i].user];
console.log(tv.comments[i]);
}
db.close(); // done with everything for this demo
}
});
});
});
I've used find and $in with an array of all userIds found in the comments for a single "tv". By using $in, it significantly reduces the number of calls needed to MongoDB to fetch single User documents. Also, using the second parameter of find, I've reduced the returned fields to just be name.
FYI -- I did simplify your structure to just be 'name' rather than 'first' and 'last'. You certainly can change it to match your exact needs.