I have a collection A that has documents in form of:
{
_id: 12345,
title: "title"
}
and document B in form of:
{
_id: 12345,
newAttribute: "newAttribute12345"
}
I want to update collection A to have documents like:
{
_id: 12345,
title: "title"
newAttribute: "newAttribute12345"
}
At this time I do it with
update({_id: doc._id}, {$set: {newAttribute: doc.newAttrubute}})
, but I need to run it 10,000 in a loop for all my documents.
How can I update multiple documents like these (by _id) in 1 db call or in most efficient way? (this is basically a join/bulk update attributes operation)
I use mongodb 2.6
consider following scenario, two collections name as title and attribute.
title collection contains following documents :
[{
_id: 12345,
title: "title"
},
{
_id: 12346,
title: "title1"
}]
and attribute collection contains following document :
[{
_id: 12345,
newAttribute: "newAttribute12345"
},
{
_id: 12346,
newAttribute: "newAttribute12346"
},
{
_id: 12347,
newAttribute: "newAttribute12347"
}]
And you want to update title collection as using this criteria title._id = attribute._id use mongo bulk update with following script :
var bulk = db.title.initializeOrderedBulkOp();
var counter = 0;
db.attribute.find().forEach(function(data) {
var updoc = {
"$set": {}
};
var updateKey = "newAttribute";
updoc["$set"][updateKey] = data.newAttribute;
bulk.find({
"_id": data._id
}).update(updoc);
counter++;
// Drain and re-initialize every 1000 update statements
if(counter % 1000 == 0) {
bulk.execute();
bulk = db.title.initializeOrderedBulkOp();
}
})
// Add the rest in the queue
if(counter % 1000 != 0) bulk.execute();
A possible/problematic answer is hacky join in mongo (maybe there is something better):
http://tebros.com/2011/07/using-mongodb-mapreduce-to-join-2-collections/
The problem with this is that I have to swap the collections later and this requires me to know the properties of my collection
var r = function(key, values){
var result = { prop1: null, prop2: null };
values.forEach(function(value){
if (result.prop1 === null && value.prop1 !== null) {
result.prop1 = value.prop1;
}
if (result.prop2 === null && value.prop2 !== null) {
result.prop2 = value.prop2;
}
})
return result;
};
var m = function(){
emit(this._id, { prop1: this.prop1, prop2: this.prop2 })
}
db.A.mapReduce(m1, r, { out: { reduce: 'C' }});
db.B.mapReduce(m1, r, { out: { reduce: 'C' }});
You can use the cursor.forEach method
db.collectionA.find().forEach(function(docA){
db.collectionB.find().forEach(function(docB){
if(docA._id === docB._id){
docA.newAttribute = docB.newAttribute;
db.collectionA.save(docA);
}
})
})
> db.collectionA.find()
{ "_id" : 12345, "title" : "title", "newAttribute" : "newAttribute12345" }
Related
This query is doing the job fine :
db.collection.update(
{ "_id": oneIdProvided },
{ $inc: { "field": 5 } },{ upsert: true }
)
Now I would like to do the same operation multiple time with different IDs, I thought the good way was to use $in and therefore I tried :
db.collection.update(
{ "_id": { $in: oneArrayOfIds} },
{ $inc: { "field": 5 } },{ upsert: true }
)
Problem is : if one of the provided ID in the array is not existing in the collection, a new document is created (which is what I want) but will be attributed an automatic ID, not using the ID I provided and was looking for.
One solution I see could be to do first an insert query with my array of ID (those already existing would not be modified) and then doing my update query with upsert: false
Do you see a way of doing that in only one query ?
We can do this by performing multiple write operations using the bulkWrite() method.
function* range(start, end, step) {
for (let val=start; val<end; val+=step)
yield val
}
let oneArrayOfIds; // For example [1, 2, 3, 4]
let bulkOp = oneArrayOfIds.map( id => {
return {
"updateOne": {
"filter": { "_id": id },
"update": { "$set": { "field": 5 } },
"upsert": true
}
};
});
const limit = 1000;
const len = bulkOp.length;
let chunks = [];
if (len > 1000) {
for (let index of range(0, len, limit)) {
db.collection.bulkWrite(bulkOp.slice(index, index+limit));
}
} else {
db.collection.bulkWrite(bulkOp);
}
I have such a schema:
doc:
{
//Some fields
visits:
[
{
userID: Int32
time: Int64
}
]
}
I want to first check if a specific userID exists, if not, push a document with that userID and system time, else just update time value. I know neither $push nor $addToSet are not able to do that. Also using $ with upsert:true doesn't work, because of official documentation advice which says DB will use $ as field name instead of operator when trying to upsert.
Please guide me about this. Thanks
You can use $addToSet to add an item to the array and $set to update an existing item in this array.
The following will add a new item to the array if the userID is not found in the array :
db.doc.update({
visits: {
"$not": {
"$elemMatch": {
"userID": 4
}
}
}
}, {
$addToSet: {
visits: {
"userID": 4,
"time": 1482607614
}
}
}, { multi: true });
The following will update the subdocument array item if it matches the userId :
db.doc.update({ "visits.userID": 2 }, {
$set: {
"visits.$.time": 1482607614
}
}, { multi: true });
const p = await Transaction.findOneAndUpdate(
{
_id: data.id,
'products.id': { $nin: [product.id] },
},
{
$inc: {
actualCost: product.mrp,
},
$push: {
products: { id: product.id },
},
},
{ new: true }
);
or
db.collection.aggregate([
{
"$match": {
"_id": 1
}
},
{
"$match": {
"sizes.id": {
"$nin": [
7
]
}
}
},
{
"$set": {
"price": 20
}
}
])
https://mongoplayground.net/p/BguFa6E9Tra
I know it's very late. But it may help others. Starting from mongo4.4, we can use $function to use a custom function to implement our own logic. Also, we can use the bulk operation to achieve this output.
Assuming the existing data is as below
{
"_id" : ObjectId("62de4e31daa9b8acd56656ba"),
"entrance" : "Entrance1",
"visits" : [
{
"userId" : 1,
"time" : 1658736074
},
{
"userId" : 2,
"time" : 1658736671
}
]
}
Solution 1: using custom function
db.visitors.updateMany(
{_id: ObjectId('62de4e31daa9b8acd56656ba')},
[
{
$set: {
visits: {
$function: {
lang: "js",
args: ["$visits"],
body: function(visits) {
let v = []
let input = {userId: 3, time: Math.floor(Date.now() / 1000)};
if(Array.isArray(visits)) {
v = visits.filter(x => x.userId != input.userId)
}
v.push(input)
return v;
}
}
}
}
}
]
)
In NodeJS, the function body should be enclosed with ` character
...
lang: 'js',
args: ["$visits"],
body: `function(visits) {
let v = []
let input = {userId: 3, time: Math.floor(Date.now() / 1000)};
if(Array.isArray(visits)) {
v = visits.filter(x => x.userId != input.userId)
}
v.push(input)
return v;
}`
...
Solution 2: Using bulk operation:
Please note that the time here will be in the ISODate
var bulkOp = db.visitors.initializeOrderedBulkOp()
bulkOp.find({ _id: ObjectId('62de4e31daa9b8acd56656ba') }).updateOne({$pull: { visits: { userId: 2 }} });
bulkOp.find({ _id: ObjectId('62de4e31daa9b8acd56656ba') }).updateOne({$push: {visits: {userId: 2, time: new Date()}}})
bulkOp.execute()
Reference link
Is there a way to insert or update/replace multiple documents in MongoDB with a single query?
Assume the following collection:
[
{_id: 1, text: "something"},
{_id: 4, text: "baz"}
]
Now I would like to add multiple documents of which some might already be in the collection. If the documents are already in the collection, I would like to update/replace them. For example, I would like to insert the following documents:
[
{_id:1, text: "something else"},
{_id:2, text: "foo"},
{_id:3, text: "bar"}
]
The query should insert the documents with _id 2 and 3. It should also update/replace the document with _id 1. After the process, the collection should look as follows:
[
{_id:1, text: "something else"},
{_id:2, text: "foo"},
{_id:3, text: "bar"},
{_id:4, text: "baz"}
]
One approach might be to use insertMany:
db.collection.insertMany(
[ {...}, {...}, {...} ],
{
ordered: false,
}
)
If duplicates occur, that query will emit a writeErrors containing an array of objects containing the indexes of the documents that failed to insert. I could go through them and update them instead.
But that process is cumbersome. Is there a way to insert or update/replace many documents in one query?
As said here, to do what you need you can put something like this in
script.js
(* warning: untested code)
use YOUR_DB
var bulk = db.collection.initializeUnorderedBulkOp();
bulk.find( { _id : 1 } ).upsert().update( { $set: { "text": "something else" } } );
bulk.find( { _id : 4 } ).upsert().update( { $set: { "text": "baz" } } );
bulk.find( { _id : 99 } ).upsert().update( { $set: { "text": "mrga" } } );
bulk.execute();
and run it with
mongo < script.js
I had to do it this way as anything I tried for updating/inserting more than 1000 documents didn't work because of the limit.
Write commands can accept no more than 1000 operations. The Bulk() operations in the mongo shell and comparable methods in the drivers do not have this limit.
source
You can also use bulkWrite api to update or insert multiple documents in a single query, here is an example
var ops = []
items.forEach(item => {
ops.push(
{
updateOne: {
filter: { _id: unique_id },
update: {
$set: { fields_to_update_if_exists },
$setOnInsert: { fileds_to_insert_if_does_not_exist }
},
upsert: true
}
}
)
})
db.collections('collection_name').bulkWrite(ops, { ordered: false });
Considering data item as follows:
interface Item {
id: string; // unique key across collection
someValue: string;
}
If you have items under the limit 1000, you can make bulk write operation like this:
public async insertOrUpdateBulk(items: Item[]) {
try {
const bulkOperation = this._itemCollection.initializeUnorderedBulkOp();
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const item = items[itemIndex];
bulkOperation.find({ id: item.id }).upsert().update(item);
}
await bulkOperation.execute();
return true;
} catch (err) {
console.log(err);
return false;
}
}
If you have items that exceed the limit 1000, you can make simultaneous promises:
public async insertOrUpdate(items: Item[]) {
try {
const promises: Array<Promise<UpdateWriteOpResult>> = [];
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const item = items[itemIndex];
const updatePromise = this.itemCollection.updateOne({ id: item.id }, item, { upsert: true });
promises.push(updatePromise);
}
await Promise.all(promises);
console.log('done...');
return true;
} catch (err) {
console.log(err);
return false;
}
}
I have documents in my db with schema:
var MySchema = new Schema({
Street: { type: String },
Age: { type: Number, default: null },
Date: { type: Date },
Stuff: [
{
_id:false,
ThisDate: { type: Date },
ThisStreet: { type: String }
}]
});
Right now it is (Stuff is empty):
db.person.findOne()
{
Street: 'TheStreet',
Age: 23,
Date: ISODate("2016-02-19T00:00:00.000Z"),
Stuff: []
}
I then want to update all documents. What I want to do is to move Street and Date fields into Stuff array and delete Street and Date fields from schema.
Like this:
db.person.findOne()
{
Age: 23,
Stuff : [
{
ThisDate : ISODate("2016-02-19T00:00:00.000Z"),
ThisStreet : "TheStreet"
}
]
}
How could I achieve this?
Best Regards
Since this is a "one off" operation I would do it in the shell rather than use any other framework.
For MongoDB 3.2.x releases and greater use, bulkWrite():
var ops = [];
db.person.find({
"Street": { "$exists": true },
"Date": { "$exists": true }
}).forEach(function(doc) {
ops.push({
"updateOne": {
"filter": { "_id": doc._id },
"update": {
"$unset": {
"Street": "",
"Date": ""
},
"$set": {
"Stuff": [{
"ThisDate": doc.Date,
"ThisStreet": doc.Street
}]
}
}
}
});
if ( ops.length == 1000 ) {
db.person.bulkWrite(ops);
ops = [];
}
})
if ( ops.length > 0 )
db.person.bulkWrite(ops);
Or for MongoDB 2.6.x and 3.0.x releases use this version of Bulk operations:
var bulk = db.person.initializeUnorderedBulkOp(),
count = 0;
db.person.find({
"Street": { "$exists": true },
"Date": { "$exists": true }
}).forEach(function(doc) {
bulk.find({ "_id": doc._id }).updateOne({
"$unset": {
"Street": "",
"Date": ""
},
"$set": {
"Stuff": [{
"ThisDate": doc.Date,
"ThisStreet": doc.Street
}]
}
});
if ( count % 1000 == 0 ) {
bulk.execute();
bulk = db.person.initializeUnorderedBulkOp();
}
});
if ( count % 1000 != 0 )
bulk.execute();
Bottom line is that you need to iterate the documents in the collection and write them back with the re-arranged content "one by one". At least the Bulk operations API in use in both cases will reduce the load of writing and responding with the server to only one in every 1000 documents in the collection to process.
Also, rather than rewriting the whole document you are using $unset to remove the desired fields and $set to write "just" the data you want
Working example
db.person.insert(
{
"Street": 'TheStreet',
"Age": 23,
"Date": ISODate("2016-02-19T00:00:00.000Z"),
"Stuff": []
}
)
Then after running either pdate above the result is:
{
"_id" : ObjectId("56e607c1ca8e7e3519b4ce93"),
"Age" : 23,
"Stuff" : [
{
"ThisDate" : ISODate("2016-02-19T00:00:00Z"),
"ThisStreet" : "TheStreet"
}
]
}
I'd suggest you to transform document using aggregation framework and update as described in code snippet below
db.person.aggregate([
{$project:{Age:1, Stuff:[{Date:"$Date", Street:"$Street"}]}}
]).forEach(function(o){
var id = o._id;
delete o._id;
db.person.update({_id:id, Street:{$exists: true}},o);
});
After successful execution, you document or documents should look like
{
"_id" : ObjectId("56e2cd45792861e14df1f0a9"),
"Age" : 23.0,
"Stuff" : [
{
"Date" : ISODate("2016-02-19T00:00:00.000+0000"),
"Street" : "TheStreet"
}
]
}
Based on the logged in user his/her preferences I want to create a collection and display in a view.
I'm in no way experienced with mongodb and now i'm ending up with this huge if/else statement and it's already slow (with 7 users in DB). But afaik it does give me the right results.
Meteor.publish('listprofiles', function () {
if ( ! this.userId ) return [];
var user = Meteor.users.findOne({ _id: this.userId }, {
fields : {
'profile.gender': 1,
'profile.preference': 1
}
}),
query;
user.gender = user.profile.gender;
user.preference = user.profile.preference;
if (user.gender === 'man') {
if (user.preference === 'straight') {
query = {
$and: [
{ 'profile.gender': 'woman' },
{
$or : [{ 'profile.preference' : 'straight' },
{ 'profile.preference' : 'bi' }]
}
]
};
} else if (user.preference === 'gay') {
query = {
$and: [
{ 'profile.gender': 'man' },
{
$or : [{ 'profile.preference' : 'gay' },
{ 'profile.preference' : 'bi' }]
},
]
};
} else if (user.preference === 'bi') {
query = {
$or: [
{
$and: [
{ 'profile.gender': 'man' },
{
$or : [{ 'profile.preference' : 'gay' },
{ 'profile.preference' : 'bi' }]
},
]
},
{
$and: [
{ 'profile.gender': 'woman' },
{
$or : [{ 'profile.preference' : 'straight' },
{ 'profile.preference' : 'bi' }]
}
]
}
]
};
}
The queries work, I tested them, but I'm unsure how to fit them dynamically. My guess is that query also shouldn't be an object, but I'm not sure how to create a valid variable..
var dbFindQuery = Meteor.users.find({
'profile.invisible': false,
queryShouldBeHereButObviouslyThisDoesNotWork
}, {
fields : {
'profile.name': 1,
'profile.city': 1,
'profile.country': 1,
'profile.gender': 1,
'profile.preference': 1,
'profile.story': 1
}
});
console.log(dbFindQuery.fetch());
return dbFindQuery;
anyone can give me a pointer in the right direction?
You can certainly factor out the common query objects. Here's one way to approach it:
Meteor.publish('listprofiles', function() {
if (!this.userId)
return [];
var user = Meteor.users.findOne(this.userId);
var gender = user.profile.gender;
var preference = user.profile.preference;
var straightOrBiWoman = {
'profile.gender': 'woman',
'profile.preference': {$in: ['straight', 'bi']}
};
var gayOrBiMan = {
'profile.gender': 'man',
'profile.preference': {$in: ['gay', 'bi']}
};
var query = {};
if (gender === 'man') {
switch (preference) {
case 'straight':
query = straightOrBiWoman;
break;
case 'gay':
query = gayOrBiMan;
break;
default:
query = {$or: [gayOrBiMan, straightOrBiWoman]};
}
}
query['profile.invisible'] = false;
return Meteor.users.find(query, {fields: {profile: 1}});
});
Here we are reusing straightOrBiWoman and gayOrBiMan based on the user's gender and preference. Note that I used the $in operator to simplify the query. I'd also suggest that you not specify 2nd-level fields in your fields modifier for reasons explained here. Finally, I'd recommend testing this code, as I may have missed something in the logic when rewriting it. Hopefully this example will help guide you in the right direction.