I have this loop:
properties.forEach(function(property) {
console.log("property: " + property);
var upsertValues = {};
upsertValues["ID"] = property.ID;
Properties.upsert(upsertValues,
{$set: property},
function(err, nbr) {
if(err)
console.log(err);
else
console.log("upsert successful" + nbr);
});
});
setTimeout(function () {
Fiber(function() {
Meteor.call("removeOldProperties", modification_date);
}).run();
}, 30000)
})
Basically, it updates a bench of documents and at the end, it removes all the once who have not been updated.
I had to use a TimeOut because without that, I removes the documents before their update, as all the Meteor.upsert statements are async.
Is there a better way to do it (without having to use this timeout) ?
Thanks,
Couple thoughts:
upserts are fast, no need for a callback
Fiber is for the server
I don't understand how your upsertValues was a valid query. Is this referring to the document _id? If so, convention is to keep using the name _id, if not, I'd use a more descriptive name. Was this code functioning??
What remains:
var upsertsCompleted = 0;
properties.forEach(function(property) {
Meteor.call("upsertProperties", property, function() {
if (++upsertsCompleted === properties.length) {
Meteor.call("removeOldProperties", modification_date);
}
}
Meteor.methods({
upsertProperties: function (property) {
return Properties.upsert(property.ID, {$set: property});
}
});
Related
Say I have a list of models:
const documents = [{}, {}, {}];
And I want to insert these into the DB, or update them all, but only if a condition is met:
Model.update({isSubscribed: {$ne: false}}, documents, {upsert:true},(err, result) => {
});
The above signature is surely wrong - what I want to do is insert/update the documents, where the condition is met.
There is this Bulk API:
https://docs.mongodb.com/manual/reference/method/Bulk.find.upsert/
but I can't tell if it will work when inserting multiple documents.
Imagine this scenario: We have a list of employees and a form of some sorts to give them all a penalty, at once, not one by one :)
On the backend side, you would have your eg addBulk function. Something like this:
Penalty controller
module.exports = {
addBulk: (req, res) => {
const body = req.body;
for (const item of body) {
Penalty.create(item).exec((err, response) => {
if (err) {
res.serverError(err);
return;
}
});
res.ok('Penalties added successfully');
}
}
Then you'll probably have an API on your frontend that directs to that route and specific function (endpoint):
penaltyApi
import axios from 'axios';
import {baseApiUrl} from '../config';
const penaltyApi = baseApiUrl + 'penalty'
class PenaltyApi {
static addBulk(penalties) {
return axios({
method: 'post',
url: penaltyApi + '/addBulk',
data: penalties
})
}
}
export default PenaltyApi;
...and now let's make a form and some helper functions. I'll be using React for demonstration, but it's all JS by the end of the day, right :)
// Lets first add penalties to our local state:
addPenalty = (event) => {
event.preventDefault();
let penalty = {
amount: this.state.penaltyForm.amount,
unit: this.state.penaltyForm.unit,
date: new Date(),
description: this.state.penaltyForm.description,
employee: this.state.penaltyForm.employee.value
};
this.setState(prevState => ({
penalties: [...prevState.penalties, penalty]
}));
}
Here we are mapping over our formData and returning the value and passing it to our saveBulkEmployees() function
save = () => {
let penaltiesData = Object.assign([], this.state.penalties);
penaltiesData.map(penal => {
penal.employeeId = penal.employee.id;
delete penal.employee;
return penaltiesData;
});
this.saveBulkEmployees(penaltiesData);
}
...and finally, let's save all of them at once to our database using the Bulk API
saveBulkEmployees = (data) => {
PenaltyApi.addBulk(data).then(response => {
this.success();
console.log(response.config.data)
this.resetFormAndPenaltiesList()
}).catch(error => {
console.log('error while adding multiple penalties', error);
throw(error);
})
}
So, the short answer is YES, you can absolutely do that. The longer answer is above :) I hope this was helpful to you. If any questions, please let me know, I'll try to answer them as soon as I can.
I'm using a MEAN stack and with Mongoose. Is there a way to query MongoDB with multiple ids to only return those specific IDs in one query e.g. /api/products/5001,5002,5003
Is this possible or would I need to query each product individually or add an additional attribute to the products and query by that.
Update: To clarify as suggested below I've managed to get it partially working using {'_id': { $in: [5001,5002,5003]} however I'm having problems figuring out how to pass the list from the api url to the find function.
Using Express.js for router
router.get('/list/:ids', controller.showByIDs);
exports.showByIDs = function(req, res) {
Product.find({'_id': { $in: [req.params.ids]}}, function (err, product) {
if(err) { return handleError(res, err); }
if(!product) { return res.send(404); }
return res.json(product);
})
};
Then trying /api/products/list/5001 works however /api/products/list/5001,5002 doesn't. I'm not sure if it's a syntax problem in the url or my router code that needs to change or the controller.
You can use the $in operator to query for multiple values at once:
Products.find({_id: {$in: [5001, 5002, 5003]}}, function (err, products) { ... });
On the Express side, you need to use a format for the ids parameter that lets you split it into an array of id values, like you had in your first example:
/api/products/5001,5002,5003
Then in your route handler, you can call the split function on the req.params.ids string to turn it into an array of id values that you can use with $in:
exports.showByIDs = function(req, res) {
var ids = req.params.ids.split(',');
Product.find({'_id': { $in: ids}}, function (err, product) {
if(err) { return handleError(res, err); }
if(!product) { return res.send(404); }
return res.json(product);
})
};
I have a fairly straight forward method below to update a document based on its ObjectId. It does not return an error but it fails to make the required updates to the document. I think it is failing because, according to my research, findByIdAndUpdate() takes only plain Javascript whereas job._id is an ObjectId from the document that I want to update. Can someone tell me how to make this work correctly?
function handleEncoderResponse(xmlResponse, job) {
var r = et.parse(xmlResponse);
var mediaID = r.findtext('./MediaID');
var message = r.findtext('./message');
EncodingJob = mongoose.model('EncodingJob');
EncodingJob.findByIdAndUpdate( job._id, {
"MediaID": mediaID,
"Status": message
}, function(err, result) {
if (err) console.log(err);
console.log(result);
});
}
Edit: Per this question Mongoose update document Fail with findByIdAndUpdate
I also tried the following code to no avail.
job.MediaID = mediaID;
job.Status = message;
job.save(function(err, res) {
if(err) console.log(err);
});
This approach yields the issue. It does not update the document and it does not return an error.
As it turns out, my mistake was forgetting to define MediaID and Status in the Schema as follows:
var encodingJobSchema = new mongoose.Schema({
...
MediaID: String,
Status: String
});
Ok, still in my toy app, I want to find out the average mileage on a group of car owners' odometers. This is pretty easy on the client but doesn't scale. Right? But on the server, I don't exactly see how to accomplish it.
Questions:
How do you implement something on the server then use it on the client?
How do you use the $avg aggregation function of mongo to leverage its optimized aggregation function?
Or alternatively to (2) how do you do a map/reduce on the server and make it available to the client?
The suggestion by #HubertOG was to use Meteor.call, which makes sense and I did this:
# Client side
Template.mileage.average_miles = ->
answer = null
Meteor.call "average_mileage", (error, result) ->
console.log "got average mileage result #{result}"
answer = result
console.log "but wait, answer = #{answer}"
answer
# Server side
Meteor.methods average_mileage: ->
console.log "server mileage called"
total = count = 0
r = Mileage.find({}).forEach (mileage) ->
total += mileage.mileage
count += 1
console.log "server about to return #{total / count}"
total / count
That would seem to work fine, but it doesn't because as near as I can tell Meteor.call is an asynchronous call and answer will always be a null return. Handling stuff on the server seems like a common enough use case that I must have just overlooked something. What would that be?
Thanks!
As of Meteor 0.6.5, the collection API doesn't support aggregation queries yet because there's no (straightforward) way to do live updates on them. However, you can still write them yourself, and make them available in a Meteor.publish, although the result will be static. In my opinion, doing it this way is still preferable because you can merge multiple aggregations and use the client-side collection API.
Meteor.publish("someAggregation", function (args) {
var sub = this;
// This works for Meteor 0.6.5
var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
// Your arguments to Mongo's aggregation. Make these however you want.
var pipeline = [
{ $match: doSomethingWith(args) },
{ $group: {
_id: whatWeAreGroupingWith(args),
count: { $sum: 1 }
}}
];
db.collection("server_collection_name").aggregate(
pipeline,
// Need to wrap the callback so it gets called in a Fiber.
Meteor.bindEnvironment(
function(err, result) {
// Add each of the results to the subscription.
_.each(result, function(e) {
// Generate a random disposable id for aggregated documents
sub.added("client_collection_name", Random.id(), {
key: e._id.somethingOfInterest,
count: e.count
});
});
sub.ready();
},
function(error) {
Meteor._debug( "Error doing aggregation: " + error);
}
)
);
});
The above is an example grouping/count aggregation. Some things of note:
When you do this, you'll naturally be doing an aggregation on server_collection_name and pushing the results to a different collection called client_collection_name.
This subscription isn't going to be live, and will probably be updated whenever the arguments change, so we use a really simple loop that just pushes all the results out.
The results of the aggregation don't have Mongo ObjectIDs, so we generate some arbitrary ones of our own.
The callback to the aggregation needs to be wrapped in a Fiber. I use Meteor.bindEnvironment here but one can also use a Future for more low-level control.
If you start combining the results of publications like these, you'll need to carefully consider how the randomly generated ids impact the merge box. However, a straightforward implementation of this is just a standard database query, except it is more convenient to use with Meteor APIs client-side.
TL;DR version: Almost anytime you are pushing data out from the server, a publish is preferable to a method.
For more information about different ways to do aggregation, check out this post.
I did this with the 'aggregate' method. (ver 0.7.x)
if(Meteor.isServer){
Future = Npm.require('fibers/future');
Meteor.methods({
'aggregate' : function(param){
var fut = new Future();
MongoInternals.defaultRemoteCollectionDriver().mongo._getCollection(param.collection).aggregate(param.pipe,function(err, result){
fut.return(result);
});
return fut.wait();
}
,'test':function(param){
var _param = {
pipe : [
{ $unwind:'$data' },
{ $match:{
'data.y':"2031",
'data.m':'01',
'data.d':'01'
}},
{ $project : {
'_id':0
,'project_id' : "$project_id"
,'idx' : "$data.idx"
,'y' : '$data.y'
,'m' : '$data.m'
,'d' : '$data.d'
}}
],
collection:"yourCollection"
}
Meteor.call('aggregate',_param);
}
});
}
If you want reactivity, use Meteor.publish instead of Meteor.call. There's an example in the docs where they publish the number of messages in a given room (just above the documentation for this.userId), you should be able to do something similar.
You can use Meteor.methods for that.
// server
Meteor.methods({
average: function() {
...
return something;
},
});
// client
var _avg = { /* Create an object to store value and dependency */
dep: new Deps.Dependency();
};
Template.mileage.rendered = function() {
_avg.init = true;
};
Template.mileage.averageMiles = function() {
_avg.dep.depend(); /* Make the function rerun when _avg.dep is touched */
if(_avg.init) { /* Fetch the value from the server if not yet done */
_avg.init = false;
Meteor.call('average', function(error, result) {
_avg.val = result;
_avg.dep.changed(); /* Rerun the helper */
});
}
return _avg.val;
});
I am in the process of changing the schema for one of my MongoDB collections. (I had been storing dates as strings, and now my application stores them as ISODates; I need to go back and change all of the old records to use ISODates as well.) I think I know how to do this using an update, but since this operation will affect tens of thousands of records I'm hesitant to issue an operation that I'm not 100% sure will work. Is there any way to do a "dry run" of an update that will show me, for a small number of records, the original record and how it would be changed?
Edit: I ended up using the approach of adding a new field to each record, and then (after verifying that the data was right) renaming that field to match the original. It looked like this:
db.events.find({timestamp: {$type: 2}})
.forEach( function (e) {
e.newTimestamp = new ISODate(e.timestamp);
db.events.save(e);
} )
db.events.update({},
{$rename: {'newTimestamp': 'timestamp'}},
{multi: true})
By the way, that method for converting the string times to ISODates was what ended up working. (I got the idea from this SO answer.)
My advice would be to add the ISODate as a new field. Once confirmed that all looks good you could then unset the the string date.
Create a test environment with your database structure. Copy a handful of records to it. Problem solved. Not the solution you were looking for, I'm sure. But, I believe, this is the exact circumstances that a 'test environment' should be used for.
Select ID of particular records that you would like to monitor. place in the update {_id:{$in:[<your monitored id>]}}
Another option which depends of the amount of overhead it will cause you -
You can consider writing a script, that performs the find operation, add printouts or run in debug while the save operation is commented out. Once you've gained confidence you can apply the save operation.
var changesLog = [];
var errorsLog = [];
events.find({timestamp: {$type: 2}}, function (err, events) {
if (err) {
debugger;
throw err;
} else {
for (var i = 0; i < events.length; i++) {
console.log('events' + i +"/"+(candidates.length-1));
var currentEvent = events[i];
var shouldUpdateCandidateData = false;
currentEvent.timestamp = new ISODate(currentEvent.timestamp);
var change = currentEvent._id;
changesLog.push(change);
// // ** Dry Run **
// currentEvent.save(function (err) {
// if (err) {
// debugger;
// errorsLog.push(currentEvent._id + ", " + currentEvent.timeStamp + ', ' + err);
// throw err;
// }
// });
}
console.log('Done');
console.log('Changes:');
console.log(changesLog);
console.log('Errors:');
console.log(errorsLog);
return;
}
});
db.collection.find({"_manager": { $exists: true, $ne: null }}).forEach(
function(doc){
doc['_managers']=[doc._manager]; // String --> List
delete doc['_manager']; // Remove "_managers" key-value pair
printjson(doc); // Debug by output the doc result
//db.teams.save(doc); // Save all the changes into doc data
}
)
In my case the collection contain _manager and I would like to change it to _managers list. I have tested it in my local working as expected.
In the several latest versions of MongoDB (at least starting with 4.2), you could do that using a transaction.
const { MongoClient } = require('mongodb')
async function main({ dryRun }) {
const client = new MongoClient('mongodb://127.0.0.1:27017', {
maxPoolSize: 1
})
const pool = await client.connect()
const db = pool.db('someDB')
const session = pool.startSession()
session.startTransaction()
try {
const filter = { id: 'some-id' }
const update = { $rename: { 'newTimestamp': 'timestamp' } }
// This is the important bit
const options = { session: session }
await db.collection('someCollection').updateMany(
filter,
update,
options // using session
)
const afterUpdate = db.collection('someCollection')
.find(
filter,
options // using session
)
.toArray()
console.debug('updated documents', afterUpdate)
if (dryRun) {
// This will roll back any changes made within the session
await session.abortTransaction()
} else {
await session.commitTransaction()
}
} finally {
await session.endSession()
await pool.close()
}
}
const _ = main({ dryRun: true })