env: Node 12.18.3, mongoose 5.10.0, Mongo 4.2.8 sharded cluster
After periods of network instability, occasionally my node/mongoose app doesn't seem to send queries to a specific db. I have mongoose's debug flag enabled and I don't see any queries in the log. It seems like they may just be getting buffered.
Here is a code snippet. I'm wondering if I'm missing something obvious in reconnecting?
var mongoose = require('mongoose');
var conn;
var connect = function () {
const dburl = config.get('activity.db');
const dbOptions = {
"native_parser": true,
"poolSize": 10
};
conn = mongoose.createConnection(dburl, dbOptions);
};
connect();
conn.on('error', err => {
console.log('Activity model mongoose connection error event caught:');
console.log(err);
});
conn.on('disconnected', connect);
conn.on('timeout', function (e) {
console.log('Activity model timeout event' + e);
});
I don't see any log lines for the error or disconnect or timeout events (although they may have happened, but my production logs wrap too quickly).
Related
I installed mongoDB localy as a service by tutorial. But if i connected to DB on sample from w3school
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/mydb";
MongoClient.connect(url, function(err, db) {
if (err) throw err;
console.log("Database created!");
db.close();
});
there is no error. When i opened Compas i didn't see new DB "mydb". But if i inserted some data to mydb across application and wrote result to console.log then are theess data returned. But in compas are still not visible. Did you meet with this problem ? Thanks.
I am calling the .save() method on a Mongo document. The success callback is triggered and I am passed the saved document, but the table in my database still has zero documents in it. db.estimates.count() when I am using the uberEstimator database returns 0.
I've closely modeled this code off other working code I have, and I just can't figure out where I've gone wrong. Am I not using .save() correctly?
const mongoose = require('mongoose');
if (process.env.MONGODB_URI) {
mongoose.connect(process.env.MONGODB_URI);
} else {
mongoose.connect('mongodb://localhost/uberEstimator')
}
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', () => console.log('mongoose connection open!'));
var estimateSchema = new mongoose.Schema({
rideTier: String,
...
});
var Estimate = mongoose.model('Estimate', estimateSchema);
const save = ({display_name, distance, high_estimate, low_estimate, duration, estimate, currency_code,
start_latitude, start_longitude, end_latitude, end_longitude, start_address, end_address}) => {
let props = {
rideTier: display_name,
...
}
var estimate = new Estimate(props);
console.log(estimate);
estimate.save((err, newEstimate) => {
if (err) console.log(err);
console.log('saved new estimate to db');
console.log(newEstimate);
})
};
var sample = JSON.parse('{"localized_display_name": "uberX","distance": 6.17,"display_name": "uberX","product_id": "a1111c8c-c720-46c3-8534-2fcdd730040d","high_estimate": 17,"low_estimate": 13,"duration": 1080,"estimate": "$13-17","currency_code": "USD"}')
sample.start_address = '300 Albany St';
save(sample);
I get these logs:
mongoose connection open!
saved new estimate to db
{ _id: 5bca37267a478b31718445f3,
rideTier: 'uberX',
distance: 6.17,
highEstimate: 17,
lowEstimate: 13,
duration: 1080,
estimateString: '$13-17',
currency: 'USD',
start_address: '300 Albany St',
__v: 0 }
But still get the following when examining the table via Mongo shell:
> use uberEstimator
switched to db uberEstimator
> show tables
estimates
> use estimates
switched to db estimates
> db.estimates.count()
0
Command use estimates switches current database to estimates and I guess that's not what you want to achieve here. Based on your code it seems like you want to browse uberEstimator database and estimates collection so you can simply try:
use uberEstimator
db.estimates.count()
When I am inserting/updating a document in a collection, is the lock applied on the database or the collection. Suppose I have two collections and they are independant of each other in the same database and wants to do write operations on them concurrently. Is this possible?
Here is the code I am using to test this:
var assert = require('assert'),
MongoClient = require('mongodb').MongoClient,
async = require('async');
var station_list = require('./station_list.json'),
trains_list = require('./trains_list.json');
var stationList = [],
trainsList = [];
var MONGO_URL = 'mongodb://localhost:27017/test';
for(var i=0; i<station_list.stations.length; i++)
stationList.push(station_list.stations[i].station_code);
for(var i=0; i<trains_list.trains.length; i++)
trainsList.push(trains_list.trains[i].code);
console.log('trains : ' + trainsList.length + ' stations : ' + stationList.length);
populateTrains();
populateStations();
function populateTrains() {
async.eachSeries(trainsList, populateTrainDb, function (err) {
assert.equal(null, err);
});
}
function populateTrainDb(code, callback) {
MongoClient.connect(MONGO_URL, function (err, db) {
assert.equal(null, err);
var jsonData = {};
jsonData.code = code;
db.collection('trainsCon').replaceOne(
{'code' : code}, jsonData, {upsert: true, w:1}, function (err, res) {
assert.equal(null, err);
db.close();
callback();
});
});
}
function populateStations() {
async.eachSeries(stationList, populateStationDb, function (err) {
assert.equal(null, err);
});
}
function populateStationDb(code, callback) {
MongoClient.connect(MONGO_URL, function (err, db) {
assert.equal(null, err);
var jsonData = {};
jsonData.code = code;
db.collection('stationsCon').replaceOne(
{'code' : code}, jsonData, {upsert:true, w:1}, function (err, res) {
assert.equal(null, err);
db.close();
callback();
});
});
}
The two json files : station_list.json and trains_list.json have around 5000 entries. So after running the given program I get this error after a while :
C:\Users\Adnaan\Desktop\hopSmart\node_modules\mongodb\lib\server.js:242
process.nextTick(function() { throw err; })
^
AssertionError: null == { [MongoError: connect EADDRINUSE 127.0.0.1:27017]
name: 'MongoError',
message: 'connect EADDRINUSE 127.0.0.1:27017' }
at C:\Users\Adnaan\Desktop\hopSmart\testing.js:52:10
at C:\Users\Adnaan\Desktop\hopSmart\node_modules\mongodb\lib\mongo_client.js:276:20
at C:\Users\Adnaan\Desktop\hopSmart\node_modules\mongodb\lib\db.js:224:14
at null.<anonymous> (C:\Users\Adnaan\Desktop\hopSmart\node_modules\mongodb\lib\server.js:240:9)
at g (events.js:273:16)
at emitTwo (events.js:100:13)
at emit (events.js:185:7)
at null.<anonymous> (C:\Users\Adnaan\Desktop\hopSmart\node_modules\mongodb-core\lib\topologies\server.js:301:68)
at emitTwo (events.js:100:13)
at emit (events.js:185:7)
When I check the number of entries entered the database, around 4000 entries had already been entered in both the collections. So what I get from the above experiment was that an error might have occured when one write was being attempted while inside other collection a document must have been getting written.
So how should I proceed to have this concurrency without conflicting locks.
The answer to this question can be quite long and depends on various factors (MongoDB version, storage engine, type of operations you are doing, sharding, etc.). I can only recommend you to read carefully the Concurrency section of the MongoDB documentation, and in particular the lock granularity part.
Make sure to choose the right version of MongoDB first as the behaviour varies greatly from one version to another (e.g. database locking in pre-3.0 vs. collection locking for most operations in post-3.0 using NMAPv1).
I don't think it's concurrency issue with MongoDB, but I could be driver or even with test itself.
I have created a sample application couple of weeks ago to stress test MongoDB while working on a nasty bug. I used C# and MongoDB 3.0 on Windows 10. I have inserted million of documents in multithreaded environment but couldn't crash MongoDB.
Parallel.For(0, 10000, (x =>
{
var lstDocs = new List<BsonDocument>();
for (var i = 0; i < 100; i++)
{
lstDocs.Add(new BsonDocument(doc));
}
collection.InsertMany(lstDocs);
lstDocs.Clear();
}));
You can find code in gist here.
You should not be calling MongoClient.connect every time. That's causing a ton of connections to open and close all the time which is overloading mongo. You should let the MongoClient manage the connection pool. Change it so that you store the db object from MongoClient.connect. Something like this:
var db
MongoClient.connect(url, function(err, database){
db = database;
}
MongoInternals.RemoteCollectionDriver("mongodb://#{server.ip}:#{server.port}/#{server.dbName}")
If I call multiple remote MongoDB methods and if there are collecitons with the same names, Meteor throws the error something like this, "collectionName/insert is already exist..."
I think Meteor creates each collection's methods internally so that control each collection, but I need to control several MongoDB at a time with some reasons.
How can I avoid this situation?
In addition,
I realize I can use Npm Mongo driver directly like this without any NPM package involving.
var MongoClient = MongoInternals.NpmModules.mongodb.module.MongoClient;
// Connection URL
var url = 'mongodb://localhost:27017/myproject';
// Use connect method to connect to the Server
MongoClient.connect(url, function(err, db) {
console.log("Connected correctly to server");
try {
var collection = db.collection('documents');
collection.find({}).toArray(function(err, docs){
console.log(err);
console.log(docs);
});
}
catch(err) {
console.log(err);
}
db.close();
});
But this still forces me to control each DB with the Node.js callback style.
Is there any idea to avoid this?
I have been checking this issue, and I found a way to do it.
The solutions that I had seen to connect several databases where:
storageServerDriver = new MongoInternals.RemoteCollectionDriver("mongodb://ip:port/dbName")
#Collection = new Mongo.Collection("collection", { _driver: storageServerDriver })
But as you mentioned before, with two collections with same name, an error was thrown (internally Meteor identifies the collections by their name, so it tries to override the structure of the collection already created).
Anyway, to fix this, you can use the following hack:
storageServerDriver = new MongoInternals.RemoteCollectionDriver("mongodb://ip:port/dbName")
#CollectionTwo = storageServerDriver.open('collection')
I am new to mongodb and node.js. I have a simple node.js app that in order to run I have to run the command
mongod
to make sure mongodb is running.
locally this works fine.
now I am moving to heroku, I createrd a Procfile where I declare :
web: node http.js
so this take care of starting node.js.
Now how can I call the command
mongod
on heroku?
Update
before the heroku requirement this is how I used to initialize my mongodb :
var mongo = require('mongodb');
var Server = mongo.Server,
Db = mongo.Db,
BSON = mongo.BSONPure,
ObjectID = mongo.ObjectID;
var server = new Server('localhost', 27017, {auto_reconnect: true});
db = new Db('sampledb', server);
db.open(function(err, db) {
if(!err) {
console.log("Connected to 'sampledb' database");
db.collection('sample', {strict:true}, function(err, collection) {
if (err) {
console.log("The 'sample' collection doesn't exist. Creating it with sample data...");
populateDB();
}
});
}
});
in order to accomodate heroku here is how I initialize mongodb :
var mongo = require('mongodb');
var Server = mongo.Server,
Db = mongo.Db,
BSON = mongo.BSONPure,
ObjectID = mongo.ObjectID;
var mongoUri = process.env.MONGOLAB_URI ||
process.env.MONGOHQ_URL ||
'mongodb://localhost/sampledb';
mongo.Db.connect(mongoUri, function (err, db) {
db.collection('sample', function(er, collection) {
if (err) {
console.log("The 'sample' collection doesn't exist. Creating it with
sample data...");
populateDB();
}
});
});
Now this throws an error because db is not recognized. I think I am missing a conceptual issue here, I am new to this world all together.
any help is appreciated.
You don't call the command mongod on heroku.
You'd add one of the mongo add-ons from addons.heroku.com to your app, then have your code connect to the credentials in the config variable exported to your app by the add-on provider.