Moongose ready state and aws lambda - cache connection - mongodb

I am trying to implement a connect to database function that uses mongoose (MongoDB), and AWS lambda functions. Best practice, as I understand it, is to cache the database so that the connection can be reused. This, I have working. However, the problem is that the mongoose.readyState variable uses some fixed amount of time to determine if the connection is valid or not.
I was wondering if there is an alternative to .readyState to check if the connection is valid. I know that I can try to run a random query against the database every time I want to reuse the connection. Thus, I could determine if an exception happens or not. However, this feels a little hackish.
Here is my connect to database function:
let cacheDb = null
let options = {
useMongoClient: true,
autoIndex: true,
autoReconnect: true,
keepAlive: true,
socketOptions: {
keepAlive: true,
autoReconnect: true,
connectTimeoutMS: 30000
},
reconnectTries: Number.MAX_VALUE,
reconnectInterval: 200,
poolSize: 1,
bufferMaxEntries: 0,
};
mongoose.Promise = global.Promise
module.exports = {
connectToDatabase: function connectToDatabase(context) {
context.callbackWaitsForEmptyEventLoop = false
if (cacheDb && mongoose.connection.readyState == 1) {
return cacheDb
}
else {
mongoose.connect(process.env['MONGODB_URI'], options, function(error){
if(error){
console.log(error.toString())
}
cacheDb = mongoose.connection
return cacheDb
})
}
}
}
Do you have any idea as to an alternative approach?

In order to check for a valid connection , you should be checking your cached db serverConfig.isConnected method.
like so:
if (cachedDb && cachedDb.serverConfig.isConnected()) { ...
This is according to mongodb, in this helpful article:
https://www.mongodb.com/blog/post/optimizing-aws-lambda-performance-with-mongodb-atlas-and-nodejs

Related

what is mongoose connection validity time?

I have custom circuit breaker logic for 3rd party services. I checked mongoose connection status like below,
mongoose.connection.on("connected", function() {
console.log("Mongoose connected");
});
mongoose.connection.on("disconnected", function() {
console.log("Mongoose disconnected");
});
while running on server logs like below,
Mongoose disconnected //Happened 12.30PM
// after sometime
Mongoose connected //Happened 12.35PM
Mongoose disconnected //Happened 12.35PM
Mongoose connected //Happened 12.35PM
My connection string,
mongoose.connect(URL, {
keepAlive: true,
keepAliveInitialDelay: 300000,
socketTimeoutMS: 300000,
poolSize: 10,
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false,
useCreateIndex: true
})
I dont know while connect why system act like above in same second of time interval ?
Why mongo connection goes to disconneted mode even i apply "keepAlive" ?
Is there any validity for mongoose connection alive status?

MongoDB Atlas connections to cluster(s) exceeded- NextJS + NextAuth

I was creating a website using NextJS and for authentication uses NextAuth. And the database is on free tier in MongoDB Atlas.
I have two versions of code for database connection. One is this:
/**
* MongoDB Connection
*
* */
import mongoose from 'mongoose'
const MONGODB_URI = process.env.MONGODB_URL
if (! process.env.MONGODB_URL) {
throw new Error(
'Please define the MONGODB_URI environment variable inside .env.local'
)
}
/**
* Global is used here to maintain a cached connection across hot reloads
* in development. This prevents connections growing exponentially
* during API Route usage.
*/
let cached = global.mongoose
if (!cached) {
cached = global.mongoose = { conn: null, promise: null }
}
async function dbConnect() {
if (cached.conn) {
return cached.conn
}
if (!cached.promise) {
const opts = {
useNewUrlParser: true,
useUnifiedTopology: true,
bufferCommands: false,
// bufferMaxEntries: 0,
// useFindAndModify: false,
// useCreateIndex: true,
}
cached.promise = mongoose.connect(process.env.MONGODB_URL, opts).then((mongoose) => {
return mongoose
})
}
cached.conn = await cached.promise
return cached.conn
}
export default dbConnect
So, before making any DB related queries via code, I call await dbConnect(). It is working fine.
But for storing the sessions in DB, in NextAuth, I was not able to use the above function. So for that, am using this custom code (/lib/mongodb.js):
/**
*
* Used only for Next-Auth
*
*/
import { MongoClient } from "mongodb"
const uri = process.env.MONGODB_URL
const options = {
useUnifiedTopology: true,
useNewUrlParser: true,
}
let client
let clientPromise
if (!process.env.MONGODB_URL) {
throw new Error("Please add your Mongo URI to .env.local")
}
if (process.env.NODE_ENV === "development") {
// In development mode, use a global variable so that the value
// is preserved across module reloads caused by HMR (Hot Module Replacement).
if (!global._mongoClientPromise) {
client = new MongoClient(uri, options)
global._mongoClientPromise = client.connect()
}
clientPromise = global._mongoClientPromise
} else {
// In production mode, it's best to not use a global variable.
client = new MongoClient(uri, options)
clientPromise = client.connect()
}
// Export a module-scoped MongoClient promise. By doing this in a
// separate module, the client can be shared across functions.
export default clientPromise
And the code in my /pages/api/auth/[...nextauth].js is like this:
import NextAuth from 'next-auth'
import { MongoDBAdapter } from "#next-auth/mongodb-adapter"
import mongodb from '../../../lib/mongodb'
//...
export default async function auth(req, res) {
return await NextAuth(req, res, {
//....
adapter: MongoDBAdapter({
db: (await mongodb).db("my_db_name_here")
}),
//....
})
}
Here's the packages am using:
"mongodb": "^4.1.2",
"mongoose": "^6.0.1",
"next": "11.0.1",
"next-auth": "^4.0.0-beta.6",
"react": "17.0.2",
"react-dom": "17.0.2",
The problem is, am receiving email notifications sometimes like the following:
My website is still in testing phase(tested by two persons only) and is hosting in Vercel server. I believe this could be because NextAuth is creating new db connections each time? Any thoughts on what went wrong?
clientPromise in next-auth is local, you create new client and 5 connections every time. Just use global.mongoose.conn.
The docs for MongoDBAdapter says it needs a promise that resolves to a client, so it must be something like this:
export default NextAuth({
adapter: MongoDBAdapter(dbConnect().then(mon => mon.connection.getClient())),
...
})
In you case you seem to use db. I couldn't find any references for MongoDBAdapter to accept something liek {db: ...} but you can get the db from mongoose as following:
await (dbConnect().then(mon => mon.connection.getClient().db("my_db_name_here")))
Or without parameter to use the same database as configured in mongoose connection.
UPDATE
The issue with number of connections from Vercel is covered in Vercel creates new DB connection for every request

Exceeded timeout of 5000 ms for a test with Jest and MongoDB

I'm trying to implement database population by using a migration function. The code works perfectly, it saves all the data into the database, but the test for the function is failing, and now I would like to know why?
I'm getting the "Exceeded timeout of 5000 ms" error for this particular test. I've written 166 tests for this app and all of them are passing.
Here is the function I want to test:
const doMigration = async ({ model, data }) => {
await model.collection.insertMany(data)
}
And here is the test:
const { Amodel } = require('../../../models/Amodel')
const { doMigration } = require('../../../database/migrations')
describe('Database Population', () => {
it ('Should populate the database using migrations', async () => {
const data = [{ name: 'A' }, { name: 'B' }]
const model = Amodel
const migration = { name: 'Amodel', model, data }
await doMigration(migration)
const countAfter = await Amodel.count()
expect(countAfter).toBe(2)
})
})
In this test I simply import the function, the model and create a migration object that then is passed to the function.
What did I try?
Tried using just the countAfter without using the doMigration function, and it still generates the same timeout error.
Tried increasing the time for this test to 30000, failed with error saying that the mongodb time exceeded the 10000 ms.
Here is the github repository: https://github.com/Elvissamir/Fullrvmovies
What is happening, how can I solve this error?
The problem was the way the mongodb connection was handled. When testing, the app created a connection to the db on startup, and then the jest tests used that connection, that caused some issues.
The solution was to connect to the database on startup only if the environment is set to testing, otherwise the connection will be handled by each set of tests.
In each set I added a beforeAll and afterAll to open and close the connection to the database.
Hope it helps anyone that finds the same problem or has similar issues.
The orientation is that the message reflect the actual reason, So i recommand to follow the following steps:
use the following code to check mongo state:
const { MongoMemoryServer } = require("mongodb-memory-server");
const mongoose = require("mongoose");
(async () => {
mongod = await MongoMemoryServer.create();
const mongoUri = mongod.getUri();
await mongoose.connect(mongoUri, {
useNewUrlParser: true,
useUnifiedTopology: true,
}).then((result) => {
console.log(result.connection.readyState)
console.log(result.connection.host)
}).catch((err) => {
});;
})();
if you are using mongodb-memory-server add "testTimeout" attribute:
"jest": {
"preset": "ts-jest",
"testEnvironment": "node",
"setupFilesAfterEnv": [
"./src/test/setup.ts"
],
"testTimeout": 15000
},
If all above still huppens check the time-out of all inter-test operation

Unable to connect to Mongodb Atlas using mongoose

I am trying to connect to a cluster created in Mongodb Atlas using mongoose in node js and I am facing below issues when doing so.
When I use the connection string that is given in the Mongo db atlasmongodb+srv://lm_dev_app:<password>#lmdev-q5biw.mongodb.net/test?retryWrites=true&w=majorityI get below error
{ Error: queryTxt EBADNAME lmdev-q5biw.mongodb.net
at QueryReqWrap.onresolve [as oncomplete] (dns.js:196:19)
errno: 'EBADNAME',
code: 'EBADNAME',
syscall: 'queryTxt',
hostname: 'lmdev-q5biw.mongodb.net'}
I cannot use this connection string in Mongodb Compass as well as I am getting the same error there.
If I try to connect using mongodb://lm_dev_app:<password>#lmdev-shard-00-01-q5biw.mongodb.net/test i get below error
MongooseServerSelectionError: connection to 54.66.221.230:27017 closed
However I am able to connect to each node using Mongodb Compass which eliminates the possibility of my ipaddress not being whitelisted.
Here is the sample code that I am using
const mongoosePromise = mongoose.connect("mongodb://lm_dev_app:<password>#lmdev-shard-00-01-q5biw.mongodb.net/test", {
useNewUrlParser: true,
useUnifiedTopology: true,
replicaSet: "LMDEV"
}, (err) => {
if (err) {
console.log(err);
} else {
console.log("Successful");
}
});
Any thoughts on what is happening here.
There are couple of things that I need to highlight here.
The default connection string that is shown in Mongodb Atlas seems to be wrong. It shows you mongodb+srv://<username>:<password>#<cluster_url>/test?retryWrites=true&w=majority. But I used mongodb://<username>:<password>#<node_url>:27017/ to make it work. You can also use mongodb://<username>:<password>#<node_url>:27017/admin.
Pass ssl:true in the options that we are passing.
Finally one of the 3 options can be used to connect to the database.
a. const mongoosePromise = mongoose.connect("mongodb://lm_dev_app:<password>#lmdev-shard-00-01-q5biw.mongodb.net:27017/", {
useNewUrlParser: true,
useUnifiedTopology: true,
authSource:"admin",
ssl: true,
}, (err) => {
if (err) {
console.log(err);
} else {
console.log("Successful");
}
});
b. const mongoosePromise = mongoose.connect("mongodb://lm_dev_app:<password>#lmdev-shard-00-01-q5biw.mongodb.net:27017/", {
useNewUrlParser: true,
useUnifiedTopology: true,
authSource:"admin",
ssl: true,
}, (err) => {
if (err) {
console.log(err);
} else {
console.log("Successful");
}
});
c. const mongoosePromise = mongoose.connect("mongodb://lm_dev_app:<password>#lmdev-shard-00-01-q5biw.mongodb.net:27017/admin", {
useNewUrlParser: true,
useUnifiedTopology: true,
ssl: true,
}, (err) => {
if (err) {
console.log(err);
} else {
console.log("Successful");
}
});
EDIT 1:
After having a chat with Atlas support team I was told that issue in point 1 is due to DNS resolution issue with my service provider. So i have changed my DNS settings to point to a public DNS server.
After trying different connection strings for several hours, I finally just copy/pasted the connection string from MongoDB Compass and it works! (first connect, then edit the connection string as it will change)
It looks like this:
mongodb+srv://username:password#fra-atlas-shard.abcde.mongodb.net/test?authSource=admin&replicaSet=atlas-abcde-shard-0&readPreference=primary&appname=MongoDB%20Compass&ssl=true

Reliably reconnect to MongoDB

UPDATE: I am using the 2.1 version on the driver, against 3.2
I have a node application that uses MongoDB. The problem I have is that if the MongoDB server goes down for any reason, the application doesn't reconnect.
To get this right, I based my tests on the code in this official tutorial.
var MongoClient = require('mongodb').MongoClient
, f = require('util').format;
MongoClient.connect('mongodb://localhost:27017/test',
// Optional: uncomment if necessary
// { db: { bufferMaxEntries: 3 } },
function(err, db) {
var col = db.collection('t');
setInterval(function() {
col.insert({a:1}, function(err, r) {
console.log("insert")
console.log(err)
col.findOne({}, function(err, doc) {
console.log("findOne")
console.log(err)
});
})
}, 1000)
});
The idea is to run this script, and then stop mongod, and then restart it.
So, here we go:
TEST 1: stopping mongod for 10 seconds
Stopping MongoDb for 10 seconds does the desired result: it will stop running the queries for those 10 seconds, and then will run all of them once the server is back ip
TEST 2: stopping mongod for 30 seconds
After exactly 30 seconds, I start getting:
{ [MongoError: topology was destroyed] name: 'MongoError', message: 'topology was destroyed' }
insert
{ [MongoError: topology was destroyed] name: 'MongoError', message: 'topology was destroyed' }
The trouble is that from this on, when I restart mongod, the connection is not re-establised.
Solutions?
Does this problem have a solution? If so, do you know what it is?
Once my app starts puking "topology was destroyed", the only way to get everything to work again is by restarting the whole app...
There are 2 connection options that control how mongo nodejs driver reconnects after connection fails
reconnectTries: attempt to reconnect #times (default 30 times)
reconnectInterval: Server will wait # milliseconds between retries
(default 1000 ms)
reference on mongo driver docs
Which means that mongo will keep trying to connect 30 times by default and wait 1 second before every retry. Which is why you start seeing errors after 30 seconds.
You should tweak these 2 parameters based on you needs like this sample.
var MongoClient = require('mongodb').MongoClient,
f = require('util').format;
MongoClient.connect('mongodb://localhost:27017/test',
{
// retry to connect for 60 times
reconnectTries: 60,
// wait 1 second before retrying
reconnectInterval: 1000
},
function(err, db) {
var col = db.collection('t');
setInterval(function() {
col.insert({
a: 1
}, function(err, r) {
console.log("insert")
console.log(err)
col.findOne({}, function(err, doc) {
console.log("findOne")
console.log(err)
});
})
}, 1000)
});
This will try 60 times instead of the default 30, which means that you'll start seeing errors after 60 seconds when it stops trying to reconnect.
Sidenote: if you want to prevent the app/request from waiting until the expiration of the reconnection period you have to pass the option bufferMaxEntries: 0. The price for this is that requests are also aborted during short network interruptions.
package.json: "mongodb": "3.1.3"
Reconnect existing connections
To fine-tune the reconnect configuration for pre-established connections, you can modify the reconnectTries/reconnectInterval options (default values and further documentation here).
Reconnect initial connection
For the initial connection, the mongo client does not reconnect if it encounters an error (see below). I believe it should, but in the meantime, I've created the following workaround using the promise-retry library (which uses an exponential backoff strategy).
const promiseRetry = require('promise-retry')
const MongoClient = require('mongodb').MongoClient
const options = {
useNewUrlParser: true,
reconnectTries: 60,
reconnectInterval: 1000,
poolSize: 10,
bufferMaxEntries: 0
}
const promiseRetryOptions = {
retries: options.reconnectTries,
factor: 1.5,
minTimeout: options.reconnectInterval,
maxTimeout: 5000
}
const connect = (url) => {
return promiseRetry((retry, number) => {
console.log(`MongoClient connecting to ${url} - retry number: ${number}`)
return MongoClient.connect(url, options).catch(retry)
}, promiseRetryOptions)
}
module.exports = { connect }
Mongo Initial Connect Error: failed to connect to server [db:27017] on first connect
By default the Mongo driver will try to reconnect 30 times, one every second. After that it will not try to reconnect again.
You can set the number of retries to Number.MAX_VALUE to keep it reconnecting "almost forever":
var connection = "mongodb://127.0.0.1:27017/db";
MongoClient.connect(connection, {
server : {
reconnectTries : Number.MAX_VALUE,
autoReconnect : true
}
}, function (err, db) {
});
With mongodb driver 3.1.10, you can set up your connection as
MongoClient.connect(connectionUrl, {
reconnectInterval: 10000, // wait for 10 seconds before retry
reconnectTries: Number.MAX_VALUE, // retry forever
}, function(err, res) {
console.log('connected')
})
You do not have to specify autoReconnect: true as that's the default.
It's happening because it might have crossed the retry connection limit. After number of retries it destroy the TCP connection and become idle. So for it increase the number of retries and it would be better if you increase the gap between connection retry.
Use below options:
retryMiliSeconds {Number, default:5000}, number of milliseconds between retries.
numberOfRetries {Number, default:5}, number of retries off connection.
For more details refer to this link https://mongodb.github.io/node-mongodb-native/driver-articles/mongoclient.html
Solution:
MongoClient.connect("mongodb://localhost:27017/integration_test_?", {
db: {
native_parser: false,
retryMiliSeconds: 100000,
numberOfRetries: 100
},
server: {
socketOptions: {
connectTimeoutMS: 500
}
}
}, callback)
Behavior may differ with different versions of driver. You should mention your driver version.
driver version : 2.2.10 (latest)
mongo db version : 3.0.7
Below code will extend the time mongod can take to come back up.
var MongoClient = require('mongodb').MongoClient
, f = require('util').format;
function connectCallback(err, db) {
var col = db.collection('t');
setInterval(function() {
col.insert({a:1}, function(err, r) {
console.log("insert")
console.log(err)
col.findOne({}, function(err, doc) {
console.log("findOne")
console.log(err)
});
})
}, 1000)
}
var options = { server: { reconnectTries: 2000,reconnectInterval: 1000 }}
MongoClient.connect('mongodb://localhost:27017/test',options,connectCallback);
2nd argument can be used to pass server options.
If you was using Mongoose for your Schemas, it would be worth considering my option below since mongoose was never retrying to reconnect to mongoDB implicitly after first attempt failed.
Kindly note I am connecting to Azure CosmosDB for MongoDB API. On yours maybe on the local machine.
Below is my code.
const mongoose = require('mongoose');
// set the global useNewUrlParser option to turn on useNewUrlParser for every connection by default.
mongoose.set('useNewUrlParser', true);
// In order to use `findOneAndUpdate()` and `findOneAndDelete()`
mongoose.set('useFindAndModify', false);
async function mongoDbPool() {
// Closure.
return function connectWithRetry() {
// All the variables and functions in here will Persist in Scope.
const COSMODDBUSER = process.env.COSMODDBUSER;
const COSMOSDBPASSWORD = process.env.COSMOSDBPASSWORD;
const COSMOSDBCONNSTR = process.env.COSMOSDBCONNSTR;
var dbAuth = {
auth: {
user: COSMODDBUSER,
password: COSMOSDBPASSWORD
}
};
const mongoUrl = COSMOSDBCONNSTR + '?ssl=true&replicaSet=globaldb';
return mongoose.connect(mongoUrl, dbAuth, (err) => {
if (err) {
console.error('Failed to connect to mongo - retrying in 5 sec');
console.error(err);
setTimeout(connectWithRetry, 5000);
} else {
console.log(`Connected to Azure CosmosDB for MongoDB API.`);
}
});
};}
You may decide to export and reuse this module everywhere you need to connect to db via Dependency Injection. But instead I will only show how to access the database connection for now.
(async () => {
var dbPools = await Promise.all([mongoDbPool()]);
var mongoDbInstance = await dbPools[0]();
// Now use "mongoDbInstance" to do what you need.
})();