My express web app uses Winston for logging, and the logs are saved to a mongolab hosted mongoDB (replica set) using winston-mongodb.
Everything was working fine for a few days, then when traffic picked up a bit the logs just stopped saving/connecting to the DB. Unfortunately this means I have no logs to check to see what went wrong - a frustrating situation. Everything else is still working fine - other collections (like users) are saving and updating correctly, the server is up.
I've tried redeploying/restarting the server to no avail, the logs won't start recording again. I suspect there is some nuanced problem with my server/logger setup, possibly a race condition related to the DB connection, but I'm really lost right now. Here's the code:
//server.js (simplified required modules - only ones possibly relevant to question)
var express = require( 'express' );
var session = require('express-session');
var methodOverride = require('method-override');
var cookieParser = require('cookie-parser');
var mongoose = require( 'mongoose' );
var uriUtil = require('mongodb-uri');
var config = require('config');
var bodyParser = require('body-parser');
var MongoStore = require('connect-mongostore')(session);
var logger = require('./logs/logger.js');
// DATABASE CONNECT
var options = { server: { socketOptions: { keepAlive: 1, connectTimeoutMS: 30000 } },
replset: { socketOptions: { keepAlive: 1, connectTimeoutMS : 30000 } } };
var mongodbUri = "mongodb://<user>:<pw>#ds012345-a0.mongolab.com:12345,ds012345-a1.mongolab.com:12345/<db>"
//use mongo-uri to make sure url is in the correct format
var mongooseUri = uriUtil.formatMongoose(mongodbUri);
mongoose.connect(mongooseUri, options);
var conn = mongoose.connection;
// __Create server__
var app = express();
conn.on('open', function(e) {
var sessionStore = new MongoStore({ mongooseConnection: mongoose.connections[0] });
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: false}));
app.use(methodOverride());
app.use(cookieParser());
app.use(session({
store: sessionStore,
secret: config.session.secret,
cookie: {
maxAge: config.session.maxage,
httpOnly: true,
secure: false
//If I set 'secure:true', it prevents the session from working properly.
},
saveUninitialized: true,
resave: true
}));
app.use(passport.initialize());
app.use(passport.session());
app.use( express.static( path.join( __dirname, 'site') ) );
//Routes
app.use('/api/forgot', controllers.forgotpw);
//etc.
});
conn.on('connected', function(){
//Start server
app.listen(config.port, function() {
logger.info( 'Express server listening on port %d in %s mode', config.port, app.settings.env );
logger.info(process.env.NODE_ENV);
});
});
//logger.js
var winston = require('winston');
var config = require('config');
require('winston-mongodb').MongoDB;
if (process.env.NODE_ENV == 'production') {
var winston = new (winston.Logger)({
transports: [
new (winston.transports.Console)({ level: 'warn' }),
new (winston.transports.MongoDB)({
//db : 'atlas_database',
level : 'debug',
//ssl : true,
dbUri : "mongodb://<user>:<pw>#ds012345-a0.mongolab.com:12345,ds012345-a1.mongolab.com:12345/<db>"
})
]
});
winston.info('Chill Winston, the logs are being captured with production settings');
}
module.exports = winston;
Any thoughts on what is causing the logs to no longer function would be greatly appreciated.
When Winston establishes a connection to a Mongo instance, it looks specifically for the replicaSet flag to determine if the connection is a replica set. Otherwise, it connects to the first host in the dbUri. ( Source: https://github.com/indexzero/winston-mongodb/blob/master/lib/winston-mongodb.js#L28 )
To have configure Winston with a connection that will properly handle failovers, use the following URI:
dbUri : "mongodb://<user>:<pw>#ds012345-a0.mongolab.com:12345,ds012345-a1.mongolab.com:12345/<db>?replicaSet=<replsetname>"
If you ever want to test your code to verify that it can handle a replica set failover gracefully, you can use a testing cluster called flip-flop described here: http://mongolab.org/flip-flop/
(Full Disclosure: I work for MongoLab.)
Related
I'm new in the MEAN developing, I'm developing a simple app, and for my first step I'm trying to connect to my mongodb, so I installed node, express, morgan,mongodb, mongoose.
So here is my code in index.js:
const express = require('express');
const morgan = require('morgan');
const app = express();
const { MongoClient } = require('./database');
// Settings
app.set('port', process.env.PORT || 3000);
// Middlewares
app.use(morgan('dev'));
app.use(express.json());
// Routes
// Starting the server
app.listen(app.get('port'), () => {
console.log('server on port', app.get('port'));
});
and then the code on my database.js:
const MongoClient = require('mongodb').MongoClient;
const uri = "mongodb+srv://duke:<password>#cluster0-dreyi.mongodb.net/test?retryWrites=true&w=majority";
const client = new MongoClient(uri, { useNewUrlParser: true });
client.connect(err => {
const collection = client.db("test").collection("devices");
console.log("horrorrrrrr");
// perform actions on the collection object
client.close();
});
module.exports = MongoClient;
I also try this code that is on the mongodb page to connect to the application:
const MongoClient = require('mongodb').MongoClient;
const uri = "mongodb+srv://duke:<password>#cluster0-dreyi.mongodb.net/test?retryWrites=true&w=majority";
const client = new MongoClient(uri, { useNewUrlParser: true });
client.connect(err => {
const collection = client.db("test").collection("devices");
// perform actions on the collection object
client.close();
});
Of course I change the password to the real one. Please keep in my today it's my first time I touch mongodb and also the MEAN full stack, and I spent too many hours stuck in this connection.
this is the error I get:
(node:5284) DeprecationWarning: current Server Discovery and Monitoring engine is deprecated, and will be removed in a future version. To use the new Server Discover and Monitoring engine, pass option { useUnifiedTopology: true } to the MongoClient constructor.
EDIT
#iLiA thanks for your reply ! I tried your code and ain't working, I will show you how I did it with the real password :
const url = 'mongodb+srv://duke:password#cluster0-dreyi.mongodb.net/test?retryWrites=true&w=majority';
const mongoose = require('mongoose');
mongoose.connect(url, {
useNewUrlParser: true,
useCreateIndex: true,
useUnifiedTopology: true,
useFindAndModify: false
})
.then(()=>{
console.log('congrats, problem solved')
})
.catch((err)=>{
console.log(`there is not a problem with ${err.message}`);
process.exit(-1)
})
module.exports = mongoose;
and the error is :
there is not a problem with Server selection timed out after 30000 ms
[nodemon] app crashed - waiting for file changes before starting...
Kind regards,
I am confused about why do you downloaded both mongodb and mongoose but here is mongoose solution
const mongoose = require('mongoose');
mongoose.connect(url, {
useNewUrlParser: true,
useCreateIndex: true,
useUnifiedTopology: true,
useFindAndModify: false
})
.then(()=>{
console.log('congrats, problem solved')
})
.catch((err)=>{
console.log(`there is a problem with ${err.message}`);
process.exit(-1)
})
EDIT:
As it seems you forgot to whitelist your IP address in mongo atlas.
Struggling to get my head around this for a week and a half, I was wondering how to get a .json endpoint that is from a query from the Sequelize ORM. Currently it logs a 404 error "GET /api/users 404 3ms". As you may have heard the documentation for Sequelize is pretty limited and I've been searching github repo after tutorial and none have worked thus far, so I'd thought I'd ask here.
A small excerpt (code on https://github.com/NatuMyers/A.M.E.N.SQL-Stack):
// VARS -----------------------------
var express = require('express')
, bodyParser = require('body-parser')
, errorHandler = require('errorhandler')
, methodOverride = require('method-override')
, morgan = require('morgan')
, http = require('http')
, path = require('path')
, db = require('./models')
var router = require('express').Router();
var app = express()
// all environments
app.set('port', process.env.PORT || 3000)
app.set('views', __dirname + '/views')
app.set('view engine', 'jade')
app.use(morgan('dev'))
app.use(bodyParser())
app.use(methodOverride())
app.use(express.static(path.join(__dirname, 'public')))
// SEQUELIZE MODELS
var userVar = require('./models/user');
// dev only
if ('development' === app.get('env')) {
app.use(errorHandler())
}
// Make db, and make it listen
db
.sequelize
.sync()
.complete(function(err) {
if (err) {
throw err
} else {
http.createServer(app).listen(app.get('port'), function() {
console.log('Express server listening on port ' + app.get('port'))
})
}
})
// HTTP GET endpoints
module.exports = function() {
router.get('/', function(req, res, next){
res.json({ message: 'This works at localhost:3000/api but getting a list of users is a pain :(' });
});
// question
router.get('/users', function(req, res, next){
res.json(/* I need to make sequelize send a part of the User db here. */);
});
return router;
};
I moved on from this by using Epilogue.js (in a vanilla way).
I added models INLINE with Sequelize (I wasted lots of time trying to import models), then add any middle ware and create the restful api based on the syntax below.
// 1. ADD SEQUELIZE MODELS ---- ---- ---- ----
var database = new Sequelize('raptroopdb', 'root', 'strongpassword');
var Employee = database.define('Employee', {
name: Sequelize.STRING,
hireDate: Sequelize.DATE
});
// Add Account model with foreign key constraint to Employee
var Account = database.define('Account', {
name: Sequelize.STRING,
managerId: {
type: Sequelize.INTEGER,
references: {
// This is a reference to model Employee
model: Employee,
// This is the column name of the referenced model
key: 'id',
}
}
});
// 2. ROOM FOR MIDDLEWARE to use for all requests
router.use(function(req, res, next) {
// do logging
console.log('In server.js');
// make sure we go to the next routes and don't stop here
next();
});
// Initialize epilogue
epilogue.initialize({
app: app,
sequelize: database
});
app.use(express.static(__dirname + "/public"));
app.get('/', function(req, res) {
res.redirect('/public/index.html');
});
// 3. Create REST resource
var employeeResource = epilogue.resource({
model: Employee,
endpoints: ['/api/employees', '/api/employees/:id']
});
var acctResource = epilogue.resource({
model: Account,
endpoints: ['/api/accounts', '/api/accounts/:id']
});
// Create database and listen
database
.sync({
force: false
})
.then(function() {
app.listen(port, function() {
console.log('listening at %s', port);
});
});
In the config/log.js file for Sails this is my code:
var express = require("express");
var app = express();
var winston = require('winston');
var logger = new(winston.Logger)({
transports: [
new(winston.transports.File)({
level: 'info',
timestamp: true,
filename: './logFile.log',
handleExceptions: true,
json: true,
colorize: false
}),
new(winston.transports.Console)({
level: 'debug',
timestamp: true,
handleExceptions: true,
json: false,
colorize: true
})
],
exitOnError: false
});
logger.stream = {
write: function (message, encoding) {
logger.verbose(message);
}
};
app.use(require('morgan')("combined", {"stream": logger.stream}));
module.exports.log = {
level: 'info',
custom: logger
};
I'm trying to use morgan along with winston to log all the HTTP requests. I found an example online that said to do it this way, and this makes sense to me, but for some reason my log file isn't showing any of the requests that are made. The winston part is fine as it is logging all the information that it should be, but I don't know how to get morgan to work with winston. Any advice or suggestions? Thanks!
Morgan is an express middleware so it should be loaded as a custom middleware to Sails.
To do that add the following to config/http.js:
customMiddleware: function(app) {
app.use(require('morgan')("combined", {"stream": sails.config.log.custom.stream}));
}
I am using self-signed certificated to encrypt traffic data.
My .crt and .key files are located at /etc/nginx/ssl/
(some_file.key and some_file.crt)
I was using socket.io over http but tried to get it over to https. Here is my actual code:
var formidable = require('formidable');
var express = require('express');
var fs = require('fs');
var privateKey = fs.readFileSync('../etc/nginx/ssl/some_file.key').toString();
var certificate = fs.readFileSync('../etc/nginx/ssl/some_file.crt').toString();
//how can I exclude this? (I have no intermediate, should I?)
var ca = fs.readFileSync('../intermediate.crt').toString();
var app = express.createServer({key:privateKey,cert:certificate,ca:ca });
var io = require('socket.io');
app.listen(3000, function(){
//wait on 3000
});
app.post('/posts', function(req, res){
//server communication
});
io.on('connection', function(socket){
//wait on connections
});
Client-side:
var socket = io(url + ":3000", { "secure": true, 'connect timeout': 5000 });
Is this the correct way to do it? I based my https code on examples, so I'm doubting on whether this is well enough (I know it isn't, but should be close). When I run the code, I also get an error no such file or directory '../etc/nginx/ssl/some_file.key'...
I use it this way, although this is very dependent on express version you are using. This is for version 3.4
var express = require('express')
, app = express()
,fs = require('fs')
,events = require('events');
...
var options = {
key: fs.readFileSync('/etc/nginx/ssl/some_file.key'),
cert: fs.readFileSync('/etc/nginx/ssl/some_file.crt')
};
/*
*Configuration
*
*/
var server = require('https').createServer(options, app), io = require("socket.io").listen(server);
var port = 8443;
var ipaddr = '0.0.0.0';
app.configure(function() {
app.set('port', port);
app.set('ipaddr', ipaddr);
app.use(express.bodyParser());
app.use(express.methodOverride());
...
});
I have the following code to set up a session store with connect-mong
var express = require('express');
var MongoStore = require('connect-mongo')(express);
var app = express();
app.use(express.cookieParser());
app.use(express.session({
secret: '1234567890QWERTY',
maxAge: new Date(Date.now() + 3600000),
store: new MongoStore({
db: 'mydb',
host: '127.0.0.1'
}),
cookie: { maxAge: 24 * 60 * 60 * 1000 }
}));
In the requests and responses from the server I can see the sessionIDs, but the database is not created or if I create it the session collection does not gets filled. I do not receive any errors on the console.
I cannot find what am I missing ( using express 3).
It happens because nobody connected yet to your express server. So there's no session to store.
Add
app.listen(3000, function() {
console.log('now listening on http://localhost:3000/');
});
At the end and connect to the server in your browser. Immediately you will get sessions collections in your Mongo db.
I had multiple use of the session and cookieParser. This caused the problem.