Store cookie information with express-rate-limit - mongodb

Is there a way by which I can store user cookies (jwt) in my mongodb database with express-rate-limit and rate-limit-mongo packages?
Code that I am currently using :
var limiter = new RateLimit({
store: new MongoStore({
uri: process.env.MONGO_URI,
expireTimeMs: 60 * 1000 * 60,
}),
max: 150,
windowMs: 10 * 60 * 1000,
message: "Too many requests in a short duration, IP Banned for an hour.",
});
I want to know the jwt cookie (if it exists) associated with the request too somehow so that I can know who the culprit was.
Basically, how can I access the request object and store it in the rate limiting database

I was able to do this with the onLimitReached function available in express-rate-limit like so:
var queslimiter = new RateLimit({
store: new MongoStore({
uri: process.env.MONGO_URI,
expireTimeMs: 60 * 1000 * 60,
collectionName: "ansForce",
}),
max: 30,
message: "Too many requests in a short duration, IP Banned for an hour.",
onLimitReached: async function (req, res) {
try {
const blacklist = new Blacklist({
team: req.cookies.team,
});
try {
const bad = await blacklist.save();
} catch (e) {
console.log(e);
}
} catch (e) {}
},
});

Related

Typesense: Firebase Function timeout using Typesense client

I have a Firebase function trigger for when an item is updated. In most instances this function completes and updates the item in the Typesense database however, occasionaly this function fails with a timeout.
This happens seemingly randomly for this and other onCreate and onUpdate Firebase Functions using the Typesense client.
Failed Function Screenshot Example (output from https://console.cloud.google.com/logs)
Logs when function times out
Succesful Function Screenshot Example
Logs when function behaves as expected
Firebase Function:
// Firebase function triggered when a program is updated
export const onUpdate = functions
// 256MB memory and timeout of 2 minutes
.runWith({ memory: '256MB', timeoutSeconds: 120 })
.firestore.document('programs/{docId}')
.onCreate(async (snap, context) => {
const id = context.params.docId;
const data = snap.data();
const client = useTypesense();
const item = { ...data, id };
console.log('Typesense: Updating Program');
await client
.collections('programs')
.documents()
.upsert(item)
.then(() => {
console.log(`Typesense: Program ${item.id} successfully updated!`);
})
.catch((error) => {
console.error('Typesense: Error creating Program: ' + item.id, error);
});
return null;
});
useTypesense() function (Initializing the client)
import Typesense from 'typesense';
const useTypesense = () => {
const host = 'api.oniworkout.app'
const port = '443'
const protocol = 'https'
const apiKey = 'b3fxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx';
console.log(
`Typesense: Initializing client with host: ${host}, port: ${port}, protocol: ${protocol}, apiKey: ${apiKey.substring(
0,
3
)}`
);
const client = new Typesense.Client({
nodes: [
{
host,
port,
protocol,
},
],
apiKey,
connectionTimeoutSeconds: 120,
retryIntervalSeconds: 120,
});
return client;
};
I have tried extending the duration of timeout on the Firebase Function from 60 seconds to 120 seconds. The issue persists.
I haved checked https://api.oniworkout.app/health which returns {"ok":true}

Node postgres - (node:43028) UnhandledPromiseRejectionWarning: error: sorry, too many clients already

I am trying to add data to my databases. There are multiple entries in an array
dataArryay.forEach((element) => {
queryData(elementData)
}
The function is called multiple times
async function queryData(data) {
const queryString = `INSERT............')`;
const query = {
text: queryString
};
const pool = await new pg.Pool({
host: 'localhost',
port: 'xxxx',
user: 'xxxxxx',
database: 'xxxxx',
max: 100,
idleTimeoutMillis: 50000,
connectionTimeoutMillis: 3000,
});
await pool.connect();
await pool.query(query)
await pool.end()
}
It does the insert but it does throw the too many connections error. I have tried .release() and .end()
When querying the connections I get
max_conn = 500
used = 6
res_for_super = 3
res_for_normal = 491
I don't really know what these men but they seem to add up to 500.

How do I set the Skipper body Limit in SailsJS v1.0?

I am currently having an issue when uploading files bigger then 10mb. I am uploading them to an s3 bucket.
I have tried to set the limit within the skipper that gets built in the middleware with the bodyparser.
order: [
'cookieParser',
'session',
'myRequestLogger',
'bodyParser',
'compress',
'poweredBy',
'router',
'www',
'favicon',
],
myRequestLogger: function (req, res, next) {
sails.log("Requested :: ", req.method, req.url);
return next();
},
/***************************************************************************
* *
* The body parser that will handle incoming multipart HTTP requests. *
* *
* https://sailsjs.com/config/http#?customizing-the-body-parser *
* *
***************************************************************************/
bodyParser: (function _configureBodyParser() {
var skipper = require('skipper');
var middlewareFn = skipper({
strict: true,
limit: '50mb'
});
return middlewareFn;
})(),
This doesn't seem to be using the limit property being set at all.
Any advice on this would help.
I'm not entirely sure where you found the limit option for Skipper, but limiting the file size of an upload is kinda documented between skipper-s3 the skipper.
Specifying the maxBytes option when receiving the upload in your action/controller should work.
If you're going to be uploading files to multiple actions/controllers then I'd keep the max file size somewhere like sails.config.custom.maxUploadFilesize so there's a single place to configure it - couldn't find any global Skipper options but I could have missed it.
const MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
req.file('avatar')
.upload({
// Required
adapter: require('skipper-s3'),
key: 'thekyehthethaeiaghadkthtekey',
secret: 'AB2g1939eaGAdesoccertournament',
bucket: 'my_stuff',
maxBytes: MAX_UPLOAD_BYTES
}, function whenDone(err, uploadedFiles) {
if (err) {
return res.serverError(err);
}
return res.ok({
files: uploadedFiles,
textParams: req.params.all()
});
});

Image returned from REST API always displays broken

I am building a content management system for an art portfolio app, with React. The client will POST to the API which uses Mongoose to insert into a MongoDB. The API then queries the DB for the newly inserted image, and returns it to the client.
Here's my code to connect to MongoDB using Mongoose:
mongoose.connect('mongodb://localhost/test').then(() =>
console.log('connected to db')).catch(err => console.log(err))
mongoose.Promise = global.Promise
const db = mongoose.connection
db.on('error', console.error.bind(console, 'MongoDB connection error:'))
const Schema = mongoose.Schema;
const ImgSchema = new Schema({
img: { data: Buffer, contentType: String }
})
const Img = mongoose.model('Img', ImgSchema)
I am using multer and fs to handle the image file. My POST endpoint looks like this:
router.post('/', upload.single('image'), (req, res) => {
if (!req.file) {
res.send('no file')
} else {
const imgItem = new Img()
imgItem.img.data = fs.readFileSync(req.file.path)
imgItem.contentType = 'image/png'
imgItem
.save()
.then(data =>
Img.findById(data, (err, findImg) => {
console.log(findImg.img)
fs.writeFileSync('api/uploads/image.png', findImg.img.data)
res.sendFile(__dirname + '/uploads/image.png')
}))
}
})
I can see in the file structure that writeFileSync is writing the image to the disk. res.sendFile grabs it and sends it down to the client.
Client side code looks like this:
handleSubmit = e => {
e.preventDefault()
const img = new FormData()
img.append('image', this.state.file, this.state.file.name)
axios
.post('http://localhost:8000/api/gallery', img, {
onUploadProgress: progressEvent => {
console.log(progressEvent.loaded / progressEvent.total)
}
})
.then(res => {
console.log('responsed')
console.log(res)
const returnedFile = new File([res.data], 'image.png', { type: 'image/png' })
const reader = new FileReader()
reader.onloadend = () => {
this.setState({ returnedFile, returned: reader.result })
}
reader.readAsDataURL(returnedFile)
})
.catch(err => console.log(err))
}
This does successfully place both the returned file and the img data url on state. However, in my application, the image always displays broken.
Here's some screenshots:
How to fix this?
Avoid sending back base64 encoded images (multiple images + large files + large encoded strings = very slow performance). I'd highly recommend creating a microservice that only handles image uploads and any other image related get/post/put/delete requests. Separate it from your main application.
For example:
I use multer to create an image buffer
Then use sharp or fs to save the image (depending upon file type)
Then I send the filepath to my controller to be saved to my DB
Then, the front-end does a GET request when it tries to access: http://localhost:4000/uploads/timestamp-randomstring-originalname.fileext
In simple terms, my microservice acts like a CDN solely for images.
For example, a user sends a post request to http://localhost:4000/api/avatar/create with some FormData:
It first passes through some Express middlewares:
libs/middlewares.js
...
app.use(cors({credentials: true, origin: "http://localhost:3000" })) // allows receiving of cookies from front-end
app.use(morgan(`tiny`)); // logging framework
app.use(multer({
limits: {
fileSize: 10240000,
files: 1,
fields: 1
},
fileFilter: (req, file, next) => {
if (!/\.(jpe?g|png|gif|bmp)$/i.test(file.originalname)) {
req.err = `That file extension is not accepted!`
next(null, false)
}
next(null, true);
}
}).single(`file`))
app.use(bodyParser.json()); // parses header requests (req.body)
app.use(bodyParser.urlencoded({ limit: `10mb`, extended: true })); // allows objects and arrays to be URL-encoded
...etc
Then, hits the avatars route:
routes/avatars.js
app.post(`/api/avatar/create`, requireAuth, saveImage, create);
It then passes through some user authentication, then goes through my saveImage middleware:
services/saveImage.js
const createRandomString = require('../shared/helpers');
const fs = require("fs");
const sharp = require("sharp");
const randomString = createRandomString();
if (req.err || !req.file) {
return res.status(500).json({ err: req.err || `Unable to locate the requested file to be saved` })
next();
}
const filename = `${Date.now()}-${randomString}-${req.file.originalname}`;
const filepath = `uploads/${filename}`;
const setFilePath = () => { req.file.path = filepath; return next();}
(/\.(gif|bmp)$/i.test(req.file.originalname))
? fs.writeFile(filepath, req.file.buffer, (err) => {
if (err) {
return res.status(500).json({ err: `There was a problem saving the image.`});
next();
}
setFilePath();
})
: sharp(req.file.buffer).resize(256, 256).max().withoutEnlargement().toFile(filepath).then(() => setFilePath())
If the file is saved, it then sends a req.file.path to my create controller. This gets saved to my DB as a file path and as an image path (the avatarFilePath or /uploads/imagefile.ext is saved for removal purposes and the avatarURL or [http://localhost:4000]/uploads/imagefile.ext is saved and used for the front-end GET request):
controllers/avatars.js (I'm using Postgres, but you can substitute for Mongo)
create: async (req, res, done) => {
try {
const avatarurl = `${apiURL}/${req.file.path}`;
await db.result("INSERT INTO avatars(userid, avatarURL, avatarFilePath) VALUES ($1, $2, $3)", [req.session.id, avatarurl, req.file.path]);
res.status(201).json({ avatarurl });
} catch (err) { return res.status(500).json({ err: err.toString() }); done();
}
Then when the front-end tries to access the uploads folder via <img src={avatarURL} alt="image" /> or <img src="[http://localhost:4000]/uploads/imagefile.ext" alt="image" />, it gets served up by the microservice:
libs/server.js
const express = require("express");
const path = app.get("path");
const PORT = 4000;
//============================================================//
// EXPRESS SERVE AVATAR IMAGES
//============================================================//
app.use(`/uploads`, express.static(`uploads`));
//============================================================//
/* CREATE EXPRESS SERVER */
//============================================================//
app.listen(PORT);
What it looks when logging requests:
19:17:54 INSERT INTO avatars(userid, avatarURL, avatarFilePath) VALUES ('08861626-b6d0-11e8-9047-672b670fe126', 'http://localhost:4000/uploads/1536891474536-k9c7OdimjEWYXbjTIs9J4S3lh2ldrzV8-android.png', 'uploads/1536891474536-k9c7OdimjEWYXbjTIs9J4S3lh2ldrzV8-android.png')
POST /api/avatar/create 201 109 - 61.614 ms
GET /uploads/1536891474536-k9c7OdimjEWYXbjTIs9J4S3lh2ldrzV8-android.png 200 3027 - 3.877 ms
What the user sees upon successful GET request:

Socket emit to specific user

I found several questions on the same subject, and none of theme worked for me. I suppose that's because I'm missing something in the process:
I want to send a message to a specific user.
I use:
"express": "^4.13.3"
"socket.io": "^1.3.7"
"socket.io-client": "^1.3.7"
"mongodb": "^2.2.2"
"mongoose": "^4.5.4"
Here is my code:
server (As you can see, I use mongoose to handle session):
const io = new SocketIo(server);
io.use(socketioJwt.authorize({
secret: configSocket.secret,
handshake: true
}));
// persistence store of our session
const MongoStore = mongoStoreFactory(session);
const sessionMiddleware = session({
store: new MongoStore({
mongooseConnection: mongoose.connection,
ttl: (1 * 60 * 60) // 1 hour
}),
secret: configSocket.secretSession,
httpOnly: true,
resave: false,
saveUninitialized: false,
// rolling: true,
// secure: true,
cookie: { maxAge: 86400000 }
});
app.use(sessionMiddleware);
...
socketRouter(io);
then the socketRouter function. I store the socketId of the user profile with mongo dataStore to aim the user with emit.
exports = module.exports = (io) => {
io.sockets.on('connection', (socket) => {
Users.findById(socket.decoded_token.sub, (err, user) => {
if (user) {
// if the user exists, save the socket Id in the user collection
Users.update({_id: user._id}, {$set: {socketId: socket.id}}, (err2, result) => {
// ------ PROTECTED ROUTES ------ //
// MOBILE CALLS
...
// DASHBOARD CALLS
socket.on('forceNotif', (data) => Notif.force(data, io, socket));
// ------------------------------ //
});
}
});
socket.on('disconnect', ()=> {
Users.update({_id: socket.decoded_token.sub}, {$set: {socketId: null}});
});
});
The function called by 'forceNotif'. Here I expect a different behavior. I want socket to send the value to a specific user (different from the one sending the request). I retrieve the socketId from MongoDb and it's exact. Then I want to use it for my purpose. Several different propositions are made on web. I tested the followings:
exports.force = (par, io, socket) => {
// retrieve the socket id of the user to aim
Users.findOne({_id: par.id}, {socketId: 1, pseudo: 1}, (err, res) => {
console.log('--------------------------------------------------');
console.log('err: ', err);
console.log('pseudo: ', res.pseudo);
console.log('socketId: ', res.socketId);
// emit the value to a specific user
// not working
io.sockets.to(res.socket).emit('notifExtUpdate', {val: 'TO'});
io.to(res.socket).emit('notifExtUpdate', {val: 'TO'});
io.broadcast.to(res.socket).emit('notifExtUpdate', {val: 'TO'});
socket.broadcast.to(res.socket).emit('notifExtUpdate', {val: 'TO'});
socket.to(res.socket).emit('notifExtUpdate', {val: 'TO'});
// working well, but not my purpose
io.emit('notifExtUpdate', {val: 'BROADCAST'});
socket.broadcast.emit('notifExtUpdate', {val: 'BROADCAST'});
});
};
I hope somebody can help me :-)
So here the solution I found:
the socket documentation says than "each socket automatically joins a room identified by the id". But for some reasons I still don't understand, when I emit in the following way (where the socketId is stored, retrieved and checked with mongoDb), nothing happens:
socket.broadcast.to(socketId).emit('blabla', msg);
But finally I joined manually the user to a personal room in the following way:
io.on('connection', (socket) => {
Users.findById(socket.decoded_token.sub, (err, user) => {
if (user) {
// create a room for every user
socket.join(user._id);
socket.on('exemple', (data) => functionExemple(data, io));
...
and then I can emit to a specific user in functionExemple like this (where the targetId is the _id of the user in the collection):
exports.functionExemple= (par, io) => {
returnEmit.to(targetId).emit('blabla', msg);
};
I hope it will help somebody :-)