How can I wrap sails-mongo db methods for profiling? - sails.js

I'm trying to setup a sails hook with miniprofiler to help profile mongo usage. I'm struggling for how to wrap the db methods in a function that will execute the profile. I'm trying to do this via a user hook:
setupMiniprofilerMongo(req, res, next) {
const adapter = sails.hooks.orm.datastores.default.adapter;
const adapterPrototype = Object.getPrototypeOf(adapter);
const originalMethod = adapter.adapter.find;
methodPrototype.find = function profiledMongoCommand(connectionName, collectionName, options, cb) {
sails.log.info(`${collectionName}.find`);
return originalMethod.call(adapter, connectionName, collectionName, options, cb);
};
}
That causes the following error to be thrown:
TypeError: Cannot read property 'collections' of undefined
at Object.module.exports.adapter.find (/Users/jgeurts/dev/platform/node_modules/sails-mongo/lib/adapter.js:349:40)
at Object.profiledMongoCommand [as find] (/Users/jgeurts/dev/platform/config/http.js:234:37)
Any help would be appreciated. I tried to wrap the methods on mongodb package, but that doesn't seem to work either. :/

I got this working by wrapping waterline query methods. There is room for improvement, though.
setupMiniprofilerWaterline(req, res, next) {
const dbOperations = [
'count',
'create',
'createEach',
'define',
'describe',
'destroy',
'drop',
'find',
'join',
// 'native',
// 'registerConnection',
'update',
];
const waterlineMethodByModels = {};
const miniprofilerWaterline = () => {
return {
name: 'mongodb',
handler(req, res, next) {
if (!req.miniprofiler || !req.miniprofiler.enabled) {
return next();
}
const profiler = req.miniprofiler;
for (const modelName of _.keys(sails.models)) {
for (const dbOperation of dbOperations) {
const model = sails.models[modelName];
if (!model[dbOperation]) {
continue;
}
if (!waterlineMethodByModels[modelName]) {
waterlineMethodByModels[modelName] = {};
}
// Prevent wrapping a method more than once
if (waterlineMethodByModels[modelName][dbOperation]) {
continue;
}
waterlineMethodByModels[modelName][dbOperation] = true;
const originalMethod = model[dbOperation];
model[dbOperation] = function profiledMongoCommand(...args) {
const query = args && args.length ? args[0] : '';
const lastArg = args && args.length ? args[args.length - 1] : null;
const modelAndMethod = `${modelName}.${dbOperation}`;
if (lastArg && typeof lastArg === 'function') {
sails.log.debug(`mongo::${modelAndMethod} - ${JSON.stringify(query)}`);
const callback = args.pop();
const timing = profiler.startTimeQuery('mongodb', query ? JSON.stringify(query || '') : '');
// In general, the callstack is kind of useless to us for these profiles
// The model/db method is more useful in the miniprofiler UI
timing.callStack = `\n\nMethod: ${modelAndMethod}`;
return originalMethod.call(this, ...args, function profiledResult(...results) {
profiler.stopTimeQuery(timing);
callback(...results);
});
}
const methodResult = originalMethod.call(this, ...args);
const methodResultPrototype = Object.getPrototypeOf(methodResult);
const isDeferred = !!methodResultPrototype.exec;
// If methodResult is a Deferred object type, then the query method will be profiled above when the deferred is executed (with a callback)
// So we only care to log this if the methodResult is not a deferred object
if (!isDeferred) {
sails.log.warn(`Was not able to profile mongo::${modelAndMethod}. Maybe its a promise? query: ${JSON.stringify(query)}`);
}
return methodResult;
};
}
}
next();
},
};
};
miniprofiler.express.for(miniprofilerWaterline())(req, res, next);
},
The code is available as miniprofiler-waterline if you want to contribute/use it in your own projects

Related

When I try to override a core function in converse.js I get an error

I am using converse.js 7.0.6 version. I am trying to set the subject topic button to be by default hidden, currently, it is visible by default. I have investigated I need to override this function toggleSubjectHiddenState, when I try to use _converse.api.user. I get an error TypeError: Cannot read properties of undefined (reading 'user')
This is how I tried, maybe I am doing wrong something.
export const addModifyHideTopicPlugin = () => {
window.converse.plugins.add('modify-hide-topic', {
overrides: {
ChatRoom: {
toggleSubjectHiddenState: async function () {
const _converse: any = this;
// const muc_jid = this.get('jid');
const jids = await _converse.api.user.settings.get(
'mucs_with_hidden_subject',
[],
);
return _converse.toggleSubjectHiddenState
.apply(_converse, arguments)
.then((response: any) => {
console.log(response, 'response');
return response;
});
},
},
},
});
};
This is the toggleSubjectHiddenState function I am trying to override because this function is handleing the show topic or hide topic.
async toggleSubjectHiddenState () {
const muc_jid = this.get('jid');
const jids = await api.user.settings.get('mucs_with_hidden_subject', []);
if (jids.includes(this.get('jid'))) {
api.user.settings.set('mucs_with_hidden_subject', jids.filter(jid => jid !== muc_jid));
} else {
api.user.settings.set('mucs_with_hidden_subject', [...jids, muc_jid]);
}
},

MessageCreate.js prefix issues with database - Discord.js v13/MongoDB

I have this code in my messageCreate event:
// MODULES
const Discord = require('discord.js');
const mongoose = require('mongoose');
const Levels = require('discord.js-leveling');
// FILES
const Guild = require('../../models/guild');
const config = require('../../files/config.json');
const swearwords = require("../../files/data.json");
const colors = require('../../files/colors.json');
// ERROR MESSAGE
const errorMain = new Discord.MessageEmbed()
.setDescription("There was an error!")
.setColor(colors.COLOR)
const addedDatabase = new Discord.MessageEmbed()
.setDescription("This server is now added to our database.")
.setColor(colors.COLOR)
module.exports = async (Discord, client, message) => {
if (!message.guild) return;
if (message.author.bot) return;
const settings = await Guild.findOne({
guildID: message.guild.id
}, (err, guild) => {
if (err) message.channel.send(errorMain);
if (!guild) {
const newGuild = new Guild({
_id: mongoose.Types.ObjectId(),
guildID: message.guild.id,
prefix: config.PREFIX,
logChannelID: String,
enableLog: false,
enableSwearFilter: true,
enableMusic: true,
enableLevel: true,
});
newGuild.save()
.catch(err => message.channel.send(errorMain));
return message.channel.send({embeds: [addedDatabase]}).then(m => m.delete({ timeout: 10000 }))
}
});
// VARIABLES
const IDGuild = message.guild.id;
const user = message.author;
const prefix = settings.prefix;
const swearFilterOn = settings.enableSwearFilter;
// LEVEL SYSTEM
if (settings.enableLevel === "true") {
const requiredXp = Levels.xpFor(parseInt(user.level) + 1)
const randomAmountOfXp = Math.floor(Math.random() * 29) + 1;
const hasLeveledUp = await Levels.appendXp(message.author.id, message.guild.id, randomAmountOfXp);
if (hasLeveledUp) {
const user = await Levels.fetch(message.author.id, message.guild.id);
const levelEmbed = new Discord.MessageEmbed()
.setTitle('New Level!')
.setColor(colors.COLOR)
.setDescription(`**GG** ${message.author}, you just leveled up to level **${user.level}**!\nContiune to chat to level up again.`)
const sendEmbed = await message.channel.send({embeds: [levelEmbed]});
}
}
// EXECUTE COMMAND AND SWEARFILTER
if (swearFilterOn === "true") {
var msg = message.content.toLowerCase();
for (let i = 0; i < swearwords["swearwords"].length; i++) {
if (msg.includes(swearwords["swearwords"][i])) {
message.delete();
return message.channel.send("Please do not swear.").then(msg => msg.delete({ timeout: 3000 }));
}
}
if (!message.content.startsWith(prefix) || message.author.bot) return;
const args = message.content.slice(prefix.length).split(/ +/);
const cmd = args.shift().toLowerCase();
const command = client.commands.get(cmd) ||
client.commands.find(a => a.aliases && a.aliases.includes(cmd));;
if (command) command.execute(client, message, args, Discord)
} else {
if (!message.content.startsWith(prefix) || message.author.bot) return;
const args = message.content.slice(prefix.length).split(/ +/);
const cmd = args.shift().toLowerCase();
const command = client.commands.get(cmd) ||
client.commands.find(a => a.aliases && a.aliases.includes(cmd));;
if (command) command.execute(client, message, args, Discord)
}
}
And when i send a message right after the bot joines the discord, it gives the error that it cannot read properties of null (reading 'prefix').
The bot crashes, and when it's restarted it works because it has been added to the Database. So how can i fix that when it sends the first message it can still read 'prefix'. its probably a database issue.
I was told that settings.prefix probaly wasn't assinged a value, is there a way to fix this?
I'm using mongodb for my database.
You really should assign the prefix right after the bot joins new server with guildCreate event. It may cause some issues if your database is down etc. When you wait till the message event

Mongoose/Mongodb, update each doc query, very slow

I have this update query in mongoose. It's 1600 posts and takes like 5 min to run.
What's the bottleneck? Am I using the wrong approach?
export const getAndStoreLatestKPI = async () => {
console.log("start kpi");
try {
const marketCaps = await getKPI();
const stocks = await mongoose.model("stock").find().exec();
for (const stock of stocks) {
const marketCap = marketCaps.find(
(marketCap) => marketCap.i === stock.insId
);
if (marketCap != null) {
const marketCapAdjustedVal =
stock.country === "Finland" ? marketCap.n * 10 : marketCap.n;
const update = {
marketCap: marketCapAdjustedVal,
};
console.log(marketCapAdjustedVal);
await mongoose
.model("stock")
.findOneAndUpdate({ insId: stock.insId }, { update });
}
}
console.log("done");
return Promise.resolve();
} catch (err) {
return Promise.reject(err);
}
};
export const getKPI = async (kpiId: number) => {
try {
const kpiFetch = await Axios.get(someurl);
return Promise.resolve(kpiFetch.data.values);
} catch (err) {
return Promise.reject(err);
}
};
So the main bottle neck is your for loop. for each stock item you perform several "expensive" actions such as data fetching from external API + a single update, and you're doing them 1 by 1.
What I would recommend you doing is looping on several items at once. similar to the idea multithreading.
There are several different solutions on how to do it in nodejs for example nodejs worker threads
However I personally use and recommend using bluebird which gives you this ability and many others straight out of the box.
Some sample code:
import Bluebird = require('bluebird');
const stocks = await mongoose.model("stock").find().exec();
await Bluebird.map(stocks, async (stock) => {
const marketCap = marketCaps.find(
(marketCap) => marketCap.i === stock.insId
);
if (marketCap != null) {
const marketCapAdjustedVal =
stock.country === "Finland" ? marketCap.n * 10 : marketCap.n;
const update = {
marketCap: marketCapAdjustedVal,
};
console.log(marketCapAdjustedVal);
await mongoose
.model("stock")
.findOneAndUpdate({ insId: stock.insId }, { update });
}
}, {concurrency: 25})
// concurrency details how many concurrent process run parallel. the heavier they are the less you want concurrent for obvious reasons.

ChangeStream#destroy is not a method?

I got this error:
app:tail-mongodb ERROR Unhandled rejection:
z.cs.destroy is not a function
TypeError: z.cs.destroy is not a function
at createChangeStream
(/Users/Olegzandr/codes/interos/read-from-mongodb/dist/main.js:74:18)
at process.internalTickCallback (internal/process/next_tick.js:77:7)
here is my code:
import {ChangeStream} from "mongodb";
const z = {
cs: null as ChangeStream,
};
const createChangeStream = () => {
if (z.cs) {
z.cs.removeAllListeners();
z.cs.destroy();
}
const changeStream = z.cs = d.db(dbName).collection(dbCollName).watch();
changeStream.on('change', next => {
if (!(next && next.fullDocument)) {
log.warn('changed doc did not have fullDocument property, or was undefined:', next);
}
bun.handleIn(next.fullDocument || next);
});
};
seems weird that destroy() is not a valid method on the ChangeStream. Does anyone know the proper way to close a ChangeStream? The TypeScript typings say that this method is available, but I guess it is not.

How to test node.js-mongodb app using Mocha, Chai and Sinon? I am having difficulty with Sinon part

I am new to testing and having difficulty with Sinon stubs and mocks.
How can I write test for 'quote.list_quote' for following senario.
Here is the routes file : quotes.js
const express = require('express');
const request = require('request');
const async = require('async');
const validator = require('validator');
const quote_router = express.Router();
const confg = require("../../confg/confg");
const quote = require("../models/mquotes");
const quotes_model = quote.quotes;
// host name - needs to be set up using the environment variable
const hostname = confg.hostname;
// route for "quotes/"
quote_router.route("/")
// get route : display the random quote
.get((req, res) => {
// display random quote
async.waterfall([
(callback) => {callback(null, {res});},
quote.count_quotes
], check_quote_exist
);
})
// post route : create a new quote
.post((req, res) => {
const doc_json = {author : validator.escape(req.body.quote_author), quote_text : validator.escape(req.body.quote_text)};
const params = {res, doc_json, quote_action : quote.create_quote};
add_edit_quote(params);
})
// put route : edit the quote
.put((req, res) => {
const doc_json = {author : validator.escape(req.body.quote_author), quote_text : validator.escape(req.body.quote_text)};
const params = {res, doc_json, quote_action : quote.update_quote, qid : req.body.quote_id};
add_edit_quote(params);
})
// delete quote : delete the quote
.delete((req, res) => {
const qid = req.body.qid;
const condition = {_id : qid};
async.waterfall([
(callback) => {callback(null, {res, condition});},
quote.delete_quote
], request_quote_list
);
});
// route for "quotes/list" : display quotes list
quote_router.get("/list/", (req, res) => {
// mention the main operation
let operation;
if(req.body.operation != 'undefined') {
operation = req.body.operation;
} else {
operation = "list_quotes";
}
async.waterfall([
(callback) => {callback(null, {res, operation});},
quote.list_quote
], display_quotes_list
);
});
// display the quotes list
const display_quotes_list = (err, params, quotes_list) => {
if (err) {return console.log(err);}
const res = params.res;
const operation = params.operation;
const header_msg = "List of all the quotes";
let alert_msg;
if(operation == "list_quotes") {
alert_msg = null;
} else if(operation == "delete_quote") {
alert_msg = "Quote has been deleted";
}
const params_out = {
page: "quote_list",
title: 'Quotes Manager',
host: hostname,
header_msg,
alert_msg,
quotes_list
};
res.render('index', params_out);
};
// send http request for quote list page
const request_quote_list = (err, params) => {
if (err) {return console.log(err);}
const res = params.res;
const operation = "delete_quote";
request.get('http://' + hostname + '/quotes/list/', {json:{operation}},
(error, response, body) => {
if (!error && response.statusCode == 200) {
res.send(body);
}
});
};
module.exports = quote_router;
This is not complete file. I have included only a portion of it.
And her is the model file : mquotes.js
const mongoose = require('mongoose');
// Define quote schema
const quoteSchema = new mongoose.Schema({
author: String,
quote_text: {type: String, required: true}
},
{timestamps: true}
);
const quote = {};
// Define quotes model
quote.quotes = mongoose.model('quotes', quoteSchema);
// error handler
error_handler = (callback, err, params, return_value) => {
if(err) { return callback(err);}
else {callback(null, params, return_value);}
};
// add quote - create
quote.create_quote = (params, callback) => {
const res = params.res;
const doc_json = params.doc_json;
quote.quotes.create(doc_json, (err, quotes_detail) => {
error_handler(callback, err, {res, operation : 'create_quote'}, quotes_detail);
});
};
// count the number of quotes
quote.count_quotes = (params, callback) => {
quote.quotes.count({}, (err, quotes_count) => {
error_handler(callback, err, params, quotes_count);
});
};
// delete quote - delete - id
quote.delete_quote = (params, callback) => {
quote.quotes.remove(params.condition, (err, query) => {
error_handler(callback, err, params);
});
};
// list quote - find
quote.list_quote = (params, callback) => {
quote.quotes.find({}, (err, quotes_list) => {
error_handler(callback, err, params, quotes_list);
});
};
// find quote by id
quote.quote_by_id = (params, callback) => {
quote.quotes.findById(params.qid, (err, quotes_detail) => {
error_handler(callback, err, params, quotes_detail);
});
};
// returns the detail of random quote
quote.random_qoute = (params, callback) => {
const random_number = params.random_number;
// select one quote after skipping random_number of times
quote.quotes.findOne({}, (err, quotes_detail) => {
error_handler(callback, err, params, quotes_detail);
}).skip(random_number);
};
// update quote - update - id
quote.update_quote = (params, callback) => {
const options = {new: true};
const qid = params.qid;
const update_json = params.doc_json;
quote.quotes.findByIdAndUpdate(qid, {$set: update_json}, options, (err, quotes_detail) => {
params.operation = 'update_quote';
error_handler(callback, err, params, quotes_detail);
});
};
module.exports = quote;
I have installed mocha globally. Now, I want to test the model. Lets take the quote.list_quote function for example.
const mongoose = require('mongoose');
const chai = require('chai');
const sinon = require('sinon');
const expect = chai.expect; // use the "expect" style of Chai
const mquotes = require('./../../app/models/mquotes');
describe('Tests for quote models', () => {
describe("List quote", () => {
it('list_quote() should return list of quotes', () => {
});
});
});
Can anyone tell me about my coding practice too. I mean the way I use functions and modules.
First of all, you should try to use statics methods. And after that, you should use sinon-mongoose and sinon-as-promised if you want to use Promise in mongoose.
And this is my sample code and test with mocha, chai, and sinon. Hope useful for you.
model.js
var Schema = new mongoose.Schema({
name: String,
created_at: {
type: Date,
default: Date.now
},
updated_at: {
type: Date,
default: Date.now
}
});
Schema.statics.findByName = function(name, cb) {
this.findOne({
name: name
})
.exec()
.then(function getTemplate(template) {
if (!template) {
var error = new Error('Not found template by name: "' + name + '"');
error.status = 404;
return cb(error);
}
return cb(null, template);
})
.catch(function catchErrorWhenFindByTemplateName(error) {
error.status = 500;
return cb(error);
});
}
module.exports = mongoose.model('model', Schema);
test.js
var expect = require('chai').expect,
sinon = require('sinon'),
mongoose = require('mongoose');
require('sinon-as-promised');
require('sinon-mongoose');
var Model = require('../../app/models/model');
describe('Model', function () {
describe('static methods', function () {
describe('#findByName', function () {
var ModelMock;
beforeEach(function () {
ModelMock = sinon.mock(Model);
});
afterEach(function () {
ModelMock.restore();
});
it('should get error status 404 if not found template', function (done) {
var name = 'temp';
ModelMock
.expects('findOne').withArgs({name: name})
.chain('exec')
.resolves(null);
Model.findByName(name, function (error) {
expect(error.status).to.eql(404);
ModelMock.verify();
done();
});
});
it('should get message not found template if name is not existed', function (done) {
var name = 'temp';
ModelMock
.expects('findOne').withArgs({name: name})
.chain('exec')
.resolves(null);
Model.findByName(name, function (error) {
expect(error.message).to.match(/Not found template by name/gi);
ModelMock.verify();
done();
});
});
it('should get template when name is existed', function (done) {
var name = 'temp';
ModelMock
.expects('findOne').withArgs({name: name})
.chain('exec')
.resolves('SUCCESS');
Model.findByName(name, function (error) {
expect(error).to.be.null;
ModelMock.verify();
done();
});
});
it('should get error status 500 when model crashed', function (done) {
var name = 'temp';
ModelMock
.expects('findOne').withArgs({name: name})
.chain('exec')
.rejects(new Error('Oops! Crashed'));
Model.findByName(name, function (error) {
expect(error.status).to.eql(500);
ModelMock.verify();
done();
});
});
});
});
});