Question
I have provided my code below for reference. I'm using MongoDB and discord.js v12. So basically, I have made a !info command which shows some general info of the user.
What this code does is, it checks through the member's roles, and regarding which role they have, it calculates their total claim time (for giveaways etc.). The problem here, is with the donator role. I can't figure out why I can't use the donates variable outside the db.findOne block. Here, data.content.length shows the total donates of the users, which means donates * 5 is +5 claim time for each donate.
My Code
const moment = require('moment');
module.exports = {
name: 'info',
async execute(client, message, args, Discord){
const member = message.mentions.members.first() || message.guild.members.cache.get(args[0]) || message.member;
const db = require('../models/d-schema');
db.findOne({ guildid: message.guild.id, user: member.user.id }, async(err, data)=>{
if(err) throw err;
if(data){
const donates = parseInt(data.content.length);
}
})
var DefaultTime = 10;
var support = 0;
var donate = 0;
var boost = 0;
const userRoles = member.roles.cache.map((r) => r.name);
if (userRoles.includes("୨・supporter")) {
support = 3;
}
if (userRoles.includes("୨・donator")) {
donate = donates * 5;
}
if (userRoles.includes("୨・booster")) {
boost = 10;
}
const TotalTime = DefaultTime + support + donate + boost;
const embed = new Discord.MessageEmbed()
.setThumbnail(member.user.displayAvatarURL( {dynamic: true} ))
.addFields(
{name: member.user.tag, value: member.user, inline: true},
{name: 'Nickname', value: `${member.nickname !== null ? member.nickname : 'None'}`, inline: true},
{name: 'Is Bot', value: member.user.bot, inline: true},
{name: 'Joined', value: `${moment.utc(member.joinedAt).format("MMMM Do YYYY")}`, inline: true},
{name: 'Created', value: `${moment.utc(member.user.createdAt).format("MMMM Do YYYY")}`, inline: true},
{name: 'Claim Time', value: `${TotalTime} seconds`, inline: true},
)
.setFooter(`ID : ${member.user.id}`)
.setTimestamp()
.setColor('00ffcc')
message.channel.send(embed)
}
}
You cannot use the donates variable because you are declaring it inside the db.findOne() block. This is called variables scope. For better understanding you can read this answer.
If you want to use it outside of that block, you have to declare it beforehand, like this:
let donates;
db.findOne({ guildid: message.guild.id, user: member.user.id }, async(err, data)=>{
if(err) throw err;
if(data){
donates = parseInt(data.content.length);
}
})
Now you're able to use that variable outside of the db.findOne() block :)
Edit:
Alternative way:
It would be easier to use the function asynchronously. That way, everything can be scoped in the same block!
Example:
These two methods will give the same results:
const data = await Model.findOne({ ... });
console.log(data);
Model.findOne({ ... }, (err, data) => {
console.log(data);
});
Suggestion from Lioness100
Related
Ok, as said in title, I have "performance issue" where I need to get all documents from a collection but it takes too long. Players collection contains around 300k documents with small size and query in service goes like this:
async getAllPlayers() {
const players = await this.playersCollection.find({}, {projection: { playerId: 1, name: 1, surname: 1, shirtNumber: 1, position: 1 }}).toArray();
return players;
}
Overall size is 6.4MB. I'm using Fastify adapter, fastify-compress and mongodb native driver. If I remove projection, it takes almost a minute.
Any idea how to improve this?
The best time I get is 8 seconds, where fast-json-stringify give me more than 10 seconds boost over 300k records:
'use strict'
// run fresh mongo
// docker run --name temp --rm -p 27017:27017 mongo
const fastify = require('fastify')({ logger: true })
const fjs = require('fast-json-stringify')
const toString = fjs({
type: 'object',
properties: {
playerId: { type: 'integer' },
name: { type: 'string' },
surname: { type: 'string' },
shirtNumber: { type: 'integer' },
}
})
fastify.register(require('fastify-mongodb'), {
forceClose: true,
url: 'mongodb://localhost/mydb'
})
fastify.get('/', (request, reply) => {
const dataStream = fastify.mongo.db.collection('foo')
.find({}, {
limit: 300000,
projection: { playerId: 1, name: 1, surname: 1, shirtNumber: 1, position: 1 }
})
.stream({
transform(doc) {
return toString(doc) + '\n'
}
})
reply.type('application/jsonl')
reply.send(dataStream)
})
fastify.get('/insert', async (request, reply) => {
const collection = fastify.mongo.db.collection('foo')
const batch = collection.initializeOrderedBulkOp();
for (let i = 0; i < 300000; i++) {
const player = {
playerId: i,
name: `Name ${i}`,
surname: `surname ${i}`,
shirtNumber: i
}
batch.insert(player);
}
const { result } = await batch.execute()
return result
})
fastify.listen(8080)
In any case, you should consider to:
paginate your output
or pushing the data into a bucket (like S3) and return to the client a URL to download the file directly, this will speed up a lot the process and will save your node.js process from this data streaming
Note that the compression in node.js is a heavy process, so it slows it down a lot the response. An nginx proxy adds it by default without the need to implement it in your business logic server.
i have Meteor App which is connected to MongoDB.
In mongo i have a table which has ~700k records.
I have a cron job each week, where i read all the records from the table (using Mongo Cursor) and in batches of 10k i want to insert them inside Elastic Search so they are indexed.
let articles = []
Collections.Articles.find({}).forEach(function(doc) {
articles.push({
index: {_index: 'main', _type: 'article', _id: doc.id }
},
doc);
if (0 === articles.length % 10000) {
client.bulk({ maxRetries: 5, index: 'main', type: 'article', body: articles })
data = []
}
})
Since for each is synchronous, goes over each record before it continues, and client.bulk is async, this is overloading the elastic search server and it crashes with Out of Memory Exception.
Is there a way to pause the forEach during the time when the insert is being done? I tried async/await but this does not seem to work as well.
let articles = []
Collections.Articles.find({}).forEach(async function(doc) {
articles.push({
index: {_index: 'main', _type: 'article', _id: doc.id }
},
doc);
if (0 === articles.length % 10000) {
await client.bulk({ maxRetries: 5, index: 'main', type: 'article', body: articles })
data = []
}
})
Any way how to achieve this?
EDIT: I am trying to achieve something like this - if i use promises
let articles = []
Collections.Articles.find({}).forEach(function(doc) {
articles.push({
index: {_index: 'main', _type: 'article', _id: doc.id }
},
doc);
if (0 === articles.length % 10000) {
// Pause FETCHING rows with forEach
client.bulk({ maxRetries: 5, index: 'main', type: 'article', body: articles }).then(() => {
console.log('inserted')
// RESUME FETCHING rows with forEach
console.log("RESUME READING");
})
data = []
}
})
Managed to get this working with ES2018 Async iteration
Got an idea from
Using async/await with a forEach loop
Here is the code that is working
let articles = []
let cursor = Collections.Articles.find({})
for await (doc of cursor) {
articles.push({
index: {_index: 'main', _type: 'article', _id: doc.id }
},
doc);
if (articles.length === 10000) {
await client.bulk({ maxRetries: 5, index: 'trusted', type: 'artikel', body: articles })
articles = []
}
}
This works correctly and it manages to insert all the records into Elastic Search without crashing.
If you are concerned with the unthrottled iteration, then may use the internal Meteor._sleepForMs method, that allows you to put a async timeout in your sync-styled code:
Collections.Articles.find().forEach((doc, index) => {
console.log(index, doc._id)
Meteor._sleepForMs(timeout)
})
Now this works fine within the Meteor environment (Meteor.startup, Meteor.methods, Meteor.publish).
You cron is likely to be not within this environment (= Fiber) so you may write a wrapper that binds the environment:
const bound = fct => Meteor.bindEnvironment(fct)
const iterateSlow = bound(function (timeout) {
Collections.Articles.find().forEach((doc, index) => {
console.log(index, doc._id)
Meteor._sleepForMs(timeout)
})
return true
})
iterateSlow(50) // iterates with 50ms timeout
Here is a complete minimal example, that you can reproduce with a fresh project:
// create a minimal collection
const MyDocs = new Mongo.Collection('myDocs')
// fill the collection
Meteor.startup(() => {
for (let i = 0; i < 100; i++) {
MyDocs.insert({})
}
})
// bind helper
const bound = fct => Meteor.bindEnvironment(fct)
// iterate docs with interval between
const iterateSlow = bound(function (timeout) {
MyDocs.find().forEach((doc, index) => {
console.log(index, doc._id)
Meteor._sleepForMs(timeout)
})
return true
})
// simulate external environment, like when cron runs
setTimeout(() => {
iterateSlow(50)
}, 2000)
I'm reviewing some old code I wrote in Express/Mongo/Mongoose (based on an online tutorial) and can't locate the portion of code which dictates which Collection in MongoDB gets written to.
I have a Database, UsersDB and within that database there are several Collections. The Collection that keeps growing every time the code executes user.save is the users Collection. But I can't find any reference in my code base to the users collection.
The only place in my code where I save a User is:
var app = require('../app');
var util = require('util');
var User = require('../models/user'),
Auth = User.Auth,
Name = User.Name,
Email= User.Email,
Phone = User.Phone,
Address = User.Address,
Company = User.Company,
PersonalData = User.PersonalData,
Id = User.Id,
Photo = User.Photo,
Member = User.Member,
CreditCard = User.CreditCard,
UserObj = User.User;
var moment = require('moment');
var async = require('async');
. . .
. . .
exports.user_create_post = [
(req,res, next) => {
console.log("Request: " + util.inspect(req.body));
},
//VALIDATE
body('mainEmail', 'Must be valid email.').isLength({min: 5}).trim(),
//SANITIZE
sanitizeBody('*').escape(),
//POPULATE NEW DOCUMENT
(req,res,next) => {
const errors = validationResult(req);
var auth = new Auth(
dateEffective: {value: moment(Date.now()).format("YYYY-MM-DD hh:mm:ss SSS"), attr: {hidden: true, label: ""}},
username: {"value": req.body.username, "attr": {hidden: false, label: "Username: "}},
password: {"value": req.body.password, "attr": {hidden: false, label: "Password: "}},
mainEmail: {"value": req.body.mainEmail, "attr": {hidden: false, label: "Email: "}}
});
var user = new UserObj(
{authData: [auth]}
);
if (!errors.isEmpty()) {
const errorFormatter = ({ location, msg, param, value, nestedErrors }) => {
// Build your resulting errors however you want! String, object, whatever - it works!
return `${location}[${param}]: ${msg}`;
};
const result = validationResult(req).formatWith(errorFormatter);
if (!result.isEmpty()) {
return res.json({ errors: result.array() });
}
}
else {
user.save(function(err){
if (err) { return next(err);}
});
res.redirect("http://localhost:4200/two-fa/"+user._id);
}
}
I also have a Models module (user.js):
. . .
. . .
. . .
module.exports = {
Auth: mongoose.model('Auth', AuthSchema),
Name: mongoose.model('Name', NameSchema),
Email: mongoose.model('Email', EmailSchema),
Phone: mongoose.model('Phone', PhoneSchema),
Address: mongoose.model('Address', AddressSchema),
Company: mongoose.model('Company', CompanySchema),
PersonalData: mongoose.model('PersonalData', PersonalDataSchema),
Id: mongoose.model('Id', IdSchema),
Photo: mongoose.model('Photo', PhoneSchema),
Member: mongoose.model('Member', MemberSchema),
CreditCard: mongoose.model('CreditCard', CreditCardSchema),
User: mongoose.model('User', UserSchema)
}
I did a search on my entire code, and nowhere is there any mention of users, which is the Collection that's getting written to.
Where should I look to try to trace how the users collection is getting written to?
Thank you!
As per how-to-access-a-preexisting-collection-with-mongoose, if a 3rd argument to mongoose.model is not provided, then Mongo automatically "pluralizes" the model name to give the name of the Collection.
I am getting inconsistent results with JEST when I try to remove items from a MongoDB Collection using the beforeEach() Hook.
My Mongoose schema and model defined as:
// Define Mongoose wafer sort schema
const waferSchema = new mongoose.Schema({
productType: {
type: String,
required: true,
enum: ['A', 'B'],
},
updated: {
type: Date,
default: Date.now,
index: true,
},
waferId: {
type: String,
required: true,
trim: true,
minlength: 7,
},
sublotId: {
type: String,
required: true,
trim: true,
minlength: 7,
},
}
// Define unique key for the schema
const Wafer = mongoose.model('Wafer', waferSchema);
module.exports.Wafer = Wafer;
My JEST tests:
describe('API: /WT', () => {
// Happy Path for Posting Object
let wtEntry = {};
beforeEach(async () => {
wtEntry = {
productType: 'A',
waferId: 'A01A001.3',
sublotId: 'A01A001.1',
};
await Wafer.deleteMany({});
// I also tried to pass in done and then call done() after the delete
});
describe('GET /:id', () => {
it('Return Wafer Sort Entry with specified ID', async () => {
// Create a new wafer Entry and Save it to the DB
const wafer = new Wafer(wtEntry);
await wafer.save();
const res = await request(apiServer).get(`/WT/${wafer.id}`);
expect(res.status).toBe(200);
expect(res.body).toHaveProperty('productType', 'A');
expect(res.body).toHaveProperty('waferId', 'A01A001.3');
expect(res.body).toHaveProperty('sublotId', 'A01A001.1');
});
}
So the error I always get is related to duplicate keys when I run my tests more than once:
MongoError: E11000 duplicate key error collection: promis_tests.promiswts index: waferId_1_sublotId_1 dup key: { : "A01A001.3", : "A01A001.1" }
But I do not understand how I can get this duplicate key error if the beforeEach() were firing properly. Am I trying to clear the collection improperly? I've tried passing in a done element to the before each callback and invoking it after delete command. I've also tried implementing the delete in beforeAll(), afterEach(), and afterAll() but still get inconsistent results. I'm pretty stumped on this one. I might just removed the schema key all together but I would like to understand what is going on here with the beforeEach(). Thanks in advance for any advice.
It might be because you are not actually using the promise API that mongoose has to offer. By default, mongooses functions like deleteMany() do not return a promise. You will have to call .exec() at the end of the function chain to return a promise e.g. await collection.deleteMany({}).exec(). So you are running into a race condition. deleteMany() also accepts a callback, so you could always wrap it in a promise. I would do something like this:
describe('API: /WT', () => {
// Happy Path for Posting Object
const wtEntry = {
productType: 'A',
waferId: 'A01A001.3',
sublotId: 'A01A001.1',
};
beforeEach(async () => {
await Wafer.deleteMany({}).exec();
});
describe('GET /:id', () => {
it('Return Wafer Sort Entry with specified ID', async () => {
expect.assertions(4);
// Create a new wafer Entry and Save it to the DB
const wafer = await Wafer.create(wtEntry);
const res = await request(apiServer).get(`/WT/${wafer.id}`);
expect(res.status).toBe(200);
expect(res.body).toHaveProperty('productType', 'A');
expect(res.body).toHaveProperty('waferId', 'A01A001.3');
expect(res.body).toHaveProperty('sublotId', 'A01A001.1');
});
}
Also, always expect the assertions with asynchronous code
https://jestjs.io/docs/en/asynchronous.html
You can read more about mongoose promises and query objects here
https://mongoosejs.com/docs/promises.html
Without deleting the schema index this seems to be the most reliable solution. Not 100% sure why it works over async await Wafer.deleteMany({});
beforeEach((done) => {
wtEntry = {
productType: 'A',
waferId: 'A01A001.3',
sublotId: 'A01A001.1',
};
mongoose.connection.collections.promiswts.drop(() => {
// Run the next test!
done();
});
});
I have a code block in my Mongoose controller which attempts to find both Projects and Levels:
exports.landing = (req, res, next) => {
console.log(req.params.projectid);
Project.findById(req.params.projectid, (err, project) => {
if (err) return res.status(500).send(err);
//find the level based on the projectid
Level.find({'projectid': req.params.projectid}, (err, level) => {
if (err) return res.status(500).send(err);
//find the level based on the projectid
res.json({
success: true,
message: 'got',
level: level.leveltempnodes
});
//res.render(path + 'project', {project: project, moment: moment, level: level});
});
});
};
Within the res.json section, If I just use 'level' without the dot notation, all the results come back as expected. When I try and get the 'levelnodes' entry, nothing comes back. The only thing I see differently with the level document compared to the other documents is that the JSON result includes a '[':
{"success":true,"message":"got","level":{"_id":"5b4205ea5b44e146b5978175" ...
The above works fine. But I am not able to use dot syntax on the below result:
{"success":true,"message":"got","level":[{"_id":"5b4202fc94855d56204c8bb7"
I am saving the level document like this:
var data = {
levelname: levelname,
leveltempnodes: leveltempnodes,
projectid: projectid};
var level = new Level(data);
level.save(function (err) {
if (err) return handleError(err);
})
My error is nothing is coming back at all:
{"success":true,"message":"got"}
Schema:
const mongoose = require('mongoose');
const LevelSchema = mongoose.Schema({
levelname: String,
leveltempnodes: String,
projectid: String
});
module.exports = mongoose.model('Level', LevelSchema);
Data is being stored on the DB without issue. I am adding it via Ajax:
var p = {
projectname : $("#projectname").val(),
levelname : 'Root',
leveltempnodes : '{"class":"go.GraphLinksModel","nodeKeyProperty":"id","nodeDataArray":[{"id":1,"loc":"226 226","text":"sensor"},{"text":"perception","loc":"426 225.99999999999997","id":-2},{"text":"planning","loc":"626 225.99999999999997","id":-3},{"text":"gate","loc":"826 225.99999999999997","id":-4}],"linkDataArray":[{"from":1,"to":-2,"text":"msg","points":[296.7874157629703,237.73538061447854,340.03133208792605,227.76937481449303,383.33478829426565,227.0952320784595,426.7981545990892,236.1401244399739]},{"from":-2,"to":-3,"text":"msg","points":[523.225709890083,236.1861908341044,558.0349502392196,229.00680324793404,592.1479459982006,228.54232080927673,626.6289592123036,236.76409981273324]},{"from":-3,"to":-4,"text":"msg","points":[709.6483081744094,237.23795381070627,748.7663709980919,229.48139598538538,787.383185499046,229.48139598538538,826.1210439041331,238.64104211943584]}]}',
}
if(p.projectname == ''){
console.log('e');
}else{
$.ajax({
type: 'POST',
contentType : "application/json",
url: 'api/project/save',
data : JSON.stringify(p),
success: function(res) {
window.location.replace("/project/"+res.id);
}
});