Why won't my insert callback run? - mongodb

I'm running the following from the mongodb console (it's in a js file called via load()), but the callback to the insert never runs even though all the database insertions happen as expected. Can anyone see why?
db.oldsets.find().forEach(function (set) {
var newSet = {
tunes: [],
keys: []
};
set.tunes.forEach(function (arr) {
if (newSet) {
var tune = db.oldtunes.findOne({
_id: arr.tune
})
var newTune = db.getSiblingDB('jnr_local').tunes.findOne({
sessionId: tune.sessionId
})
if (newTune) {
newSet.tunes.push(newTune._id);
newSet.keys.push(arr.root + tune.mode);
} else {
newSet = null;
}
}
})
print('out') // outputs for every iteration
if (newSet) {
db.sets.insert(newSet, function (err, doc) {
print('in') // never outputs
db.practices.insert({
type: 'set',
// srcId: doc[0]._id,
stickyness: 0
});
});
} else {
print('else') // never outputs
db.dodgySets.insert(set);
}
});

It looks like you are mixing node.js and mongodb shell. In mongodb shell all code is synchronous and run line by line.
So the db.sets.insert will simply return inserted document.
So, try to rewrite it as follow:
if (newSet) {
insertedSet = db.sets.insert(newSet);
print('in') // never outputs
db.practices.insert({
type: 'set',
srcId: insertedSet._id,
stickyness: 0
});
}
Hope this helps!

Related

Mongodb collection not updating properly after webscraping new values

Working on a webscraping bot, that when a user uses !stats in discord chat, it displays their stats that are scraped from a site using cheerio. Everytime the command is called, the site is scraped and the new stats are pulled. However the issue is I am having difficulty updating the new values in mongodb Please see commented console.logs for the issue. Any help would be appreciated, feel like i am missing something super simple. Ive tried find, findOne, and findOneAndUpdate and all have the same issue.
Stats.find({}, 'userId', { '_id': 0 }, function (err, docs) {
for (i = 0; i < docs.length; i++) {
ids.push(docs[i].userId);
}
/////
ids.forEach(function (entry) {
var userUrl = 'https://popflash.site/user/' + entry;
rp(userUrl)
.then(function (html) {
const arr = [];
var e = 0;
$('.stat-container', html).each(function (key, value) {
arr[e++] = $(this).find(".stat").text();
});
var results = arr.map(Number)
console.log(results); //this is printing the newly scraped stats from the site which is working fine.
var query = { userId: entry };
Stats.find(query, {
$set: {
HLTV: results[0],
ADR: results[1],
HS: results[2],
W: results[3],
L: results[4],
T: results[5],
totalGames: results[3] + results[4],
win_percent: results[6]
}
})
.then(function (result) {
console.log(result) //this is displaying old stats that are stored in the db, seems to not be updating.
})
})
});
});
You're running a find query, of course the result will be what is already in the database. If you want to update the db, you need to use update method.
Stats.update(query, {
$set: {
HLTV: results[0],
ADR: results[1],
HS: results[2],
W: results[3],
L: results[4],
T: results[5],
totalGames: results[3] + results[4],
win_percent: results[6]
}
})

Unable to get documents when I use ES6 in my node js

I'm working on MEAN stack to create some web services.
I thought of using ES6 for synchronizing mongodb find operations.
Here is the code(UserService):
var Todo = require('../models/user.js');
var db = mongoose.createConnection('mongodb://localhost:27017/abc');
var Users = db.model('User');
function *myGenerator() {
return yield Todo.find({});//Throwing Undefined function Todo.find
//return yield Users.find({}); //DOes not returns documents but returns a json object which has full mongodb database details
}
function getDocs(){
var iterator = myGenerator();
var firstYield = iterator.next();
}
return yield Todo.find({}) is throwing exception Undefined function Todo.find
return yield Users.find({}); does not return documents but returns a JSON object which has full mongodb database details
return yield Users.find({}).exec() returns following output
{ value:
Promise {
emitter:
EventEmitter {
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined },
emitted: {},
ended: false },
done: false }
PS: I used --harmony node js option as well.
Could you please help me to get User rows/documents?
Todo.find({}); returns a Query object. You must call exec function on that object to execute the query. e.g.
Todo.find({}).exec(function (error, docs) {
if (error) {
// handle
}
if (docs) {
// yeah !!!
}
})
Also mongoose database connection is asynchronous. So any queries made before the connection is established obviously wont work. Here's a working example..
var mongoose = require('mongoose');
mongoose.Promise = Promise;
var db = mongoose.connect('mongodb://localhost:27017/test', (error) => {
if (error) {
throw error
}
console.log('DB Connected');
var Todo = require('./models/user.js');
var Users = db.model('User');
function *myGenerator() {
yield Todo.find({}); // Returns a Query object
//yield Users.find({}); // Returns a Query object
}
function getDocs(){
var iterator = myGenerator();
var firstYield = iterator.next();
// firstYield is a `Query` object
firstYield.value.exec((error, users) => {
if (error) {
throw error;
}
console.log(users);
})
}
getDocs();
});
var Todo = requires('models/user.js'); produces ReferenceError: requires is not defined
should be var Todo = require('models/user.js');
maybe even
var Todo = require('./models/user.js'); because 'models/user.js' is relative to the node_modules directory
return yield Todo.find({});
should be yield Todo.find({});
As far as I can see this code will throw an Exception.
Please provide the actual code and some more info like what version of node.js you are currently running ?
p.S I wrote this in the answers section because I have yet to earned the comment everywhere priveledge

Mongodb query.update method does not update array

Previously I had tried something like (with mongoose and promises):
var cursor = User.find({email: from.address, token: tokenMatches[1]});
and then
return cursor.update(
{'votes.title': b},
{'$set': { 'votes.$.category': a }}
).then(function (result) {
if(result.nModified == 0) {
return cursor.update({}, {'$push': { votes: { category: a, title: b }}}).then(function (res) {
ServerLog('updatePush', res);
return res;
});
}
});
But it always returned nModified = 0 for the first and second call. Until I found out that the cursor object actually has no update function. So why is it so? And why did it not throw an exception?
Model.find returns a Query object, not a cursor. Query does have an update method that lets you execute the query as an update operation.

Is there a way to perform a "dry run" of an update operation?

I am in the process of changing the schema for one of my MongoDB collections. (I had been storing dates as strings, and now my application stores them as ISODates; I need to go back and change all of the old records to use ISODates as well.) I think I know how to do this using an update, but since this operation will affect tens of thousands of records I'm hesitant to issue an operation that I'm not 100% sure will work. Is there any way to do a "dry run" of an update that will show me, for a small number of records, the original record and how it would be changed?
Edit: I ended up using the approach of adding a new field to each record, and then (after verifying that the data was right) renaming that field to match the original. It looked like this:
db.events.find({timestamp: {$type: 2}})
.forEach( function (e) {
e.newTimestamp = new ISODate(e.timestamp);
db.events.save(e);
} )
db.events.update({},
{$rename: {'newTimestamp': 'timestamp'}},
{multi: true})
By the way, that method for converting the string times to ISODates was what ended up working. (I got the idea from this SO answer.)
My advice would be to add the ISODate as a new field. Once confirmed that all looks good you could then unset the the string date.
Create a test environment with your database structure. Copy a handful of records to it. Problem solved. Not the solution you were looking for, I'm sure. But, I believe, this is the exact circumstances that a 'test environment' should be used for.
Select ID of particular records that you would like to monitor. place in the update {_id:{$in:[<your monitored id>]}}
Another option which depends of the amount of overhead it will cause you -
You can consider writing a script, that performs the find operation, add printouts or run in debug while the save operation is commented out. Once you've gained confidence you can apply the save operation.
var changesLog = [];
var errorsLog = [];
events.find({timestamp: {$type: 2}}, function (err, events) {
if (err) {
debugger;
throw err;
} else {
for (var i = 0; i < events.length; i++) {
console.log('events' + i +"/"+(candidates.length-1));
var currentEvent = events[i];
var shouldUpdateCandidateData = false;
currentEvent.timestamp = new ISODate(currentEvent.timestamp);
var change = currentEvent._id;
changesLog.push(change);
// // ** Dry Run **
// currentEvent.save(function (err) {
// if (err) {
// debugger;
// errorsLog.push(currentEvent._id + ", " + currentEvent.timeStamp + ', ' + err);
// throw err;
// }
// });
}
console.log('Done');
console.log('Changes:');
console.log(changesLog);
console.log('Errors:');
console.log(errorsLog);
return;
}
});
db.collection.find({"_manager": { $exists: true, $ne: null }}).forEach(
function(doc){
doc['_managers']=[doc._manager]; // String --> List
delete doc['_manager']; // Remove "_managers" key-value pair
printjson(doc); // Debug by output the doc result
//db.teams.save(doc); // Save all the changes into doc data
}
)
In my case the collection contain _manager and I would like to change it to _managers list. I have tested it in my local working as expected.
In the several latest versions of MongoDB (at least starting with 4.2), you could do that using a transaction.
const { MongoClient } = require('mongodb')
async function main({ dryRun }) {
const client = new MongoClient('mongodb://127.0.0.1:27017', {
maxPoolSize: 1
})
const pool = await client.connect()
const db = pool.db('someDB')
const session = pool.startSession()
session.startTransaction()
try {
const filter = { id: 'some-id' }
const update = { $rename: { 'newTimestamp': 'timestamp' } }
// This is the important bit
const options = { session: session }
await db.collection('someCollection').updateMany(
filter,
update,
options // using session
)
const afterUpdate = db.collection('someCollection')
.find(
filter,
options // using session
)
.toArray()
console.debug('updated documents', afterUpdate)
if (dryRun) {
// This will roll back any changes made within the session
await session.abortTransaction()
} else {
await session.commitTransaction()
}
} finally {
await session.endSession()
await pool.close()
}
}
const _ = main({ dryRun: true })

Auto increment document number in Mongo / Mongoose

My app has several users, each user has documents. Each documents needs to have a sequence number, that may look something like this: 2013-1, 2013-2 (year and sequence number), or perhaps just a simple number: 1, 2, 3...
Currently, I am assigning the sequence number from user's settings when the Mongoose docuemnt is created. Based on that sequence number and the number format from user's settings, I am generating the final document number.
What I realized is that when 2 documents are created at the same time, they will get exactly the same number, because I am incrementing the sequence number in settings just after I have saved a document. But I am assigning the sequence number when I am creating (not saving yet) the document so the sequence number will be exactly the same for both documents.
I obviously need a way to handle this sequence number auto-incrementing at the moment of saving...
How can I assure that this number is unique and automatically incremented/generated?
#emre and #WiredPraire pointed me to the right direction, but I wanted to provide a full Mongoose-compatible answer to my question. I ended up with the following solution:
var Settings = new Schema({
nextSeqNumber: { type: Number, default: 1 }
});
var Document = new Schema({
_userId: { type: Schema.Types.ObjectId, ref: "User" },
number: { type: String }
});
// Create a compound unique index over _userId and document number
Document.index({ "_userId": 1, "number": 1 }, { unique: true });
// I make sure this is the last pre-save middleware (just in case)
Document.pre('save', function(next) {
var doc = this;
// You have to know the settings_id, for me, I store it in memory: app.current.settings.id
Settings.findByIdAndUpdate( settings_id, { $inc: { nextSeqNumber: 1 } }, function (err, settings) {
if (err) next(err);
doc.number = settings.nextSeqNumber - 1; // substract 1 because I need the 'current' sequence number, not the next
next();
});
});
Please note that with this method there is no way to require the number path in the schema, and there is no point as well, because it is automatically added.
You can achieve that through:
create sequence generator, which is just another document that keeps a counter of the last number.
Use a mongoose middleware to update the auto increment the desired field.
Here is a working and tested example with the todo app.
var mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/todoApp');
// Create a sequence
function sequenceGenerator(name){
var SequenceSchema, Sequence;
SequenceSchema = new mongoose.Schema({
nextSeqNumber: { type: Number, default: 1 }
});
Sequence = mongoose.model(name + 'Seq', SequenceSchema);
return {
next: function(callback){
Sequence.find(function(err, data){
if(err){ throw(err); }
if(data.length < 1){
// create if doesn't exist create and return first
Sequence.create({}, function(err, seq){
if(err) { throw(err); }
callback(seq.nextSeqNumber);
});
} else {
// update sequence and return next
Sequence.findByIdAndUpdate(data[0]._id, { $inc: { nextSeqNumber: 1 } }, function(err, seq){
if(err) { throw(err); }
callback(seq.nextSeqNumber);
});
}
});
}
};
}
// sequence instance
var sequence = sequenceGenerator('todo');
var TodoSchema = new mongoose.Schema({
name: String,
completed: Boolean,
priority: Number,
note: { type: String, default: '' },
updated_at: { type: Date, default: Date.now }
});
TodoSchema.pre('save', function(next){
var doc = this;
// get the next sequence
sequence.next(function(nextSeq){
doc.priority = nextSeq;
next();
});
});
var Todo = mongoose.model('Todo', TodoSchema);
You can test it out in the node console as follows
function cb(err, data){ console.log(err, data); }
Todo.create({name: 'hola'}, cb);
Todo.find(cb);
With every newly created object the you will see the priority increasing. Cheers!
This code is taken from MongoDB manual and it actually describes making the _id field auto increment. However, it can be applied to any field. What you want is to check whether the inserted value exists in database just after you inserted your document. If it is allready inserted, re increment the value then try to insert again. This way you can detect dublicate values and re-increment them.
while (1) {
var cursor = targetCollection.find( {}, { f: 1 } ).sort( { f: -1 } ).limit(1);
var seq = cursor.hasNext() ? cursor.next().f + 1 : 1;
doc.f = seq;
targetCollection.insert(doc);
var err = db.getLastErrorObj();
if( err && err.code ) {
if( err.code == 11000 /* dup key */ )
continue;
else
print( "unexpected error inserting data: " + tojson( err ) );
}
break;
}
In this example f is the field in your document that you want to auto increment. To make this work you need to make your field UNIQUE which can be done with indexes.
db.myCollection.ensureIndex( { "f": 1 }, { unique: true } )
You can use mongoose-auto-increment package as follows:
var mongoose = require('mongoose');
var autoIncrement = require('mongoose-auto-increment');
/* connect to your database here */
/* define your DocumentSchema here */
autoIncrement.initialize(mongoose.connection);
DocumentSchema.plugin(autoIncrement.plugin, 'Document');
var Document = mongoose.model('Document', DocumentSchema);
You only need to initialize the autoIncrement once.