Sails js bulk upload and insert into MongoDB - mongodb

Hi In my Sailsjs application there is bulk upload feature where a admin can upload csv file. I am using csv-parse for parsing the csv content.
Now i want to insert the data into Mongo DB where each row has Vehicle Info and Test Info. These two models have one to many relationship. I am not able to figure out how to insert them into DB. Below is my code for inserting the vehicle.
Below is the code sample i was thinking to create VehicleTest model once Vehicle model is created but i don't have access to vehicle to get info for vechileTest model
Vehicle Model
attributes: {
make: {
type: "string",
required: true
},
tests :{
collection: 'VehicleTest',
via : 'vehicleTested'
},
VechileTest model
attributes: {
vehicleTested :{
modal :'Vehicle',
required:true
},
Below lines of code from the function reads the file and parse it
var parse = require('csv-parse');
var fs = require("fs")
var createdVehicle =[];
var stream = files[0].fd;
fs.readFile(stream, 'utf8', function (err, data) {
// Print the contents of the file as a string here
// and do whatever other string processing you want
parse(data, {columns: true,delimiter:'|'}, function(err, output){
if(output.length>0)
{
for (var i = 0, len = output.length; i < len; i++) {
var vehicle_TestInfo = output[i];
//console.log(vehicle_TestInfo);
//
Vehicle.create({make : vehicle_TestInfo.make,model:vehicle_TestInfo.model,year:vehicle_TestInfo.year ,engineFuel:vehicle_TestInfo.engineFuel,obdZone:vehicle_TestInfo.obdZone,generation:vehicle_TestInfo.generation,protocol:vehicle_TestInfo.protocol,onStar:vehicle_TestInfo.onStar}).exec(function(err,vehCreated){
console.log(vehCreated);
//VehicleTest.Create({vehicleTested:vehCreated._id,overAllComp:vehicle_TestInfo.overAllComp,deviceGeneration:vehicle_TestInfo.deviceGeneration}).exec(function(err,testCreated){
//
// console.log(testCreated);
//
//})
});
}
}

this is how i implemented
for (var i = 0, len = output.length; i < len; i++) {
var vehicle_TestInfo = output[i];
//console.log(vehicle_TestInfo);
//
Vehicle.create({make : vehicle_TestInfo.make,model:vehicle_TestInfo.model,year:vehicle_TestInfo.year ,trim:vehicle_TestInfo.trim ,engineFuel:vehicle_TestInfo.engineFuel,obdZone:vehicle_TestInfo.obdZone,generation:vehicle_TestInfo.generation,protocol:vehicle_TestInfo.protocol,onStar:vehicle_TestInfo.onStar,
tests : {deviceGeneration:vehicle_TestInfo.deviceGeneration,overAllComp:vehicle_TestInfo.overAllComp,isTested:vehicle_TestInfo.isTested,testingInferred:vehicle_TestInfo.testingInferred,vinRead:vehicle_TestInfo.vinRead,
odoRead:vehicle_TestInfo.odoRead,pidRead:vehicle_TestInfo.pidRead,doorLocked:vehicle_TestInfo.doorLocked,SeatBelt:vehicle_TestInfo.seatBelt,fuelLevel:vehicle_TestInfo.fuelLevel,issueType:vehicle_TestInfo.issueType,degreeOfERGIntr:vehicle_TestInfo.degreeOfERGIntr,dataLoggerModel:vehicle_TestInfo.dataLoggerModel,
numberOfVehicle:vehicle_TestInfo.numberOfVehicle,remarks:vehicle_TestInfo.remarks}}).exec(function(err,vehCreated){
if(err && err.originalError && err.originalError.code===50)
{
Vehicle.update({id:vehCreated.id})
}
/// console.log(vehCreated);
// vehCreated.tests.add({})
createdVehicle.push(vehCreated);
});

Related

Mongodb collection not updating properly after webscraping new values

Working on a webscraping bot, that when a user uses !stats in discord chat, it displays their stats that are scraped from a site using cheerio. Everytime the command is called, the site is scraped and the new stats are pulled. However the issue is I am having difficulty updating the new values in mongodb Please see commented console.logs for the issue. Any help would be appreciated, feel like i am missing something super simple. Ive tried find, findOne, and findOneAndUpdate and all have the same issue.
Stats.find({}, 'userId', { '_id': 0 }, function (err, docs) {
for (i = 0; i < docs.length; i++) {
ids.push(docs[i].userId);
}
/////
ids.forEach(function (entry) {
var userUrl = 'https://popflash.site/user/' + entry;
rp(userUrl)
.then(function (html) {
const arr = [];
var e = 0;
$('.stat-container', html).each(function (key, value) {
arr[e++] = $(this).find(".stat").text();
});
var results = arr.map(Number)
console.log(results); //this is printing the newly scraped stats from the site which is working fine.
var query = { userId: entry };
Stats.find(query, {
$set: {
HLTV: results[0],
ADR: results[1],
HS: results[2],
W: results[3],
L: results[4],
T: results[5],
totalGames: results[3] + results[4],
win_percent: results[6]
}
})
.then(function (result) {
console.log(result) //this is displaying old stats that are stored in the db, seems to not be updating.
})
})
});
});
You're running a find query, of course the result will be what is already in the database. If you want to update the db, you need to use update method.
Stats.update(query, {
$set: {
HLTV: results[0],
ADR: results[1],
HS: results[2],
W: results[3],
L: results[4],
T: results[5],
totalGames: results[3] + results[4],
win_percent: results[6]
}
})

How to give configurable URL in tableau WDC

I am trying to build a tabeau WDC.
this is my code
(function () {
var myConnector = tableau.makeConnector();
myConnector.getSchema = function (schemaCallback) {
var cols = [{
id: "month",
dataType: tableau.dataTypeEnum.string
}, {
id: "value1",
alias: "value1",
dataType: tableau.dataTypeEnum.float
}, {
id: "value2",
alias: "value2",
dataType: tableau.dataTypeEnum.float
}];
var tableSchema = {
id: "testfeed",
alias: "test Feed",
columns: cols
};
schemaCallback([tableSchema]);
};
myConnector.getData = function (table, doneCallback) {
$.getJSON('http://test.com/view?name=test&filters=[{"type":"number","id_equals":["123"]}]', function (resp) {
var feat = resp.DATA,
tableData = [];
// Iterate over the JSON object
for (var i = 0, len = feat.length; i < len; i++) {
tableData.push({
"MONTH": feat[I].month,
"ChargeEntryLag_NUMERATOR": feat[i]. value1,
"ChargeEntryLag_DENOMINATOR": feat[i]. value2
});
}
table.appendRows(tableData);
doneCallback();
});
};
tableau.registerConnector(myConnector);
$(document).ready(function () {
$("#submitButton").click(function () {
tableau.connectionName = "testFeed";
tableau.submit();
});
});
})();
my URL contains some filters as shown in the above code, so if U want to get data for a particular filter I have to hardcode it in URL and the use it.
In other word my URL is static , Is there a way to make it dynamic.
suppose I want the value of 'id' to be 10in my filter, for that I have to go the the WDC code and change it. can it be made configurable.
use tableau.connectionData to pass data. There is an example in this tutorial:
https://tableau.github.io/webdataconnector/docs/wdc_multi_table_tutorial
Typically you'd create a form. When you connect with the WDC in tableau desktop, you put in the URL of your form. The form will store the form vars in tableau.connectData. Your getData can then take those and create a custom Data Source inside tableau desktop for you.
- Mike

Where is Schema data saved in MongoDB?

I am saving some data with Mongoose but unsure where it is saved. Where is this data saved in MongoDb? In which table or collection?
var Chat = mongoose.model('Chat', ChatSchema);
console.log('Message Received: ', msg);
var data = [{
sent: new Date(),
room: 'General',
username: 'Chris',
text: msg
}]
//Loop through each of the chat data and insert into the database
for (var c = 0; c < data.length; c++) {
//Create an instance of the chat model
var entry = new Chat(data[c]);
//Call save to insert the chat
entry.save(function(err, savedChat) {
console.log(savedChat);
});
}
Mongoose by default produces a collection name by passing the model name to the utils.toCollectionName method. This method pluralizes the name.
Check the documentation: http://mongoosejs.com/docs/guide.html (option: collection)
In your case if you connect to the database you will see a collection: chats

Selective specific columns in SailsJS via API

I want to select certain columns be returned back from sailsjs / waterline / API
Given the search parameters:
var searchParams = {
"select":["ClientTypeID"]
, "where": {
"or" :
[{"ClientType": {"contains": "MAINT"}}]
}};
I make API call :
/api/reference/client_type?select=ClientTypeID&where=%7B%22or%22:%5B%7B%22ClientType%22:%7B%22contains%22:%22MAINT%22%7D%7D%5D%7D
based on
Select specific fields from database
I believe I am making the correct query, however, the JSON object that is returned has the all the columns of entity vs the 1 I wish to request ClientTypeID?
This is known issue but I've found workaround. You can override default find blueprint with your own. You need to create api/blueprints/find.js file with content:
var _ = require('lodash');
var Promise = require('bluebird');
var actionUtil = require('sails/lib/hooks/blueprints/actionUtil');
var takeAliases = _.partial(_.pluck, _, 'alias');
var populateAliases = function (model, alias) {
return model.populate(alias);
};
module.exports = function (req, res) {
_.set(req.options, 'criteria.blacklist', ['limit', 'skip', 'sort', 'populate', 'fields']);
var fields = req.param('fields') ? req.param('fields').replace(/ /g, '').split(',') : [];
var populate = req.param('populate') ? req.param('populate').replace(/ /g, '').split(',') : [];
var Model = actionUtil.parseModel(req);
var where = actionUtil.parseCriteria(req);
var limit = actionUtil.parseLimit(req);
var skip = actionUtil.parseSkip(req);
var sort = actionUtil.parseSort(req);
var findQuery = _.reduce(_.intersection(populate, takeAliases(Model.associations)), populateAliases, Model.find().where(where).limit(limit).skip(skip).sort(sort));
var countQuery = Model.count(where);
Promise.all([findQuery, countQuery])
.spread(function (_records, _count) {
var records = fields.length > 0 ? _.map(_records, _.partial(_.pick, _, fields)) : _records;
return [records, null, null, {
criteria: where,
limit: limit,
start: skip,
end: skip + limit,
total: _count
}];
})
.spread(res.ok)
.catch(res.serverError);
};
This is general blueprint I'm using in all of my projects. For all fixed blueprints you can take a look here - https://github.com/ghaiklor/generator-sails-rest-api/tree/dev/generators/app/templates/api/blueprints

Auto increment document number in Mongo / Mongoose

My app has several users, each user has documents. Each documents needs to have a sequence number, that may look something like this: 2013-1, 2013-2 (year and sequence number), or perhaps just a simple number: 1, 2, 3...
Currently, I am assigning the sequence number from user's settings when the Mongoose docuemnt is created. Based on that sequence number and the number format from user's settings, I am generating the final document number.
What I realized is that when 2 documents are created at the same time, they will get exactly the same number, because I am incrementing the sequence number in settings just after I have saved a document. But I am assigning the sequence number when I am creating (not saving yet) the document so the sequence number will be exactly the same for both documents.
I obviously need a way to handle this sequence number auto-incrementing at the moment of saving...
How can I assure that this number is unique and automatically incremented/generated?
#emre and #WiredPraire pointed me to the right direction, but I wanted to provide a full Mongoose-compatible answer to my question. I ended up with the following solution:
var Settings = new Schema({
nextSeqNumber: { type: Number, default: 1 }
});
var Document = new Schema({
_userId: { type: Schema.Types.ObjectId, ref: "User" },
number: { type: String }
});
// Create a compound unique index over _userId and document number
Document.index({ "_userId": 1, "number": 1 }, { unique: true });
// I make sure this is the last pre-save middleware (just in case)
Document.pre('save', function(next) {
var doc = this;
// You have to know the settings_id, for me, I store it in memory: app.current.settings.id
Settings.findByIdAndUpdate( settings_id, { $inc: { nextSeqNumber: 1 } }, function (err, settings) {
if (err) next(err);
doc.number = settings.nextSeqNumber - 1; // substract 1 because I need the 'current' sequence number, not the next
next();
});
});
Please note that with this method there is no way to require the number path in the schema, and there is no point as well, because it is automatically added.
You can achieve that through:
create sequence generator, which is just another document that keeps a counter of the last number.
Use a mongoose middleware to update the auto increment the desired field.
Here is a working and tested example with the todo app.
var mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/todoApp');
// Create a sequence
function sequenceGenerator(name){
var SequenceSchema, Sequence;
SequenceSchema = new mongoose.Schema({
nextSeqNumber: { type: Number, default: 1 }
});
Sequence = mongoose.model(name + 'Seq', SequenceSchema);
return {
next: function(callback){
Sequence.find(function(err, data){
if(err){ throw(err); }
if(data.length < 1){
// create if doesn't exist create and return first
Sequence.create({}, function(err, seq){
if(err) { throw(err); }
callback(seq.nextSeqNumber);
});
} else {
// update sequence and return next
Sequence.findByIdAndUpdate(data[0]._id, { $inc: { nextSeqNumber: 1 } }, function(err, seq){
if(err) { throw(err); }
callback(seq.nextSeqNumber);
});
}
});
}
};
}
// sequence instance
var sequence = sequenceGenerator('todo');
var TodoSchema = new mongoose.Schema({
name: String,
completed: Boolean,
priority: Number,
note: { type: String, default: '' },
updated_at: { type: Date, default: Date.now }
});
TodoSchema.pre('save', function(next){
var doc = this;
// get the next sequence
sequence.next(function(nextSeq){
doc.priority = nextSeq;
next();
});
});
var Todo = mongoose.model('Todo', TodoSchema);
You can test it out in the node console as follows
function cb(err, data){ console.log(err, data); }
Todo.create({name: 'hola'}, cb);
Todo.find(cb);
With every newly created object the you will see the priority increasing. Cheers!
This code is taken from MongoDB manual and it actually describes making the _id field auto increment. However, it can be applied to any field. What you want is to check whether the inserted value exists in database just after you inserted your document. If it is allready inserted, re increment the value then try to insert again. This way you can detect dublicate values and re-increment them.
while (1) {
var cursor = targetCollection.find( {}, { f: 1 } ).sort( { f: -1 } ).limit(1);
var seq = cursor.hasNext() ? cursor.next().f + 1 : 1;
doc.f = seq;
targetCollection.insert(doc);
var err = db.getLastErrorObj();
if( err && err.code ) {
if( err.code == 11000 /* dup key */ )
continue;
else
print( "unexpected error inserting data: " + tojson( err ) );
}
break;
}
In this example f is the field in your document that you want to auto increment. To make this work you need to make your field UNIQUE which can be done with indexes.
db.myCollection.ensureIndex( { "f": 1 }, { unique: true } )
You can use mongoose-auto-increment package as follows:
var mongoose = require('mongoose');
var autoIncrement = require('mongoose-auto-increment');
/* connect to your database here */
/* define your DocumentSchema here */
autoIncrement.initialize(mongoose.connection);
DocumentSchema.plugin(autoIncrement.plugin, 'Document');
var Document = mongoose.model('Document', DocumentSchema);
You only need to initialize the autoIncrement once.