ihave been using mongoose to create models and exporting schema, now i have toupload file in mongo db, Please check the code below.
errmsg: 'cannot do raw queries on admin in atlas',
const util = require("util");
const multer = require("multer");
//const { GridFsStorage } = require("multer-gridfs-storage");
const dbConfig = require("../config/db.config");
const GridFsStorage = require('multer-gridfs-storage');
var storage = new GridFsStorage({
url: dbConfig.url + dbConfig.database,
options: { useNewUrlParser: true, useUnifiedTopology: true },
file: (req, file) => {
const match = ["image/png", "image/jpeg" ,"application/pdf","text/plain"];
if (match.indexOf(file.mimetype) === -1) {
const filename = `${file.originalname}`;
return filename;
}
return {
bucketName: dbConfig.imgBucket,
filename: `${file.originalname}`
// filename: `${Date.now()}-srikanth-${file.originalname}`
};
}
});
var uploadFiles = multer({ storage: storage }).array("file", 10);
// var uploadFiles = multer({ storage: storage }).single("file");
var uploadFilesMiddleware = util.promisify(uploadFiles);
module.exports = uploadFilesMiddleware;
db.config
module.exports = {
url: "mongodb+srv://username:password#cluster0.fatnc.mongodb.net/",
database: "students",
imgBucket: "photos",
};
Related
I currently have the following code, which saves the temp file to public/files I have tried to understand the MongoDB GridFS documentation but with no success.
I am wondering how do I get the files to save inside MongoDB GridFS instead of my public/file directory
I am aware I am missing the part where I need to send the uploaded file to mongodb - this is the part I don't know how to do.
In mongodb example they say to do something like:
fs.createReadStream('./myFile').pipe(
bucket.openUploadStream('myFile', {
chunkSizeBytes: 1048576,
metadata: { field: 'myField', value: 'myValue' },
})
);
however I am not using FS or do I need to upload the file to the temp and then do the fs
import formidable from 'formidable';
import { MongoClient, ObjectId } from 'mongodb';
var Grid = require('gridfs-stream');
export const config = {
api: {
bodyParser: false,
},
};
export default async (req, res) => {
const uri = process.env.MONGODB_URI;
let client;
let clientPromise;
const options = {};
client = new MongoClient(uri, options);
clientPromise = client.connect();
const clients = await clientPromise;
const database = clients.db('AdStitchr');
var gfs = Grid(database, client);
gfs.collection('uploads');
const form = new formidable.IncomingForm();
form.uploadDir = 'public/files';
form.keepExtensions = true;
form.parse(req, (err, fields, files) => {
var file = files.file;
console.log(JSON.stringify(file));
try {
const newFile = File.create({
name: `files\${file.newFilename}.mp3`,
});
res.status(200).json({ status: 'success' });
} catch (error) {
res.send(error);
}
});
};
i'm stuck at displaying an image that i uploaded to mongodb with multer-gridfs-storage.
the problem is, i'm supposed to have binary data in every chunks and i want to join them to make the final file and encode it to base64
mongodb tell it's binary data but i'm not very sure
mongodb chunks
here is my code:
multer.js:
const multer = require("multer");
const { GridFsStorage } = require("multer-gridfs-storage");
const db = require("../utils/database").db;
const storage = new GridFsStorage({ db });
module.exports = {
upload: multer({
storage: storage,
})
};
my function to reassemble the file:
const getImage = (fileName) => {
return files
.find({ filename: fileName })
.then((file) => {
const id = mongoose.Types.ObjectId(file[0]._id);
return chunks
.find({ files_id: id })
.then((chunks) => {
if (!chunks || chunks.length === 0) {
console.log("No data found");
}
let fileData = [];
for (let i = 0; i < chunks.length; i++) {
//This is in Binary JSON or BSON format, which is stored
//in fileData array in base64 endocoded string format
fileData.push(chunks[i].data.toString('base64');
}
//Display the chunks using the data URI format
return (finalFile =
"data:" + file[0].contentType + ";base64," + fileData.join(""));
})
.catch((err) => {
console.log(err);
});
})
.catch((err) => {
console.log(err);
});
};
and i have this kind of result with .toString('base64')
���R3\x17�\x18��bC4%\x19�rS5&��cDT6\'7\x1A�s�E���dUte�89\x11\x00\x02\x01\x02\x04\x03\x05\x07\x02\x04\x04\x05\x02\x03\x01\x11\x00\x01\x02\x11\x03!1\x12\x04AQ\x05aq"\x13\x06������2�\x07�B#\x14�R3\x15br$4\b�\x16���C%�S\x17D5&�c��s�Td6\t��\x00\f\x03\x01\x00\x02\x
and without .toString('base64')
77+977+977+9DR7vv70D0IJO77+9NSVq0qEu77+9I++/vRl477+9D++/vS3vv70GUE7vv71H77+977+9UO+/vS8eKO+/vVLvv73vv73vv73vv73vv70d77+9XO+/vXjvv71t77+977+9f++/vW3vv73vv73vv73vv71Q77+977+9Nu+/vSVD77+977+977+9FXnvv70277+9d++/vRE3Zu+/vQcgST1P77+92orVle+/vX5lIu+/vVPvv70o77+9Cu+/ve+/vTvvv73bnu+/vR3vv70a77+9VEZqTu+/vXkR77+977+9cyjvv73vv70kVT/vv73vv73vv70Hbe+/vW3vv73oiJzvv71FeGtS77+9Xu+/ve+/vT9xWcSGAO+/ve+/ve+/ve+/ve+/vX3vv73vv73vv73vv73vv71zSsKcVF5VOw7vv73vv73vv71M77+9CGPvv73Mo++/ve+/vWDvv712Ie+/ve+/vXbvv71g77+9S++/ve+/vWJRWNGU77+977+9GSw5AO+/vRIzdu+/ve+/vQkb77+9Se+/vTrvv70277+977+977+977+977+977+9RVVn77
thanks for your help !
Ok, i figured out what was the problem,
in my model chunks.js
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const chunks = new Schema({
files_id: {
type: mongoose.Schema.Types.ObjectId,
},
n: {
type: Number,
},
data: {
type: String,
},
});
module.exports = mongoose.model("fs.chunks", chunks);
i had the type of data in 'String" instead of "Buffer"
it just work as expected with this
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const chunks = new Schema({
files_id: {
type: mongoose.Schema.Types.ObjectId,
},
n: {
type: Number,
},
data: {
type: Buffer,
},
});
module.exports = mongoose.model("fs.chunks", chunks);
I am building a full-stack application using MERN(MongoDB, Express, React, Node.js).
I have a .CSV file that I uploaded to MongoDB. However I can't get my routes to return any data.
I can see the database collection on my MongoDB cluster, But it's not showing up in my mongo shell.
My thought process is that's why I can't get my routes to work. How can I fix that?
sample .csv file data
Date,Client,Project,Project Code,Hours,Billable?,First Name,Last Name,Billable Rate
,,,,,,,,
4/3/17,Anil,Ethereum,RD001,3.84,No,Robert,Powell,0
4/3/17,Olith,Pharos,DV002,7.69,Yes,Hubert,Green,80
4/3/17,Olith,Pharos,DV002,4.46,Yes,Bradley,Hale,80
4/3/17,Olith,Pharos,DV002,7.57,Yes,Rudy,Parker,80
4/3/17,Prosaria,Business Development,GM001,0.92,No,Walter,Silva,0
As you can see the data exists in my mongo-atlas
But my mongo-shell is empty when I call db.clients.find()
server.js
const express = require('express');
const app = express();
const mongodb = require('mongodb').MongoClient;
const fs = require('fs');
const fastcsv = require('fast-csv');
const csvtojson = require('csvtojson');
require("dotenv").config()
const PORT = process.env.PORT || 3003;
//middleware
app.use(express.json());
app.use(express.urlencoded({extended: false}));
app.use(express.static(__dirname + 'public'));
// Global Configuration
const MONGODBURI = process.env.MONGODBURI || 'mongodb://localhost:27017/projectData';
let stream = fs.createReadStream('./gm_data.csv');
let csvData = [];
let csvStream = fastcsv
.parse()
.on('data', (data) => {
csvData.push({
date: data[0],
client: data[1],
project: data[2],
projectCode: data[3],
hours: data[4],
billable: data[5],
firstName: data[6],
lastName: data[7],
billableRate: data[8]
})
})
.on('end', () => {
csvData.shift();
console.log(csvData);
mongodb.connect(
MONGODBURI, {
useNewUrlParser: true,
useUnifiedTopology: true
},
(err, client) => {
if (err) throw err;
client
.db('gm-clients')
.collection('clients')
.insertMany(csvData, (err, res) => {
if (err) throw err;
console.log(`Inserted: ${res.insertedCount} rows`);
client.close
});
}
);
});
stream.pipe(csvStream);
//TEST ROUTE --- // http://localhost:3003/
app.get('/', (req, res) => {
res.send('Say Hello to Zee Server')
})
//routes
const customerRoutes = require('./routes/routes.js');
app.use('/customer', customerRoutes);
//Listening
app.listen(PORT, () => {
console.log('Listening on port', PORT)
})
routes.js
const express = require('express')
const router = express.Router();
const path = require('path');
let fs = require('fs');
let parse = require('csv-parse');
const csv = require('csvtojson');
const Customer = require('../models/customer.js');
//Index Route -- //http://localhost:3003/customer/allcustomers
router.get('/allcustomers', (req, res) => {
Customer.find({}, 'client', (err, data) => {
res.json({data: data});
})
})
module.exports = router;
models.js
const mongoose = require('mongoose')
const customerSchema = new mongoose.Schema({
name: String, //project col in CSV
clients: String, //client col in csv
hours: Number, // hours col in csv
billable_hours: Number, // billable col in csv
billable_amount: Number // billable rate in csv
})
// name | clients | hours | billable hours | billable amount
module.exports = mongoose.model('Customer', customerSchema);
My approach was like this below command
npx sequelize-cli model:generate --name User --attributes name:string,email:string,password:string
then it's generated a model file named user.js & file structure like below
'use strict';
const {
Model
} = require('sequelize');
module.exports = (sequelize, DataTypes) => {
class User extends Model {
/**
* Helper method for defining associations.
* This method is not a part of Sequelize lifecycle.
* The `models/index` file will call this method automatically.
*/
static associate(models) {
// define association here
User.hasMany(models.Course, {
foreignKey: 'userId'
})
}
};
User.init({
name: DataTypes.STRING
}, {
sequelize,
modelName: 'User',
});
return User;
};
also, created the Users table with columns.
And the index.js file like below which automatically generated
'use strict';
// main model file
const fs = require('fs');
const path = require('path');
const Sequelize = require('sequelize');
const basename = path.basename(__filename);
const env = process.env.NODE_ENV || 'development';
const config = require(__dirname + '/../config/config.json')[env];
const db = {};
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
fs
.readdirSync(__dirname)
.filter(file => {
return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
})
.forEach(file => {
const model = require(path.join(__dirname, file))(sequelize, Sequelize.DataTypes);
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;
and when I try to create a user like the below codes
import { User } from '../../../../models'
try {
await User.create({
name: 'john doe',
email: 'john#gmail.com'
})
res.json({"message": "create"})
} catch (error) {
res.send(error.message)
}
it's showing
Cannot read property 'create' of undefined
what I have done wrong here?
Can you please help me out with that?
Thanks
replace this code in models/index.js for printing database successfully connected or not.
'use strict';
const fs = require('fs');
const path = require('path');
const Sequelize = require('sequelize');
const basename = path.basename(__filename);
const env = process.env.NODE_ENV || 'development';
const config = require(__dirname + '/../../config/config.js')[env];
const db = {};
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
//for checking database successfully connected or not
sequelize
.authenticate()
.then(() => {
console.log('Connection has been established successfully.');
})
.catch(err => {
console.error('Unable to connect to the database:', err);
});
fs
.readdirSync(__dirname)
.filter(file => {
return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
})
.forEach(file => {
const model = require(path.join(__dirname, file))(sequelize, Sequelize.DataTypes);
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;
Then try to importing like this.
const admin = require("../../database/models").tbl_admin;
try {
await admin.create({
name: 'john doe',
email: 'john#gmail.com'
})
res.json({"message": "create"})
} catch (error) {
res.send(error.message)
}
As you can see, I am connecting to the database, reading multiple JSON files from a directory and import them into the MongoDB server with help of mongoose.
After this job is done, I would like to close the connection and print out "job is done"
How do I do that?
var mongoose = require('mongoose'),
_ = require('lodash'),
fs = require('fs'),
path = require('path');
mongoose.Promise = require('bluebird'),
mongoose.connect('mongodb://localhost/eclass');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
// we're connected!
// create db schema
var EclassSchema = new mongoose.Schema({
xsi: {
xsitype: 'string',
id: 'string'
},
date_of_original_definition: 'string',
date_of_current_version: 'string',
date_of_current_revision: 'string',
revision: 'string',
status: 'string',
source_language: {
country_code: 'string',
language_code: 'string'
},
preferred_name: 'string',
definition: 'string',
its_superclass: 'string',
hierarchical_position: 'string',
//keywords: 'string'
});
// Create model
var Eclass = mongoose.model('Eclass', EclassSchema);
const pjsons = path.join(__dirname, '/../', 'file-operations', 'json-files');
console.log(pjsons);
function readFiles(pjsons, onError) {
fs.readdir(pjsons, function(err, filenames) {
if(err) {
onError(err);
return;
}
filenames.forEach(function(filename) {
fs.readFile(pjsons + '/' + filename, 'utf-8', function(err, data) {
if(err) {
onError(err);
return;
}
data = JSON.parse(data);
// Digging down into the json code
const ontomlOntoml = data['dic:eclass_dictionary']['ontoml:ontoml'];
const onto = _.first(ontomlOntoml);
const dictionary = onto['dictionary'];
const contClasses = _.first(dictionary);
const containedClasses = contClasses['contained_classes'];
const ontClass = _.first(containedClasses);
const ontomlClass = _.find(ontClass);
//Arrays
const xsiArray = _.map(ontomlClass, '$');
const date_of_original_definitionArray = _.map(ontomlClass, 'date_of_original_definition');
const date_of_current_versionArray = _.map(ontomlClass, 'date_of_current_version');
const date_of_current_revisionArray = _.map(ontomlClass, 'date_of_current_revision');
const revisionArray = _.map(ontomlClass, 'revision');
const statusArray = _.map(ontomlClass, 'status');
const sourceLanguageArray = _.map(ontomlClass, 'source_language');
const preferredNameArray = _.map(ontomlClass, 'preferred_name');
const definitionArray = _.map(ontomlClass, 'definition');
const itsSuperclassArray = _.map(ontomlClass, 'its_superclass');
const hierarchical_positionArray = _.map(ontomlClass, 'hierarchical_position');
//const keywordsArray = _.map(ontomlClass, 'keywords');
// Looping and storing the data into mongodb
//for (var i = 0; i < hierarchical_positionArray.length; i++) {
for (var i = 0; i < 2; i++) {
//console.log(hierarchical_positionArray[i]);
var newEclass = new Eclass();
newEclass.xsi.xsitype = xsiArray[i]['xsi:type'];
newEclass.xsi.id = xsiArray[i]['id'];
newEclass.date_of_original_definition = date_of_original_definitionArray[i];
newEclass.date_of_current_version = date_of_current_versionArray[i];
newEclass.date_of_current_revision = date_of_current_revisionArray[i];
newEclass.revision = revisionArray[i];
newEclass.status = statusArray[i];
newEclass.source_language.country_code = sourceLanguageArray[i][0].$.country_code;
newEclass.source_language.language_code = sourceLanguageArray[i][0].$.language_code;
newEclass.preferred_name = preferredNameArray[i][0].label[0]._;
newEclass.definition = definitionArray[i][0].text[0]._;
newEclass.its_superclass = itsSuperclassArray[i][0].$.class_ref;
newEclass.hierarchical_position = hierarchical_positionArray[i];
//newEclass.keywords = keywordsArray[i][0].label[0]._;
newEclass.save(function (err) {});
}
});
});
});
}
readFiles(pjsons);
});
mongoose.disconnect() closes all opened connections. for more
http://mongoosejs.com/docs/api.html#index_Mongoose-disconnect
I solved it like this:
newEclass.save()
.then(function() {
mongoose.disconnect();
})
.catch(function(err) {
console.log('There was an error', err);
});
...
const mongoose = require('mongoose'),
parse = require('csv-parse'),
path = require('path'),
fs = require('fs');
mongoose.Promise = require('bluebird');
mongoose.connect('mongodb://localhost/eclassCSV');
const db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
// we're connected!
// create db schema
const EclassSchema = new mongoose.Schema({
codedName: { type: String, min: 10, max: 10 },
preferredName: { type: String, max: 80 },
definition: { type: String, max: 1023 },
level: { type: String, min: 1, max: 1 },
mkSubclass: { type: String, min: 1, max: 1 },
mkKeyword: { type: String, min: 1, max: 1 }
});
// Create MongoDB model with mongoose
const Eclass = mongoose.model('Eclass', EclassSchema);
const p = path.join(__dirname, '/../', 'file-operations', 'csv-files');
//console.log(p);
const parser = parse({delimiter: ';'}, function(err, data){
//console.log(data);
//const supplier = data[0][0];
const codedName = data.map((item,i) => data[i][6]);
const preferredName = data.map((item,i) => data[i][7]);
const definition = data.map((item,i) => data[i][8]);
const level = data.map((item,i) => data[i][13]);
const mkSubclass = data.map((item,i) => data[i][14]);
const mkKeyword = data.map((item,i) => data[i][15]);
// Looping and storing the data into mongodb
//console.log(ontomlClass.length);
for (let i = 0; i < data.length; i++) {
//console.log(hierarchical_positionArray[i]);
const newEclass = new Eclass();
newEclass.codedName = codedName[i];
newEclass.preferredName = preferredName[i];
newEclass.definition = definition[i];
newEclass.level = level[i];
newEclass.mkSubclass = mkSubclass[i];
newEclass.mkKeyword = mkKeyword[i];
newEclass.save()
.then(function() {
mongoose.disconnect();
})
.catch(function(err) {
console.log('There was an error', err);
});
}
});
fs.createReadStream(p + '/One-eClass-10_0_CC_en.csv').pipe(parser);
});