Import CSV file data into MongoDB using mongoose - mongodb

I am building a full-stack application using MERN(MongoDB, Express, React, Node.js).
I have a .CSV file that I uploaded to MongoDB. However I can't get my routes to return any data.
I can see the database collection on my MongoDB cluster, But it's not showing up in my mongo shell.
My thought process is that's why I can't get my routes to work. How can I fix that?
sample .csv file data
Date,Client,Project,Project Code,Hours,Billable?,First Name,Last Name,Billable Rate
,,,,,,,,
4/3/17,Anil,Ethereum,RD001,3.84,No,Robert,Powell,0
4/3/17,Olith,Pharos,DV002,7.69,Yes,Hubert,Green,80
4/3/17,Olith,Pharos,DV002,4.46,Yes,Bradley,Hale,80
4/3/17,Olith,Pharos,DV002,7.57,Yes,Rudy,Parker,80
4/3/17,Prosaria,Business Development,GM001,0.92,No,Walter,Silva,0
As you can see the data exists in my mongo-atlas
But my mongo-shell is empty when I call db.clients.find()
server.js
const express = require('express');
const app = express();
const mongodb = require('mongodb').MongoClient;
const fs = require('fs');
const fastcsv = require('fast-csv');
const csvtojson = require('csvtojson');
require("dotenv").config()
const PORT = process.env.PORT || 3003;
//middleware
app.use(express.json());
app.use(express.urlencoded({extended: false}));
app.use(express.static(__dirname + 'public'));
// Global Configuration
const MONGODBURI = process.env.MONGODBURI || 'mongodb://localhost:27017/projectData';
let stream = fs.createReadStream('./gm_data.csv');
let csvData = [];
let csvStream = fastcsv
.parse()
.on('data', (data) => {
csvData.push({
date: data[0],
client: data[1],
project: data[2],
projectCode: data[3],
hours: data[4],
billable: data[5],
firstName: data[6],
lastName: data[7],
billableRate: data[8]
})
})
.on('end', () => {
csvData.shift();
console.log(csvData);
mongodb.connect(
MONGODBURI, {
useNewUrlParser: true,
useUnifiedTopology: true
},
(err, client) => {
if (err) throw err;
client
.db('gm-clients')
.collection('clients')
.insertMany(csvData, (err, res) => {
if (err) throw err;
console.log(`Inserted: ${res.insertedCount} rows`);
client.close
});
}
);
});
stream.pipe(csvStream);
//TEST ROUTE --- // http://localhost:3003/
app.get('/', (req, res) => {
res.send('Say Hello to Zee Server')
})
//routes
const customerRoutes = require('./routes/routes.js');
app.use('/customer', customerRoutes);
//Listening
app.listen(PORT, () => {
console.log('Listening on port', PORT)
})
routes.js
const express = require('express')
const router = express.Router();
const path = require('path');
let fs = require('fs');
let parse = require('csv-parse');
const csv = require('csvtojson');
const Customer = require('../models/customer.js');
//Index Route -- //http://localhost:3003/customer/allcustomers
router.get('/allcustomers', (req, res) => {
Customer.find({}, 'client', (err, data) => {
res.json({data: data});
})
})
module.exports = router;
models.js
const mongoose = require('mongoose')
const customerSchema = new mongoose.Schema({
name: String, //project col in CSV
clients: String, //client col in csv
hours: Number, // hours col in csv
billable_hours: Number, // billable col in csv
billable_amount: Number // billable rate in csv
})
// name | clients | hours | billable hours | billable amount
module.exports = mongoose.model('Customer', customerSchema);

Related

trying to get uploads saving in MongoDB

I currently have the following code, which saves the temp file to public/files I have tried to understand the MongoDB GridFS documentation but with no success.
I am wondering how do I get the files to save inside MongoDB GridFS instead of my public/file directory
I am aware I am missing the part where I need to send the uploaded file to mongodb - this is the part I don't know how to do.
In mongodb example they say to do something like:
fs.createReadStream('./myFile').pipe(
bucket.openUploadStream('myFile', {
chunkSizeBytes: 1048576,
metadata: { field: 'myField', value: 'myValue' },
})
);
however I am not using FS or do I need to upload the file to the temp and then do the fs
import formidable from 'formidable';
import { MongoClient, ObjectId } from 'mongodb';
var Grid = require('gridfs-stream');
export const config = {
api: {
bodyParser: false,
},
};
export default async (req, res) => {
const uri = process.env.MONGODB_URI;
let client;
let clientPromise;
const options = {};
client = new MongoClient(uri, options);
clientPromise = client.connect();
const clients = await clientPromise;
const database = clients.db('AdStitchr');
var gfs = Grid(database, client);
gfs.collection('uploads');
const form = new formidable.IncomingForm();
form.uploadDir = 'public/files';
form.keepExtensions = true;
form.parse(req, (err, fields, files) => {
var file = files.file;
console.log(JSON.stringify(file));
try {
const newFile = File.create({
name: `files\${file.newFilename}.mp3`,
});
res.status(200).json({ status: 'success' });
} catch (error) {
res.send(error);
}
});
};

MERN stack : Express api returning empty data , but the data is already present in mongodb

I am new to the MERN stack, and I have been trying to access my collections in MongoDB.
Here is the code for the router, view bookings:
/*This is router file*/
var mongoose = require('mongoose');
const express = require('express');
const bodyParser = require('body-parser')
let book = require('../models/BookTravel');
const router = require('express').Router()
router.use(express.json())
router.route('/').get((req, res) => {
// Company.aggregate({companyId})
book.find()
.then((result) => {
console.log(result)
return res.status(200).json(result)
})
})
module.exports = router;
/*
* this is for model
*/
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const TravelSchema = new Schema({
firstname:{
type: String,
required: true
},
bookingId:{
type: String,
required: true
},
lastname:{
type: String,
required: true
},
startcity:{
type: String,
required: true
}
})
const travel = mongoose.model('travel', TravelSchema)
module.exports = travel;
////in app.js file
const viewBookings = require('./routes/viewBookings');
app.use('/viewBookings', viewBookings)
The postman is also giving empty result.
What am I missing out ? Is it not possible to access the already existing collection with this method ?
You are missing some code in the router file.
for example! If you want to get data from a database
you can simply use like below this
.......
router.get("/",async (req,res)=>
{
try{
const result = await book.find();
res.status(200).json({"message" : result})
}
catch(error)
{
console.log(error)
}
})
......

How to use Mongoose find query in express router?

I am trying to use Mongoose find query to get the complete list of documents held in my mongodb. However for some reason, the query stopped working and hasn't worked since. I believe all my code is correct, and its not giving me any error messages either.
Routes\image.js
const router = require('express').Router();
let imageModel = require('../Models/image');
router.get('/collection',(req,res) => {
imageModel.find({},(err,data) => {
if(err) {
res.status(500).json({msg:"Cant find collection",err})
} else {
res.send(data);
}
})
})
module.exports = router;
Models \ image
let mongoose = require('mongoose');
const Schema = mongoose.Schema;
let imageSchema = new Schema({
name:String,
desc:String,
imgUrl:String
});
let model = new mongoose.model('Image',imageSchema);
module.exports = model;
index.js
const express = require('express');
const app = express();
const cors = require("cors");
const PORT = process.env.PORT || 5001;
// Routes
const imageRoute = require('./Routes/image');
app.use(express.json());
app.use(cors());
require('dotenv').config()
require('./db.config');
app.use('/image',imageRoute);
app.get('/',(req,res) => {
res.send('First Route!')
})
app.listen(PORT,() => {
console.log(`listening on port ${PORT}`)
})

Express Sequelize Error - Cannot read property 'findAll' of undefined

I have successfully connected Sequelize and Express using Sequelize's github example with a few changes. I am now trying to do a simple Sequelize query to test the connection, but continue to receive an error stating that the model I have queried is not defined.
// ./models/index.js
...
const sequelize = new Sequelize(process.env.DB, process.env.DB_USER, process.env.DB_PASS, {
host: 'localhost',
dialect: 'postgres'
});
// Test SEQUELIZE connection
sequelize
.authenticate()
.then(() => {
console.log('Database connection has been established successfully.');
})
.catch(err => {
console.error('Unable to connect to the database:', err);
});
fs
.readdirSync(__dirname)
.filter(file => {
return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
})
.forEach(file => {
var model = sequelize['import'](path.join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;
// ./routes/index.js
const models = require('../models');
const express = require('express');
const router = express.Router();
router.get('/contacts', (req, res) => {
models.Contact.findAll().then(contacts => {
console.log("All users:", JSON.stringify(contacts, null, 4));
});
});
module.exports = router;
// ./models/contact.js
const Sequelize = require('sequelize');
var Model = Sequelize.Model;
module.exports = (sequelize, DataTypes) => {
class Contact extends Model {}
Contact.init({
// attributes
firstName: {
type: Sequelize.STRING,
allowNull: false
},
lastName: {
type: Sequelize.STRING,
allowNull: false
}
}, {
sequelize,
modelName: 'contact'
// options
});
return Contact;
};
The error I am getting when using postman to hit /contacts with a GET request is:
[nodemon] starting `node server.js`
The server is now running on port 3000!
Executing (default): SELECT 1+1 AS result
Database connection has been established successfully.
TypeError: Cannot read property 'findAll' of undefined
at router.get (C:\Users\username\desktop\metropolis\metropolis-backend\routes\index.js:6:20)
You are not requiring the model properly.
In ./routes/index.js add the next line:
const Contact = require('./models/contact.js');
And then call Contact.findAll()...
Second approach:
You can gather all your models by importing them into a loader.js file which you will store in the models directory. The whole job of this module is to import the modules together to the same place and then export them from a single place.
It will look something like that:
// loader.js
const modelA = require('./modelA');
const modelB = require('./modelB');
const modelC = require('./modelC');
...
module.exports = {
modelA,
modelB,
modelC,
...
}
And then you can require it in the following way:
in router/index.js:
const Models = require('./models');
const contact = Models.Contact;

Mongoose is returning an empty array from my database when doing a Model.find() query

I looked at this popular question, but it didn't seem to fix my issue, so I'm going to post this.
I currently have an express.js server file using mongoose, that keeps returning an empty array. I have no idea if it might by an async issue, and I don't know what I can use to indicate that I'm connected to my database.
const express = require('express');
const app = express();
const bodyParser = require('body-parser');
const mongoose = require('mongoose');
const PORT = process.env.PORT || 8080;
//Mongoose stuff
mongoose.connect('mongodb+srv://excelsiorAdmin:Mysecretpassword#excelsiorcluster-zakfd.mongodb.net/test?retryWrites=true', { useNewUrlParser: true, dbName: 'excelsiorDB'});
const dbConnection = mongoose.connection;
dbConnection.on('error', console.error.bind(console, 'connection error:'));
dbConnection.once('open', function() {
console.log('connected to the database');
let charSchema = new mongoose.Schema({
imageURL: String,
company: String,
name: String,
civName: String,
alignment: String,
firstDebut: String,
abilities: Array,
teams: Array,
desc: String
});
let Char = mongoose.model('Char', charSchema, 'chars');
//root
app.get('/', (req, res, next) => res.send('Welcome to the API!'));
//get all characters
app.get('/chars', (req, res, next) => {
console.log('getting all characters');
Char.find(function (err, chars) {
if (err) {
res.status(404).send(err);
console.log('there was an error');
};
console.log(chars);
res.send(chars);
});
});
//get heroes
app.get('/chars/heroes', (req, res, next) => {
Char.find({alignment: "Hero"}, function (err, chars) {
if (err) {
res.status(404).send(err);
};
res.send(chars);
});
});
});
app.listen(PORT, () => console.log(`This API is listening on port ${PORT}!`));
The mongoose.model will set the collection it's looking for equal to the lowercase, pluralized form of the name of the model.
let Char = mongoose.model('Char', charSchema);
This will look for the "chars" collection. However, if the database you're connecting to doesn't have a collection with the same name as the mongoose default, it will return results from a collection that doesn't exist. To make sure it hits the right collection if they don't match, you'll have to manually enter the collection's name as a third parameter:
let Char = mongoose.model('Char', charSchema, "excelsiorCollection");