I have a route set up to receive a webhook from SendGrid, which sends MultiPart/form data. I can get the various fields to output in the console with busboy, but I'm struggling to fill in the final piece of the puzzle: getting this parsed data into a Collection object (or just into MongoDB if not familiar with meteor).
I thought something like the following would work, but the data arrays in the db are always blank, i'm guessing i'm missing a crucial step in knowing when the stream has finished?
WebApp.connectHandlers.use('/applicants', (req, res, next) => {
let body = '';
req.on('data', Meteor.bindEnvironment(function (data) {
body += data;
let bb = new Busboy({ headers: req.headers });
let theEmail = [];
bb.on('field', function(fieldname, val) {
console.log('Field [%s]: value: %j', fieldname, val);
let theObject = [];
theObject[fieldname] = val;
theEmail.push(theObject);
}).on('error', function(err) {
console.error('oops', err);
}).on('finish', Meteor.bindEnvironment(function() {
console.log('Done parsing form!');
// try to add data to database....
Meteor.call('applicants.add', theEmail);
}));
return req.pipe(bb);
}));
req.on('end', Meteor.bindEnvironment(function () {
res.writeHead(200);
res.end();
}));
Related
It's fairly simple to make one call to Adobe PDF Services, get the result, and save it, for example:
// more stuff above
exportPdfOperation.execute(executionContext)
.then(result => result.saveAsFile(output))
But if I want to do two, or more, operations, do I need to keep saving the result to the file system and re-providing it (is that even a word ;) to the API?
So this tripped me up as well. In most demos, you'll see:
result => result.saveAsFile()
towards the end. However, the object passes to the completed promise, result, is a FileRef object that can then be used as the input to another call.
Here's a sample that takes an input Word doc and calls the API method to create a PDF. It then takes that and runs OCR on it. Both methods that wrap the API calls return FileRefs, so at the end I saveAsFile on it. (Note, this demo is using v1 of the SDK, it would work the same w/ v2.)
const PDFToolsSdk = require('#adobe/documentservices-pdftools-node-sdk');
const fs = require('fs');
//clean up previous
(async ()=> {
// hamlet.docx was too big for conversion
const input = './hamlet2.docx';
const output = './multi.pdf';
const creds = './pdftools-api-credentials.json';
if(fs.existsSync(output)) fs.unlinkSync(output);
let result = await createPDF(input, creds);
console.log('got a result');
result = await ocrPDF(result, creds);
console.log('got second result');
await result.saveAsFile(output);
})();
async function createPDF(source, creds) {
return new Promise((resolve, reject) => {
const credentials = PDFToolsSdk.Credentials
.serviceAccountCredentialsBuilder()
.fromFile(creds)
.build();
const executionContext = PDFToolsSdk.ExecutionContext.create(credentials),
createPdfOperation = PDFToolsSdk.CreatePDF.Operation.createNew();
// Set operation input from a source file
const input = PDFToolsSdk.FileRef.createFromLocalFile(source);
createPdfOperation.setInput(input);
let stream = new Stream.Writable();
stream.write = function() {
}
stream.end = function() {
console.log('end called');
resolve(stream);
}
// Execute the operation and Save the result to the specified location.
createPdfOperation.execute(executionContext)
.then(result => resolve(result))
.catch(err => {
if(err instanceof PDFToolsSdk.Error.ServiceApiError
|| err instanceof PDFToolsSdk.Error.ServiceUsageError) {
reject(err);
} else {
reject(err);
}
});
});
}
async function ocrPDF(source, creds) {
return new Promise((resolve, reject) => {
const credentials = PDFToolsSdk.Credentials
.serviceAccountCredentialsBuilder()
.fromFile(creds)
.build();
const executionContext = PDFToolsSdk.ExecutionContext.create(credentials),
ocrOperation = PDFToolsSdk.OCR.Operation.createNew();
// Set operation input from a source file.
//const input = PDFToolsSdk.FileRef.createFromStream(source);
ocrOperation.setInput(source);
let stream = new Stream.Writable();
stream.end = function() {
console.log('end called');
resolve(stream);
}
// Execute the operation and Save the result to the specified location.
ocrOperation.execute(executionContext)
.then(result => resolve(result))
.catch(err => reject(err));
});
}
I have been trying to reduce my NextJS bundle size by moving my XLSX parsing to an API route. It uses the npm xlsx (sheetjs) package, and extracts JSON from a selected XLSX.
What I am doing in the frontend is
let res;
let formData = new FormData();
formData.append("file", e.target.files[0]);
try {
res = await axios.post("/api/importExcel", formData);
} catch (e) {
createCriticalError(
"Critical error during file reading from uploaded file!"
);
}
On the API route I am unable to to read the file using XLSX.read()
I believe NextJS uses body-parser on the incoming requests but I am unable to convert the incoming data to an array buffer or any readable format for XLSX.
Do you have any suggestions about how to approach this issue?
I tried multiple solutions, the most viable seemed this, but it still does not work
export default async function handler(req, res) {
console.log(req.body);
let arr;
let file = req.body;
let contentBuffer = await new Response(file).arrayBuffer();
try {
var data = new Uint8Array(contentBuffer);
var workbook = XLSX.read(data, { type: "array" });
var sheet = workbook.Sheets[workbook.SheetNames[0]];
arr = XLSX.utils.sheet_to_json(sheet);
} catch (e) {
console.error("Error while reading the excel file");
console.log({ ...e });
res.status(500).json({ err: e });
}
res.status(200).json(arr);
}
Since you're uploading a file, you should start by disabling the body parser to consume the body as a stream.
I would also recommend using a third-party library like formidable to handle and parse the form data. You'll then be able to read the file using XLSX.read() and convert it to JSON.
import XLSX from "xlsx";
import formidable from "formidable";
// Disable `bodyParser` to consume as stream
export const config = {
api: {
bodyParser: false
}
};
export default async function handler(req, res) {
const form = new formidable.IncomingForm();
try {
// Promisified `form.parse`
const jsonData = await new Promise(function (resolve, reject) {
form.parse(req, async (err, fields, files) => {
if (err) {
reject(err);
return;
}
try {
const workbook = XLSX.readFile(files.file.path);
const sheet = workbook.Sheets[workbook.SheetNames[0]];
const jsonSheet = XLSX.utils.sheet_to_json(sheet);
resolve(jsonSheet);
} catch (err) {
reject(err);
}
});
});
return res.status(200).json(jsonData);
} catch (err) {
console.error("Error while parsing the form", err);
return res.status(500).json({ error: err });
}
}
I am having hard times trying to write data received from a api to db.
I successfully got data and then have to write it to db. The point is to check whether the quote is already exists in my collection.
The problem I am dealing with is that every value gets inserted in my collection, not regarding if it exists or not.
const { MongoClient } = require('mongodb')
const mongoUrl = 'mongodb://localhost/kanye_quotes'
async function connectToDb() {
const client = new MongoClient(mongoUrl, { useNewUrlParser: true })
await client.connect()
db = client.db()
}
async function addQuote(data) {
await connectToDb()
try {
const collection = db.collection('quotes')
let quotes = [];
quotes = await collection.find({}).toArray()
if (quotes = []) { // I added this piece of code because if not check for [], no values will be inserted
collection.insertOne(data, (err, result) => {
if (err) {
return
}
console.log(result.insertedId);
return
})
}
quotes.forEach(quote => {
if (quote.quote !== data.quote) { // I compare received data with data in collection, it actually works fine(the comparison works as it supposed to)
collection.insertOne(data, (err, result) => {
if (err) {
return
}
console.log(result.insertedId);
})
} else console.log('repeated value found'); // repeated value gets inserted. Why?
})
}
catch (err) {
console.log(err)
}
}
Hi it's probably better to set unique: true indexing on your schema. That way you won't have duplicated values.
Sorry if the question sounds a little bit weird, but i don't know how to explain it the best way.
I'm building a simple fullstack app with Vue, Vuex, Express and Postgresql.
Now I'm fetching data from my database and displaying it on my view. Everything works fine, but I noticed that I could do it in two different ways so I wanted to ask you, what is the best solution.
The First way:
I define two separate routes in my express app. The first returns all customer objects and the second return only a specific customer. Then I can call a action in my vuex to fetch the data and display the data through getters in my template
Express.app:
router.route('/')
.get(async (req, res) => {
try {
const allcustomer = await db.query(`select * from Kunde`);
res.json(allcustomer.rows);
} catch (err) {
console.log(err);
res.json('An error occurred!');
}
})
router.route('/:id')
.get(async (req, res) => {
try {
const { id } = req.params;
const onecustomer = await db.query(`Select * from Kunde where kunde.id = ${id}`)
console.log(onecustomer.rows[0]);
res.send(onecustomer.rows[0]);
} catch (err) {
res.send("An error occurred!");
}
})
module.exports = router;
Vuex:
const state = {
customers: [],
onecustomer: {}
}
const getters = {
allcustomers: (state) => state.customers,
onecustomer: (state) => state.onecustomer,
}
const actions = {
async fetchcustomers({ commit }) {
const response = await axios.get('http://localhost:3000/customer');
commit('setcustomers', response.data);
},
async fetchonecustomer({ commit }, id) {
const response = await axios.get(`http://localhost:3000/customer/${id}`);
console.log(response.data)
commit('setonecustomer', response.data);
}
}
The second way:
I only have one route in my express app, who return all customers. In my Vuex I only have on actions who fetch all customers from this route and I have one getter method who filters the state array by a id property and through this getter I display my data on my view.
Express app:
router.route('/')
.get(async (req, res) => {
try {
const allcustomer = await db.query(`select * from Kunde`);
res.json(allcustomer.rows);
} catch (err) {
console.log(err);
res.json('An error occurred!');
}
})
Vuex:
const state = {
customers: [],
}
const getters = {
allcustomers: (state) => state.customers,
onecustomer: (state) => (id) => this.state.customers.filter(customer => customer.id = id)
}
const actions = {
async fetchcustomers({ commit }) {
const response = await axios.get('http://localhost:3000/customer');
commit('setcustomers', response.data);
},
}
const mutations = {
setcustomers: (state, customers) => (state.customers = customers),
}
Both methods are working and I get the same results, but which of these ways are the best one or is it just personally preference?
but I noticed on the second solution I get an error message on my console in my Browser that the object what my getter response is undefined but the data is displayed as I wanted.
I personally would take the first solution because of the error message and its feeling some kind of natural to me.
But what do you think?
First of all read up on sql injection. Your second endpoint is currently unsafe and can be used to compromise any and all data in your database.
Besides that, the following line does not do what you think it does:
onecustomer: (state) => (id) => this.state.customers.filter(customer => customer.id = id)
In this line you assign id to customer.id for each customer, and return an array. You probably want something like this
onecustomer: (state) => (id) => this.state.customers.find(customer => customer.id === id)
In general, doing network requests is an expensive operation. You need to send a request, wait for the response, then parse the response. Your second option is better in the sense that you only send the data once. In most cases however you paginate such queries and you might not have the data available when you load a single customer.
If you preload all the data, then having one request to load all customers is all you need. If you do not preload all data, just load a single customer when you do not have the data already. In most cases, you need the two endpoints you have in your first option.
I am attempting to build a Vue.js App that synthesizes properties of AWS, MongoDB, and Express. I built an authentication page for the app using aws-amplify and aws-amplify-vue. After logging into the app, metadata containing the username for the logged in AWS user is passed into data object property this.name like so:
async beforeCreate() {
let name = await Auth.currentAuthenticatedUser()
this.name = name.username
}
this.name is then added to MongoDB via Axios:
async addName() {
let uri = 'http://localhost:4000/messages/add';
await this.axios.post(uri, {
name: this.name,
})
this.getMessage()
}
I also have a getName() method that I am using to retrieve that data from MongoDB:
async getData () {
let uri = 'http://localhost:4000/messages';
this.axios.get(uri).then(response => {
this.userData = response.data;
});
},
This method, however, returns data for ALL users. I want to reconfigure this method to ONLY return data for .currentAuthenticatedUser(). In my previous experience with Firebase, I would set up my .getData() method with something like:
let ref = db.collection('users')
let snapshot = await ref.where('user_id', '==', firebase.auth().currentUser.uid).get()
...in order to return currentUser information on the condition that 'user_id' in the collection matches the currently logged-in Firebase user.
To achieve this with MongoDB, I attempted to configure the above method like so:
async getData () {
let uri = 'http://localhost:4000/messages';
let snapshot = await uri.where('name', '==', this.name);
this.axios.get(snapshot).then(response => {
this.userData = response.data;
});
},
My thought here was to try and return current user data by comparing 'name' in the MongoDB collection with the logged-in user stored in this.name...but I understand that this might not work because the .where() method is probably unique to Firebase. Any recommendations on how to configure this .getData() to return ONLY data associated with the currentAuthenticatedUser? Thanks!
EXPRESS ROUTES:
const express = require('express');
const postRoutes = express.Router();
// Require Post model in our routes module
let Post = require('./post.model');
// Defined store route
postRoutes.route('/add').post(function (req, res) {
let post = new Post(req.body);
post.save()
.then(() => {
res.status(200).json({'business': 'business in added successfully'});
})
.catch(() => {
res.status(400).send("unable to save to database");
});
});
// Defined get data(index or listing) route
postRoutes.route('/').get(function (req, res) {
Post.find(function(err, posts){
if(err){
res.json(err);
}
else {
res.json(posts);
}
});
});
module.exports = postRoutes;
It is not possible to apply a where clause to a uri AFAIK. What you should do is adding a where clause to the actual query you are making in your backend and, to do that, send the username you want to filter the query with through a query parameter like this: /messages?name=JohnDoe.
So basically if you are using a Node/Express backend, as you suggested, and using Mongoose as the ODM for MongoDB your request would probably be looking something like this:
const Users = require('../models/users.model');
Users.find({}, function (e, users) {
if (e) {
return res.status(500).json({
'error': e
})
}
res.status(200).json({
'data': users
});
})
What you should do is getting the username query parameter through req.query and add it to the options in the first parameter of the find function.
const Users = require('../models/users.model');
let params = {},
name = req.query.name;
if (name) {
params.name = name
}
Users.find(params, function (e, users) {
if (e) {
return res.status(500).json({
'error': e
})
}
res.status(200).json({
'data': users.slice
});
})
That way if you point to /messages?name=John you will get the users with "John" as their name.
Edit:
If your backend is configured in the following way
postRoutes.route('/').get(function (req, res) {
Post.find(function(err, posts){
if(err){
res.json(err);
}
else {
res.json(posts);
}
});
});
what you should do is get the query parameters from inside the get method
postRoutes.route('/').get(function (req, res) {
let params = {},
name = req.query.name
if (name) {
params.name = name
}
Post.find(params, function(err, posts){
if(err){
res.json(err);
}
else {
res.json(posts);
}
});
});