Puppeteer and express can not load new data using REST API - rest

I'm using puppeteer to scrape page that has contents that change periodically and use express to present data in rest api.
If I turn on headless chrome to see what is being shown in the browser, the new data is there, but the data is not showing up in get() and http://localhost:3005/api-weather. The normal browser only shows the original data.
const express = require('express');
const server = new express();
const cors = require('cors');
const morgan = require('morgan');
const puppeteer = require('puppeteer');
server.use(morgan('combined'));
server.use(
cors({
allowHeaders: ['sessionId', 'Content-Type'],
exposedHeaders: ['sessionId'],
origin: '*',
methods: 'GET, HEAD, PUT, PATCH, POST, DELETE',
preflightContinue: false
})
);
const WEATHER_URL = 'https://forecast.weather.gov/MapClick.php?lat=40.793588904953985&lon=-73.95738513173298';
const hazard_url2 = `file://C:/Users/xdevtran/Documents/vshome/wc_api/weather-forecast-nohazard.html`;
(async () => {
try {
const browser = await puppeteer.launch({ headless: true });
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on("request", request => {
console.log(request.url());
request.continue();
});
await page.goto(hazard_url2, { timeout: 0, waitUntil: 'networkidle0' });
hazard = {
"HazardTitle": "stub",
"Hazardhref": "stub"
}
let forecast = await page.evaluate(() => {
try {
let forecasts = document.querySelectorAll("#detailed-forecast-body.panel-body")[0].children;
let weather = [];
for (var i = 0, element; element = forecasts[i]; i++) {
period = element.querySelector("div.forecast-label").textContent;
forecast = element.querySelector("div.forecast-text").textContent;
weather.push(
{
period,
forecast
}
)
}
return weather;
} catch (err) {
console.log('error in evaluate: ', err);
res.end();
}
}).catch(err => {
console.log('err.message :', err.message);
});
weather = forecast;
server.get('/api-weather', (req, res) => {
try {
res.end(JSON.stringify(weather, null, ' '));
console.log(weather);
} catch (err) {
console.log('failure: ', err);
res.sendStatus(500);
res.end();
return;
}
});
} catch (err) {
console.log('caught error :', err);
}
browser.close();
})();
server.listen(3005, () => {
console.log('http://localhost:3005/api-weather');
});
I've tried several solutions WaitUntil, WaitFor, .then and sleep but nothing seems to work.
I wonder if it has something to do with express get()? I'm using res.end() instead of res.send() is because the json looks better when I use res.end(). I don't really know the distinction.
I'm also open to using this reload solution. But I received errors and didn't use it.
I also tried waitForNavigation(), but I don't know how it works, either.
Maybe I'm using the wrong search term to find the solution. Could anyone point me in the right direction? Thank you.

Related

SocketCluster - `procedure` and `receiver` in a single loop

I am trying out SocketCluster for an app where I will need both RPC style as well as One way Event based communication. I am following the Basic Usage section https://socketcluster.io/docs/basic-usage/
If I include socket.procedure and socket.receiver in the same listener loop it won't invoke the event. I want to understand why it is like that.
Really appreciate any input that will clear-up the concept.
Thanks!
My Server Side/Node:
const http = require("http");
const finalHandler = require("finalhandler");
const socketCluster = require("socketcluster-server");
const serveStatic = require("serve-static");
let serve = serveStatic("public", {index: ["index.html"]});
let httpServer = http.createServer(function onfRequest(req, res) {
serve(req, res, finalHandler(req, res))
});
let agServer = socketCluster.attach(httpServer);(async () => {
for await (let {socket} of agServer.listener("connection")) {
// RPC
(async () => {
for await (let req of socket.procedure("customProc1")) {
console.log(`Received RPC request`);
req.end("Success");
}
})();
// Event (One Way)
(async () => {
for await (let data of socket.receiver("customEvent1")) {
console.log(`Received Invoke data: ${data}`);
}
})();
}
})()
httpServer.listen(8000, () => {
console.log(`Server Started On : ${new Date().getTime()}`);
});
On the Client side Index.html/JavaScript:
function fireAnEvent() {
socket.transmit("customEvent1", "My Data");
}
async function callRpc() {
await socket.invoke("customProc1", {greetings: "Hey there function!"});
}

Socket.io, Mongodb returning undefined to frontend

I want to use socekt.io for a new project I am building. I am using socket.io for a login component and will be using socket.io in the future to update pages like a chat app. I am also using mongoose to handle my mongodb connection. I am taking in a username, and returning a password to my front end to be bcryptjs compareSync hashed. The problem I am having is that whatever is returned to the front end is undefined. When I print out what is returned to the front end, it prints out the value I am looking for though. Something is going on between the backend emitting something, and the frontend receiving something but I don't know what is it exactly. Here is my code for the back end:
const express = require('express')
const socket = require('socket.io');
const http = require('http');
const router = require('./router');
const mongoose = require('mongoose');
let Player = require('../models/player.model');
require('dotenv').config();
const PORT = process.env.PORT || 5000;
const app = express();
const server = http.createServer(app);
const uri = process.env.ATLAS_URI;
mongoose.connect(uri, {useNewUrlParser: true, useCreateIndex: true,
useUnifiedTopology: true });
const connection = mongoose.connection;
connection.once('open',() => {
console.log('MongoDB database connection established successfully')
});
const io = socket(server);
io.on('connection', (socket) => {
console.log('We have a new connection');
socket.on('login', ({ username }, callback) => {
console.log(username);
Player.find({"username": username}, function (err, player) {
if(err) {
console.log("there has been an error"), {player: null}
}
socket.emit('id', { password: player[0]['password'].toString(), id : player[0]['_id']})
}) })})
app.use(router);
server.listen(PORT, () => console.log('Server is working'))
Here is my code for the front end:
const ENDPOINT = 'localhost:5000';
async function submitAccount (e) {
e.preventDefault();
socket.emit('login', { username });
socket.on("id", (response) => {
setPassword2(String(response['password']));
id = response['id']; console.log(id);
console.log(password2)
});
try {
if (bcrypt.compareSync(password, password2) == true) {
props.setCookie("id", id);
setAccess(true);
access2 = true;
console.log(access2)
console.log('works')
}
else {
setErrorType('Invalid Password')
setErrorMsg('There is an issue with your password. Please try again')
setOpenModal(true)
console.log(password);
console.log(password2);
}
}
catch {
setErrorType('Invalid Username')
setErrorMsg('This username does not exist. Please try another')
setOpenModal(true)
}
Thanks for the help!
When you do the socket.on, it should include the whole statement you are looking to change with the socket.io output. See below:
async function submitAccount (e) {
e.preventDefault();
socket.emit('login', { username });
socket.on("id", (response) => {
setPassword2(String(response['password']));
id = response['id']; console.log(id);
console.log(password2)
if (password2 != undefined) {
try {
if (bcrypt.compareSync(password, password2) == true) {
props.setCookie("id", id);
setAccess(true);
access2 = true;
console.log(access2)
console.log('works')
}
}

My mongoDB api is getting a webtask.io token error

I'm learning about webtask.io and so I've coded a simple REST api (I'm not using Express.js here but maybe I should). It's a little webtask.io app that connects to an mlab MongoDB database and retrieves todos from a tasklist collection. The issue is that I'm getting this error:
{"code":404,"message":"unable to resolve jtn to webtask token","req_id":"1504385487318.83712"}"
Any idea how to fix this error? Here is a snippet of my code:
var MongoClient = require('mongodb').MongoClient;
...
module.exports =
function (ctx, req, res) {
// write the header and set the response type as a json
res.writeHead(200, { 'Content-Type': 'application/json' });
MongoClient.connect(ctx.data.MONGO_URL, function (err, db) {
if (err) {
res.writeHead(400, { 'Content-Type': 'application/json'});
res.end(JSON.stringify(ERROR_RESPONSE.CONNECT_ERROR));
} else {
switch(req.method) {
case 'GET':
db.collection('tasklist').find({}).sort({"dateAdded" : -1}).toArray(function(err, docs) {
if (err) {
res.writeHead(400, { 'Content-Type': 'application/json'});
res.end(JSON.stringify(ERROR_RESPONSE.GET_ERROR));
} else {
res.end(JSON.stringify(docs));
}
}); //toArray
break;
//post, delete, and put are in here
} //switch
} //else no error
db.close();
}); //Mongo connect
res.end();
} //export function
I decided to try using Express and now I'm able to run my little webtask.io without having to have a web token. I'm not sure why my first try required one and if I find that answer I will post it. Here is my working version:
/* express app as a webtask */
var MongoClient = require('mongodb').MongoClient;
var Express = require('express');
var wt = require('webtask-tools');
var app = Express();
var assert = require('assert');
var ObjectId = require('mongodb').ObjectId;
app.use(require('body-parser').json());
function doCRUD (crudType,req,res) {
MongoClient.connect(req.webtaskContext.secrets.MONGO_URL,function (err, db) {
if (err) {
res.send(JSON.stringify(err));
} else {
switch(crudType) {
case 'GET':
db.collection('tasklist').find({}).sort({"dateAdded" : -1}).toArray(function(err, docs) {
if (err) {
res.send(JSON.stringify(err));
} else {
res.end(JSON.stringify(docs));
}
}); //toArray
break;
case 'POST':
db.collection('tasklist').insertOne({"tasklist" : req.query.todo, "dateAdded" : new Date()}, function(err, r) {
assert.equal(null, err);
assert.equal(1, r.insertedCount);
});
break;
case 'DELETE':
db.collection('tasklist').deleteOne({_id: new ObjectId(req.query.id)},function(err){assert.equal(null,err)});
break;
case 'PUT':
//not implemented for this hack
break;
}
}
});
}
// GET
app.get('*', function (req, res) {
doCRUD('GET',req,res);
});
// POST
app.post('*', function (req, res) {
doCRUD('POST',req,res);
res.end();
});
// DELETE
app.delete('*', function (req, res) {
doCRUD('DELETE',req,res);
res.end();
});
// expose this express app as a webtask-compatible function*/
module.exports = wt.fromExpress(app);
This error appears if you do not specify a valid path.
Try this one for example :
https://wt-666ohgod666ohgod666ohgod666ohgod-0.run.webtask.io/antidisestablishmentarianism666

How to translate superagent to axios?

I have some upload working for superagent. It involves posting to an api for cloudinary. My question is how do I do the same thing with axios. I'm not sure what superagent.attach and superagent.field relate to in axios.
Basically when I make the post request I need to attach all these fields to the request or else I get bad request and I want to do this in axios not superagent as I am switching over to axios.
Here are all the params:
const image = files[0];
const cloudName = 'tbaustin';
const url = `https://api.cloudinary.com/v1_1/${cloudName}/image/upload`;
const timestamp = Date.now()/1000;
const uploadPreset = 'cnh7rzwp';
const paramsStr = `timestamp=${timestamp}&upload_preset=${uploadPreset}ORor-6scjYwQGpNBvMW2HGMkc8k`;
const signature = sha1(paramsStr);
const params = {
'api_key': '177287448318217',
'timestamp': timestamp,
'upload_preset': uploadPreset,
'signature': signature
}
Here is the superagent post request:
let uploadRequest = superagent.post(url)
uploadRequest.attach('file', image);
Object.keys(params).forEach((key) => {
uploadRequest.field(key, params[key]);
});
uploadRequest.end((err, res) => {
if(err) {
alert(err);
return
}
You would need to use FromData as follows:
var url = `https://api.cloudinary.com/v1_1/${cloudName}/upload`;
var fd = new FormData();
fd.append("upload_preset", unsignedUploadPreset);
fd.append("tags", "browser_upload"); // Optional - add tag for image admin in Cloudinary
fd.append("signature", signature);
fd.append("file", file);
const config = {
headers: { "X-Requested-With": "XMLHttpRequest" },
onUploadProgress: function(progressEvent) {
// Do something with the native progress event
}
};
axios.post(url, fd, config)
.then(function (res) {
// File uploaded successfully
console.log(res.data);
})
.catch(function (err) {
console.error('err', err);
});
See full example here

ES6 Promises in express app not properly resolving data

I'm writing an a async function with ES6 promises, that 1) saves the query parameters for a user 2) fetches data from mongodb using mongoose, 3) manipulates the json into a DSL, 4) and queries another db with it.
mongoose": "^4.7.7"
//myController.js
const myQuery = require('../models/myQuery_model');
require('mongoose').Promise = global.Promise
const uuidV4 = require('uuid/v4');
exports.saveNewQuery = function(req, res, next) {
const rawQuery = req.body;
const queryToStore = new myQuery(rawQuery);
const uid = uuidV4();
const queryToStore.uid = uid
queryToStore.save().then(() => {
fetchQueryFromMongo(uid);
}).then((storedQuery) => {
compileQueryToString(storedQuery);
}).then((queryString) => {
fetchResultsFromOtherDb(queryString);
}).then((results) => {
res.json({ results });
}).catch((error) => {
console.log(error)
})
}
Currently I'm not able to resolve the response from mongodb step 2. Still, the controllter goes on to compileQueryToString rather than catch the error from fetchQueryFromMongo
// fetchQueryFromMongo.js
const myQuery = require('../models/myQuery');
require('mongoose').Promise = global.Promise
module.exports = (uid) => {
return new Promise(
(resolve, reject) => {
myQuery.find({ uid }).then((err, res) => {
if (err) {
reject(err);
}
console.log('response success!')
resolve(res);
});
}
);
};
I'm new to promises so any tips / suggestions would be appreciated!
Make sure to return a value from your then handlers. The code below does this by using the concise body form of arrow functions.
queryToStore.save()
.then(() => fetchQueryFromMongo(uid))
.then(storedQuery => compileQueryToString(storedQuery))
.then(queryString => fetchResultsFromOtherDb(queryString))
.then(results => res.json({ results }))
.catch(console.log);