I am getting request failed with status code 502. I am trying to web scrape this page to get data from the home page and display it to a frontend website that I am building but I can't get the raw html for the life of me.
const express = require("express");
const cheerio = require("cheerio");
const axios = require("axios");
const app = express();
const PORT = process.env.PORT || 3000;
const website = "https://und.com";
try {
axios(website).then((res) => {
const data = res.data;
const $ = cheerio.load(data);
let content = [];
$(".sdc-site-tile__headline", data).each(function () {
const title = $(this).text();
const url = $(this).find("a").attr("href");
content.push({
title,
url
});
app.get("/", (req, res) => {
res.json(content);
});
});
});
} catch (error) {
console.log(error, error.message);
}
app.listen(PORT, () => {
console.log(`server is running on PORT:${PORT}`);
});
This question needs more information. A 502 HTTP error means the following:
"The 502 error is one of the status codes, which indicates that the web server you've connected to is serving as a proxy for data relayed from another server, but that server has returned an invalid response."
Since you are trying to scrape a website it is likely that you are being blocked. This might be due to your user agent containing flags that are sniffed by their servers and thus blocked.
You can change your user agent to something that shows you are an actual browser. Try passing a user agent in your http header.
Here's a link for adding header options with Axios:
Force to use different user agent with js or axios
Based on your comments and the existing answer which suggests a user agent to avoid the 502 response, it sounds like you want the selector ".post__meta h3 a:last-child":
const axios = require("axios"); // ^0.21.4
const cheerio = require("cheerio"); // 1.0.0-rc.12
const website = "https://und.com";
const options = {
headers: {
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
},
};
axios(website, options)
.then(({data}) => {
const $ = cheerio.load(data);
const result = $(".post__meta h3 a:last-child")
.map((_, e) => $(e).text().trim())
.get();
console.log(result);
})
.catch((err) => console.error(err));
Note that try/catch doesn't work with .then:
// wrong (Uncaught (in promise) TypeError: Failed to fetch)
try {
fetch("wrong")
.then(res => res.text())
.then(text => console.log(text));
}
catch (err) {
console.error(err);
}
// correct
fetch("wrong")
.then(res => res.text())
.then(text => console.log(text))
.catch(err => console.error(err));
// correct
(async () => {
try {
const res = await fetch("wrong");
const text = await res.text();
console.log(text);
}
catch (err) {
console.error(err);
}
})();
Related
I am really new to the industry and have this error when trying to check the database connection via API reuests with postman..... Please help me to settle this issue...
I just want to check the mongodb database by sendng API requests. Still I cannot identify the error and I am following a set of tutorials and occure this issue... Anyone can help me to identify the mistake it's highly appreciated....
{ this is dummy text to avoid please add more details...
Here is my code...
const app = express();
const { MongoClient } = require('mongodb');
const PORT = process.env.PORT || 8000;
// Initialize middleware
// we used to install body parser but now it's a built in middleware
// Function of express. It parses incoming JSONpayload
// app.use(express.json({extended:false}));
app.use(express.json({ extended: false }));
// Test Routs
// app.get("/", (req,res)=>res.send("Hello Aruna !!!"));
// app.post("/", (req,res)=>res.send(`Hello ${req.body.name} `));
// app.get("/hello/:name", (req.res)=>res.send(`Hello ${req.params.name}`))
app.get('/api/articles/:name', async (req, res) => {
try {
const articleName = req.params.name;
const client = await MongoClient.connect('mongodb://localhost:27017');
const db = client.db('mernblog');
const articlesinfo = db
.collection('articles')
.findOne({ name: articleName });
res.status(200).jason(articlesinfo);
client.close();
} catch (error) {
res.status(500).jason({ message: 'Error connecting to database', error });
}
});
app.post('/api/articles/:name/add-comments', (req, res) => {
const { username, text } = req.body;
const articleName = req.params.name;
articlesinfo[articleName].comments.push({ username, text });
res.status(200).send(articlesinfo[articleName]);
});
app.post('/', (req, res) => res.send(`Hello ${req.body.name}`));
app.get('/hello/:name', (req, res) => res.send(`Hello ${req.params.name}`));
app.listen(PORT, () => console.log(`Server is running at port ${PORT}`));
Server.js
Terminal
Error and API request in Postman
You have a typo in your code: jason should be json.
Other tips, you should handle your DB connection in a separate method and change your post request since articlesinfo is not a global variable:
const app = express();
const { MongoClient } = require('mongodb');
const PORT = process.env.PORT || 8000;
const client = new MongoClient('mongodb://localhost:27017');
const connectDB = async () => {
try {
await client.connect();
console.log('Successfully connected to DB')
} catch (err) {
await client.close();
console.log('Error connecting to DB');
process.exit(1);
}
}
// Initialize middleware
// we used to install body parser but now it's a built in middleware
// Function of express. It parses incoming JSONpayload
// app.use(express.json({extended:false}));
app.use(express.json({ extended: false }));
// Test Routs
// app.get("/", (req,res)=>res.send("Hello Aruna !!!"));
// app.post("/", (req,res)=>res.send(`Hello ${req.body.name} `));
// app.get("/hello/:name", (req.res)=>res.send(`Hello ${req.params.name}`))
app.get('/api/articles/:name', async (req, res) => {
try {
const articleName = req.params.name;
const db = client.db('mernblog');
const articlesinfo = db
.collection('articles')
.findOne({ name: articleName });
res.status(200).json(articlesinfo);
client.close();
} catch (error) {
res.status(500).json({ message: 'Error connecting to database', error });
}
});
app.post('/api/articles/:name/add-comments', (req, res) => {
const { username, text } = req.body;
const articleName = req.params.name;
const db = client.db('mernblog');
const articlesinfo = db
.collection('articles')
.updateOne({ name: articleName }, { $push: { comments: { username, text } } });
res.status(200).send(articlesinfo);
});
app.post('/', (req, res) => res.send(`Hello ${req.body.name}`));
app.get('/hello/:name', (req, res) => res.send(`Hello ${req.params.name}`));
connectDB();
app.listen(PORT, () => console.log(`Server is running at port ${PORT}`));
I am new to socket.io. I have basic understanding of how it works, but I am struggling to find proper setup for it within MERN app. If there is any article, or guidance that you can give me, I would appretiate it. I am building social network app, and I need to have live notifications and messages. So I am not sure how to setup socket.io client in react. Should I instanciate it in helper file, like mongoose in express, or is there any other way? Thanks
Install socket.io for server app
Install socket.io - client for client app
import socket.io in server page
const express = require('express');
const http = require('http');
const socketio = require('socket.io');
const cors = require('cors');
const app = express();
const server = http.createServer(app)
const io = socketio(server, { cors: { origin: '*' } }) //for omit cors error
const PORT = 2900;
app.use(express.json());
app.use(cors());
io.on('connect', (socket) => {
console.log("user connected")
socket.on('valor', ({ id, name, }, callback) => {
console.log('data::', id, name)
socket.emit('receiveGreet', { data: 'This message from server' }, (error) => {
console.log('error::', error)
})
callback()
})
socket.on('disconnect', () => {
console.log('user disconnected')
})
})
app.get('/', (req, res) => {
res.json('api running')
})
server.listen(PORT, console.log(`server running in node on port ${PORT}`));
Client side Code May look like this
import io from 'socket.io-client';
let socket: any;
const serverUrl = 'http://localhost:2900';
const MyComponent = () => {
useEffect(() => {
socket = io(serverUrl);
socket.on('receiveGreet', (data) => {
console.log('data::', data);
});
}, []);
return () => {
socket.disconnect();
socket.off();
};
};
This is not so much of a question but more of a consult request. I couldn't find resources to check my method's validity so I would like to hear MongoDB experts' opinion.
I was playing around with MongoDB and came up with this middleware method to pass client to my routes. I have this Express middleware:
const addClientToRequest = async (req, _, next) => {
const client = new MongoClient(uri);
await client.connect();
req.client = client;
next();
};
app.use(addClientToRequest);
After that, I use req.client in my routes to access my database.
app.get("/:id", async (req, res) => {
const client = req.client;
const id = req.params.id;
try {
const data = await client.db("mydb").collection("mycollection").findOne({ id });
if (data) return res.status(200).json(data);
} catch (error) {
return res
.status(500)
.json({ message: "Error fetching requested data", error });
}
return res.status(404).json({ message: "Requested data cannot be found" });
});
What would be a problem in this approach? Is it okay to use MongoDB client like this?
In my experience, we have always defined a separate utility to load a connection pool at the app startup and then reused those connections.
In the above approach, you seem to be creating a new connection for every HTTP request that is made and then not terminating (or) closing the connection. This may be expensive for a large app.
db.util.js
const { MongoClient } = require("mongodb");
const uri = `mongodb://${process.env.DB_USER}:${process.env.DB_PASSWORD}#localhost:27017/${process.env.DATABASE}?maxPoolSize=2-&w=majority`;
const client = new MongoClient(uri);
const init = async () => {
try {
await client.connect();
console.log("Connected");
} catch (error) {
console.log(error);
}
};
const getClient = () => {
return client;
};
module.exports.init = init;
module.exports.getClient = getClient;
app.js
//Import modules
require("dotenv").config({ path: __dirname + "/.env" });
const express = require("express");
const dogRoutes = require("./routes/dog.routes");
const db = require("./utils/db.util");
// Define PORT for HTTP Server
const PORT = 9900;
// Initialize Express
const app = express();
app.use(express.json());
app.use(dogRoutes);
(async () => {
await db.init();
app.listen(PORT, (err) => {
console.log(`Server is up at localhost ${PORT}`);
});
})();
I think that what you could do is to put the client outside of the middleware, so you doesn't re define it and re connect to it each time a request is done.
To do so, simply define it and connect before the middleware, and in the middleware, set the client as req.mongoClient or how you want to name it.
const client = new MongoClient(uri);
await client.connect(); // if this is outside of an async function, either use an async function like (async () => {..script..})(), either define a variable isClientReady and set it on true after the promise resolved.
const addClientToRequest = (req, _, next) => {
req.client = client;
next();
};
app.use(addClientToRequest);
I am using dio to make a network request. In testing phases I was using local host port 3000. I was using a javascript file and node to run it in testing mode. I would simply run node on the javascript file it would fire up the port at it would work. This was great but whenever I run it on a real device it does not work. So I am assuming I need to change it to something else for release...? I am new bare with me. Any suggestion or guidance would be helpful thank you.
const muxServerUrl = 'http://localhost:3000';
initializeDio() {
BaseOptions options = BaseOptions(
baseUrl: muxServerUrl,
connectTimeout: 8000,
receiveTimeout: 5000,
headers: {
"Content-Type": contentType, // application/json
},
);
_dio = Dio(options);
}
Implementation
late Response response;
try {
// print(response);
response = await _dio.post(
"/assets",
data: {
"videoUrl": videoUrl,
},
);
} catch (e) {
print('ran 2');
throw Exception('Failed to store video on MUX');
}
if (response.statusCode == 200) {
print('ran 4');
VideoData videoData = VideoData.fromJson(response.data);
String status = videoData.data!.status;
while (status == 'preparing') {
await Future.delayed(Duration(seconds: 1));
videoData = (await checkPostStatus(videoId: videoData.data!.id))!;
status = videoData.data!.status;
}
print('Video READY, id: ${videoData.data!.id}');
return videoData;
}
That Node Temp JS file
require("dotenv").config();
const express = require("express");
const bodyParser = require("body-parser");
const Mux = require("#mux/mux-node");
const { Video } = new Mux(
process.env.MUX_TOKEN_ID,
process.env.MUX_TOKEN_SECRET
);
const app = express();
const port = 3000;
var jsonParser = bodyParser.json();
app.post("/assets", jsonParser, async (req, res) => {
console.log("BODY: " + req.body.videoUrl);
const asset = await Video.Assets.create({
input: req.body.videoUrl,
playback_policy: "public",
});
res.json({
data: {
id: asset.id,
status: asset.status,
playback_ids: asset.playback_ids,
created_at: asset.created_at,
},
});
});
app.get("/assets", async (req, res) => {
const assets = await Video.Assets.list();
res.json({
data: assets.map((asset) => ({
id: asset.id,
status: asset.status,
playback_ids: asset.playback_ids,
created_at: asset.created_at,
duration: asset.duration,
max_stored_resolution: asset.max_stored_resolution,
max_stored_frame_rate: asset.max_stored_frame_rate,
aspect_ratio: asset.aspect_ratio,
})),
});
});
app.get("/asset", async (req, res) => {
let videoId = req.query.videoId;
const asset = await Video.Assets.get(videoId);
console.log(asset);
res.json({
data: {
id: asset.id,
status: asset.status,
playback_ids: asset.playback_ids,
created_at: asset.created_at,
duration: asset.duration,
max_stored_resolution: asset.max_stored_resolution,
max_stored_frame_rate: asset.max_stored_frame_rate,
aspect_ratio: asset.aspect_ratio,
},
});
});
app.listen(port, () => {
console.log(`Mux API listening on port ${port}`);
});
localhost is what's called your loopback address and that's only working because you are running the application on your machine. When you release the app you have to host your Nodejs app in some server and use the IP address of that server instead. Before you host that app I encourage you to spend more time making sure that it secure.
If you just want to run the app on an Android emulator you can use 10.0.2.2 to reach the hosting machine loopback
I am following a tutorial from Coding Garden. There he writes to a database and sends it then back to the client.
When I try to do it, I do not get a respond from the server. I guess there has been a mix up in my code.
When I go to localhost/5000/posts there is no database. Why do I not get an errormessage, or a database?
Best regards
Expected Result:
https://youtu.be/JnEH9tYLxLk?t=3060
client code
const form = document.querySelector('form');
const loadingElement = document.querySelector(".loading");
const API_URL = "http://localhost:5000/posts";
loadingElement.style.display = "none";
form.addEventListener('submit', (event) => {
event.preventDefault();
const formData = new FormData(form);
const name = formData.get('name');
const content = formData.get('content');
const post = {
name,
content
};
form.style.display = "none";
loadingElement.style.display= "";
fetch(API_URL, {
method: "POST",
body: JSON.stringify(post),
headers: {
"content-type": "application/json"
}
}).then(response => response.json())
.then(createdPost => {
console.log(createdPost);
});
});
server code
const express = require("express");
const cors = require('cors');
const monk = require("monk");
const app = express();
const db = monk("localhost/posts");
const posts = db.get("posts");
app.use(cors());
app.use(express.json());
app.get("/", (req, res) => {
res.json({
message: "Post"
});
});
function isValidPost(post){
return post.name && post.name.toString().trim() !== "" &&
post.content && post.content.toString().trim() !=="";
}
app.post("/posts", (req, res) => {
if (isValidPost(req.body)){
const post = {
name: req.body.name.toString(),
content: req.body.content.toString(),
created: new Date()
};
//console.log(post);
posts
.insert(post)
.then(createdPost => {
res.json(createdPost);
});
}else {
res.status(422);
res.json({
message: "Hey, Titel und Inhalt werden benötigt!"
});
}
});
app.listen(5000, () => {
console.log('Listening on http://localhost:5000');
});
You forgot to handle the case when post.insert(...) fails and rejects. In this case no response is sent from your server and the request will hang. Add the following:
posts
.insert(post)
.then(createdPost => {
res.json(createdPost);
})
.catch(err => {
console.log(err);
res.status(500).json({errorMessage: err.message});
});
handle the fetch method with catch. It probably goes to catch.
fetch().then().catch(err => ...)