I am trying to get change my request url and see the new url in the response
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', interceptedRequest => {
if (interceptedRequest.url().includes('some-string')) {
interceptedRequest.respond({
status: 302,
headers: {
url: 'www.new.url.com'
},
})
}
interceptedRequest.continue()
});
page.on('response', response => {
console.log(response.url())
})
await page.goto('www.orginal.url.com')
// some code omitted
})();
In the interceptedRequest.respond method I'm trying to update the value of the url. Originally I was trying:
interceptedRequest.continue({url: 'www.new.url.com'})
but that way is not long supported in the current version of Puppeteer.
I was expecting to get www.new.url.com in the response, but I actually get the orignial url with www.new.url.com appended to the end.
Thanks in advance for any help.
It helped me. You need to change url to location
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', interceptedRequest => {
if (interceptedRequest.url().includes('some-string')) {
interceptedRequest.respond({
status: 302,
headers: {
location: 'www.new.url.com'
},
})
}
});
page.on('response', response => {
console.log(response.url())
})
await page.goto('www.orginal.url.com')
// some code omitted
})();
Related
I don't understand how to use axios to fetch data from an array of urls. But I can do it with fetch. The following code works perfectly:
const url = 'https://vimeo.com/api/oembed.json?url='
async index(videoUrls = []) {
try {
const response = await Promise.all(
// videoUrls.map(videoUrl => axios.$get(`${url}${encodeURIComponent(videoUrl)}`))
videoUrls.map(videoUrl => fetch(`${url}${encodeURIComponent(videoUrl)}`))
)
const results = await Promise.all(response.map(r => r.json()));
return results;
} catch (e) {
console.error(e)
}
}
When I make a call like index(["https://vimeo.com/216850224", "https://vimeo.com/642263700"]), my console shows an array with all the video meta details vimeo has to give me. This is perfect.
But the moment I comment out the line that uses fetch and use axios, I get a CORS error.
What is the idiomatic way to fetch data from a bunch of urls in axios?
EDIT
I also tried this, but the .all() function doesn't seem to exist
async index(videoUrls = []) {
try {
const response = await axios.all(videoUrls.map(videoUrl => `${url}${encodeURIComponent(videoUrl)}`));
return response;
} catch (e) {
console.error(e)
}
}
You can easily do it like below:
(async function getAll() {
const axiosrequest1 = axios.get('https://jsonplaceholder.typicode.com/posts');
const axiosrequest2 = axios.get('https://jsonplaceholder.typicode.com/posts');
const axiosrequest3 = axios.get('https://jsonplaceholder.typicode.com/posts');
const [res1, res2, res3] = await Promise.all([axiosrequest1, axiosrequest2, axiosrequest3]);
console.log('request1', res1.data);
console.log('request2', res2.data);
console.log('request3', res3.data);
})();
<script src="https://unpkg.com/axios/dist/axios.min.js"></script>
The Axios version would be slightly different because it automatically decodes and embeds the response body into the response.data property (no need for res.json())
const baseUrl = "https://vimeo.com/api/oembed.json"
const index = async (videoUrls = []) => {
// create an array of responses and wait for them to resolve
const responses = await Promise.all(
videoUrls.map(url => axios.get(baseUrl, { params: { url } })
)
// extract the `data` properties and return them as an array
return responses.map(({ data }) => data)
}
Exactly when you extract response.data is totally up to you. It could also look like this
const index = (videoUrls = []) => Promise.all(
videoUrls.map(async (url) => (
await axios.get(baseUrl, { params: { url } })
).data)
)
FYI, your fetch() version could be a little cleaner too...
const baseUrl = "https://vimeo.com/api/oembed.json"
const index = (videoUrls = []) => Promise.all(
videoUrls.map(async (url) => {
const params = new URLSearchParams({ url })
const res = await fetch(`${baseUrl}?${params}`)
if (!res.ok) { // check for bad response
throw new Error(`${res.status}: ${await res.text()}`)
}
return res.json()
})
)
I am using dio to make a network request. In testing phases I was using local host port 3000. I was using a javascript file and node to run it in testing mode. I would simply run node on the javascript file it would fire up the port at it would work. This was great but whenever I run it on a real device it does not work. So I am assuming I need to change it to something else for release...? I am new bare with me. Any suggestion or guidance would be helpful thank you.
const muxServerUrl = 'http://localhost:3000';
initializeDio() {
BaseOptions options = BaseOptions(
baseUrl: muxServerUrl,
connectTimeout: 8000,
receiveTimeout: 5000,
headers: {
"Content-Type": contentType, // application/json
},
);
_dio = Dio(options);
}
Implementation
late Response response;
try {
// print(response);
response = await _dio.post(
"/assets",
data: {
"videoUrl": videoUrl,
},
);
} catch (e) {
print('ran 2');
throw Exception('Failed to store video on MUX');
}
if (response.statusCode == 200) {
print('ran 4');
VideoData videoData = VideoData.fromJson(response.data);
String status = videoData.data!.status;
while (status == 'preparing') {
await Future.delayed(Duration(seconds: 1));
videoData = (await checkPostStatus(videoId: videoData.data!.id))!;
status = videoData.data!.status;
}
print('Video READY, id: ${videoData.data!.id}');
return videoData;
}
That Node Temp JS file
require("dotenv").config();
const express = require("express");
const bodyParser = require("body-parser");
const Mux = require("#mux/mux-node");
const { Video } = new Mux(
process.env.MUX_TOKEN_ID,
process.env.MUX_TOKEN_SECRET
);
const app = express();
const port = 3000;
var jsonParser = bodyParser.json();
app.post("/assets", jsonParser, async (req, res) => {
console.log("BODY: " + req.body.videoUrl);
const asset = await Video.Assets.create({
input: req.body.videoUrl,
playback_policy: "public",
});
res.json({
data: {
id: asset.id,
status: asset.status,
playback_ids: asset.playback_ids,
created_at: asset.created_at,
},
});
});
app.get("/assets", async (req, res) => {
const assets = await Video.Assets.list();
res.json({
data: assets.map((asset) => ({
id: asset.id,
status: asset.status,
playback_ids: asset.playback_ids,
created_at: asset.created_at,
duration: asset.duration,
max_stored_resolution: asset.max_stored_resolution,
max_stored_frame_rate: asset.max_stored_frame_rate,
aspect_ratio: asset.aspect_ratio,
})),
});
});
app.get("/asset", async (req, res) => {
let videoId = req.query.videoId;
const asset = await Video.Assets.get(videoId);
console.log(asset);
res.json({
data: {
id: asset.id,
status: asset.status,
playback_ids: asset.playback_ids,
created_at: asset.created_at,
duration: asset.duration,
max_stored_resolution: asset.max_stored_resolution,
max_stored_frame_rate: asset.max_stored_frame_rate,
aspect_ratio: asset.aspect_ratio,
},
});
});
app.listen(port, () => {
console.log(`Mux API listening on port ${port}`);
});
localhost is what's called your loopback address and that's only working because you are running the application on your machine. When you release the app you have to host your Nodejs app in some server and use the IP address of that server instead. Before you host that app I encourage you to spend more time making sure that it secure.
If you just want to run the app on an Android emulator you can use 10.0.2.2 to reach the hosting machine loopback
I am following a tutorial from Coding Garden. There he writes to a database and sends it then back to the client.
When I try to do it, I do not get a respond from the server. I guess there has been a mix up in my code.
When I go to localhost/5000/posts there is no database. Why do I not get an errormessage, or a database?
Best regards
Expected Result:
https://youtu.be/JnEH9tYLxLk?t=3060
client code
const form = document.querySelector('form');
const loadingElement = document.querySelector(".loading");
const API_URL = "http://localhost:5000/posts";
loadingElement.style.display = "none";
form.addEventListener('submit', (event) => {
event.preventDefault();
const formData = new FormData(form);
const name = formData.get('name');
const content = formData.get('content');
const post = {
name,
content
};
form.style.display = "none";
loadingElement.style.display= "";
fetch(API_URL, {
method: "POST",
body: JSON.stringify(post),
headers: {
"content-type": "application/json"
}
}).then(response => response.json())
.then(createdPost => {
console.log(createdPost);
});
});
server code
const express = require("express");
const cors = require('cors');
const monk = require("monk");
const app = express();
const db = monk("localhost/posts");
const posts = db.get("posts");
app.use(cors());
app.use(express.json());
app.get("/", (req, res) => {
res.json({
message: "Post"
});
});
function isValidPost(post){
return post.name && post.name.toString().trim() !== "" &&
post.content && post.content.toString().trim() !=="";
}
app.post("/posts", (req, res) => {
if (isValidPost(req.body)){
const post = {
name: req.body.name.toString(),
content: req.body.content.toString(),
created: new Date()
};
//console.log(post);
posts
.insert(post)
.then(createdPost => {
res.json(createdPost);
});
}else {
res.status(422);
res.json({
message: "Hey, Titel und Inhalt werden benötigt!"
});
}
});
app.listen(5000, () => {
console.log('Listening on http://localhost:5000');
});
You forgot to handle the case when post.insert(...) fails and rejects. In this case no response is sent from your server and the request will hang. Add the following:
posts
.insert(post)
.then(createdPost => {
res.json(createdPost);
})
.catch(err => {
console.log(err);
res.status(500).json({errorMessage: err.message});
});
handle the fetch method with catch. It probably goes to catch.
fetch().then().catch(err => ...)
I can get access to the entire HTML for any URL by opening dev-tools and typing:
document.documentElement
I am trying to replicate the same behavior using puppeteer, however, the snippet below returns {}
const puppeteer = require('puppeteer'); // v 1.1.0
const iPhone = puppeteer.devices['Pixel 2 XL'];
async function start(canonical_url) {
const browserURL = 'http://127.0.0.1:9222';
const browser = await puppeteer.connect({browserURL});
const page = await browser.newPage();
await page.emulate(iPhone);
await page.goto(canonical_url, {
waitUntil: 'networkidle2',
});
const data = await page.evaluate(() => document.documentElement);
console.log(data);
}
returns:
{}
Any idea on what I could be doing wrong here?
I am using Puppeteer to generate PDF files from HTML strings.
Reading the documentation, I found two ways of generating the PDF files:
First, passing an url and call the goto method as follows:
page.goto('https://example.com');
page.pdf({format: 'A4'});
The second one, which is my case, calling the method setContent as follows:
page.setContent('<p>Hello, world!</p>');
page.pdf({format: 'A4'});
The thing is that I have 3 different HTML strings that are sent from the client and I want to generate a single PDF file with 3 pages (in case I have 3 HTML strings).
I wonder if there exists a way of doing this with Puppeteer? I accept other suggestions, but I need to use chrome-headless.
I was able to do this by doing the following:
Generate 3 different PDFs with puppeteer. You have the option of saving the file locally or to store it in a variable.
I saved the files locally, because all the PDF Merge plugins that I found only accept URLs and they don't accept buffers for instance. After generating synchronously the PDFs locally, I merged them using PDF Easy Merge.
The code is like this:
const page1 = '<h1>HTML from page1</h1>';
const page2 = '<h1>HTML from page2</h1>';
const page3 = '<h1>HTML from page3</h1>';
const browser = await puppeteer.launch();
const tab = await browser.newPage();
await tab.setContent(page1);
await tab.pdf({ path: './page1.pdf' });
await tab.setContent(page2);
await tab.pdf({ path: './page2.pdf' });
await tab.setContent(page3);
await tab.pdf({ path: './page3.pdf' });
await browser.close();
pdfMerge([
'./page1.pdf',
'./page2.pdf',
'./page3.pdf',
],
path.join(__dirname, `./mergedFile.pdf`), async (err) => {
if (err) return console.log(err);
console.log('Successfully merged!');
})
I was able to generate multiple PDF from multiple URLs from below code:
package.json
{
............
............
"dependencies": {
"puppeteer": "^1.1.1",
"easy-pdf-merge": "0.1.3"
}
..............
..............
}
index.js
const puppeteer = require('puppeteer');
const merge = require('easy-pdf-merge');
var pdfUrls = ["http://www.google.com","http://www.yahoo.com"];
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
var pdfFiles=[];
for(var i=0; i<pdfUrls.length; i++){
await page.goto(pdfUrls[i], {waitUntil: 'networkidle2'});
var pdfFileName = 'sample'+(i+1)+'.pdf';
pdfFiles.push(pdfFileName);
await page.pdf({path: pdfFileName, format: 'A4'});
}
await browser.close();
await mergeMultiplePDF(pdfFiles);
})();
const mergeMultiplePDF = (pdfFiles) => {
return new Promise((resolve, reject) => {
merge(pdfFiles,'samplefinal.pdf',function(err){
if(err){
console.log(err);
reject(err)
}
console.log('Success');
resolve()
});
});
};
RUN Command: node index.js
pdf-merger-js is another option. page.setContent should work just the same as a drop-in replacement for page.goto below:
const PDFMerger = require("pdf-merger-js"); // 3.4.0
const puppeteer = require("puppeteer"); // 14.1.1
const urls = [
"https://news.ycombinator.com",
"https://en.wikipedia.org",
"https://www.example.com",
// ...
];
const filename = "merged.pdf";
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
const merger = new PDFMerger();
for (const url of urls) {
await page.goto(url);
merger.add(await page.pdf());
}
await merger.save(filename);
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;