axios - requests pool implementation - axios

I'm looking for a way to create a pool of HTTP requests and have a timer that will send them every x seconds.
I'm using axios as my HTTP client and wonder if there is a hook in axios to handle a request before it being sent, save it in this pool and resolve immediately, then my timer will do the actual request asynchronously.
I imagine to have some ability like that like that:
const pool = [];
setInterval(() => {
const promises = [];
while(pool.length){
const { url, data, config } = pool.pop();
delete config.usePool;
// This request will be send to the server
promises.push(axios.post(url, data, config);
}
await Promise.all(promises);
}, 5000);
// this `axios.hook` not really exists, im looking for a way to implement it (this is my actual question)
axios.hook = (url, data, config) => {
if(config?.usePool){
pool.push({ url, data, config });
}
};
// This request won't send but save in the pool
const res = axios.post('/my/url', { a: 1, b: 1 }, { usePool: true });

For a request pool of size 8, you can use the following implementation:
const axios = require('axios');
const { Agent } = require('https');
export default axios.create({
httpsAgent: new Agent({
maxSockets: 8
})
});
You can find more configuration options for the https agent here:
https://nodejs.org/api/http.html#new-agentoptions
There isn't an option available to space out the requests in time as you desired, although the usefulness of that approach is debatable.

Related

How to update React-Query data after socket signal?

I have app using React-Query, and I need to trigger data update after socket signal. That is, if I am on the user's page and I received a notification that the user's data has changed, then I need to send a request to receive updated data and write it. How to listen to socket events using useQuery hook?
You wouldn't do that with useQuery, but you would setup a socket subscription separately and then either write to the cache when you receive a message, or just invalidate the query to trigger a refetch.
I have a full blog post on this topic:
https://tkdodo.eu/blog/using-web-sockets-with-react-query
But the gist is:
const useReactQuerySubscription = () => {
const queryClient = useQueryClient()
React.useEffect(() => {
const websocket = new WebSocket('wss://echo.websocket.org/')
websocket.onopen = () => {
console.log('connected')
}
websocket.onmessage = (event) => {
const data = JSON.parse(event.data)
const queryKey = [...data.entity, data.id].filter(Boolean)
queryClient.invalidateQueries(queryKey)
}
return () => {
websocket.close()
}
}, [queryClient])
}

Send message and leave server on ready event if not whitelisted (Discord.JS + MongoDB)

I'm coding a whitelist system for my discord bot that, on ready event (and after a 3 seconds delay), checks if every server it is in has it's ID added to the whitelist database on MongoDB. If not, the bot sends an embed and leaves the server. I managed to get it working on the guildCreate event, but on ready event it performs the message and leave actions on every single server without filtering conditions, even though those are added to the list. I cannot figure out why. Also, I'm still new to JavaScript, so it could be just a minor mistake.
//VARIABLES
const { Client, MessageEmbed } = require("discord.js")
const config = require('../../Files/Configuration/config.json');
const DB = require("../../Schemas/WhitelistDB");
//READY EVENT
module.exports = {
name: "ready",
once: false,
async execute(client) {
//[ ... ] <--- OTHER UNNECESSARY CODE IN BETWEEN
setTimeout(function() { // <--- 3 SECONDS DELAY
client.guilds.cache.forEach(async (guild) => { // <--- CHECK EVERY SERVER
await DB.find({}).then(whitelistServers => { // <--- CHECK MONGODB ID LIST
if(!whitelistServers.includes(guild.id)) {
const channel = guild.channels.cache.filter(c => c.type === 'GUILD_TEXT').random(1)[0]; // <--- SEND MESSAGE TO RANDOM TEXT CHANNEL (It is sending to every server, when it should be sending only to the not whitelisted ones)
if(channel) {
const WhitelistEmbed = new MessageEmbed()
WhitelistEmbed.setColor(config.colors.RED)
WhitelistEmbed.setDescription(`${config.symbols.ERROR} ${config.messages.SERVER_NOT_WHITELISTED}`)
channel.send({embeds: [WhitelistEmbed]});
}
client.guilds.cache.get(guild.id).leave(); // <--- LEAVE SERVER (It is leaving every server, when it should be leaving only the not whitelisted ones)
} else { return }
});
});
}, 1000 * 3);
}
}
I found the solution myself!
Instead of finding the array of whitelisted ID's for each guild, find one at a time and instead of checking the content of the array, check if the array exists. This is the updated code:
//WHITELIST
setTimeout(function() {
client.guilds.cache.forEach(async (guild) => {
await DB.findOne({ GuildID: guild.id }).then(whitelistServers => {
if(!whitelistServers) {
const channel = guild.channels.cache.filter(c => c.type === 'GUILD_TEXT').random(1)[0];
if(channel) {
const WhitelistEmbed = new MessageEmbed()
WhitelistEmbed.setColor(config.colors.RED)
WhitelistEmbed.setDescription(`${config.symbols.ERROR} ${config.messages.SERVER_NOT_WHITELISTED}`)
channel.send({embeds: [WhitelistEmbed]});
}
client.guilds.cache.get(guild.id).leave();
} else { return }
});
});
}, 1000 * 3);

Google Action Webhook Inline Editor Returns Before the API call

This is my first Google Action project. I have a simple slot after the invocation. User enters the value on prompt and slot invokes the webhook and make a call to API using the user input. All works fine. However the webhook returns to users even before the API call finish processing and returns the value (line 1 conv.add). I do see in the logs that everything from API is logged fine after the webhook returns to user. Below is the code I am using. I am using inline editor. What am I missing? Thanks for help in advance.
const { conversation } = require('#assistant/conversation');
const functions = require('firebase-functions');
var https = require('https');
const fetch = require('node-fetch');
const app = conversation({debug: true});
app.handle('SearchData', conv => {
const body = JSON.stringify({
val: "this is my body"
});
// prepare the header
var postheaders = {
'Content-Type' : 'application/json',
'Auth' : 'MyAuthCreds'
};
fetch('https://host.domain.com/data', {
method: 'post',
body: body,
headers: postheaders,
})
.then(res => res.json())
.then(d => {
console.log(d);
var profile = d;//JSON.parse(d);
console.log(d.entries);
console.log("Length: "+ d.entries.length);
if(d.entries.length > 0)
{
console.log("Data found");
conv.add("Data found"); //line 1
}
else
{
console.log("no data found");
conv.add("no data found"); //line 1
}
})
.catch(function (err) {
// POST failed...
console.log(err);
});
});
exports.ActionsOnGoogleFulfillment = functions.https.onRequest(app);
Your issue is that your handler is making API calls which are asynchronous, but the Assistant Conversation library doesn't know that you're doing so. So as soon as the handler finishes, it tries to send back a response, but your asynchronous responses (the stuff in the then() blocks) haven't executed yet.
To address this, you need to return a Promise object so the library knows to wait till the Promise is fulfilled before it returns.
Fortunately, in your case, this should be pretty straightforward. fetch and all the .then() blocks return a Promise. So all you need to do is add a return statement in front of the call to fetch. So something like this:
return fetch('https://host.domain.com/data', {

How to trigger Google Composer Airflow dag using appscript?

I want to trigger a Google Composer airflow dag using Appscript. Is there any way to do it via rest API or another way.
If it is possible please suggest the solution.
Airflow has an endpoint that allows to trigger a DAG through its REST API, however it’s not possible to access it directly, since within the Cloud Composer architecture, the Airflow web server is located under an App Engine flexible environment. By default, the Airflow web server is integrated with Identity-Aware Proxy (IAP) and authentication is required.
Based on that, I found an example in the Cloud Composer documentation, that guides you to trigger a DAG using Cloud Functions, although the code is in JavaScript I don’t think it’s possible to execute it by Google App Script.
On the other hand, a workaround is to follow the Triggering DAGs guide changing some settings as follows.
In the creation of the function instead of setting the trigger type as Cloud Storage set it as HTTP, and check the “Allow unauthenticated invocations” for test purpose. An URL will be displayed, the goal is that every time that URL is accessed the DAG is executed.
Modify the first part of the index.js file, since no data would be passed as parameters and also the makeIapPostRequest function to return the response of the API call.
exports.triggerDag = async (req, res) => { // Modification
// Fill in your Composer environment information here.
// The project that holds your function
const PROJECT_ID = 'your-project-id';
// Navigate to your webserver's login page and get this from the URL
const CLIENT_ID = 'your-iap-client-id';
// This should be part of your webserver's URL:
// {tenant-project-id}.appspot.com
const WEBSERVER_ID = 'your-tenant-project-id';
// The name of the DAG you wish to trigger
const DAG_NAME = 'composer_sample_trigger_response_dag';
// Other constants
const WEBSERVER_URL = `https://${WEBSERVER_ID}.appspot.com/api/experimental/dags/${DAG_NAME}/dag_runs`;
const USER_AGENT = 'gcf-event-trigger';
const BODY = {conf: ‘’}; // Modification
// Make the request
try {
const iap = await authorizeIap(CLIENT_ID, PROJECT_ID, USER_AGENT);
const apiReponse = await makeIapPostRequest(WEBSERVER_URL, BODY, iap.idToken, USER_AGENT); // Modification
res.status(200).send('DAG_running!'); // Modification
} catch (err) {
console.error('Error authorizing IAP:', err.message);
throw new Error(err);
}
};
const makeIapPostRequest = async (url, body, idToken, userAgent) => {
const res = await fetch(url, {
method: 'POST',
headers: {
'User-Agent': userAgent,
Authorization: `Bearer ${idToken}`,
},
body: JSON.stringify(body),
});
if (!res.ok) {
const err = await res.text();
console.error('Error making IAP post request:', err.message);
throw new Error(err);
}
return {
apiRes: res.ok, // Modification
};
};
At this point, anything else has to be changed, so in your Script file execute the next instructions in order to trigger the DAG.
function myFunction() {
var response = UrlFetchApp.fetch("Cloud-function-URL");
Logger.log(response.getAllHeaders());
}
Finally, verify in the Airflow web interface if the DAG was triggered.

What is correct way to respond from webhook running nodejs?

Trying to implement web-hook (with V2 dialogflow) running nodejs. Received response "MalformedResponse 'final_response' must be set.". Below is the code. To the end of POST (app.post) code block was expecting conv.close would send SimpleResponse. But that's not happening. Need help understand why this error is seen and probable direction to solve it.
Thanks
const express = require('express');
const {
dialogflow,
Image,
SimpleResponse,
} = require('actions-on-google')
const bodyParser = require('body-parser');
const request = require('request');
const https = require("https");
const app = express();
const Map = require('es6-map');
// Pretty JSON output for logs
const prettyjson = require('prettyjson');
const toSentence = require('underscore.string/toSentence');
app.use(bodyParser.json({type: 'application/json'}));
// http://expressjs.com/en/starter/static-files.html
app.use(express.static('public'));
// http://expressjs.com/en/starter/basic-routing.html
app.get("/", function (request, response) {
console.log("Received GET request..!!");
//response.sendFile(__dirname + '/views/index.html');
response.end("Response from my server..!!");
});
// Handle webhook requests
app.post('/', function(req, res, next) {
console.log("Received POST request..!!");
// Log the request headers and body, to aide in debugging. You'll be able to view the
// webhook requests coming from API.AI by clicking the Logs button the sidebar.
console.log('======Req HEADERS================================================');
logObject('Request headers: ', req.headers);
console.log('======Req BODY================================================');
logObject('Request body: ', req.body);
console.log('======Req END================================================');
// Instantiate a new API.AI assistant object.
const assistant = dialogflow({request: req, response: res});
// Declare constants for your action and parameter names
//const PRICE_ACTION = 'price'; // The action name from the API.AI intent
const PRICE_ACTION = 'revenue'; // The action name from the API.AI intent
var price = 0.0
// Create functions to handle intents here
function getPrice(assistant) {
console.log('** Handling action: ' + PRICE_ACTION);
let requestURL = 'https://blockchain.info/q/24hrprice';
request(requestURL, function(error, response) {
if(error) {
console.log("got an error: " + error);
next(error);
} else {
price = response.body;
logObject('the current bitcoin price: ' , price);
// Respond to the user with the current temperature.
//assistant.tell("The demo price is " + price);
}
});
}
getPrice(assistant);
var reponseText = 'The demo price is ' + price;
// Leave conversation with SimpleResponse
assistant.intent(PRICE_ACTION, conv => {
conv.close(new SimpleResponse({
speech: responseText,
displayText: responseText,
}));
});
}); //End of app.post
// Handle errors.
app.use(function (err, req, res, next) {
console.error(err.stack);
res.status(500).send('Oppss... could not check the price');
})
// Pretty print objects for logging.
function logObject(message, object, options) {
console.log(message);
console.log(prettyjson.render(object, options));
}
// Listen for requests.
let server = app.listen(process.env.PORT || 3000, function () {
console.log('Your app is listening on ' + JSON.stringify(server.address()));
});
In general, The "final_response" must be set error is because you didn't send anything back. You have a lot going on in your code, and while you're on the right track, there are a few things in the code that could be causing this error.
First - in the code, it looks like you are confused about how to send a response. You have both a call to conv.close() and the commented out assistant.tell(). The conv.close() or conv.ask() methods are the way to send a reply using this version of the library. The tell() method was used by a previous version and is no longer supported.
Next, your code looks like it is only setting up the assistant object when the routing function is called. While this can be done, it is not the usual way to do it. Typically you'll create the assistant object and setup the Intent handlers (using assistant.intent()) as part of the program initialization. This is a rough equivalent to setting up the express app and the routes for it before the request itself comes in.
The portion that sets up the Assistant and then hooks it into a route might look something like this:
const assistant = dialogflow();
app.post('/', assistant);
If you really wanted to examine the request and response objects first, you might do this as something like
const assistant = dialogflow();
app.post('/', function( req, res ){
console.log(JSON.stringify(req.body,null,1));
assistant( req, res );
});
Related to this appears to be that you're trying to execute code in the route handler and then trying to call the intent handler. Again, this might be possible, but isn't the suggested way to use the library. (And I haven't tried to debug your code to see if there are problems in how you're doing it to see if you're doing it validly.) More typical would be to call getPrice() from inside the Intent handler instead of trying to call it from inside the route handler.
But this leads to another problem. The getPrice() function calls request(), which is an asynchronous call. Async calls are one of the biggest problems that causes an empty response. If you are using an async call, you must return a Promise. The easiest way to use a Promise with request() is to use the request-promise-native package instead.
So that block of code might look something (very roughly) like this:
const rp = require('request-promise-native');
function getPrice(){
return rp.get(url)
.then( body => {
// In this case, the body is the value we want, so we'll just return it.
// But normally we have to get some part of the body returned
return body;
});
}
assistant.intent(PRICE_ACTION, conv => {
return getPrice()
.then( price => {
let msg = `The price is ${price}`;
conv.close( new SimpleResponse({
speech: msg,
displayText: msg
});
});
});
The important thing to note about both getPrice() and the intent handler are that they both return a Promise.
Finally, there are some odd aspects in your code. Lines such as res.status(500).send('Oppss... could not check the price'); probably won't do what you think they will do. It won't, for example, send a message to be spoken. Instead, the Assistant will just close the connection and say that something went wrong.
Many thanks to #Prisoner. Below is the V2 working solution based on above comments. Same has been verified on nodejs webhook (without firebase). V1 version of the code was referenced from https://glitch.com/~aog-template-1
Happy coding..!!
// init project pkgs
const express = require('express');
const rp = require('request-promise-native');
const {
dialogflow,
Image,
SimpleResponse,
} = require('actions-on-google')
const bodyParser = require('body-parser');
const request = require('request');
const app = express().use(bodyParser.json());
// Instantiate a new API.AI assistant object.
const assistant = dialogflow();
// Handle webhook requests
app.post('/', function(req, res, next) {
console.log("Received POST request..!!");
console.log('======Req HEADERS============================================');
console.log('Request headers: ', req.headers);
console.log('======Req BODY===============================================');
console.log('Request body: ', req.body);
console.log('======Req END================================================');
assistant(req, res);
});
// Declare constants for your action and parameter names
const PRICE_ACTION = 'revenue'; // The action name from the API.AI intent
var price = 0.0
// Invoke http request to obtain blockchain price
function getPrice(){
console.log('getPrice is invoked');
var url = 'https://blockchain.info/q/24hrprice';
return rp.get(url)
.then( body => {
// In this case, the body is the value we want, so we'll just return it.
// But normally we have to get some part of the body returned
console.log('The demo price is ' + body);
return body;
});
}
// Handle AoG assistant intent
assistant.intent(PRICE_ACTION, conv => {
console.log('intent is triggered');
return getPrice()
.then(price => {
let msg = 'The demo price is ' + price;
conv.close( new SimpleResponse({
speech: msg,
}));
});
});
// Listen for requests.
let server = app.listen(process.env.PORT || 3000, function () {
console.log('Your app is listening on ' + JSON.stringify(server.address()));
});