Perl rest client declaration causes failure of user agent call with custom headers with another end point - rest

I have 2 subroutines called in a single perl program .
First one (get_secrets) I am using the perl REST client directly with custom header and second one (app_restart) I am using LWP user agent and make and HTTP call .
my second subroutine fails when the $client header declaration is available in the first subroutine , as soon as i remove the that subroutine or comment the lines those lines app_restart subroutine works fine .
use REST::Client;
use Data::Dumper;
use JSON; #use strict;
use MIME::Base64 qw( decode_base64 );
use POSIX 'strftime';
use Date::Parse;
use DateTime;
use Date::Calc qw(:all);
use LWP::UserAgent;
#use IO::Socket::SSL 'debug4';
use Data::Dumper qw(Dumper);
use Getopt::Long;
sub toList {
my $data = shift;
my $key = shift;
if ( ref( $data->{$key} ) eq 'ARRAY' ) {
$data->{$key};
}
elsif ( ref( $data->{$key} ) eq 'HASH' ) {
[ $data->{$key} ];
}
else {
[];
}
}
$ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = 0;
$endpoint = $ENV{'ENDPOINT'};
$token = `cat /var/run/secrets/kubernetes.io/serviceaccount/token`;
$namespace = $ENV{'NAMESPACE'};
$apikey = $ENV{'APIKEY'};
$instance = $ENV{'INSTANCE'};
$appid = $ENV{'APPID'};
$storeid = "checker";
sub get_secrets {
my $client = REST::Client->new();
$client->setHost("https://${endpoint}");
#$client->addHeader('Authorization', "Bearer ${token}");
$client->addHeader( 'Accept', "application/json" );
$client->GET("/api/v1/namespaces/${namespace}/secrets?labelSelector=true");
}
get_secrets();
$ENV{PERL_NET_HTTPS_SSL_SOCKET_CLASS} = "Net::SSL";
my $ua = LWP::UserAgent->new( 'send_te' => '0' );
$ua->ssl_opts(
SSL_verify_mode => IO::Socket::SSL::SSL_VERIFY_NONE,
SSL_hostname => '',
verify_hostname => 0
);
sub app_restart {
$ENV{PERL_NET_HTTPS_SSL_SOCKET_CLASS} = "Net::SSL";
$ua = LWP::UserAgent->new( 'send_te' => '0' );
$ua->ssl_opts(
SSL_verify_mode => IO::Socket::SSL::SSL_VERIFY_NONE,
verify_hostname => 0
print "$instance\n";
print "$apikey\n";
print "$cstoreid\n";
print "$appid\n";
my $r = HTTP::Request->new(
'PUT' =>
"https://api.service.intranet.com/rest/application/$appid/instance/$instance/action?action=restart&config=$storeid&deploy=0",
[
'Accept' => '*/*',
'Authorization' => "Realm $apikey",
'Host' => 'api.service.intranet.com:443',
'User-Agent' => 'curl/7.55.1',
],
);
my $res = $ua->request( $r, );
#$response = $res->decoded_content;
$json = JSON->new->allow_nonref;
$response_decoded = $json->decode( $res->decoded_content );
$actionID = $response_decoded->{'action_id'};
print "$actionID\n";
}
app_restart();

Related

connect to google spreadsheet

i have a web app created using perl, and i have a table on a spreadsheet document, i want to connect my app to the spreadsheet, i tried the documentation but i can't get the token
here is what i did:
sub authenticate {
my $oauth2 = Net::Google::DataAPI::Auth::OAuth2->new(
client_id => 'my client id',
client_secret => 'my secret code',
scope => ['http://spreadsheets.google.com/feeds/'],
);
my $url = $oauth2->authorize_url(access_type => 'offline', approval_prompt => 'force');
use Data::Dumper;
Core::Global::Logger::debug(Dumper($url));
#you will need to put code here and receive token
print "OAuth URL, get code: \n$url\n";
use Term::Prompt;
my $code = prompt('x', 'my code', '', '');
my $token = $oauth2->get_access_token($code) or die;
#save token for future use
my $session = $token->session_freeze;
store( $session, 'google_spreadsheet.session' );
}
I was not able to make Net::Google::Spreadsheets::V4 work with a service account and using Net::Google::DataAPI::Auth::OAuth2 to get the access token. But you should be able to get the access token from WWW::Google::Cloud::Auth::ServiceAccount instead, and then use that token to authorize requests to the google drive rest api. The following worked for me:
use feature qw(say);
use strict;
use warnings;
use WWW::Google::Cloud::Auth::ServiceAccount;
use LWP::UserAgent;
use URI;
use HTTP::Request;
use Mojolicious::Lite;
use Data::Dumper;
use JSON;
use experimental qw(declared_refs refaliasing signatures);
{
my #scopes = ('https://www.googleapis.com/auth/spreadsheets',
'https://www.googleapis.com/auth/drive');
my $auth = WWW::Google::Cloud::Auth::ServiceAccount->new(
credentials_path => 'credentials.json',
scope => join ' ', #scopes
);
get '/' => sub {
my $c = shift;
my $token = $auth->get_token();
my ($return_code, $files) = get_all_spreadsheets($token);
$c->render(template => 'index', return_code => $return_code);
};
app->start;
}
sub get_all_spreadsheets($token) {
my $url = 'https://www.googleapis.com/drive/v3/files';
my $query = 'mimeType="application/vnd.google-apps.spreadsheet"';
my $params = {
"q" => $query,
"pageSize" => 1000,
"supportsAllDrives" => 1,
"includeItemsFromAllDrives" => 1,
"fields" => "kind,nextPageToken,"
. "files(id,name,createdTime,modifiedTime)",
};
my $more_pages = 1;
my $page_token = '';
my $status_line;
my #files;
while ($more_pages) {
$params->{pageToken} = $page_token if $page_token;
my $result = send_google_drive_get_request($url, $params, $token);
$status_line = $result->status_line;
if (!$result->is_success) {
return $status_line;
}
my $hash = decode_json($result->content);
push #files, $hash->{files};
if (exists $hash->{nextPageToken}) {
$page_token = $hash->{nextPageToken};
}
else {
$more_pages = 0;
}
}
return $status_line, \#files;
}
sub send_google_drive_get_request( $url, $params, $token ) {
my $uri = URI->new( $url );
$uri->query_form($params);
my $str = $uri->as_string();
my #headers = get_headers($token);
my $req = HTTP::Request->new(
'GET',
$uri->as_string(),
\#headers,
);
my $ua = LWP::UserAgent->new();
my $res = $ua->request($req);
return $res;
}
sub get_headers($token) {
return 'Accept-Encoding' => 'gzip, deflate',
'Accept' => '*/*',
'Connection' => 'keep-alive',
"Authorization" => "Bearer $token";
}
__DATA__
## index.html.ep
<!DOCTYPE html>
<html>
<head><title>Testing access to google sheets...</title></head>
<body>
<h1>Return code = <%= $return_code %></h1>
</body>
</html>
Edit:
To get the value of a cell for a given sheet with a given id, you can use the following url: https://sheets.googleapis.com/v4/spreadsheets/%s/values/%s where the first %s is replaced by the id of the sheet and the second %s represents the cell range to extract. Here is an example:
use feature qw(say);
use strict;
use warnings;
use WWW::Google::Cloud::Auth::ServiceAccount;
use LWP::UserAgent;
use URI;
use URI::Encode;
use HTTP::Request;
use Mojolicious::Lite;
use Data::Dumper;
use JSON;
use experimental qw(declared_refs refaliasing signatures);
{
my #scopes = ('https://www.googleapis.com/auth/spreadsheets',
'https://www.googleapis.com/auth/drive');
my $auth = WWW::Google::Cloud::Auth::ServiceAccount->new(
credentials_path => 'credentials.json',
scope => join " ", #scopes
);
get '/' => sub {
my $c = shift;
my $token = $auth->get_token();
my $sheet_id = '1FPyDuIPPzwUeLNpLbdI-RzfouKcm-2duOJ9Jio-Z-Qw';
my $sheet_cell = 'B1';
my ($return_code, $cell_value) =
read_spreadsheet_cell($token, $sheet_id, $sheet_cell);
app->log->debug(app->dumper( { cell_value => $cell_value } ));
$c->render(template => 'index', return_code => $return_code);
};
app->start;
}
sub read_spreadsheet_cell($token, $id, $cell) {
my $encoder = URI::Encode->new();
my $value_range = $encoder->encode(sprintf "'Sheet1'!%s", $cell);
my $url = sprintf 'https://sheets.googleapis.com/v4/spreadsheets/%s/values/%s',
$id, $value_range;
my $result = send_google_drive_get_request($url, $token);
my $status_line = $result->status_line;
if (!$result->is_success) {
return $status_line;
}
my $result_hash = decode_json( $result->content );
#app->log->debug(app->dumper( $result_hash ));
return $status_line, $result_hash->{values}[0][0];
}
sub send_google_drive_get_request( $url, $token ) {
my #headers = get_headers($token);
my $req = HTTP::Request->new('GET', $url, \#headers);
my $ua = LWP::UserAgent->new();
my $res = $ua->request($req);
return $res;
}
sub get_headers($token) {
return
'User-Agent' => 'Mozilla/8.0',
'Accept-Encoding' => 'gzip, deflate',
'Accept' => '*/*',
'Connection' => 'keep-alive',
"Authorization" => "Bearer $token";
}
__DATA__
## index.html.ep
<!DOCTYPE html>
<html>
<head><title>Testing access to google sheets...</title></head>
<body>
<h1>Return code = <%= $return_code %></h1>
</body>
</html>

mod_perl and CGI behavior

This has got to be something silly I'm doing wrong. It's such a newbie type problem.
The original script is something that sits and waits for a 3rd party to connect and POST some xml to it, it takes that xml, does some validation, and stores it in a db. That part is fine. The problem is my response. I'm trying to use the header() function from CGI and it's just not behaving. It comes up blank. Obviously I could just do this manually and just print the header string, but now I'm really curious why this is behaving so strangely.
Here is a stripped down test version of the cgi script:
use strict;
use warnings;
use Data::Dumper::Names;
use CGI qw(:standard);
use Apache2::Connection ();
use Apache2::RequestRec ();
$| = 1;
# Grab the request object provided by mod_perl.
our $request_obj = shift;
our $connection = $request_obj->connection;
our $remote_ip = $connection->client_ip();
my $cgi = CGI->new($request_obj->args());
print STDERR Dumper($cgi);
my $input = $cgi->param('POSTDATA');
print STDERR Dumper($input);
my $cgi_header = $cgi->header();
print STDERR Dumper($cgi_header);
my $cgi_full_header = $cgi->header(-type => 'application/xml');
print STDERR Dumper($cgi_full_header);
my $q = CGI->new({});
print STDERR Dumper($q);
my $q_header = $q->header();
print STDERR Dumper($q_header);
my $q_full_header = $q->header(-type => 'application/xml' );
print STDERR Dumper($q_full_header);
And the output:
$cgi = bless( {
'.r' => bless( do{\(my $o = '94118860562256')}, 'Apache2::RequestRec' ),
'param' => {
'POSTDATA' => [
'test'
],
'XForms:Model' => [
'test'
]
},
'use_tempfile' => 1,
'.fieldnames' => {},
'.charset' => 'ISO-8859-1',
'escape' => 1,
'.parameters' => [
'XForms:Model',
'POSTDATA'
]
}, 'CGI' );
$input = 'test';
$cgi_header = '';
$cgi_full_header = '';
$q = bless( {
'.parameters' => [
'XForms:Model',
'POSTDATA'
],
'escape' => 1,
'.fieldnames' => {},
'.charset' => 'ISO-8859-1',
'use_tempfile' => 1,
'.r' => bless( do{\(my $o = '94118860562256')}, 'Apache2::RequestRec' ),
'param' => {
'POSTDATA' => [
''
],
'XForms:Model' => [
''
]
}
}, 'CGI' );
$q_header = '';
$q_full_header = '';
And here is the simple test script I'm using to send the POST.
#!/perl/bin/perl
use strict;
use warnings;
use DBI;
use URI;
use LWP::UserAgent;
use Data::Dumper::Names;
my $ua = LWP::UserAgent->new;
$ua->max_size( 131072 );
$ua->agent('test_xml_pusher');
$ua->ssl_opts(verify_hostname => 0);
my $url = URI->new;
$url->scheme('https');
$url->host('xxxxxxxxxxxxxxxxxxxxxxxxx');
$url->port(443);
$url->path_segments('test.cgi');
# Yes, I know... it's not valid xml... don't care for the purposes of this test.
#
my $xml = 'test';
my $response = $ua->post( $url, Content => $xml, 'Content-Type' => 'application/xml' );
print Dumper($response);
my $status_line = $response->status_line;
print Dumper($status_line);
my $content = $response->content;
print Dumper($content);
So why is $cgi_header empty? And why does $q end up being a reference to the same thing as $cgi even though I tried initializing it as my $q = CGI->new({});? (I also tried empty quotes instead of empty brackets.)
Any thoughts?
Thanks!
My environment is a centos 7 server running apache httpd 2.4.34 with mod_perl 2.0.11 and perl 5.22.4. (httpd is installed from from SCL, but perl and mod_perl are installed from source.)
--
Andy

Sequentioal requests with Mojo::IOLoop::Delay

I need to get list of URLs in non-blocking mode, but not in parallel. It should be sequential requests one by one. How can I realize that?
I cannot find examples. Documentation and articles highlight only parallel execution.
Now my code looks like the following (simplified):
my $delay = Mojo::IOLoop::Delay->new;
$delay->steps(
sub {
build_report();
say "done";
}
);
sub parse_data {
...;
my $url = shift #urls;
my $end = $delay->begin;
$ua->get( $url => \&parse_data );
$end->();
}
my $end = $delay->begin;
$ua->get( $url => \&parse_data );
$end->();
$delay->wait;
I want to avoid multiple closures by using Mojo::IOLoop::Delay.
I have just started looking at Mojo::IOLoop myself for a project and I'm not au fait with it yet. But I think the simplest way is to build an array of closures and pass it to $delay->steps.
use strict;
use warnings 'all';
use feature 'say';
use Mojo;
use constant URL => 'http://stackoverflow.com/questions/tagged/perl';
STDOUT->autoflush;
my $ua = Mojo::UserAgent->new;
my $delay = Mojo::IOLoop::Delay->new;
my #urls = ( URL ) x 10;
my #steps = map {
my $url = $_;
sub {
my $end = $delay->begin;
$ua->get( $url => sub {
my ( $ua, $txn ) = #_;
$end->();
if ( my $err = $txn->error ) {
say $err->message;
}
else {
my $res = $txn->success;
say "#{$res}{qw/ code message /}";
}
});
}
} #urls;
$delay->steps( #steps, sub { say 'Done' } );
Mojo::IOLoop->start unless Mojo::IOLoop->is_running;
output
200 OK
200 OK
200 OK
200 OK
200 OK
200 OK
200 OK
200 OK
200 OK
Done
200 OK
Note that Done is printed before the status line of the final get, because I have called $end->() immediately the callback arrives, assuming that any handling of the response doesn't need to be synchronised
If you don't want that then just move $end->() to the end of the callback. Then the delay will wait until the output has been generated before sending another request
Sequential requests very easy to implement.
Web server
#!/usr/bin/perl
use Mojo::Base -strict;
use Mojolicious::Lite;
get '/' => sub {
my $c = shift->render_later;
$c->delay(sub {
my $delay = shift;
$c->ua->get('https://google.ru' => $delay->begin);
}, sub {
my ($delay, $tx) = #_;
$tx->res->body; # body of the google.ru
$c->ua->get('https://twitter.com' => $delay->begin);
}, sub {
my ($delay, $tx) = #_;
$tx->res->body; # body of the twitter.com
$c->render(text => 'Done');
});
};
app->start;
Script
#!/usr/bin/perl
use Mojo::Base -strict;
use Mojo::IOLoop;
use Mojo::UserAgent;
my $ua = Mojo::UserAgent->new;
Mojo::IOLoop->delay(sub {
my $delay = shift;
$ua->get('https://www.google.ru' => $delay->begin);
}, sub {
my ($delay, $tx) = #_;
$tx->res->body; # body of the google.ru
$ua->get('https://twitter.com' => $delay->begin);
}, sub {
my ($delay, $tx) = #_;
$tx->res->body; # body of the twitter.com
warn 'DONE';
})->wait;
Script (dynamic requests)
Example here

How can I send an incorrect Content-Length header for an HTTP request using Perl?

I'm trying to debug a weird warning that is showing up in server logs when a Plack::Request is being parsed. In some cases, a broken UserAgent will send a Content-Length header that looks something like "6375, 6375", which is obviously wrong.
To fix this properly, I need to be able to reproduce the warning. I'd like to include this in a unit test so that I can ensure there are no regressions after the warning is silenced. However, I'm having trouble doing this with Perl. I know this can be done using netcat and socat, but I don't want the unit test to have to rely on other binaries to be installed.
Here is what I've tried:
#!/usr/bin/env perl
use strict;
use warnings;
use JSON::XS qw( encode_json );
use WWW::Mechanize;
my $mech = WWW::Mechanize->new;
$mech->add_handler(
request_prepare => sub {
my ( $req, $ua, $h ) = #_;
$req->headers->header( 'Content-Length' => 9999 );
return;
}
);
my $json = encode_json( { foo => 'bar' } );
$mech->post(
'http://example.com'/url,
'Content-Length' => 999,
Content => $json
);
Output is:
Content-Length header value was wrong, fixed at /opt/perl5.16.3/lib/site_perl/5.16.3/LWP/Protocol/http.pm line 260.
200
That's entirely too helpful for me. :)
If I use HTTP::Request and LWP::UserAgent, it's the same end result.
So, I tried HTTP::Tiny.
#!/usr/bin/env perl
use strict;
use warnings;
use DDP;
use HTTP::Tiny;
use JSON::XS qw( encode_json );
my $http = HTTP::Tiny->new;
my $json = encode_json( { foo => 'bar' } );
my $response = $http->request(
'POST',
'http://example.com'/url',
{ headers => { 'Content-Length' => 999, },
content => $json,
}
);
p $response;
The output is:
{ content => "Content-Length missmatch (got: 13 expected: 999)
",
headers => {
content
-length => 49,
content-type => "text/plain",
},
reason => "Internal Exception",
status => 599,
success => "",
url => "http://example.com'/url",
}
Again, too helpful. At this point, I could use a few suggestions.
Seems like the higher level API's are fixing your error; Here's an example using raw sockets that overcomes this;
#!/usr/bin/env perl
use strict 'vars';
use warnings;
use Socket;
# initialize host and port
my $host = 'www.example.com';
my $port = 80;
# contact the server
open_tcp(F, $host, $port)
or die 'Could not connect to server';
# Send request data
while ( my $request = <DATA> ) {
print F $request;
}
# Get Response
while ( my $response = <F> ) {
print "Response:> $response";
}
close(F);
# TCP Helper
sub open_tcp
{
# get parameters
my ($FS, $dest, $port) = #_;
my $proto = getprotobyname('tcp');
socket($FS, PF_INET, SOCK_STREAM, $proto);
my $sin = sockaddr_in($port,inet_aton($dest));
connect($FS,$sin);
my $old_fh = select($FS);
$| = 1; # don't buffer output
select($old_fh);
}
__DATA__
GET / HTTP/1.1
Host: example.com
Content-Length: 999
-END-

how to get session id from cookie jar in perl?

My question is very simple.. It is how to get session id from cookie jar ... I have tried below code :-
use warnings;
use HTTP::Cookies;
use HTTP::Request::Common;
use LWP::UserAgent;
$ua = new LWP::UserAgent;
if ( !$ua ) {
print "Can not get the page :UserAgent fialed \n";
return 0;
}
my $cookies = new HTTP::Cookies( file => './cookies.dat', autosave => 1 );
$ua->cookie_jar($cookies);
# push does all magic to exrtact cookies and add to header for further reqs. useragent should be newer
push #{ $ua->requests_redirectable }, 'POST';
$result = $ua->request(
POST "URL",
{ Username => 'admin',
Password => 'admin',
Submit => 'Submit',
}
);
my $session_id = $cookies->extract_cookies($result);
print $session_id->content;
print "\n\n";
$resp = $result->content;
#print "Result is \n\n\n $resp \n";
$anotherURI = URL;
$requestObject = HTTP::Request::Common::GET $anotherURI;
$result = $ua->request($requestObject);
$resp = $result->content;
#print $resp."\n";
I am not getting where the session id is stored and how to fetch it ?
Note:- URL contains the URL of the page.
I wrote HTTP::CookieMonster to make this kind of thing a bit easier. If you don't know which cookie you're looking for, you can do something like this:
use strict;
use warnings;
use HTTP::CookieMonster;
use WWW::Mechanize;
my $mech = WWW::Mechanize->new;
my $monster = HTTP::CookieMonster->new( $mech->cookie_jar );
my $url = 'http://www.nytimes.com';
$mech->get( $url );
my #all_cookies = $monster->all_cookies;
foreach my $cookie ( #all_cookies ) {
printf( "key: %s value: %s\n", $cookie->key, $cookie->val);
}
If you already know the cookie's key, you can something like:
my $cookie = $monster->get_cookie('RMID');
my $session_id = $cookie->val;
Have a look at HTTP::Cookies->scan.
Something like this should do the trick (should add a constraint on the domain at least):
my $session_id;
$cookie_jar->scan(
sub {
my ($key, $val, $path, $domain, $port,
$path_spec, $secure, $expires, $discard, $hash
) = #_;
if ( $key eq "session_id" ) {
$session_id = $val;
}
}
);