Using cookie authentication with bitbucket and curl - perl

I'm trying to login with curl to Bitbucket server and use the created cookie to make another request, but somehow it is not working.
#!/usr/bin/perl
use strict;
use Data::Dumper;
my $user = 'user';
my $password = 'pass';
my $base_url = 'https://bitbucket.company.com/bitbucket';
my $project = 'PROJ';
my $repository = 'REPO';
my $login = `curl -s -u $user:$password --cookie-jar \"cookie.txt\" -H \"Content-Type: application/json\" \"$base_url/rest/api/1.0/projects\"`;
print $login;
my $url = $base_url.'/projects/'.$project.'/repos/'.$repository.'/settings/pull-requests';
my $pr_page = `curl -s --cookie \"cookie.txt\" -H \"Content-Type: application/json\" -H \"X-Atlassian-Token: no-check\" \"$url\"";
print $pr_page;
The login succeeds and I do get a cookie, it states:
Netscape HTTP Cookie File
http://curl.haxx.se/docs/http-cookies.html
This file was generated by libcurl! Edit at your own risk.
HttpOnly_bitbucket.mycompany.com FALSE /bitbucket/ TRUE 0 JSESSIONID 8079B4AC59C823137D7A78E4414C7CB3
But the script does not return the second page. I think it has something to do with not having the remember-me-cookie, but I can't generate it.
Thanks,
Rudy

or you could skip the login part and just steal the session from your UI browser and save it to a cookies file and passed that one to the curl obj conf calls
you could easily "steal" your current ui session in Chrome with cookies export chrome extension
For the php junkies out there - the syntax is almost the same ...
#
# performs post or get http request , returns
# usage:
# ( $ret , $response_code , $response_body , $response_content )
# = $objUrlRunner->doRunURL( 'GET' , $url , $headers );
#
sub doRunURL {
my $self = shift ;
my $http_method_type = shift ;
my $url = shift ;
my $headers = shift ;
my $cookies_file = $appConfig->{'COOKIES_FILE'} ;
$objLogger->doLogInfoMsg ( "cookies_file: " . $cookies_file ) ;
my $curl = WWW::Curl::Easy->new();
## Set up the standard GET/POST request options
$curl->setopt(WWW::Curl::Easy::CURLOPT_COOKIEFILE, $cookies_file ); # set where the cookies are stored
$curl->setopt(WWW::Curl::Easy::CURLOPT_HEADER(),1);
$curl->setopt(WWW::Curl::Easy::CURLOPT_MAXREDIRS(),3);
$curl->setopt(WWW::Curl::Easy::CURLOPT_URL(), "$url" );
$curl->setopt(WWW::Curl::Easy::CURLOPT_VERBOSE, 0); # Disable verbosity
$curl->setopt(WWW::Curl::Easy::CURLOPT_HEADER, 1); # Don't include header in body
$curl->setopt(WWW::Curl::Easy::CURLOPT_NOPROGRESS, 1); # Disable internal progress meter
$curl->setopt(WWW::Curl::Easy::CURLOPT_FOLLOWLOCATION, 0); # Disable automatic location redirects
$curl->setopt(WWW::Curl::Easy::CURLOPT_FAILONERROR, 1); # Setting this to true fails on HTTP error
$curl->setopt(WWW::Curl::Easy::CURLOPT_SSL_VERIFYPEER, 0); # Ignore bad SSL
$curl->setopt(WWW::Curl::Easy::CURLOPT_SSL_VERIFYHOST, 0); # Ignore bad SSL
$curl->setopt(WWW::Curl::Easy::CURLOPT_NOSIGNAL, 1); # To make thread safe, disable signals
$curl->setopt(WWW::Curl::Easy::CURLOPT_ENCODING, 'gzip'); # Allow gzip compressed pages
if ( $headers ) {
for my $key ( sort ( keys %$headers )) {
my $header_name = $key ;
my $header_val = $headers->{ "$key" } ;
$curl->setopt(WWW::Curl::Easy::CURLOPT_HTTPHEADER() , [ $header_name . $header_val ] );
}
}
if ( $http_method_type eq 'POST' ) {
$curl->setopt(WWW::Curl::Easy::CURLOPT_POST(), 1);
}
# A filehandle, reference to a scalar or reference to a typeglob can be used here.
my $response_body = q{} ;
my $response_code = q{} ;
my $response_content = q{} ;
$curl->setopt(WWW::Curl::Easy::CURLOPT_WRITEDATA(),\$response_body);
# Starts the actual request
my $ret = $curl->perform;
if ($ret == 0) {
my $msg = "OK for the curl transfer for the url: $url " ;
$objLogger->doLogInfoMsg ( $msg ) ;
$response_code = $curl->getinfo(CURLINFO_HTTP_CODE);
$response_content = HTTP::Response->parse( "$response_body" ) ;
$response_content = $response_content->content;
} else {
my $msg = "An error happened: $ret ".$curl->strerror($ret)." ".$curl->errbuf."\n" ;
$objLogger->doLogErrorMsg ( $msg ) ;
# Error code, type of error, error message
}
return ( $ret , $response_code , $response_body , $response_content ) ;
}

Related

using Net::LDAPs with Net::LDAP::Control::Paged

I'm trying to use Net::LDAPs with Net::LDAP::CONTROL::PAGED to return many records via a privlidged bind, but so far I have failed, miserably. I've used this Net::LDAPs extensively in the past, but I've never been able to find any documentation suggesting that it is compatible with Net::LDAP:Control::Paged. Everything I find is related to Net::LDAP.
The error message I get is: Undefined subroutine &main::process_entry called at /usr/local/share/perl/5.20.2/Net/LDAP/Search.pm line 55, line 755
Here is my code:
sub Ldap636{
my ($filter) = $_[0];
my $USERNAME = 'username';
my $PASSWORD = 'password';
my $LDAP_SERVER = 'directory.domain.edu';
my $LDAP_SSL_PORT = '636';
my $LDAP_BASE = 'ou=people,dc=domain,dc=edu';
my $userDN = "uid=$USERNAME,ou=identities,ou=special,dc=domain,dc=edu";
my $ldap = Net::LDAPS->new($LDAP_SERVER, port => $LDAP_SSL_PORT) or die "Could not create LDAP object because:\n$!";
my $ldapMsg = $ldap->bind($userDN, password => $PASSWORD);
die $ldapMsg->error if $ldapMsg->is_error;
my $page = Net::LDAP::Control::Paged->new( size => 100 );
#args = (base => "$LDAP_BASE",
callback => \&process_entry,
filter => $filter,
control => [ $page ],
);
my $cookie;
while (1) {
my $result = $ldap->search(#args);
"LDAP error: server says ",$result->error,"\n" if $result->code;
foreach my $entry ($result->entries ) {
my $cn = $entry->get_value('cn');
my $desc = $entry->get_value('description');
print "$cn - $desc\n";
}
# Get cookie from paged control
my($resp) = $result->control( LDAP_CONTROL_PAGED ) or last;
$cookie = $resp->cookie or last;
$page->cookie($cookie);
}
$ldap->unbind;
}
The error message I get is: Undefined subroutine &main::process_entry
called at /usr/local/share/perl/5.20.2/Net/LDAP/Search.pm line 55,
line 755
You have written process_entry as a callback but you didn't write that subroutine. That's why you are getting the above error.

Why is the server returned the result for a different submit than selected by perl HTML::Form and LWP::UserAgent?

I want to process a number of files with http://2struc.cryst.bbk.ac.uk/twostruc; to automate this I wrote a perl script using perl's HTML::Form.
This server has a two step submit process: first, upload a file or enter an id; second, select the methods to be used and the output (by chosing one of five submits).
The first step works, but for the second step I seem to be unable to chose any submit button other than the first, even though my script output confirms that I selected the one I want (different from the first).
The two core parts of the code are below, the request function:
sub create_submit_request
{
my $form_arrayref = shift;
my $form_action = shift;
my $value_hashref = shift;
my $submit_name = shift;
my $submit_index = shift;
my $found_form = 0;
my $form;
foreach my $this_form( #$form_arrayref)
{
printf( "# Found form with action=%s\n", $this_form->action);
if( $this_form->action eq $form_action)
{
$found_form = 1;
$form = $this_form;
}
}
die( "# Error: No form with action $form_action") if( $found_form == 0);
my #inputs = $form->inputs;
my $inputs_string;
foreach my $input( #inputs)
{
my $input_name = defined( $input->name) ? $input->name : "<unnamed_input>";
my $input_value = defined( $input->value) ? $input->value : "";
$inputs_string .= $input_name.( length( $input_value) > 0 ? "=".$input_value : "")." (".$input->type."); ";
}
printf( "# Available input names: %s\n", $inputs_string);
printf( "# Filling in form data\n");
while( my( $key, $value) = each( %$value_hashref))
{
$form->value( $key, $value);
}
my #submit_buttons = $form->find_input( $submit_name, "submit", $submit_index); # 1-based counting for the index
die( "# Error: Can only handle a single submit, but found ".scalar( #submit_buttons)) if( scalar( #submit_buttons) != 1);
my %submit_hash = %{ $submit_buttons[ 0]};
# DEBUG
printf( "# Use submit: %s\n", Data::Dumper->Dump( [ \%submit_hash ]));
return $form->click( %submit_hash);
}
and the code using it:
my $request = HTTP::Request->new( GET => $url_server);
my $response = $useragent->request( $request);
# the first page contains the pdb id input and file upload inputs
my #forms = HTML::Form->parse( $response);
my %value_hash = ( "file" => $pdb_file);
# the submit buttons have no name, use undef; chose the first one (w/o javascript)
$request = create_submit_request( \#forms, $form_action1, \%value_hash, undef, 1);
printf( "# Submitting to server\n");
$response = $useragent->request( $request);
# the first page contains the pdb id input and file upload inputs
#forms = HTML::Form->parse( $response);
%value_hash =( "dsspcont" => "on", "stride" => "on");
# this form has 5 submit buttons; select the 5th
$request = create_submit_request( \#forms, $form_action2, \%value_hash, undef, 5);
printf( "# Submitting to server\n");
$response = $useragent->request( $request);
my $response_content = $response->content;
printf( "# Response content: %s\n", $response_content);
Even though the script prints
# Use submit: $VAR1 = {
'name' => 'function_sequenceStructureAlignment',
'onclick' => 'this.form.target=\'_blank\';return true;',
'type' => 'submit',
'value' => 'Sequence Structure Alignments',
'value_name' => ''
};
which is the 5th submit button in the second step, the response is equivalent to pressing the first submit button.
To test the server itself, the file 1UBI.pdb can be downloaded from http://www.rcsb.org/pdb/files/1UBI.pdb and uploaded to the server. The full script is at http://pastebin.com/bSJLvNfc and can be run with
perl 2struc.pl --pdb 1UBI.pdb
Why is the server returning a different output/submit that I seem to select in the script?
(It seems it's not dependend on cookies, because I can clear them after the first step, and still get the correct result for the second step in a web browser.)
You gave a hash as selector for click, which is wrong (see documentation how to specify the selector). But because you have already found the correct submit element you could simply call click directly on it:
--- orig.pl
+++ fixed.pl
## -87,7 +87,7 ##
# DEBUG
printf( "# Use submit: %s\n", Data::Dumper->Dump( [ \%submit_hash ]));
- return $form->click( %submit_hash);
+ return $submit_buttons[0]->click($form);
}
sub predict_pdb

Perl: How can i test for a URL ( https ) accepting GET requests using "login" parameter

I have a CGI server side script that accepts GET and POST, with login parameters.
I want to test it to make sure it is not vulnerable. So the plan is to use Perl LWP, and send login parameters in GET and POST, and compare the results. the interface has been changed, so that only in POST we can send user-name and password in session cookies ( not sure if that is a great idea ) , so how do i test it ? Here is what i have so far:
#!/usr/bin/perl
use LWP;
print "This is libwww-perl-$LWP::VERSION\n";
# Create a user agent object
use LWP::UserAgent;
my $ua = LWP::UserAgent->new;
$ua->agent("MyApp/0.1 ");
# Create a request
#my $req = HTTP::Request->new(POST => 'http://search.cpan.org/search');
#my $req = HTTP::Request->new(GET => 'https://qa.co.net:443/cgi-bin/n-cu.cgi');
my $req = HTTP::Request->new(GET => 'https://qa.co.net:443/cgi-bin/n-cu.cgi?mode=frameset&JScript=1&remote_user&login=foo&password=foo HTTP/1.1');
$req->content_type('application/x-www-form-urlencoded');
$req->content('query=libwww-perl&mode=dist');
# Pass request to the user agent and get a response back
my $res = $ua->request($req);
# Check the outcome of the response
if ($res->is_success) {
print $res->content;
#print $res->code;
#print $res->message;
}
else {
print $res->status_line, "\n";
}
This is not going to do it, since it does not have the session cookie stuff. But might be a good start though. Is this the right way to test the GET and POST ?
Here is what was implemented in the cgi:
#cr_login for POST && login for GET -- leave GET param as it used to be.
if ($m eq 'GET' && defined($req->param('login'))) {
$msg = 'parameter "login" is invalid for this request type.';
+ my $seclog = $event_logging_directory . '/invalid_request.log';
+ open(S, ">>$seclog") or die $!;
+ my $logmsg = sprintf("%4d-%02d-%02d %02d:%02d:%02d",Today_and_Now())
+ . "|mode:" . $req->param('mode')
+ . "|login:" . $req->param('login')
+ . "|remote_addr:" . $ENV{REMOTE_ADDR}
+ . "|$msg\n";
+ print S $logmsg;
and :
POST request to n-cu.cgi should use parameter "cr_login". If the parameter "login" is passed in a post request, it should throw error and return to login screen.
GET request to n-cu.cgi should use the parameter "login". If the parameter "cr_login" is passed in a post request, it should throw error and return to login screen.
so here is how we do it:
Keep the session cookie and context alive :
my $browser = LWP::UserAgent->new(keep_alive => 10);
$browser->cookie_jar( {} );
$browser->agent('Mozilla/8.0');
#$browser->ssl_opts({ verify_hostname => 0 });
$browser->show_progress(1);
and later: print the response
print "Cookies:\n", Dumper($browser->cookie_jar()), "\n\n";
my $content = $response->as_string;
print "$content\n";
Sending password in a cookie? Nope.
Disallow GET for /login.
POST username and password to /login, over SSL.
In CGI, the GET/POST is indicated via the REQUEST_METHOD environment variable.
You cannot stop determined people from issuing a GET request to your server, but you can refuse to process it like so (untested code - you have to fill in details):
if ($ENV{REQUEST_METHOD} ne 'POST') {
# issue a redirect to a suitable error page, then return.
}
my $q = CGI->new();
my $user = $q->params('username');
my $password = $q->params('password');
my $encrypted_password = my_password_encryptor($password);
unless ( can_log_in($user, $encrypted_password) ) {
# issue an error message - redirect&return or fall-through...
}
else {
$session->set_user_logged_in();
}
Most people do not roll their own authentication or session handling. They mostly use one from CPAN, or one included with the larger app framework. If you're doing CGI, you can use CGI::Session.
You might give CGI::Application and/or its offspring a look. Those authors have already solved a bunch of the problems that you're encountering.

www::curl - how to upload (post) large files

I use WWW::Curl to upload files:
use WWW::Curl::Easy 4.14;
use WWW::Curl::Form;
my $url = 'http://example.com/backups/?sid=12313qwed323';
my $params = {
name => 'upload',
action => 'keep',
backup1 => [ '/tmp/backup1.zip' ], # 1st file for upload
};
my $form = WWW::Curl::Form->new();
foreach my $k (keys %{$params}) {
if (ref $params->{$k}) {
$form->formaddfile(#{$params->{$k}}[0], $k, 'multipart/form-data');
} else {
$form->formadd($k, $params->{$k});
}
}
my $curl = WWW::Curl::Easy->new() or die $!;
$curl->setopt(CURLOPT_HTTPPOST, $form);
$curl->setopt(CURLOPT_URL, $url);
my $body;
$curl->setopt(CURLOPT_WRITEDATA, \$body);
my $retcode = $curl->perform();
my $response_code = $curl->getinfo(CURLINFO_HTTP_CODE);
nothing special here and this code works well.
I want to upload large files and I don't want to preload everything in the memory. At least that is what I heard that libcurl is doing.
CURLOPT_READFUNCTION accepts callbacks which returns parts of the content. That means that I cannot use WWW::Curl::Form to set POST parameters but that I have to return the whole content through this callback. Is that right?
I think that the code could look like this:
use WWW::Curl::Easy 4.14;
my $url = 'http://example.com/backups/?sid=12313qwed323'
my $params = {
name => 'upload',
action => 'keep',
backup1 => [ '/tmp/backup1.zip' ], # 1st file for upload
};
my $fields;
foreach my $k (keys %{$params}) {
$fields .= "$k=".(ref $params->{$k} ? '#'.#{$params->{$k}}[0] : uri_escape_utf8($params->{$k}))."&";
}
chop($fields);
my $curl = WWW::Curl::Easy->new() or die $!;
$curl->setopt(CURLOPT_POST, 1);
$curl->setopt(CURLOPT_POSTFIELDS, $fields); # is it needed with READFUNCTION??
$curl->setopt(CURLOPT_URL, $url);
my #header = ('Content-type: multipart/form-data', 'Transfer-Encoding: chunked');
$curl->setopt(CURLOPT_HTTPHEADER, \#header);
#$curl->setopt(CURLOPT_INFILESIZE, $size);
$curl->setopt(CURLOPT_READFUNCTION, sub {
# which data to return here?
# $params (without file) + file content?
return 0;
});
Which data does CURLOPT_READFUNCTION callback have to return? $params + File(s) content? In which format?
Do I really have to create the data (returned by CURLOPT_READFUNCTION) by myself or is there a simple way to create it in the right format?
Thanks
Test 16formpost.t is relevant. As you can see, it's completely disabled. This fact and my fruitless experiments with various return values for the callback function lets me believe the CURLOPT_READFUNCTION feature is known broken in the Perl binding.
I have to return the whole content through this callback. Is that right?
No, you can feed it the request body piecewise, suitable for chunked encoding. The callback will be necessarily called several times, according to the limit set in CURLOPT_INFILESIZE.
Which data does CURLOPT_READFUNCTION callback have to return?
A HTTP request body. Since you do a file upload, this means Content-Type multipart/form-data. Following is an example using HTTP::Message. CURLOPT_HTTPPOST is another way to construct this format.
use HTTP::Request::Common qw(POST);
use WWW::Curl::Easy 4.14;
my $curl = WWW::Curl::Easy->new or die $!;
$curl->setopt(CURLOPT_POST, 1);
$curl->setopt(CURLOPT_URL, 'http://localhost:5000');
$curl->setopt(CURLOPT_HTTPHEADER, [
'Content-type: multipart/form-data', 'Transfer-Encoding: chunked'
]);
$curl->setopt(CURLOPT_READFUNCTION, sub {
return POST(undef, Content_Type => 'multipart/form-data', Content => [
name => 'upload',
action => 'keep',
backup1 => [ '/tmp/backup1.zip' ], # 1st file for upload
])->content;
});
my $r = $curl->perform;
The CURLOPT_READFUNCTION callback is only used for chunked tranfer encoding. It may work, but I haven't been able to get it to and found that doing so wasn't necessary anyway.
My use case was for upload of data to AWS, where it's not ok to upload the data as multi-part form data. Instead, it's a straight POST of the data. It does require that you know how much data you're sending the server, though. This seems to work for me:
my $infile = 'file-to-upload.json';
my $size = -s $infile;
open( IN, $infile ) or die("Cannot open file - $infile. $! \n");
my $curl = WWW::Curl::Easy->new;
$curl->setopt(CURLOPT_HEADER, 1);
$curl->setopt(CURLOPT_NOPROGRESS, 1);
$curl->setopt(CURLOPT_POST, 1);
$curl->setopt(CURLOPT_URL, $myPostUrl);
$curl->setopt(CURLOPT_HTTPHEADER,
['Content-Type: application/json']); #For my use case
$curl->setopt(CURLOPT_POSTFIELDSIZE_LARGE, $size);
$curl->setopt(CURLOPT_READDATA, \*IN);
my $retcode = $curl->perform;
if ($retcode == 0) {
print("File upload success\n");
}
else {
print("An error happened: $retcode ".$curl->strerror($retcode)."\n");
}
The key is providing an open filehandle reference to CURLOPT_READDATA. After that, the core curl library handles the reads from it without any need for callbacks.

How to post non-latin1 data to non-UTF8 site using perl?

I want to post russian text on a CP1251 site using LWP::UserAgent and get following results:
# $text="Русский текст"; obtained from command line
FIELD_NAME => $text # result: Г?в г'В?г'В?г'В?г?вєг?вёг?в? Г'В'Г?вчг?вєг'В?г'В'
$text=Encode::decode_utf8($text);
FIELD_NAME => $text # result: Р с?с?с?рєрёр? С'Рчрєс?с'
FIELD_NAME => Encode::encode("cp1251", $text) # result: Г?гіг+г+гЄгёгЏ ГІгҐгЄг+гІ
FIELD_NAME => URI::Escape::uri_escape_utf8($text) # result: D0%a0%d1%83%d1%81%d1%81%d0%ba%d0%b8%d0%b9%20%d1%82%d0%b5%d0%ba%d1%81%d1%82
How can I do this? Content-Type must be x-www-form-urlencoded. You can find similar form here, but there you can just escape any non-latin character using &#...; form, trying to escape it in FIELD_NAME results in 10561091108910891 10901077108210891 (every &, # and ; stripped out of the string) or 1056;усский текст (punctuation characters at the beginning of the string are stripped out) depending on what the FIELD_NAME actually is.
UPDATE: Anybody knows how to convert the following code so that it will use LWP::UserAgent::post function?
my $url=shift;
my $fields=shift;
my $request=HTTP::Request->new(POST => absURL($url));
$request->content_type('application/x-www-form-urlencoded');
$request->content_encoding("UTF-8");
$ua->prepare_request($request);
my $content="";
for my $k (keys %$fields) {
$content.="&" if($content ne "");
my $c=$fields->{$k};
eval {$c=Encode::decode_utf8($c)};
$c=Encode::encode("cp1251", $c, Encode::FB_HTMLCREF);
$content.="$k=".URI::Escape::uri_escape($c);
}
$request->content($content);
my $response=$ua->simple_request($request);
This code actually solves the problem, but I do not want to add the third request wrapper function (alongside with get and post).
One way around it appears to be (far from the best, I think) to use recode system command if you have it avialable. From http://const.deribin.com/files/SignChanger.pl.txt
my $boardEncoding="cp1251"; # encoding used by the board
$vals{'Post'} = `fortune $forunePath | recode utf8..$boardEncoding`;
$res = $ua->post($formURL,\%vals);
Another approach seems to be in http://mail2lj.nichego.net/lj.txt
my $formdata = $1 ;
my $hr = ljcomment_string2form($formdata) ;
my $req = new HTTP::Request('POST' => $ljcomment_action)
or die "new HTTP::Request(): $!\n" ;
$hr->{usertype} = 'user' ;
$hr->{encoding} = $mh->mime_attr('content-type.charset') ||
"cp1251" ;
$hr->{subject} = decode_mimewords($mh->get('Subject'));
$hr->{body} = $me->bodyhandle->as_string() ;
$req->content_type('application/x-www-form-urlencoded');
$req->content(href2string($hr)) ;
my $ljres = submit_request($req, "comment") ;
if ($ljres->{'success'} eq "OK") {
print STDERR "journal updated successfully\n" ;
} else {
print STDERR "error updating journal: $ljres->{errmsg}\n" ;
send_bounce($ljres->{errmsg}, $me, $mh->mime_attr("content-type.charset")) ;
}
Use WWW::Mechanize, it takes care of encoding (both character encoding and form encoding) automatically and does the right thing if a form element's accept-charset attribute is set appropriately. If it's missing, the form defaults to UTF-8 and thus needs correction. You seem to be in this situation. By the way, your example site's encoding is KOI8-R, not Windows-1251. Working example:
use utf8;
use WWW::Mechanize qw();
my $message = 'Русский текст';
my $mech = WWW::Mechanize->new(
cookie_jar => {},
agent => 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.9 SUSE/6.0.401.0-2.1 (KHTML, like Gecko)',
);
$mech->get('http://zhurnal.lib.ru/cgi-bin/comment?COMMENT=/z/zyx/index_4-1');
$mech->current_form->accept_charset(scalar $mech->response->content_type_charset);
$mech->submit_form(with_fields => { TEXT => $message });
HTTP dump (essential parts only):
POST /cgi-bin/comment HTTP/1.1
Content-Length: 115
Content-Type: application/x-www-form-urlencoded
FILE=%2Fz%2Fzyx%2Findex_4-1&MSGID=&OPERATION=store_new&NAME=&EMAIL=&URL=&TEXT=%F2%D5%D3%D3%CB%C9%CA+%D4%C5%CB%D3%D
These functions solve the issue (first for posting application/x-www-form-urlencoded data and second for multipart/form-data):
#{{{2 postue
sub postue($$;$) {
my $url=shift;
my $fields=shift;
my $referer=shift;
if(defined $referer and $referer eq "" and defined $fields->{"DIR"}) {
$referer=absURL($url."?DIR=".$fields->{"DIR"}); }
else {
$referer=absURL($referer); }
my $request=HTTP::Request->new(POST => absURL($url));
$request->content_type('application/x-www-form-urlencoded');
$request->content_encoding("UTF-8");
$ua->prepare_request($request);
my $content="";
for my $k (keys %$fields) {
$content.="&" if($content ne "");
my $c=$fields->{$k};
if(not ref $c) {
$c=Encode::decode_utf8($c) unless Encode::is_utf8($c);
$c=Encode::encode("cp1251", $c, Encode::FB_HTMLCREF);
$c=URI::Escape::uri_escape($c);
}
elsif(ref $c eq "URI::URL") {
$c=$c->canonical();
$c=URI::Escape::uri_escape($c);
}
$content.="$k=$c";
}
$request->content($content);
$request->referer($referer) if(defined $referer);
my $i=0;
print STDERR "Doing POST request to url $url".
(($::o_verbose>2)?(" with fields:\n".
::YAML::dump($fields)):("\n"))
if($::o_verbose>1);
REQUEST:
my $response=$ua->simple_request($request);
$i++;
my $code=$response->code;
if($i<=$o_maxtries and 500<=$code and $code<600) {
print STDERR "Failed to request $url with code $code... retrying\n"
if($::o_verbose>2);
sleep $o_retryafter;
goto REQUEST;
}
return $response;
}
#{{{2 postfd
sub postfd($$;$) {
my $url=absURL(shift);
my $content=shift;
my $referer=shift;
$referer=absURL($referer) if(defined $referer);
my $i=0;
print STDERR "Doing POST request (form-data) to url $url".
(($::o_verbose>2)?(" with fields:\n".
::YAML::dump($content)):("\n"))
if($::o_verbose>1);
my $newcontent=[];
while(my ($f, $c)=splice #$content, 0, 2) {
if(not ref $c) {
$c=Encode::decode_utf8($c) unless Encode::is_utf8($c);
$c=Encode::encode("cp1251", $c, Encode::FB_HTMLCREF);
}
push #$newcontent, $f, $c;
}
POST:
my $response=$ua->post($url, $newcontent,
Content_type => "form-data",
((defined $referer)?(referer => $referer):()));
$i++;
my $code=$response->code;
if($i<=$o_maxtries and 500<=$code and $code<600) {
print STDERR "Failed to download $url with code $code... retrying\n"
if($::o_verbose>2);
sleep $o_retryafter;
goto POST;
}
return $response;
}