when I am using "--disk-cache=true" in phantomjs_arg then it's getting error In this line:
my $form = $self->{obj_mech}->form_number( 2 );
No elements found for form number 2 at modules/TestLogin.pm line 1129.
at /usr/local/share/perl/5.22.1/WWW/Mechanize/PhantomJS.pm line 796.
WWW::Mechanize::PhantomJS::signal_condition(WWW::Mechanize::PhantomJS=HASH(0x4cfa120),
"No elements found for form number 2") called at
/usr/local/share/perl/5.22.1/WWW/Mechanize/PhantomJS.pm line 1732
WWW::Mechanize::PhantomJS::xpath(WWW::Mechanize::PhantomJS=HASH(0x4cfa120),
"(//form)[2]", "user_info", "form number 2", "single", 1) called at
/usr/local/share/perl/5.22.1/WWW/Mechanize/PhantomJS.pm line 2102
WWW::Mechanize::PhantomJS::form_number(WWW::Mechanize::PhantomJS=HASH(0x4cfa120),
2) called at modules/TestLogin.pm line 1129
TestLogin::TestLogin_login(TestLogin=HASH(0x4f5c8a8)) called at collectBets.pl line 20 Debugged program terminated. Use q to quit
or R to restart, use o inhibit_exit to avoid stopping after program
termination, h q, h R or h o to get additional info.
without disk-cashe it's working fine.
This is my sample code for better understanding.
#!/usr/bin/perl
use strict;
use warnings;
use Helper;
use WWW::Mechanize::PhantomJS;
use DataBase;
use MyConfig;
use JSON;
use DateTime;
use HTML::Entities;
sub new($$) {
my ($class,$params) = #_;
my $self = $params || {};
bless $self, $class;
$self->{obj_mech} = WWW::Mechanize::PhantomJS -> new( phantomjs_arg => ['--ssl-protocol=any','--disk-cache=true','--max-disk-cache-size=1024'], ignore_ssl_errors => 1);
$self->{obj_helper} = new Helper();
#$self->{obj_db} = new DataBase();
$self->{logged_in} = 0;
#$self->setTorProxy();
#$self->init_market_master();
return $self;
}
Login();
print "\nlogin done...\n";
exit;
sub Login {
my ($self) = #_;
my $html = $self->{obj_mech}->get( "https://www.gmail.com/" );
sleep(25);
$html = $self->{obj_mech}->content;
$self->{obj_mech}->viewport_size({ width => 1366, height => 768 });
my $form = $self->{obj_mech}->form_number( 2 );
my $user_name = '*****';
my $password = '******';
$self->{obj_mech}->set_fields('InputEmail' =>$user_name);
$self->{obj_mech}->set_fields('InputPassword' =>$password);
$self->{obj_mech}->click({ xpath => '//button[#class="PrimaryButton"]' });
sleep(20);
my $test_html=$self->{obj_mech}->content;
$self->{obj_helper}->writeFileNew( "TestLoginPage.html" , $test_html );
my $png = $self->{obj_mech}->content_as_png();
$self->{obj_helper}->writeFileNew( "LoginPage.png" , $png );
return 1;
}
Well, before looking at the disk-cache arguments, I found that there are no such elements.
# There is only 1 form. If you want to keep this line,
# you need to change the form number to 1
my $form = $self->{obj_mech}->form_number( 2 );
# I didn't find input field named 'InputEmail'
# The actual field name is 'Email'
$self->{obj_mech}->set_fields('InputEmail' =>$user_name);
# You have to click 'Next' button firstly then the password
# input box is shown. And the field name should be 'Passwd'
$self->{obj_mech}->set_fields('InputPassword' =>$password);
# The xpath of 'Sign in' button is //input[#value="Sign in"]
$self->{obj_mech}->click({ xpath => '//button[#class="PrimaryButton"]' });
A simple working script either with disk cache or without disk cache:
#! /usr/bin/perl
use strict;
use warnings;
use WWW::Mechanize::PhantomJS;
use open ':std', ':encoding(UTF-8)';
#my $p = WWW::Mechanize::PhantomJS->new(phantomjs_arg=>['--ssl-protocol=any','--disk-cache=false','--max-disk-cache-size=1024'],ignore_ssl_errors=>1);
my $p = WWW::Mechanize::PhantomJS->new(phantomjs_arg=>['--ssl-protocol=any','--disk-cache=true','--max-disk-cache-size=1024'],ignore_ssl_errors=>1);
my $html = $p->get("https://www.gmail.com/");
sleep(5);
write_html('first-page.html', $p->content);
$p->viewport_size({width=>1366,height=>768});
my $form = $p->form_number(1);
my $user_name = '*****';
my $password = '*****';
$p->set_fields('Email'=>$user_name);
sleep(5);
$p->click({xpath=>'//input[#value="Next"]'});
sleep(5);
write_html('after-click-next.html', $p->content);
$p->set_fields('Passwd'=>$password);
sleep(5);
$p->click({xpath=>'//input[#value="Sign in"]'});
sleep(5);
write_html('after-login.html', $p->content);
sub write_html {
my ($file, $content) = #_;
open my $fh, '>', $file or die;
print $fh $content;
close $fh;
}
Related
I have a requirement to fetch many http urls and I use AnyEvent::HTTP to do this
For every URL I need to measure the time taken how can I do this ?
My code (stripped down) is here
#!/usr/bin/perl
use strict;
use AnyEvent::HTTP;
use AnyEvent::Socket;
use Data::Dumper;
my $internal_ip=v192.168.2.103; #Use this ip to bind instead of default ip. Harcoding necessary :-( using v$ip
sub prep_cb {
my ($socket)=#_;
my $bind = AnyEvent::Socket::pack_sockaddr undef, $internal_ip;
# I need to start the time here
bind $socket, $bind
or die "bind: $!";
}
my $url="http://192.168.2.105/echo.php";
my $anyevent = AnyEvent->condvar;
$anyevent->begin;
http_request(
"GET" => $url,
on_prepare =>\&prep_cb,
sub {
my ($data, $hdr) = #_;
$anyevent->end;
# I need to measure the time taken
print Dumper([$data,$hdr]);
}
);
$anyevent->recv;
What if you replace your http_request() with the following:
my $timer;
http_request(
"GET" => $url,
on_prepare => sub {$timer = time; prep_cb},
sub {
my ($data, $hdr) = #_;
$anyevent->end;
print "Took " . (time - $timer) . " seconds.\n";
print Dumper([$data,$hdr]);
}
);
Simpler way is to have a variable and update it on on_prepare and log it after $anyevent->end as mentioned by TheAmigo
A general way to profile/time any function:
Assuming your function is fetchHttpUrl($url),
you could call it like this
profile(\&fetchHttpUrl, $url);
sub profile {
my($function, #arguments) = #_;
my $startTime = currentTimeInMilliseconds();
$function->(#arguments);
my $durationInMs = currentTimeInMilliseconds() - $startTime;
print"{".getMethodNameFromPointer($function)."(".join(",", #arguments).")"."} : $durationInMs ms";
}
In this example:
$logger->debug({
filter => \&Data::Dumper::Dumper,
value => $ref
});
I can pretty print my references instead of ARRAY(0xFFDFKDJ). But it's too boring to type that long code every time. I just want:
$logger->preprocessor({
filter => \&Data::Dumper::Dumper,
value => $ref
});
$logger->debug( $ref, $ref2 );
$logger->info( $array );
And $ref, $ref2, and $array will be dumped by Data::Dumper.
It there a way to do this?
UPD
With help of your answers I do the patch
Now you just:
log4perl.appender.A1.layout=FallbackLayout
log4perl.appender.A1.layout.chain=PatternLayout
log4perl.appender.A1.layout.chain.ConversionPattern=%m%n
log4perl.appender.A1.warp_message = sub { $#_ = 2 if #_ > 3; \
return #_; }
# OR
log4perl.appender.A1.warp_message = main::warp_my_message
sub warp_my_message {
my( #chunks ) = #_;
use Data::Dump qw/ pp /;
for my $msg ( #chunks ) {
$msg = pp $msg if ref $msg;
}
return #chunks;
}
UPD2
Or you can use this small module
log4perl.appender.SomeAPP.warp_message = Preprocess::Messages::msg_filter
log4perl.appender.SomeAPP.layout = Preprocess::Messages
package Preprocess::Messages;
sub msg_filter {
my #chunks = #_;
for my $msg ( #chunks ) {
$msg = pp $msg if ref $msg;
}
return #chunks;
};
sub render {
my $self = shift;
my $layout = Log::Log4perl::Layout::PatternLayout->new(
'%d %P %p> %c %F:%L %M%n %m{indent=2}%n%n'
);
$_[-1] += 1; # increase level of the caller
return $layout->render( join $Log::Log4perl::JOIN_MSG_ARRAY_CHAR, #{ shift() }, #_ );
}
sub new {
my $class = shift;
$class = ref ($class) || $class;
return bless {}, $class;
}
1;
Yes, of course you can set 'warp_message = 0' and combine msg_filter and render together.
log4perl.appender.SomeAPP.warp_message = 0
log4perl.appender.SomeAPP.layout = Preprocess::Messages
sub render {
my($self, $message, $category, $priority, $caller_level) = #_;
my $layout = Log::Log4perl::Layout::PatternLayout->new(
'%d %P %p> %c %F:%L %M%n %m{indent=2}%n%n'
);
for my $item ( #{ $message } ) {
$item = pp $item if ref $item;
}
$message = join $Log::Log4perl::JOIN_MSG_ARRAY_CHAR, #$message;
return $layout->render( $message, $category, $priority, $caller_level+1 );
}
The easy way: use warp_message
The easiest way to do this is to create a custom appender and set the warp_message parameter so you can get the original references that were passed to the logger:
package DumpAppender;
use strict;
use warnings;
use Data::Dumper;
$Data::Dumper::Indent = 0;
$Data::Dumper::Terse = 1;
sub new {
bless {}, $_[0];
}
sub log {
my($self, %params) = #_;
print ref($_) ? Dumper($_) : $_ for #{ $params{message} };
print "\n";
}
package main;
use strict;
use warnings;
use Log::Log4perl;
Log::Log4perl->init(\q{
log4perl.rootLogger=DEBUG,Dump
log4perl.appender.Dump=DumpAppender
log4perl.appender.Dump.layout=NoopLayout
log4perl.appender.Dump.warp_message=0
});
my $logger = Log::Log4perl->get_logger;
$logger->debug(
'This is a string, but this is a reference: ',
{ foo => 'bar' },
);
Output:
This is a string, but this is a reference: {'foo' => 'bar'}
Unfortunately, if you take this approach, you're stuck writing your own code to handle layouts, open files, etc. I wouldn't take this approach except for very simple projects that only need to print to screen.
A better way: composite appender
A better approach is to write your own composite appender. A composite appender forwards messages on to another appender after manipulating them somehow, e.g. filtering or caching them. With this approach, you can write only the code for dumping the references and let an existing appender do the heavy lifting.
The following shows how to write a composite appender. Some of this is explained in the docs for Log::Log4perl::Appender, but I copied much of it from Mike Schilli's Log::Log4perl::Appender::Limit:
package DumpAppender;
use strict;
use warnings;
our #ISA = qw(Log::Log4perl::Appender);
use Data::Dumper;
$Data::Dumper::Indent = 0;
$Data::Dumper::Terse = 1;
sub new {
my ($class, %options) = #_;
my $self = {
appender => undef,
%options
};
# Pass back the appender to be limited as a dependency to the configuration
# file parser.
push #{ $options{l4p_depends_on} }, $self->{appender};
# Run our post_init method in the configurator after all appenders have been
# defined to make sure the appenders we're connecting to really exist.
push #{ $options{l4p_post_config_subs} }, sub { $self->post_init() };
bless $self, $class;
}
sub log {
my ($self, %params) = #_;
# Adjust call stack so messages are reported with the correct caller and
# file
local $Log::Log4perl::caller_depth = $Log::Log4perl::caller_depth + 2;
# Dump all references with Data::Dumper
$_ = ref($_) ? Dumper($_) : $_ for #{ $params{message} };
$self->{app}->SUPER::log(
\%params,
$params{log4p_category},
$params{log4p_level}
);
}
sub post_init {
my ($self) = #_;
if(! exists $self->{appender}) {
die "No appender defined for " . __PACKAGE__;
}
my $appenders = Log::Log4perl->appenders();
my $appender = Log::Log4perl->appenders()->{$self->{appender}};
if(! defined $appender) {
die "Appender $self->{appender} not defined (yet) when " .
__PACKAGE__ . " needed it";
}
$self->{app} = $appender;
}
package main;
use strict;
use warnings;
use Log::Log4perl;
Log::Log4perl->init(\q{
log4perl.rootLogger=DEBUG, Dump
log4perl.appender.Dump=DumpAppender
log4perl.appender.Dump.appender=SCREEN
log4perl.appender.SCREEN=Log::Log4perl::Appender::Screen
log4perl.appender.SCREEN.layout=PatternLayout
log4perl.appender.SCREEN.layout.ConversionPattern=%d %p %m%n
});
my $logger = Log::Log4perl->get_logger;
$logger->debug(
'This is a string, but this is a reference: ',
{ foo => 'bar' },
);
Output:
2015/09/14 13:38:47 DEBUG This is a string, but this is a reference: {'foo' => 'bar'}
Note that you have to take some extra steps if you initialize Log::Log4perl via the API instead of via a file. This is documented in the composite appenders section of the Log::Log4perl::Appender documentation.
I'm trying to debug a cgi script that reports no errors but the browser displays the generated text as opposed to rendering the page. I called cgi from a container object (of sorts) to see if I'm sending the header twice.
package debugcgi;
use CGI qw(:standard);
use CGI qw(:standard Vars);
use CGI::Carp qw(warningsToBrowser fatalsToBrowser);
sub new {
my ($class,$glob) = #_;
open(lls,">process-cgi.txt");
return bless{'cgi'=>CGI->new($glob),'glob'=>\*lls,'headers'=>0},$class;
}
sub header {
my $self = shift;
my $tmp = shift->{'cgi'}->header(#_);
print $tmp;
my $t = $self->{'glob'};
print $t $tmp;
$self->{'headers'}++;
}
...
sub myclose {
my $self = shift;
my $t = $self->{'glob'};
my $tmp = $self->{'headers'};
print $t "\nnumber of headers: ";
print $t $tmp;
close $t;
}
1;
used as a simple replacement for the real cgi in the bad script:
use debugcgi;
...
#my $cgi = CGI->new(\*STDIN);
my $cgi = debugcgi->new(\*STDIN);
...
print $cgi->header(Referer => $cgi->url());
oh.
but nothing gets printed to the file except "number of headers: 0" and I still get a full HTML document to show up. What did I do wrong, how can I improve on this?
You have a problem here:
sub header {
my $self = shift;
my $tmp = shift->{'cgi'}->header(#_);
...
$self is your hash, which contains the cgi object. So, you need to do this:
sub header {
my $self = shift;
my $cgi = $self->{'cgi'};
my $header_str = $cgi->header(#_);
shift
Shifts the first value of the array off and returns it, shortening the array by 1 and moving everything down.
http://perldoc.perl.org/functions/shift.html
This is more like what modern perl code looks like:
DebugCGI.pm:
package DebugCGI;
use strict;
use warnings;
use 5.016;
use Data::Dumper;
use CGI;
use CGI::Carp qw(warningsToBrowser fatalsToBrowser);
sub new {
my ($class, $PARAMFILE) = #_;
my $fname = 'process-cgi.txt';
open my $OUTFILE, '>', $fname
or die "Couldn't read from $fname: $!";
my $obj_attributes = {
'cgi_obj' => CGI->new($PARAMFILE),
'outfile' => $OUTFILE,
'header_count' => 0,
};
return bless $obj_attributes, $class;
}
sub header {
my ($self, #headers) = #_;
my $cgi = $self->{'cgi_obj'};
my $header_str = $cgi->header(#headers);
print {$self->{outfile}} $header_str;
$self->{'header_count'}++;
return;
}
sub close {
my ($self) = #_;
my $count = $self->{'header_count'};
my $OUTFILE = $self->{'outfile'};
say {$OUTFILE} "number of headers: $count";
close $OUTFILE;
return;
}
1;
Test it out:
use strict;
use warnings;
use 5.016;
use Data::Dumper;
use DebugCGI;
my $fname = 'params.txt';
open my $PARAMFILE, '<', $fname
or die "Couldn't open $fname: $!";
my $debug_cgi = DebugCGI->new($PARAMFILE);
close $PARAMFILE;
$debug_cgi->header(
'-type' => 'text/html; charset=UTF-8',
);
$debug_cgi->header(
'-type' => 'text/plain: charset=UTF-8',
);
$debug_cgi->close;
params.txt:
x=3
y=4
Output:
$ cat process-cgi.txt
Content-Type: text/html; charset=UTF-8
Content-Type: text/plain: charset=UTF-8
number of headers: 2
Note the double newline that $cgi->header() adds after its output. A double newline is a signal to the browser, that the headers have ended, and that any subsequent text is to be considered the body of the response. Therefore, you can't print $cgi->header() twice because the second time the text won't be considered a header. If for some reason you want to print $cgi->header() twice, then you can strip off the trailing newlines with s/\s+\z//xms.
I am trying to gather data from a website. Some anti-patterns make looking finding the right form objects difficult but I have this solved. I am using a post method to get around some javascript acting as a wrapper to submit the form. My problem seems to be in getting the results from the mechanize->post method.
Here's a shortened version of my code.
use strict;
use warnings;
use HTML::Tree;
use LWP::Simple;
use WWW::Mechanize;
use HTTP::Request::Common;
use Data::Dumper;
$| = 1;
my $site_url = "http://someURL";
my $mech = WWW::Mechanize->new( autocheck => 1 );
foreach my $number (#numbers)
{
my $content = get($site_url);
$mech->get ($site_url);
my $tree = HTML::Tree->new();
$tree->parse($content);
my ($title) = $tree->look_down( '_tag' , 'a' );
my $atag = "";
my $atag1 = "";
foreach $atag ( $tree->look_down( _tag => q{a}, 'class' => 'button', 'title' => 'SEARCH' ) )
{
print "Tag is ", $atag->attr('id'), "\n";
$atag1 = Dumper $atag->attr('id');
}
# Enter permit number in "Number" search field
my #forms = $mech->forms;
my #fields = ();
foreach my $form (#forms)
{
#fields = $form->param;
}
my ($name, $fnumber) = $fields[2];
print "field name and number is $name\n";
$mech->field( $name, $number, $fnumber );
print "field $name populated with search data $number\n" if $mech->success();
$mech->post($site_url ,
[
'$atag1' => $number,
'internal.wdk.wdkCommand' => $atag1,
]) ;
print $mech->content; # I think this is where the problem is.
}
The data I get from my final print statement is the data from teh original URL not the page the POST command should take me to. What have I done wrong?
Many Thanks
Update
I don't have Firefox installed so I'm avoiding WWW::Mechanize::Firefox intentionally.
Turns out I was excluding some required hidden fields from my POST command.
I am trying to have a pool of shared connections that can be accessed by Net::Server instances. Unfortunately IPC::Shareable does not allow me to store the connections as they are code references. This is a stripped down version of the code:
use IPC::Shareable (':lock');
use parent 'Net::Server::Fork';
use MyConnectClass;
sub login {
return MyConnectClass->new();
};
my %connection;
tie %connection, 'IPC::Shareable', 'CONN', {
'create' => 1,
'exclusive' => 0,
'mode' => 0666,
'destroy' => 'yes',
}
or croak 'Can not tie connection variable';
sub add_connection {
my $id = shift(#_);
my $con = shift(#_);
$connection{$id} = $con;
};
sub get_connection {
my $id = # .. find unused connection
return $connection{$id};
}
sub process_request {
my $self = shift(#_);
eval {
my $connection = get_connection();
my $line = <STDIN>;
# .. use $connection to fetch data for user
};
};
for (my $i=0; $i<10; $i++) {
add_connection($i, &login);
};
main->run(
'host' => '*',
'port' => 7000,
'ipv' => '*',
'max_server' => 3,
};
Unfortunately the program dies after the first login: 'Can't store CODE items at ../../lib/Storable.pm'. This happens even when hiding $connection in an anonymous array. I am looking for an alternative to utilize the pool.
I appreciate your support
I am unable to propose an alternative module, but make a suggestion which may or not be of use. While you cannot store CODE, you can store strings which can be evaluated to run. would it be possible to pass a reference to the string q!&login! which you can dereference call after being assigned to $connection. ?
#!/usr/bin/perl
use warnings;
use strict;
use Storable;
my $codestring = q'sub { q^japh^ };' ;
#my $codestring = q'sub { return MyConnectClass->new(); }';
#
# for (0..9){ add_connection($i, $codestring) }
open my $file, '>', '.\filestore.dat' or die $!;
store \ $codestring, $file;
close $file;
open $file, '<', '.\filestore.dat' or die " 2 $!";
my $stringref = retrieve $file; # my $con = get_connection()
close $file;
print &{ eval $$stringref } ; # &{eval $$con} ;
exit 0; # my $line = <STDIN>; ...