'm trying to execute this code but it's already give me the following error :
Use of uninitialized value $site in string at C:\Users\USER\Desktop\script.pl line 35.
#!/usr/bin/perl
use LWP::UserAgent;
use File::Slurp;
use warnings;
use strict;
use HTTP::Request;
open (THETARGET, "<list.txt") || die "[-] Can't open the file";
my #TARGETS = <THETARGET>;
close THETARGET;
my $link=$#TARGETS + 1;
OUTER: foreach my $site(#TARGETS){
chomp($site);
}
my $ua = LWP::UserAgent->new;
$ua->timeout(10);
my $url = my $site;
my $picture = "teest.png";
my %args;
my $field_name = "file[]";
my $buf ;
my $buf_ref = $args{'buf'} || \$buf ;
my $value = read_file( $picture , binmode => ':raw' , scalar_ref => 1 );
my $response = $ua->post( $url,
Content_Type => 'form-data',
Content => [ $field_name => ["$picture"] ]
);
print "$site";
I think what you are expecting is the value of $site to carry over from the for loop, but because it is defined in the for loop, its scope is limited to that loop. Perhaps what you intended is for all of the following code to also be inside that loop. Otherwise, that value for $site ceases to exist at the closing-brace two lines after it was created. Your line
my $url = my $site;
then creates a new variable $site (which is undef), and that undef value is used to set $url, so it is also undef. I suspect you initially had
my $url = $site;
then added the extra "my" to resolve an earlier error.
I hope this helps you.
Related
im using this perl code to transform JSON into other form with some regular expressions:
use strict;
use warnings;
use feature 'say';
use JSON;
use utf8;
my %IDs = ( 'User awx01 logged in.' => 1001 );
my %levels = ( INFO => 4 );
my $data = <DATA>;
my $json = data2json($data);
my $record = decode_json($json);
say rec2msg($record);
sub data2json {
my $json = shift;
$json =~ s/[""]/"/g;
$json =~ s/\\//g;
$json =~ s/"(\{.*?\})"/$1/;
return $json;
}
sub rec2msg {
my $r = shift;
$r->{Message}{message} =~ /(\w+) (\w+) (.+)/;
my($user,$msg) = ($2,"$1 $3");
my $ID = $IDs{$r->{Message}{message}};
my $level = $levels{$r->{Message}{level}};
my $out = "$r->{Message}{'#timestamp'} host CEF:0|OpenSource|AWX|7.0.0|$ID|$msg|$level|src=127.0.0.1 dst=$r->{MessageSourceAddress} duser=$user";
return $out;
}
__DATA__
{"MessageSourceAddress":"192.168.81.20","EventReceivedTime":"2020-02-06 11:55:14","SourceModuleName":"udp","SourceModuleType":"im_udp","SyslogFacilityValue":1,"SyslogFacility":"USER","SyslogSeverityValue":5,"SyslogSeverity":"NOTICE","SeverityValue":2,"Severity":"INFO","EventTime":"2020-02-06 11:55:14","Hostname":"192.168.81.20","Message":"{\"#timestamp\": \"2020-02-06T08:55:52.907Z\", \"message\": \"User awx01 logged in.\", \"host\": \"awxweb\", \"level\": \"INFO\", \"logger_name\": \"awx.api.generics\", \"stack_info\": null, \"type\": \"other\", \"cluster_host_id\": \"awx-contr-01\", \"tower_uuid\": \"333b4131-495f-4460-8e4b-890241a9d73d\"}"}
But im getting this error:
2020-03-31 20:48:50 ERROR perl subroutine rec2msg failed with an error: 'Can't use string ("140511667030448") as a HASH ref while "strict refs" in use at /usr/libexec/nxlog/modules/extension/perl/event1.pl line 21.;'
What im doing wrong? How could i solve it?
You have JSON embedded in JSON, so you need to decode it twice. This often happens when you have one service passing through the response for another service.
Your data2json wasn't decoding that second level, so the value for the Message name was still a string. Since that value wasn't a hash reference, you get the error you reported.
You don't want to use a bunch of substitutions on the entire thing because you can inadvertently change things you shouldn't be messing with. Decode the top level just as you did, but then do the same thing for the Message value:
# read in all the data, even though it looks like a single line. Maybe it won't be later.
my $data = do { local $/; <DATA> };
# decode the first layer
my $decoded = decode_json( $data );
# decode the Message value:
$decoded->{Message} = decode_json( $decoded->{Message} );
Now, when you call rec2msg it should work out.
Note that this has the opposite problem to reverse it. You can't merely encode the entire thing to JSON again. The value for Message still needs to be a string, so you have to encode that first if you want to send it somewhere else. If you are doing that, you probably want to work on a copy. I use dclone to make a deep copy so whatever I do to $encoded does not show up in $decoded:
# make a deep copy so nested references aren't shared
use Storable qw(dclone);
my $encoded = dclone( $decoded );
$encoded->{Message} = encode_json( $encoded->{Message} );
my $new_data = encode_json( $encoded );
Then $new_data will have the same escaping as the original input.
Here it is altogether:
use strict;
use warnings;
use feature 'say';
use JSON;
use utf8;
my %IDs = ( 'User awx01 logged in.' => 1001 );
my %levels = ( INFO => 4 );
# read in all the data, even though it looks
my $data = do { local $/; <DATA> };
my $decoded = decode_json( $data );
$decoded->{Message} = decode_json( $decoded->{Message} );
say rec2msg($decoded);
sub rec2msg {
my $r = shift;
$r->{Message}{message} =~ /(\w+) (\w+) (.+)/;
my($user,$msg) = ($2,"$1 $3");
my $ID = $IDs{$r->{Message}{message}};
my $level = $levels{$r->{Message}{level}};
my $out = "$r->{Message}{'#timestamp'} host CEF:0|OpenSource|AWX|7.0.0|$ID|$msg|$level|src=127.0.0.1 dst=$r->{MessageSourceAddress} duser=$user";
return $out;
}
I have a doubt I've been trying to solve myself using CPAN modules documentation, but I'm a bit new and I'm confused with some terminology and sections within the different modules.
I'm trying to create the object in the code below, and get the absolute URL for relative links extracted from a website.
#!/usr/bin/perl
use strict;
use warnings;
use LWP::UserAgent;
use Digest::MD5 qw(md5_hex);
use URI;
my $url = $ARGV[0];
if ($url !~ m{^https?://[^\W]+-?\.com/?}i) {
exit(0);
}
my $ua = LWP::UserAgent->new;
$ua->timeout( 10 );
my $response = $ua->get( $url );
my $content = $response->decoded_content();
my $links = URI->new($content);
my $abs = $links->abs('http:', $content);
my $abs_links = $links->abs($abs);
while ($content =~ m{<a[^>]\s*href\s*=\s*"?([^"\s>]+)}gis) {
$abs_links = $1;
print "$abs_links\n";
print "Digest for the above URL is " . md5_hex($abs_links) . "\n";
}
The problem is when I try to add that part outside the While loop (the 3-line block preceding the loop), it does not work, whereas if I add the same part in the While loop, it will work fine. This one just gets the relative URLs from a given website, but instead of printing "Http://..." it prints "//...".
The script that works fine for me is the following:
#!/usr/bin/perl
use strict;
use warnings;
use LWP::UserAgent;
use Digest::MD5 qw(md5_hex);
use URI::URL;
my $url = $ARGV[0]; ## Url passed in command
if ($url !~ m{^https?://[\w]+-?[\w]+\.com/?}i) {
exit(0); ## Program stops if not valid URL
}
my $ua = LWP::UserAgent->new;
$ua->timeout( 10 );
my $response = $ua->get( $url ); ## Get response, not content
my $content = $response->decoded_content(); ## Now let's get the content
while ($content =~ m{<a[^>]\s*href\s*=\s*"?([^"\s>]+)}gis) { ## All links
my $links = $1;
my $abs = new URI::URL "$links";
my $abs_url = $abs->abs('http:', $links);
print "$abs_url\n";
print "Digest for the above URL is " . md5_hex($abs_url) . "\n";
}
Any ideas? Much appreciated.
I don't understand your code. There are a few weird bits:
[^\W] is the same as \w
The regex allows an optional - before and an optional / after .com, i.e. http://bitwise.complement.biz matches but http://cool-beans.com doesn't.
URI->new($content) makes no sense: $content is random HTML, not a URI.
$links->abs('http:', $content) makes no sense: $content is simply ignored, and $links->abs('http:') tries to make $links an absolute URL relative to 'http:', but 'http:' is not a valid URL.
Here's what I think you're trying to do:
#!/usr/bin/perl
use strict;
use warnings;
use LWP::UserAgent;
use HTML::LinkExtor;
use Digest::MD5 qw(md5_hex);
#ARGV == 1 or die "Usage: $0 URL\n";
my $url = $ARGV[0];
my $ua = LWP::UserAgent->new(timeout => 10);
my $response = $ua->get($url);
$response->is_success or die "$0: " . $response->request->uri . ": " . $response->status_line . "\n";
my $content = $response->decoded_content;
my $base = $response->base;
my #links;
my $p = HTML::LinkExtor->new(
sub {
my ($tag, %attrs) = #_;
if ($tag eq 'a' && $attrs{href}) {
push #links, "$attrs{href}"; # stringify
}
},
$base,
);
$p->parse($content);
$p->eof;
for my $link (#links) {
print "$link\n";
print "Digest for the above URL is " . md5_hex($link) . "\n";
}
I don't try to validate the URL passed in $ARGV[0]. Leave it to LWP::UserAgent. (If you don't like this, just add the check back in.)
I make sure $ua->get($url) was successful before proceeding.
I get the base URL for absolutifying relative links from $response->base.
I use HTML::LinkExtor for parsing the content, extracting links, and making them absolute.
I think your biggest mistake is trying to parse links out of HTML using a regular expression. You would be far better advised to use a CPAN module for this. I'd recommend WWW::Mechanize, which would make your code look something like this:
#!/usr/bin/perl
use strict;
use warnings;
use feature 'say';
use WWW::Mechanize;
use Digest::MD5 qw(md5_hex);
use URI;
my $url = $ARGV[0];
if ($url !~ m{^https?://[^\W]+-?\.com/?}i) {
exit(0);
}
my $ua = WWW::Mechanize->new;
$ua->timeout( 10 );
$ua->get( $url );
foreach ($ua->links) {
say $_->url;
say "Digest for the above URL is " . md5_hex($_->url) . "\n";
}
That looks a lot simpler to me.
Below is what I have. If I change my $url = #_ to a working URL, it works, but the example is supposed to read in from what's defined in my #URLs using the foreach, I believe. Could someone show me or tell me why this doesn't work so I can correct it?
#!/bin/perl
use IO::Async::Loop;
use Net::Async::HTTP;
use Future::Utils qw(fmap_void);
use URI;
use feature 'say';
use strict;
use warnings;
my #URLs = ( "http://example.com, http://example2.com" );
my $loop = IO::Async::Loop->new();
my $http = Net::Async::HTTP->new();
$loop->add($http);
my $future = fmap_void {
my $url = #_;
$http->GET($url)->on_done(
sub {
my $response = shift;
say $response->content;
}
)->on_fail(
sub {
my $fail = shift;
say $fail;
}
);
}
foreach => \#URLs;
$loop->await($future);
You are assigning the number of items in #_ to $url because that's what arrays do in list context.
my ( $url ) = #_;
The parenthesis will tell Perl that the left-hand-side of the assignment is a list.
(my $url) = #_;
This would work as well, but looks stupid.
when I am using "--disk-cache=true" in phantomjs_arg then it's getting error In this line:
my $form = $self->{obj_mech}->form_number( 2 );
No elements found for form number 2 at modules/TestLogin.pm line 1129.
at /usr/local/share/perl/5.22.1/WWW/Mechanize/PhantomJS.pm line 796.
WWW::Mechanize::PhantomJS::signal_condition(WWW::Mechanize::PhantomJS=HASH(0x4cfa120),
"No elements found for form number 2") called at
/usr/local/share/perl/5.22.1/WWW/Mechanize/PhantomJS.pm line 1732
WWW::Mechanize::PhantomJS::xpath(WWW::Mechanize::PhantomJS=HASH(0x4cfa120),
"(//form)[2]", "user_info", "form number 2", "single", 1) called at
/usr/local/share/perl/5.22.1/WWW/Mechanize/PhantomJS.pm line 2102
WWW::Mechanize::PhantomJS::form_number(WWW::Mechanize::PhantomJS=HASH(0x4cfa120),
2) called at modules/TestLogin.pm line 1129
TestLogin::TestLogin_login(TestLogin=HASH(0x4f5c8a8)) called at collectBets.pl line 20 Debugged program terminated. Use q to quit
or R to restart, use o inhibit_exit to avoid stopping after program
termination, h q, h R or h o to get additional info.
without disk-cashe it's working fine.
This is my sample code for better understanding.
#!/usr/bin/perl
use strict;
use warnings;
use Helper;
use WWW::Mechanize::PhantomJS;
use DataBase;
use MyConfig;
use JSON;
use DateTime;
use HTML::Entities;
sub new($$) {
my ($class,$params) = #_;
my $self = $params || {};
bless $self, $class;
$self->{obj_mech} = WWW::Mechanize::PhantomJS -> new( phantomjs_arg => ['--ssl-protocol=any','--disk-cache=true','--max-disk-cache-size=1024'], ignore_ssl_errors => 1);
$self->{obj_helper} = new Helper();
#$self->{obj_db} = new DataBase();
$self->{logged_in} = 0;
#$self->setTorProxy();
#$self->init_market_master();
return $self;
}
Login();
print "\nlogin done...\n";
exit;
sub Login {
my ($self) = #_;
my $html = $self->{obj_mech}->get( "https://www.gmail.com/" );
sleep(25);
$html = $self->{obj_mech}->content;
$self->{obj_mech}->viewport_size({ width => 1366, height => 768 });
my $form = $self->{obj_mech}->form_number( 2 );
my $user_name = '*****';
my $password = '******';
$self->{obj_mech}->set_fields('InputEmail' =>$user_name);
$self->{obj_mech}->set_fields('InputPassword' =>$password);
$self->{obj_mech}->click({ xpath => '//button[#class="PrimaryButton"]' });
sleep(20);
my $test_html=$self->{obj_mech}->content;
$self->{obj_helper}->writeFileNew( "TestLoginPage.html" , $test_html );
my $png = $self->{obj_mech}->content_as_png();
$self->{obj_helper}->writeFileNew( "LoginPage.png" , $png );
return 1;
}
Well, before looking at the disk-cache arguments, I found that there are no such elements.
# There is only 1 form. If you want to keep this line,
# you need to change the form number to 1
my $form = $self->{obj_mech}->form_number( 2 );
# I didn't find input field named 'InputEmail'
# The actual field name is 'Email'
$self->{obj_mech}->set_fields('InputEmail' =>$user_name);
# You have to click 'Next' button firstly then the password
# input box is shown. And the field name should be 'Passwd'
$self->{obj_mech}->set_fields('InputPassword' =>$password);
# The xpath of 'Sign in' button is //input[#value="Sign in"]
$self->{obj_mech}->click({ xpath => '//button[#class="PrimaryButton"]' });
A simple working script either with disk cache or without disk cache:
#! /usr/bin/perl
use strict;
use warnings;
use WWW::Mechanize::PhantomJS;
use open ':std', ':encoding(UTF-8)';
#my $p = WWW::Mechanize::PhantomJS->new(phantomjs_arg=>['--ssl-protocol=any','--disk-cache=false','--max-disk-cache-size=1024'],ignore_ssl_errors=>1);
my $p = WWW::Mechanize::PhantomJS->new(phantomjs_arg=>['--ssl-protocol=any','--disk-cache=true','--max-disk-cache-size=1024'],ignore_ssl_errors=>1);
my $html = $p->get("https://www.gmail.com/");
sleep(5);
write_html('first-page.html', $p->content);
$p->viewport_size({width=>1366,height=>768});
my $form = $p->form_number(1);
my $user_name = '*****';
my $password = '*****';
$p->set_fields('Email'=>$user_name);
sleep(5);
$p->click({xpath=>'//input[#value="Next"]'});
sleep(5);
write_html('after-click-next.html', $p->content);
$p->set_fields('Passwd'=>$password);
sleep(5);
$p->click({xpath=>'//input[#value="Sign in"]'});
sleep(5);
write_html('after-login.html', $p->content);
sub write_html {
my ($file, $content) = #_;
open my $fh, '>', $file or die;
print $fh $content;
close $fh;
}
I am trying to gather data from a website. Some anti-patterns make looking finding the right form objects difficult but I have this solved. I am using a post method to get around some javascript acting as a wrapper to submit the form. My problem seems to be in getting the results from the mechanize->post method.
Here's a shortened version of my code.
use strict;
use warnings;
use HTML::Tree;
use LWP::Simple;
use WWW::Mechanize;
use HTTP::Request::Common;
use Data::Dumper;
$| = 1;
my $site_url = "http://someURL";
my $mech = WWW::Mechanize->new( autocheck => 1 );
foreach my $number (#numbers)
{
my $content = get($site_url);
$mech->get ($site_url);
my $tree = HTML::Tree->new();
$tree->parse($content);
my ($title) = $tree->look_down( '_tag' , 'a' );
my $atag = "";
my $atag1 = "";
foreach $atag ( $tree->look_down( _tag => q{a}, 'class' => 'button', 'title' => 'SEARCH' ) )
{
print "Tag is ", $atag->attr('id'), "\n";
$atag1 = Dumper $atag->attr('id');
}
# Enter permit number in "Number" search field
my #forms = $mech->forms;
my #fields = ();
foreach my $form (#forms)
{
#fields = $form->param;
}
my ($name, $fnumber) = $fields[2];
print "field name and number is $name\n";
$mech->field( $name, $number, $fnumber );
print "field $name populated with search data $number\n" if $mech->success();
$mech->post($site_url ,
[
'$atag1' => $number,
'internal.wdk.wdkCommand' => $atag1,
]) ;
print $mech->content; # I think this is where the problem is.
}
The data I get from my final print statement is the data from teh original URL not the page the POST command should take me to. What have I done wrong?
Many Thanks
Update
I don't have Firefox installed so I'm avoiding WWW::Mechanize::Firefox intentionally.
Turns out I was excluding some required hidden fields from my POST command.