Downloading data from another site - scrape

I would like the data from https://ucp.paradise-rpg.pl/group/684 for this data:
SCREENSHOOT
and then display them on my website but I don't even know where to start. If anyone could help me, I would be grateful

would like to download data from https://ucp.paradise-rpg.pl/group/684
I was able to make such a code in CURL `<?php
$lurl=get_fcontent("https://ucp.paradise-rpg.pl/");
print_r($lurl[0]);
function get_fcontent( $url, $javascript_loop = 0, $timeout = 5 ) {
$url = str_replace( "&", "&", urldecode(trim($url)) );
$cookie = tempnam ("/tmp", "CURLCOOKIE");
$ch = curl_init();
curl_setopt( $ch, CURLOPT_USERAGENT, "Mozilla / 5.0 (Windows NT 5.1; rv: 31.0) Gecko / 20100101 Firefox / 31.0");
curl_setopt( $ch, CURLOPT_URL, $url );
curl_setopt( $ch, CURLOPT_COOKIEJAR, $cookie );
curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, true );
curl_setopt( $ch, CURLOPT_ENCODING, "" );
curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true );
curl_setopt( $ch, CURLOPT_AUTOREFERER, true );
curl_setopt( $ch, CURLOPT_SSL_VERIFYPEER, false ); # required for https urls
curl_setopt( $ch, CURLOPT_CONNECTTIMEOUT, $timeout );
curl_setopt( $ch, CURLOPT_TIMEOUT, $timeout );
curl_setopt( $ch, CURLOPT_MAXREDIRS, 10 );
$content = curl_exec( $ch );
$response = curl_getinfo( $ch );
curl_close ( $ch );
if ($response['http_code'] == 301 || $response['http_code'] == 302) {
ini_set("user_agent", "Mozilla / 5.0 (Windows NT 5.1; rv: 31.0) Gecko / 20100101 Firefox / 31.0");
if ( $headers = get_headers($response['url']) ) {
foreach( $headers as $value ) {
if ( substr( strtolower($value), 0, 9 ) == "location:" )
return get_url( trim( substr( $value, 9, strlen($value) ) ) );
}
}
}
if ( ( preg_match("/>[[:space:]]+window\.location\.replace\('(.*)'\)/i", $content, $value) || preg_match("/>[[:space:]]+window\.location\=\"(.*)\"/i", $content, $value) ) && $javascript_loop < 5) {
return get_url( $value[1], $javascript_loop+1 );
} else {
return array( $content, $response );
}
}
?>`, it works correctly on other websites and on https://ucp.paradise-rpg.pl/group/684 such an error pops up from cloudflare, what should I do?
ERROR SS

You can just look for the the API's other than downloading. I guess using API's will be the best option here

Related

How do we attach multiple images with JIRA rest attachments api via PHP Curl?

I am able to attach a single image via Jira Rest Api but it fails when i attempt to send multiple images through it. This is my code for single attachment. Need help to make multiple attachments work.
Reference:
Jira attach file to issue with PHP and CURL
$cfile = new CURLFile($attachment['tmp_name'],$attachment['type'], $attachment['name']);
$data = array('file' => $cfile);
$url = "{$uriapi}"."issue/"."{$bugid}"."/attachments";
curl_setopt_array(
$ch,
array(
CURLOPT_URL=>$url,
CURLOPT_POST=>true,
CURLOPT_VERBOSE=>1,
CURLOPT_POSTFIELDS=>$data,
CURLOPT_INFILESIZE => 10,
CURLOPT_SSL_VERIFYHOST=> 0,
CURLOPT_SSL_VERIFYPEER=> 0,
CURLOPT_RETURNTRANSFER=>true,
CURLOPT_HTTPHEADER=> $headers,
CURLOPT_USERPWD=>"$Jirausername:$Jirapassword"
)
);
$result=curl_exec($ch);
$ch_error = curl_error($ch);
if ($ch_error) {
echo "cURL Error: $ch_error";
return "Error Opening file. Failed to add Attachment";
}
elseif(isset($result)){
//return "Attachment added";
return "";
}
else{
return "Failed to add Attachment";
}
curl_close($ch);
}
The following code works fine for me, I hope it helps.
$username = "xxxxx";
$password = "xxxxx";
$url = "https://YourUrl/rest/api/latest/issue/YourKey/attachments";
$attachments = array("attachment1", "attachment2", "attachment3");
$curl = curl_init();
for ($i = 0; $i < count($attachments); $i++) {
$attachmentPath = "/your/attachment/path/$attachments[$i]";
$filename = array_pop(explode('/', $attachmentPath));
$cfile = new CURLFile($attachmentPath);
$cfile->setPostFilename($filename);
$data = array('file' => $cfile);
$headers = array(
'Content-Type: multipart/form-data',
'X-Atlassian-Token: nocheck'
);
curl_setopt($curl, CURLOPT_USERPWD, "$username:$password");
curl_setopt($curl, CURLOPT_URL, $url);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($curl, CURLOPT_VERBOSE, 1);
curl_setopt($curl, CURLOPT_HTTPHEADER, $headers);
curl_setopt($curl, CURLOPT_POSTFIELDS, $data);
$result = curl_exec($curl);
$ch_error = curl_error($curl);
if ($ch_error) {
echo "cURL Error: $ch_error";
} else {
echo $result;
}
}
curl_close($curl);

Get the href configured in the comments plugin from the facebook webhook callback

I'm trying to retrieve the href attribute configured in the Facebook comments plugin from the plugin_comment webhook, in that way I can know the comment origin and trigger a notification to the owner of the content where the comment was made.
I've reviewed the facebook documentation of the comments webhook and the comments data returned by the graph API, but I couldn't find a hint about how to get the source URL of the comment.
Is it possible to get that value?
Note: The owners of the content are not part of our company, so I can not use the moderation tool unfortunately.
FINAL UPDATE:
Ok, now! here it is! lol...
you can have a webhook file that looks something like this:
<?php
if ( $_GET['hub_verify_token'] === 'MyPl#ylist.FunRulez!')
{
echo $_GET['hub_challenge'];
}
$filename = dirname( dirname( __FILE__ ) ) . '/fb_webook/log.json';
if ( file_exists( $filename ) )
{
$data = file_get_contents( $filename );
$data = json_decode( $data , true );
}
else
{
$data = array();
}
$new = file_get_contents( 'php://input' );
$new = json_decode( $new , true );
$new['href'] = get_fb_comment_url( $new );
$data[] = $new;
file_put_contents(
$filename , json_encode( $data , JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES )
);
exit();
?>
Now... for the get_fb_comment_url() function, here it is:
<?php
function get_fb_comment_url( $data )
{
$comment_id = $data['entry'][0]['changes'][0]['value']['id'];
$comment_id = explode( '_' , $comment_id );
$comment_id = $comment_id[1];
$client_id = 'yourappclientid';
$client_secret = 'yourappclientsecret';
//GET ACCESS TOKEN
$url = 'https://graph.facebook.com/oauth/access_token?client_id=' .
$client_id . '&client_secret=' . $client_secret . '&grant_type=client_credentials';
$ch = curl_init();
curl_setopt( $ch , CURLOPT_URL , $url );
curl_setopt( $ch , CURLOPT_RETURNTRANSFER , 1 );
$result = curl_exec( $ch );
curl_close( $ch );
//RESPONSE LOOKS LIKE THIS
//{"access_token":"[access-token]","token_type":"bearer"}
$access_token_data = json_decode( $result , true );
$url = 'https://graph.facebook.com/' .
$comment_id . '?fields=permalink_url&access_token=' . $access_token_data['access_token'];
$ch = curl_init();
curl_setopt( $ch , CURLOPT_URL , $url );
curl_setopt( $ch , CURLOPT_RETURNTRANSFER , 1 );
$result = curl_exec( $ch );
curl_close( $ch );
/*RESPONSE LOOKS LIKE THIS
{
"permalink_url": "https://l.facebook.com/l.php?u=https\u00253A\u00252F\u00252Fmiplaylist.fun\u00252FyRz3RyE8314Q\u00252F\u00253Ffb_comment_id\u00253D2249651171827020_2250200588438745\u002526comment_id\u00253D2250200588438745&h=AT1z4LqCDM_CUg_0zvt1m5fKEPeCEQqrjvH8t27Wepuy3y_gFbwG6FaFY-bSHBH1-Ypfji7R-59HL0yBIeRrXuqBCRVWosvRGURvc3j55gG1iu4ClZQ51oFqouxWbh3-CNupERMQ-NvmstLXF3N_d_vejS2NzXXCZJE&s=1",
"id": "2250200588438745"
}
*/
$data = json_decode( $result , true );
$permalink_url = explode( '?' , $data['permalink_url'] );
parse_str( $permalink_url[1] , $data );
$href = explode( '?' , $data['u'] );
$href = $href[0];
return $href;
}
?>
ENJOY! :D
ORIGINAL ANSWER:
Ok, here it is...
Was looking for the answer myself I think I figured it out.
Webhook sends an object like this:
{"object": "application", "entry": [{"id": "325011391698942", "time":
1569257207, "changes": [{"value": {"id":
"2249651171827020_2250207455104725", "from": {"name": "Daniel Morales
Lira", "id": "10161627783545224"}, "message": "New Reply",
"created_time": "2019-09-23T16:46:46+0000", "parent": {"created_time":
"2019-09-23T16:46:17+0000", "from": {"name": "Daniel Morales Lira",
"id": "10161627783545224"}, "message": "New Comment", "id":
"2249651171827020_2250206608438143"}}, "field":
"plugin_comment_reply"}]}]}
In the changes field, the value id is: 2249651171827020_2250207455104725
take the number after the underscore: 2250207455104725
And run a FB API Call like this:
FB.api(
"/2250207455104725/" ,
{
'fields' : 'permalink_url'
} ,
function ( response )
{
if( window.console ) console.log( response );
myfb.page_info = response;
}
);
This is from the javascript sdk. It might look different on the server. I will update later on that.
You get a response like this:
{"permalink_url":"https://l.facebook.com/l.php?u=https%3A%2F%2Fmiplaylist.fun%2FyRz3RyE8314Q%2F%3Ffb_comment_id%3D2249651171827020_2250207455104725%26comment_id%3D2250206608438143%26reply_comment_id%3D2250207455104725&h=AT3eO0qIkDRuNpTWhP650IQ9DKVJREdcQyYUknRo2CJdNFtnDDT8YgvOqfFxVph04FWkTI80rBaiehHllACgoZdSqilIFPlA-Jq8FK4Hy6YlrelNUtQRKIXCqNPmRgp_HsT0GbovLfV45uhI&s=1","id":"2250207455104725"}
Embeded in the permalink_url is the url where the comment was posted, url encoded in the u parameter.
UPDATE:
You can make an API call from the server side to this URL:
https://graph.facebook.com/2250200588438745?fields=permalink_url&access_token=youraccesstoken
(Replace 2250200588438745 for the number you got after the underscore from the webhook notification)

Convert Php Curl Function to Perl

It would be great if someone could help convert this function from Php to Perl, basically this function is meant to get contents of a https url. Right now I'm not good with Perl, I haven't learnt on how to use cURL just yet.
function curl_get_content($url)
{
$handle = curl_init();
curl_setopt($handle, CURLOPT_URL, $url);
curl_setopt($handle, CURLOPT_RETURNTRANSFER, 1);
curl_setopt ($handle, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($handle, CURLOPT_SSLVERSION, 3);
$data = curl_exec($handle);
curl_close($handle);
return $data;
}
Thanks!
EDIT:
I attempted to use this
my $ua = LWP::UserAgent->new(ssl_opts => { verify_hostname => 1 });
my $header = HTTP::Request->new(GET => $URL);
my $request = HTTP::Request->new('GET', $URL, $header);
my $response = $ua->request($request);
if ($response->is_success){
print "URL:$URL\nHeaders:\n";
print $response->headers_as_string;
}elsif ($response->is_error){
print "Error:$URL\n";
print $response->error_as_HTML;
}
But it keeps giving me server error: 500.
My suspicion is that you're not setting the headers correctly so when the server is receiving your request and trying to process the headers, it's not understanding the content and just giving up (giving you the 500 error.) In your code, you're setting $header to a HTTP::Request object but that should actually be a HTTP::Headers object. Set that to:
my $header = HTTP::Headers->new();
$header->header( 'Content-Type' => 'text/plain' ); # set content-type to what you need
# you can add additional headers if needed

How to make individual share buttons for facebook, twitter and google plus

Like this:
See exmaple under the article:
http://www.golem.de/news/server-prozessor-intel-bestaetigt-broadwell-als-xeon-fuer-sockel-1150-1311-102622.html
The design is not the problem, but how to get the number of shares for this url for twitter, facebook and google plus?
Found the solution. Here is how to get the counts by API:
<?php
if ($network == 'facebook')
{
$string = file_get_contents('http://graph.facebook.com/?id=' . $current_url);
$array = json_decode($string, true);
if (isset($array['shares']))
$number = intval($array['shares']);
else
$number = 0;
}
elseif ($network == 'twitter')
{
$string = file_get_contents('http://urls.api.twitter.com/1/urls/count.json?url=' . $current_url);
$array = json_decode($string, true);
if (isset($array['count']))
$number = intval($array['count']);
else
$number = 0;
}
elseif ($network == 'googleplus')
{
$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, "https://clients6.google.com/rpc");
curl_setopt($curl, CURLOPT_POST, true);
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($curl, CURLOPT_POSTFIELDS, '[{"method":"pos.plusones.get","id":"p","params":{"nolog":true,"id":"'.$current_url.'","source":"widget","userId":"#viewer","groupId":"#self"},"jsonrpc":"2.0","key":"p","apiVersion":"v1"}]');
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl, CURLOPT_HTTPHEADER, array('Content-type: application/json'));
$curl_results = curl_exec($curl);
curl_close ($curl);
$array = json_decode($curl_results, true);
if (isset($array[0]['result']['metadata']['globalCounts']['count']))
$number = intval($array[0]['result']['metadata']['globalCounts']['count']);
else
$number = 0;
}
?>

check if facebook URL redirected or not?

i want to check if the url's in my database are reaching the facebook page they should or redirected to "www.facebook.com".
this is the code i use:
<?php
$conn = mysql_connect('localhost', 'user', 'pass');
mysql_select_db('database');
?>
<?php
$query = "SELECT data_txt FROM jos_sobi2_fields_data WHERE fieldid=8 ";
$result = mysql_query($query);
$row = mysql_fetch_array($result);
$url = $row['data_txt'];
$ch = curl_init();
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
foreach($row as $url) {
curl_setopt($ch, CURLOPT_URL, $url);
$out = curl_exec($ch);
$out = str_replace("\r", "", $out);
$headers_end = strpos($out, "\n\n");
if( $headers_end !== false ) {
$out = substr($out, 0, $headers_end);
}
$headers = explode("\n", $out);
foreach($headers as $header) {
if( substr($header, 0, 10) == "Location: " ) {
$target = substr($header, 10);
echo "[$url] redirects to [$target]<br>";
continue 2;
}
}
echo "[$url] does not redirect<br>";
}
?>
the result is this:
[http://www.facebook.com/shanibakshi.grooming.dogtraining] redirects to [http://www.facebook.com/common/browser.php]
[http://www.facebook.com/shanibakshi.grooming.dogtraining] redirects to [http://www.facebook.com/common/browser.php]
and this url -> http://www.facebook.com/common/browser.php is a facebook page that says my browser is old...probably because of some function in the code.....
anyway all i want to do is to check if the url in my database stays in their place with any redirection.
thanks :)
ronen.
Are you saying that you want to detect redirection, but the problem is you are always getting redirected to browser.php so you get nothing but "false positives"? In that case you probably just need to set the USERAGENT option, something like:
curl_setopt($ch, CURLOPT_USERAGENT,'Mozilla/5.0 (Windows NT 6.2; rv:9.0.1) Gecko/20100101 Firefox/9.0.1');