Can someone please help with this. Thanks in advance.
I send request to twitter with nonce, callback_url, consumer_key, etc. Twitter then prompts the user to enter login and password. User successfully enters login credentials. After that Twitter sends oauth_token and oauth_verifier to the callback URl specified in the earlier call. But now I always get 'HTTP Error 401: Unauthorized' error. Can someone please help with this? Have spent days on it. Thank you.
"GET /account/twitter/oauthacc?oauth_token=ererererer&oauth_verifier=ereret43rwererererer"
HTTP Error 401: Unauthorized
Traceback (most recent call last):
File
"/base/python_runtime/python_lib/versions/1/google/appengine/ext/webapp/_webapp25.py", line 714, in __call__handler.get(*groups)
File "/base/data/home/apps/s~test/2-7.3629731/oauth.py", line 74, in get
self.mOauthRequestToken = vOauthApi.getRequestToken(self.ACCESS_TOKEN_URL)
File "/base/data/home/apps/s~-test/2-7.369731/libs/oauthtwitter.py", line 206, in getRequestToken
resp = self._FetchUrl(url, no_cache=True)
File "/base/data/home/apps/s~test/2-7.3639731/libs/oauthtwitter.py", line 118, in _FetchUrl
url_data = opener.open(url).read()
File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 387, in open
response = meth(req, response)
File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 498, in http_response
'http', request, response, code, msg, hdrs)
File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 425, in error
return self._call_chain(*args)
File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 360, in _call_chain
result = func(*args)
File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 506, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
HTTPError: HTTP Error 401: Unauthorized
Related
I am trying to trigger DAG from Airflow API through python script.
DAG is triggering from curl command but its not working from API.
import requests
url ='http://localhost:8080/api/experimental/dags/document_validation/dag_runs'
myobj = {''}
x =requests.post(url, data=myobj, headers={"Content-Type": "application/json"})
print(x.text)
Error I am getting
File "run_dag_api.py", line 6, in <module>
`x = requests.post(url, data = myobj)`
File "/…/lib/python3.7/site-packages/requests/api.py", line 119, in post
`return request('post', url, data=data, json=json, **kwargs)`
File "/…/lib/python3.7/site-packages/requests/api.py", line 61, in request
`return session.request(method=method, url=url, **kwargs)`
File "/…/lib/python3.7/site-packages/requests/sessions.py", line 530, in request
`resp = self.send(prep, **send_kwargs)`
File "/…/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
`r = adapter.send(request, **kwargs)`
File "/…/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
`raise ConnectionError(err, request=request)`
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
google.api_core.exceptions.ServiceUnavailable: 503 Deadline Exceeded
using python 3.7 ,google-cloud-pubsub ==1.1.0 publishing data on topic. In my local machine it's working perfectly fine and able to publish data on that topic and also able to pull data from that topic through subscriber.
but don't understand it's not working when i deploy the code on server and it's failing with INLINE ERROR however when i explicitly call the publisher method on server it's publishing fine over server box also.code which is failing at below line while publishing:
future = publisher.publish(topic_path, data=data)
**ERROR:2020-02-20 14:24:42,714 ERROR Failed to publish 1 messages.**
Trackback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
return callable_(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/grpc/_channel.py", line 826, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.7/site-packages/grpc/_channel.py", line 729, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Deadline Exceeded"
debug_error_string = "{"created":"#1582208682.711481693","description":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":69,"grpc_status":14}"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/google/api_core/retry.py", line 184, in retry_target
return target()
File "/usr/local/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
six.raise_from(exceptions.from_grpc_error(exc), exc)
File "<string>", line 3, in raise_from
google.api_core.exceptions.ServiceUnavailable: 503 Deadline Exceeded
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/google/cloud/pubsub_v1/publisher/_batch/thread.py", line 219, in _commit
response = self._client.api.publish(self._topic, self._messages)
File "/usr/local/lib/python3.7/site-packages/google/cloud/pubsub_v1/gapic/publisher_client.py", line 498, in publish
request, retry=retry, timeout=timeout, metadata=metadata
File "/usr/local/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in call
return wrapped_func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/google/api_core/retry.py", line 286, in retry_wrapped_func
on_error=on_error,
File "/usr/local/lib/python3.7/site-packages/google/api_core/retry.py", line 206, in retry_target
last_exc,
File "", line 3, in raise_from
google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(.error_remapped_callable at 0x7f67d064e950>
You should try to chunk your data in reasonable sized chunks (max_messages) and don't forget to add a done callback.
# Loop over json containing records/rows
for idx, row in enumerate(rows_json):
publish_json(row, idx, rowmax=len(rows_json), topic_name)
# Publish messages asynchronous
def publish_json(msg, rowcount, rowmax, topic_project_id, topic_name):
batch_settings = pubsub_v1.types.BatchSettings(max_messages=100)
publisher = pubsub_v1.PublisherClient(batch_settings)
topic_path = publisher.topic_path(topic_project_id, topic_name)
future = publisher.publish(
topic_path, bytes(json.dumps(msg).encode('utf-8')))
future.add_done_callback(
lambda x: logging.info(
'Published msg with ID {} ({}/{} rows).'.format(
future.result(), rowcount, rowmax))
)
I've made a spider using Scrapy that first solves a CAPTCHA in a redirected address before accessing the main website I intend to scrape. It says that I have an HTTP error causing an infinite loop but I can't find which part of the script is causing this.
In the middleware:
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
class ProtectRedirectMiddleware(RedirectMiddleware):
def __init__(self, settings):
super().__init__(settings)
self.source = urllib.request.urlopen('http://sampleurlname.com/')
soup = BeautifulSoup(source, 'lxml')
def _redirect(self, redirected, request, spider, reason):
# act normally if this isn't a CAPTCHA redirect
if not self.is_protected(redirected.url):
return super()._redirect(redirected, request, spider, reason)
# if this is a CAPTCHA redirect
logger.debug(f'The protect URL is triggered for {request.url}')
request.cookies = self.bypass_protection(redirected.url)
request.dont_filter = True
return request
def is_protected(self, url):
return 'sampleurlname.com/protect' in url
def bypass_protection(self, url=None):
# only navigate if any explicit url is provided
if url:
url = url or self.source.geturl(url)
img = soup.find_all('img')[0]
imgurl = img['src']
urllib.request.urlretrieve(imgurl, "captcha.png")
return self.solve_captcha(imgurl)
# wait for the redirect and try again
self.wait_for_redirect()
return self.bypass_protection()
def wait_for_redirect(self, url = None, wait = 0.1, timeout=10):
url = self.url
for i in range(int(timeout//wait)):
time.sleep(wait)
if self.response.url() != url:
return self.response.url()
logger.error(f'Maybe {self.response.url()} isn\'t a redirect URL')
raise Exception('Timed out')
def solve_captcha(self, img, width=150, height=50):
# open image
self.img = 'captcha.png'
img = Image.open("captcha.png")
# image manipulation - simplified
# input the captcha text - simplified
# click the submit button - simplified
# save the URL
url = self.response.url()
# try again if wrong
if self.is_protected(self.wait_for_redirect(url)):
return self.bypass_protection()
# return the cookies as a dict
cookies = {}
for cookie_string in self.response.css.cookies():
if 'domain=sampleurlname.com' in cookie_string:
key, value = cookie_string.split(';')[0].split('=')
cookies[key] = value
return cookies
Then, this is the error I get when I run the scrapy crawl of my spider:
Unhandled error in Deferred:
2018-08-06 16:34:33 [twisted] CRITICAL: Unhandled error in Deferred:
2018-08-06 16:34:33 [twisted] CRITICAL:
Traceback (most recent call last):
File "/username/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
result = g.send(result)
File "/username/anaconda/lib/python3.6/site-packages/scrapy/crawler.py", line 80, in crawl
self.engine = self._create_engine()
File "/username/anaconda/lib/python3.6/site-packages/scrapy/crawler.py", line 105, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/username/anaconda/lib/python3.6/site-packages/scrapy/core/engine.py", line 69, in __init__
self.downloader = downloader_cls(crawler)
File "/username/anaconda/lib/python3.6/site-packages/scrapy/core/downloader/__init__.py", line 88, in __init__
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
File "/username/anaconda/lib/python3.6/site-packages/scrapy/middleware.py", line 58, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/username/anaconda/lib/python3.6/site-packages/scrapy/middleware.py", line 36, in from_settings
mw = mwcls.from_crawler(crawler)
File "/username/anaconda/lib/python3.6/site-packages/scrapy/downloadermiddlewares/redirect.py", line 26, in from_crawler
return cls(crawler.settings)
File "/username/...../scraper/myscraper/myscraper/middlewares.py", line 27, in __init__
self.source = urllib.request.urlopen('http://sampleurlname.com/')
File "/username/anaconda/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/username/anaconda/lib/python3.6/urllib/request.py", line 532, in open
response = meth(req, response)
File "/username/anaconda/lib/python3.6/urllib/request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "/username/anaconda/lib/python3.6/urllib/request.py", line 564, in error
result = self._call_chain(*args)
File "/username/anaconda/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/username/anaconda/lib/python3.6/urllib/request.py", line 756, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "/username/anaconda/lib/python3.6/urllib/request.py", line 532, in open
It basically repeats the bottom part of these over and over: open, http_response, error, _call_chain, and http_error_302, until these show at the end:
File "/username/anaconda/lib/python3.6/urllib/request.py", line 746, in http_error_302
self.inf_msg + msg, headers, fp)
urllib.error.HTTPError: HTTP Error 307: The HTTP server returned a redirect error that would lead to an infinite loop.
The last 30x error message was:
Temporary Redirect
In setting.py is:
DOWNLOADER_MIDDLEWARES = {
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': None,
'myscrape.middlewares.ProtectRedirectMiddleware': 600}
Your issue has nothing to do with scrapy itself. You are using blocking requests in your middleware initiation.
This request seems to be stuck in a redirect loop. This usually happens when websites do not act appropriately and require cookies to allow you through:
First you connect and get a redirect response 30x and some setCokies headers
You redirect again but not with Cookies headers and the page lets you through
Python urllib doesn't handle cookies, so try this:
import urllib
from http.cookiejar import CookieJar
def __init__(self):
try:
req=urllib.request.Request(url)
cj = CookieJar()
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
response = opener.open(req)
source = response.read().decode('utf8', errors='ignore')
response.close()
except urllib.request.HTTPError as e:
logging.error(f"couldn't initiate middleware: {e}")
return
# you should use scrapy selectors instead of beautiful soup here
#soup = BeautifulSoup(source, 'lxml')
selector = Selector(text=source)
Alternatively you should use requests package that handles cookies by itself.
Problem:
Trying to connect to Softlayer Swift Object Storage using apache libcloud and I can't get it to work. I have tried passing in different options to the provider but no matter what I pass in I get the same error. I would appreciate any and all pointers to help me resolve this issue.
Code:
from pprint import pprint
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
_swift = get_driver(Provider.OPENSTACK_SWIFT)
_authurl = "https://lon02.objectstorage.softlayer.net/auth/v1.0"
_user = "IBMOS<redacted>"
_key = "redacted"
driver = _swift(_user, _key,
region='lon02',
ex_force_auth_url=_authurl,
ex_force_auth_version='1.0',
ex_force_service_type='object-store',
ex_force_service_name='cloudFiles')
#pprint(driver.__dict__)
pprint(driver.list_containers())
Stack Trace:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py", line 441, in wrap_socket
cnx.do_handshake()
File "/usr/local/lib/python3.6/site-packages/OpenSSL/SSL.py", line 1806, in do_handshake
self._raise_ssl_error(self._ssl, result)
File "/usr/local/lib/python3.6/site-packages/OpenSSL/SSL.py", line 1539, in _raise_ssl_error
raise SysCallError(-1, "Unexpected EOF")
OpenSSL.SSL.SysCallError: (-1, 'Unexpected EOF')
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 346, in _make_request
self._validate_conn(conn)
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 850, in _validate_conn
conn.connect()
File "/usr/local/lib/python3.6/site-packages/urllib3/connection.py", line 326, in connect
ssl_context=context)
File "/usr/local/lib/python3.6/site-packages/urllib3/util/ssl_.py", line 329, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "/usr/local/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py", line 448, in wrap_socket
raise ssl.SSLError('bad handshake: %r' % e)
ssl.SSLError: ("bad handshake: SysCallError(-1, 'Unexpected EOF')",)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/local/lib/python3.6/site-packages/urllib3/util/retry.py", line 388, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='lon02.objectstorage.softlayer.net', port=443): Max retries exceeded with url: /auth/v1.0 (Caused by SSLError(SSLError("bad handshake: SysCallError(-1, 'Unexpected EOF')",),))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "swift.py", line 19, in <module>
pprint(driver.list_containers())
File "/usr/local/lib/python3.6/site-packages/libcloud/storage/base.py", line 209, in list_containers
return list(self.iterate_containers())
File "/usr/local/lib/python3.6/site-packages/libcloud/storage/drivers/cloudfiles.py", line 275, in iterate_containers
response = self.connection.request('')
File "/usr/local/lib/python3.6/site-packages/libcloud/storage/drivers/cloudfiles.py", line 165, in request
raw=raw)
File "/usr/local/lib/python3.6/site-packages/libcloud/common/openstack.py", line 223, in request
raw=raw)
File "/usr/local/lib/python3.6/site-packages/libcloud/common/base.py", line 536, in request
action = self.morph_action_hook(action)
File "/usr/local/lib/python3.6/site-packages/libcloud/common/openstack.py", line 290, in morph_action_hook
self._populate_hosts_and_request_paths()
File "/usr/local/lib/python3.6/site-packages/libcloud/common/openstack.py", line 324, in _populate_hosts_and_request_paths
osa = osa.authenticate(**kwargs) # may throw InvalidCreds
File "/usr/local/lib/python3.6/site-packages/libcloud/common/openstack_identity.py", line 761, in authenticate
resp = self.request('/v1.0', headers=headers, method='GET')
File "/usr/local/lib/python3.6/site-packages/libcloud/common/base.py", line 603, in request
headers=headers, stream=stream)
File "/usr/local/lib/python3.6/site-packages/libcloud/http.py", line 213, in request
verify=self.verification
File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/adapters.py", line 506, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='lon02.objectstorage.softlayer.net', port=443): Max retries exceeded with url: /auth/v1.0 (Caused by SSLError(SSLError("bad handshake: SysCallError(-1, 'Unexpected EOF')",),))
To answer my own question, the specific issue seen above was due to the system I was using. Once I switched to another system it proceeded even further however there is a bug I identified and filed in apache libcloud with a documented workaround inside the jira ticket here https://issues.apache.org/jira/browse/LIBCLOUD-972.
I'm new to Python, and have been using different tutorials/guides to build a program for getting data from a Facebook page, and write it into an AzureSQL database. In the Facebook Developer page I've set up an app, and created the AppID and AppSecret.
Here's the relevant part of the code I think the issue is with:
#create authenticated post URL
def create_post_url(graph_url, APP_ID, APP_SECRET):
post_args = "/posts/?key=value&access_token=" + APP_ID + "|" + APP_SECRET
post_url = graph_url + post_args
return post_url
#render graph url call to JSON
def render_to_json(graph_url):
web_response = urllib2.urlopen(graph_url)
readable_page = web_response.read()
json_data = json.loads(readable_page)
return json_data
def main():
#define facebook app secret and app id
APP_SECRET = "xyz"
APP_ID = "abc"
#define page username
pagelist = ["porsche"]
graph_url = "https://graph.facebook.com/"
In the Facebook Graph API Explorer, running a GET request will work great and return data.
Facebook Graph API Explorer results
Howerver if I run my code in the CLI, the resulting error is as follows:
[vagrant#localhost SomeAnalysis]$ python src/collectors/facebook/facebookScaper.py
Traceback (most recent call last):
File "src/collectors/facebook/facebookScaper.py", line 92, in <module>
main()
File "src/collectors/facebook/facebookScaper.py", line 56, in main
json_fbpage = render_to_json(current_page)
File "src/collectors/facebook/facebookScaper.py", line 25, in render_to_json
web_response = urllib2.urlopen(graph_url)
File "/usr/lib64/python2.7/urllib2.py", line 154, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib64/python2.7/urllib2.py", line 437, in open
response = meth(req, response)
File "/usr/lib64/python2.7/urllib2.py", line 550, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib64/python2.7/urllib2.py", line 475, in error
return self._call_chain(*args)
File "/usr/lib64/python2.7/urllib2.py", line 409, in _call_chain
result = func(*args)
File "/usr/lib64/python2.7/urllib2.py", line 558, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 400: Bad Request
I've been checking SO and have found similar posts but none explain how to use the Access Token. I've tried changing the part in the URL formation with APP_ID and APP_SECRET to just using the long ACCESS_TOKEN from the Graph API Explorer but that yields the same 400 error.
Any help is appreciated.
/K