I'm trying to scrape data from atp ranking site:
https://www.atptour.com/en/rankings/singles
So I tried to do a GET on this URL, but I have this error code: "403 Forbidden" with Postman. I don't have this error code on other websites like https://www.worldometers.info/coronavirus/.
ATP do not have API, so I want to scrape the website but it seems scraping is not allowed on this website, do you have a solution ?
Thanks
error code
The problem is that I don't know the headers to be included in the request.
Sometimes it works in Postman but i still get the error in my express JS application:
config: {
adapter: [Function: httpAdapter],
transformRequest: { '0': [Function: transformRequest] },
transformResponse: { '0': [Function: transformResponse] },
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/0.18.1'
},
method: 'get',
url: 'https://www.atptour.com/en/rankings/singles',
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype] {
socket: [Function (anonymous)],
abort: [Function (anonymous)],
aborted: [Function (anonymous)],
error: [Function (anonymous)],
timeout: [Function (anonymous)],
prefinish: [Function: requestOnPrefinish]
},
_eventsCount: 6,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: TLSSocket {
_tlsOptions: [Object],
_secureEstablished: true,
_securePending: false,
_newSessionPending: false,
_controlReleased: true,
secureConnecting: false,
_SNICallback: null,
servername: 'www.atptour.com',
alpnProtocol: false,
authorized: true,
authorizationError: null,
encrypted: true,
_events: [Object: null prototype],
_eventsCount: 10,
connecting: false,
_hadError: false,
_parent: null,
_host: 'www.atptour.com',
_readableState: [ReadableState],
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: undefined,
_server: null,
ssl: [TLSWrap],
_requestCert: true,
_rejectUnauthorized: true,
parser: null,
_httpMessage: [Circular *1],
[Symbol(res)]: [TLSWrap],
[Symbol(verified)]: true,
[Symbol(pendingSession)]: null,
[Symbol(async_id_symbol)]: 13,
[Symbol(kHandle)]: [TLSWrap],
[Symbol(kSetNoDelay)]: false,
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0,
[Symbol(connect-options)]: [Object],
[Symbol(RequestTimeout)]: undefined
},
_header: 'GET /en/rankings/singles HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.18.1\r\n' +
'Host: www.atptour.com\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object],
requests: {},
sockets: [Object],
freeSockets: {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'fifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
maxCachedSessions: 100,
_sessionCache: [Object],
[Symbol(kCapture)]: false
},
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/en/rankings/singles',
_ended: true,
res: IncomingMessage {
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 3,
_maxListeners: undefined,
socket: [TLSSocket],
httpVersionMajor: 1,
httpVersionMinor: 1,
httpVersion: '1.1',
complete: true,
headers: [Object],
rawHeaders: [Array],
trailers: {},
rawTrailers: [],
aborted: false,
upgrade: false,
url: '',
method: null,
statusCode: 403,
statusMessage: 'Forbidden',
client: [TLSSocket],
_consuming: true,
_dumped: false,
req: [Circular *1],
responseUrl: 'https://www.atptour.com/en/rankings/singles',
redirects: [],
[Symbol(kCapture)]: false,
[Symbol(RequestTimeout)]: undefined
},
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'www.atptour.com',
protocol: 'https:',
_redirectable: Writable {
_writableState: [WritableState],
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
_options: [Object],
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: [Circular *1],
_currentUrl: 'https://www.atptour.com/en/rankings/singles',
[Symbol(kCapture)]: false
},
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype] {
accept: [Array],
'user-agent': [Array],
host: [Array]
}
},
response: {
status: 403,
statusText: 'Forbidden',
headers: {
date: 'Fri, 21 May 2021 18:27:42 GMT',
'content-type': 'text/html; charset=UTF-8',
'transfer-encoding': 'chunked',
connection: 'close',
'cf-chl-bypass': '1',
'permissions-policy': 'accelerometer=(),autoplay=(),camera=(),clipboard-read=(),clipboard-write=(),fullscreen=(),geolocation=(),gyroscope=(),hid=(),interest-cohort=(),magnetometer=(),microphone=(),payment=(),publickey-credentials-get=(),screen-wake-lock=(),serial=(),sync-xhr=(),usb=()',
'cache-control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0',
expires: 'Thu, 01 Jan 1970 00:00:01 GMT',
'x-frame-options': 'SAMEORIGIN',
'cf-request-id': '0a31c7fa700000ee0f36b38000000001',
'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
'set-cookie': [Array],
server: 'cloudflare',
'cf-ray': '652fdc3d7c82ee0f-CDG'
},
config: {
adapter: [Function: httpAdapter],
transformRequest: [Object],
transformResponse: [Object],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers: [Object],
method: 'get',
url: 'https://www.atptour.com/en/rankings/singles',
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype],
_eventsCount: 6,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [TLSSocket],
_header: 'GET /en/rankings/singles HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.18.1\r\n' +
'Host: www.atptour.com\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/en/rankings/singles',
_ended: true,
res: [IncomingMessage],
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'www.atptour.com',
protocol: 'https:',
_redirectable: [Writable],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
While trying to assert the date entered in the following web page getAttribute('value') returning "Object (browser_, then, ...)" instead of value .
https://material.angular.io/components/datepicker/examples
following is the snippet of code giving error
expect(element(by.xpath('//input[#id="mat-input-2"]')).getAttribute('value')).to.eventually.equal(element(by.xpath('//input[#id="mat-input-2"]')).getAttribute('value')).and.notify(callback);
For that i'm getting the following error
AssertionError: expected '1/6/2019' to equal { Object (browser_, then, ...) }
var date=element(by.xpath('//input[#id="mat-input-2"]')).getAttribute('value').getText();
console.log(date);
and the console.log for the above code is returning some thing like this.
ElementFinder {
browser_:
ProtractorBrowser {
controlFlow: [Function],
schedule: [Function],
setFileDetector: [Function],
getExecutor: [Function],
getSession: [Function],
getCapabilities: [Function],
quit: [Function],
actions: [Function],
touchActions: [Function],
executeScript: [Function],
executeAsyncScript: [Function],
call: [Function],
wait: [Function],
sleep: [Function],
getWindowHandle: [Function],
getAllWindowHandles: [Function],
getPageSource: [Function],
close: [Function],
getCurrentUrl: [Function],
getTitle: [Function],
findElementInternal_: [Function],
findElementsInternal_: [Function],
takeScreenshot: [Function],
manage: [Function],
switchTo: [Function],
driver:
Driver {
flow_: [ControlFlow],
session_: [ManagedPromise],
executor_: [Executor],
fileDetector_: null,
onQuit_: undefined,
getNetworkConnection: [Function],
setNetworkConnection: [Function],
toggleAirplaneMode: [Function],
toggleWiFi: [Function],
Please provide some solutions for this
Try that:
const elementOne = element(by.xpath('//input[#id="mat-input-2"]');
const elementTwo = element(by.xpath('//input[#id="mat-input-2"]');
return elementOne.getAttribute('value').then((elementOneValue) => {
return expect(elementTwo.getAttribute('value')).to.eventually.equal(elementOneValue).and.notify(callback);
});
Btw. Both elements got same selectors - they have to be equal.
I've encountered read:ECONNRESET error while I was trying to some data through axios post.
A configuration for Axios is
const http = require('http');
const https = require('https');
const axios = require('axios').create({
//keepAlive pools and reuses TCP connections, so it's faster
httpAgent: new http.Agent({ keepAlive: true }),
httpsAgent: new https.Agent({ keepAlive: true }),
});
function to send post request is
axios.post(`${HOST}/sendEmail/`, qs.stringify(params))
.then(() => {
if (conversation.managerUnreadCount) {
console.log('[todo] conversation.managerUnreadCount ', conversation.managerUnreadCount);
}
})
.catch((err) => {
console.error('err in axios', err, url);
});
Error is below..
{ Error: read ECONNRESET
at exports._errnoException (util.js:1020:11)
at TLSWrap.onread (net.js:580:26)
code: 'ECONNRESET',
errno: 'ECONNRESET',
syscall: 'read',
config:
{ adapter: [Function: httpAdapter],
transformRequest: { '0': [Function: transformRequest] },
transformResponse: { '0': [Function: transformResponse] },
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers:
{ Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'axios/0.16.2',
'Content-Length': 737 },
httpAgent:
Agent {
domain: null,
_events: [Object],
_eventsCount: 1,
_maxListeners: undefined,
defaultPort: 80,
protocol: 'http:',
options: [Object],
requests: {},
sockets: {},
freeSockets: {},
keepAliveMsecs: 1000,
keepAlive: true,
maxSockets: Infinity,
maxFreeSockets: 256 },
httpsAgent:
Agent {
domain: null,
_events: [Object],
_eventsCount: 1,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object],
requests: {},
sockets: [Object],
freeSockets: [Object],
keepAliveMsecs: 1000,
keepAlive: true,
maxSockets: Infinity,
maxFreeSockets: 256,
maxCachedSessions: 100,
_sessionCache: [Object] },
method: 'post',
url: 'https://us-central1-customurl.cloudfunctions.net/sendEmail/',
data: 'fromEmail=testerA11%40mailinator.com&fromEmailName=tester%20A11&replyTo=testerA11%40msg.dev.hostthere.email&replyToName=tester%20A11&toEmail=hostthere.auth.test%40gmail.com&subject=test%20...*(custom data string here)*' },
request:
Writable {
_writableState:
WritableState {
objectMode: false,
highWaterMark: 16384,
needDrain: false,
ending: false,
ended: false,
finished: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function],
writecb: null,
writelen: 0,
bufferedRequest: null,
lastBufferedRequest: null,
pendingcb: 0,
prefinished: false,
errorEmitted: false,
bufferedRequestCount: 0,
corkedRequestsFree: [Object] },
writable: true,
domain: null,
_events:
{ response: [Function: handleResponse],
error: [Function: handleRequestError] },
_eventsCount: 2,
_maxListeners: undefined,
_options:
{ protocol: 'https:',
maxRedirects: 21,
maxBodyLength: 10485760,
hostname: 'us-central1-customurl.cloudfunctions.net',
port: null,
path: '/sendEmail/',
method: 'post',
headers: [Object],
agent: [Object],
auth: undefined,
nativeProtocols: [Object],
pathname: '/sendEmail/' },
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 737,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function],
_currentRequest:
ClientRequest {
domain: null,
_events: [Object],
_eventsCount: 6,
_maxListeners: undefined,
output: [],
outputEncodings: [],
outputCallbacks: [],
outputSize: 0,
writable: true,
_last: false,
upgrading: false,
chunkedEncoding: false,
shouldKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedHeader: [Object],
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
socket: [Object],
connection: [Object],
_header: 'POST /sendEmail/ HTTP/1.1\r\nAccept: application/json, text/plain, */*\r\nContent-Type: application/x-www-form-urlencoded\r\nUser-Agent: axios/0.16.2\r\nContent-Length: 737\r\nHost: us-central1-customurl.cloudfunctions.net\r\nConnection: keep-alive\r\n\r\n',
_headers: [Object],
_headerNames: [Object],
_onPendingData: null,
agent: [Object],
... }
And in my system many different post requests are made at same time.
(not same url with above)
I thought if I added keep-alive options where axios created, but it did not help.
Any other ideas to resolve this error?
Thank you for reading.
I had the same problem. Turns out I was running script from a local machine that had intermittent Internet connection. Fixing the connection solved the issue.
check error log at api side.
In my case, it was error from api side which was throwing exception and its leading to this error.
I'm receiving this error event from Watson Speech to Text when I try to pip the microphone input to it. Using NodeJS app based on this TJBot Code https://github.com/ibmtjbot/tjbot/tree/master/recipes/conversation
This is the error object that I get
=== Watson Speech to Text : An Error has occurred =====
_Event {
type: 'error',
isTrusted: false,
_yaeti: true,
target:
W3CWebSocket {
_listeners: {},
addEventListener: [Function: _addEventListener],
removeEventListener: [Function: _removeEventListener],
dispatchEvent: [Function: _dispatchEvent],
_url: 'wss://stream.watsonplatform.net/speech-to-text/api/v1/recognize?model=en-US_BroadbandModel',
_readyState: 3,
_protocol: undefined,
_extensions: '',
_bufferedAmount: 0,
_binaryType: 'arraybuffer',
_connection: undefined,
_client:
WebSocketClient {
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined,
config: [Object],
_req: null,
protocols: [],
origin: null,
url: [Object],
secure: true,
base64nonce: 'XU/MDifsB9QqtrWn4vPFgw==' },
onerror: [Function],
onopen: [Function],
onclose: [Function],
onmessage: [Function] },
cancelable: true,
stopImmediatePropagation
: [Function] }
Press +C to exit.
This was working fine until some hours ago but suddenly stopped working. Any idea on what this error means ?
I just found my credentials were wrong. It seems that during migration to DIEGO the credentials at VCAP_SERVICES got deleted or something like that. I connected the services to the app again on Bluemix and the credentials got recreated
I want to extends query method of model, this is what i wrote:
store.query.expose('todos', 'forGroup', function(group) {
return this.where('group').equals(group);
});
but when i run the app, console says "Cannot call method expose of undefined", which means store.query is undefined.Why is that?
the store variable is:
var store = derby.createStore({
db: liveDbMongo(mongoUrl + '?auto_reconnect', {safe: true})
, redis: redis
});
And i also output the store using console.log:
{ domain: null,
_events: { client: [Function] },
_maxListeners: 10,
racer:
{ get: [Function],
_plugins: [ [Function: derbyServer] ],
run: [Function: run],
createApp: [Function: createApp],
createStatic: [Function: createStatic] },
modelOptions: undefined,
shareClient:
{ options: { db: [Object], redis: [Object] },
preValidate: undefined,
validate: undefined,
backend:
{ snapshotDb: [Object],
getOps: [Function],
publish: [Function],
submit: [Function],
_subscribeChannels: [Function],
subscribe: [Function],
fetch: [Function],
bulkFetchCached: [Function],
fetchAndSubscribe: [Function],
queryFetch: [Function],
query: [Function],
collection: [Function],
destroy: [Function] },
extensions: { '': [] },
docFilters: [],
opFilters: [] },
logger: undefined }
and console.log(store.query) outputs undefined.
The Access Control Api was changed in the 0.5 version of Racer, which is mentioned here http://blog.derbyjs.com/2013/06/04/migrating-from-derby-0-dot-3-to-0-dot-5/
I have not played around that much with authentication, but a pointer is to look through the Auth example on GitHub - https://github.com/codeparty/derby-examples/tree/master/auth.