Related
I am new to e2e tests and nestjs.
I am trying to perform a simple e2e test but I keep getting this error:
AppController (e2e) › / (POST - create new user preferences)
expect(received).toBe(expected) // Object.is equality
Expected: 201
Received: 404
46 | console.log('res: ', res);
47 | // expect(res.body._id).toBeDefined();
> 48 | expect(res.status).toBe(201);
| ^
49 | });
50 | });
51 |
as far as I understand, and after reading some answers here. the main cause for this issue is that the URL that was provided was wrong.
of course I have an env.test and I have configured the test process on e2e run.
Furthermore I see in mongo atlas that a new collection is created but it contains no data even when I remove the mongoose.connection.db.dropDatabase();.
this is the e2e test file content:
import { Test, TestingModule } from '#nestjs/testing';
import { INestApplication } from '#nestjs/common';
import * as request from 'supertest';
import { AppModule } from './../src/app.module';
import * as mongoose from 'mongoose';
describe('AppController (e2e)', () => {
let app: INestApplication;
beforeEach(async () => {
const moduleFixture: TestingModule = await Test.createTestingModule({
imports: [AppModule],
}).compile();
app = moduleFixture.createNestApplication();
await app.init();
});
beforeAll(() => {
mongoose.connect(
`mongodb+srv://${process.env.DB_USER}:${process.env.DB_PASS}#${process.env.DB_NAME}.hucjifz.mongodb.net/${process.env.DB_COLLECTION}?retryWrites=true&w=majority`,
function () {
mongoose.connection.db.dropDatabase();
},
);
});
afterAll(() => mongoose.disconnect());
const eUserIdeMock = '123456';
const userPreferencesMock = {
uiTheme: 'dark',
panelWidth: 300,
editingHandles: false,
enableLightboxInEditor: true,
hiddenElements: true,
defaultDeviceView: 'mobile',
exitTo: 'dashboard',
};
const data = {
eUserIdeMock,
userPreferencesMock,
};
it('/ (POST - create new user preferences)', () => {
return request(app.getHttpServer())
.post('/api/v1/user-preferences')
.send(data)
.expect(201)
.then((res) => {
console.log('res: ', res);
// expect(res.body._id).toBeDefined();
expect(res.status).toBe(201);
});
});
it('/ (GET)', () => {
return request(app.getHttpServer())
.get('/api/v1/user-preferences')
.send(eUserIdeMock)
.expect(200);
});
});
this is the API I use to in postman:
{{DOMAIN}}/api/v1/user-preferences
and I perform CRUD tasks without any issue.
maybe this can help, this is the entire output of the res:
<ref *2> Response {
_events: [Object: null prototype] {},
_eventsCount: 0,
_maxListeners: undefined,
res: <ref *1> IncomingMessage {
_readableState: ReadableState {
objectMode: false,
highWaterMark: 16384,
buffer: BufferList { head: null, tail: null, length: 0 },
length: 0,
pipes: [],
flowing: true,
ended: true,
endEmitted: true,
reading: false,
sync: true,
needReadable: false,
emittedReadable: false,
readableListening: false,
resumeScheduled: false,
errorEmitted: false,
emitClose: true,
autoDestroy: false,
destroyed: false,
errored: null,
closed: false,
closeEmitted: false,
defaultEncoding: 'utf8',
awaitDrainWriters: null,
multiAwaitDrain: false,
readingMore: true,
dataEmitted: true,
decoder: [StringDecoder],
encoding: 'utf8',
[Symbol(kPaused)]: false
},
_events: [Object: null prototype] {
end: [Array],
data: [Array],
error: [Array],
close: [Function: bound emit]
},
_eventsCount: 4,
_maxListeners: undefined,
socket: Socket {
connecting: false,
_hadError: false,
_parent: null,
_host: null,
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: null,
_server: null,
parser: null,
_httpMessage: [ClientRequest],
[Symbol(async_id_symbol)]: 232,
[Symbol(kHandle)]: [TCP],
[Symbol(kSetNoDelay)]: true,
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0,
[Symbol(RequestTimeout)]: undefined
},
httpVersionMajor: 1,
httpVersionMinor: 1,
httpVersion: '1.1',
complete: true,
headers: {
'x-powered-by': 'Express',
'content-type': 'application/json; charset=utf-8',
'content-length': '87',
etag: 'W/"57-fJKdAAcKjvI7+1q+kvS8il37LfM"',
date: 'Mon, 27 Jun 2022 07:40:21 GMT',
connection: 'close'
},
rawHeaders: [
'X-Powered-By',
'Express',
'Content-Type',
'application/json; charset=utf-8',
'Content-Length',
'87',
'ETag',
'W/"57-fJKdAAcKjvI7+1q+kvS8il37LfM"',
'Date',
'Mon, 27 Jun 2022 07:40:21 GMT',
'Connection',
'close'
],
trailers: {},
rawTrailers: [],
aborted: false,
upgrade: false,
url: '',
method: null,
statusCode: 404,
statusMessage: 'Not Found',
client: Socket {
connecting: false,
_hadError: false,
_parent: null,
_host: null,
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: null,
_server: null,
parser: null,
_httpMessage: [ClientRequest],
[Symbol(async_id_symbol)]: 232,
[Symbol(kHandle)]: [TCP],
[Symbol(kSetNoDelay)]: true,
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0,
[Symbol(RequestTimeout)]: undefined
},
_consuming: false,
_dumped: false,
req: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 3,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 208,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [Socket],
_header: 'POST /api/v1/user-preferences HTTP/1.1\r\n' +
'Host: 127.0.0.1:52789\r\n' +
'Accept-Encoding: gzip, deflate\r\n' +
'Content-Type: application/json\r\n' +
'Content-Length: 208\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/api/v1/user-preferences',
_ended: true,
res: [Circular *1],
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: '127.0.0.1',
protocol: 'http:',
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
text: '{"statusCode":404,"message":"Cannot POST /api/v1/user-preferences","error":"Not Found"}',
[Symbol(kCapture)]: false,
[Symbol(RequestTimeout)]: undefined
},
request: Test {
_events: [Object: null prototype] { abort: [Function (anonymous)] },
_eventsCount: 1,
_maxListeners: undefined,
_enableHttp2: false,
_agent: false,
_formData: null,
method: 'POST',
url: 'http://127.0.0.1:52789/api/v1/user-preferences',
_header: { 'content-type': 'application/json' },
header: { 'Content-Type': 'application/json' },
writable: true,
_redirects: 0,
_maxRedirects: 0,
cookies: '',
qs: {},
_query: [],
qsRaw: [],
_redirectList: [],
_streamRequest: false,
_lookup: undefined,
_buffer: true,
app: Server {
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
_connections: 0,
_handle: null,
_usingWorkers: false,
_workers: [],
_unref: false,
allowHalfOpen: true,
pauseOnConnect: false,
httpAllowHalfOpen: false,
timeout: 0,
keepAliveTimeout: 5000,
maxHeadersCount: null,
headersTimeout: 60000,
requestTimeout: 0,
_connectionKey: '6::::0',
[Symbol(IncomingMessage)]: [Function: IncomingMessage],
[Symbol(ServerResponse)]: [Function: ServerResponse],
[Symbol(kCapture)]: false,
[Symbol(async_id_symbol)]: 230
},
_asserts: [],
_server: Server {
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
_connections: 0,
_handle: null,
_usingWorkers: false,
_workers: [],
_unref: false,
allowHalfOpen: true,
pauseOnConnect: false,
httpAllowHalfOpen: false,
timeout: 0,
keepAliveTimeout: 5000,
maxHeadersCount: null,
headersTimeout: 60000,
requestTimeout: 0,
_connectionKey: '6::::0',
[Symbol(IncomingMessage)]: [Function: IncomingMessage],
[Symbol(ServerResponse)]: [Function: ServerResponse],
[Symbol(kCapture)]: false,
[Symbol(async_id_symbol)]: 230
},
_data: { eUserIdeMock: '123456', userPreferencesMock: [Object] },
req: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 3,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 208,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [Socket],
_header: 'POST /api/v1/user-preferences HTTP/1.1\r\n' +
'Host: 127.0.0.1:52789\r\n' +
'Accept-Encoding: gzip, deflate\r\n' +
'Content-Type: application/json\r\n' +
'Content-Length: 208\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/api/v1/user-preferences',
_ended: true,
res: [IncomingMessage],
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: '127.0.0.1',
protocol: 'http:',
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
protocol: 'http:',
host: '127.0.0.1:52789',
_endCalled: true,
_callback: [Function (anonymous)],
_fullfilledPromise: Promise { [Circular *2] },
res: <ref *1> IncomingMessage {
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 4,
_maxListeners: undefined,
socket: [Socket],
httpVersionMajor: 1,
httpVersionMinor: 1,
httpVersion: '1.1',
complete: true,
headers: [Object],
rawHeaders: [Array],
trailers: {},
rawTrailers: [],
aborted: false,
upgrade: false,
url: '',
method: null,
statusCode: 404,
statusMessage: 'Not Found',
client: [Socket],
_consuming: false,
_dumped: false,
req: [ClientRequest],
text: '{"statusCode":404,"message":"Cannot POST /api/v1/user-preferences","error":"Not Found"}',
[Symbol(kCapture)]: false,
[Symbol(RequestTimeout)]: undefined
},
_resBuffered: true,
response: [Circular *2],
called: true,
[Symbol(kCapture)]: false
},
req: <ref *3> ClientRequest {
_events: [Object: null prototype] {
drain: [Function],
error: [Function (anonymous)],
prefinish: [Function: requestOnPrefinish]
},
_eventsCount: 3,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 208,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: Socket {
connecting: false,
_hadError: false,
_parent: null,
_host: null,
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: null,
_server: null,
parser: null,
_httpMessage: [Circular *3],
[Symbol(async_id_symbol)]: 232,
[Symbol(kHandle)]: [TCP],
[Symbol(kSetNoDelay)]: true,
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0,
[Symbol(RequestTimeout)]: undefined
},
_header: 'POST /api/v1/user-preferences HTTP/1.1\r\n' +
'Host: 127.0.0.1:52789\r\n' +
'Accept-Encoding: gzip, deflate\r\n' +
'Content-Type: application/json\r\n' +
'Content-Length: 208\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 80,
protocol: 'http:',
options: [Object],
requests: {},
sockets: [Object],
freeSockets: {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'lifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
[Symbol(kCapture)]: false
},
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/api/v1/user-preferences',
_ended: true,
res: <ref *1> IncomingMessage {
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 4,
_maxListeners: undefined,
socket: [Socket],
httpVersionMajor: 1,
httpVersionMinor: 1,
httpVersion: '1.1',
complete: true,
headers: [Object],
rawHeaders: [Array],
trailers: {},
rawTrailers: [],
aborted: false,
upgrade: false,
url: '',
method: null,
statusCode: 404,
statusMessage: 'Not Found',
client: [Socket],
_consuming: false,
_dumped: false,
req: [Circular *3],
text: '{"statusCode":404,"message":"Cannot POST /api/v1/user-preferences","error":"Not Found"}',
[Symbol(kCapture)]: false,
[Symbol(RequestTimeout)]: undefined
},
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: '127.0.0.1',
protocol: 'http:',
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype] {
host: [Array],
'accept-encoding': [Array],
'content-type': [Array],
'content-length': [Array]
}
},
text: '{"statusCode":404,"message":"Cannot POST /api/v1/user-preferences","error":"Not Found"}',
files: undefined,
buffered: true,
headers: {
'x-powered-by': 'Express',
'content-type': 'application/json; charset=utf-8',
'content-length': '87',
etag: 'W/"57-fJKdAAcKjvI7+1q+kvS8il37LfM"',
date: 'Mon, 27 Jun 2022 07:40:21 GMT',
connection: 'close'
},
header: {
'x-powered-by': 'Express',
'content-type': 'application/json; charset=utf-8',
'content-length': '87',
etag: 'W/"57-fJKdAAcKjvI7+1q+kvS8il37LfM"',
date: 'Mon, 27 Jun 2022 07:40:21 GMT',
connection: 'close'
},
statusCode: 404,
status: 404,
statusType: 4,
info: false,
ok: false,
redirect: false,
clientError: true,
serverError: false,
error: Error: cannot POST /api/v1/user-preferences (404)
at Response.Object.<anonymous>.Response.toError (/Users/davids/Developmet/editor-sync-user-preferences/node_modules/superagent/src/node/response.js:110:17)
at Response.toError [as _setStatusProperties] (/Users/davids/Developmet/editor-sync-user-preferences/node_modules/superagent/src/response-base.js:107:48)
at new _setStatusProperties (/Users/davids/Developmet/editor-sync-user-preferences/node_modules/superagent/src/node/response.js:41:8)
at Test.Object.<anonymous>.Request._emitResponse (/Users/davids/Developmet/editor-sync-user-preferences/node_modules/superagent/src/node/index.js:952:20)
at _emitResponse (/Users/davids/Developmet/editor-sync-user-preferences/node_modules/superagent/src/node/index.js:1153:38)
at IncomingMessage.fn (/Users/davids/Developmet/editor-sync-user-preferences/node_modules/superagent/src/node/parsers/json.js:19:7)
at IncomingMessage.emit (events.js:412:35)
at endReadableNT (internal/streams/readable.js:1334:12)
at processTicksAndRejections (internal/process/task_queues.js:82:21) {
status: 404,
text: '{"statusCode":404,"message":"Cannot POST /api/v1/user-preferences","error":"Not Found"}',
method: 'POST',
path: '/api/v1/user-preferences'
},
created: false,
accepted: false,
noContent: false,
badRequest: false,
unauthorized: false,
notAcceptable: false,
forbidden: false,
notFound: true,
unprocessableEntity: false,
type: 'application/json',
charset: 'utf-8',
links: {},
setEncoding: [Function: bound ],
redirects: [],
_body: {
statusCode: 404,
message: 'Cannot POST /api/v1/user-preferences',
error: 'Not Found'
},
pipe: [Function (anonymous)],
[Symbol(kCapture)]: false
}
The problem was that the my api contain two parts:
globalPrefix:/api/v1
controller endpoint: /user-preferences
apperantley the test can read only the controller endpoint.
now the code looks like this:
it('/ (POST - create new user preferences)', () => {
return request(app.getHttpServer())
.post('/user-preferences') // this is the change that was made!
.send(data)
.expect(201)
.then((res) => {
console.log('res: ', res);
// expect(res.body._id).toBeDefined();
expect(res.status).toBe(201);
});
});
instead of this:
it('/ (POST - create new user preferences)', () => {
return request(app.getHttpServer())
.post('/api/v1/user-preferences')
.send(data)
.expect(201)
.then((res) => {
console.log('res: ', res);
// expect(res.body._id).toBeDefined();
expect(res.status).toBe(201);
});
});
When I tried running code on my local mission by npm run dev, I am getting these error, tried changing code as per error. When I tried to console data it showing unreachable to call, tried with some code changes but did not work. Initially there was autoprefixer error rectified it but now this error pops up not able to solve any suggestions will be appreciated
import {comps as newsComps} from 'ui-comps-news'
import {useEffect} from 'react'
import pageLoad from '../lib/pageLoad'
import UI from '../components/UI'
import PCMPreview from '../containers/PCMPreview'
import { useDispatch } from 'react-redux'
import {loadData} from '../actions';
export default function Index(props) {
const dispatch = useDispatch();
const elements = [];
Object.keys(newsComps).forEach(key => {
elements.push({key, value: newsComps[key]});
});
const layout = JSON.parse(props.dataStr);
const dataObj = {layout, elements, options: {}, styles: {}}
dispatch(loadData({layout: dataObj.layout}));
useEffect(() => {
PCMPreview.init({dispatch, data: dataObj});
}, []);
return <UI {...dataObj} />
}
export async function getServerSideProps(context) {
const { variant } = context.query;
const data = await pageLoad(variant);
return { props: { dataStr : console.log(JSON.stringify(data.layout)) } }
}
Error I am getting when running on localhost:3000:
Error: connect ECONNREFUSED 127.0.0.1:9001
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1133:16) {
errno: -4078,
code: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 9001,
config: {
url: 'http://localhost:9001/page?variant=undefined',
method: 'get',
headers: {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/0.21.1'
},
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
adapter: [Function: httpAdapter],
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
validateStatus: [Function: validateStatus],
data: undefined
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError]
},
_eventsCount: 2,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'http:',
path: '/page?variant=undefined',
method: 'GET',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'localhost',
port: '9001',
nativeProtocols: [Object],
pathname: '/page',
search: '?variant=undefined'
},
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
_closed: false,
socket: [Socket],
_header: 'GET /page?variant=undefined HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.21.1\r\n' +
'Host: localhost:9001\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/page?variant=undefined',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'localhost',
protocol: 'http:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: 'http://localhost:9001/page?variant=undefined',
[Symbol(kCapture)]: false
},
response: undefined,
isAxiosError: true,
toJSON: [Function: toJSON]
}
TypeError: Cannot read property 'layout' of undefined
at _callee$ (C:\Users\computer\Desktop\news-web\.next\server\pages\index.js:3664:58)
at tryCatch (C:\Users\computer\Desktop\news-web\node_modules\regenerator-runtime\runtime.js:63:40)
at Generator.invoke [as _invoke] (C:\Users\computer\Desktop\news-web\node_modules\regenerator-runtime\runtime.js:293:22)
at Generator.next (C:\Users\computer\Desktop\news-web\node_modules\regenerator-runtime\runtime.js:118:21)
at asyncGeneratorStep (C:\Users\computer\Desktop\news-web\.next\server\pages\index.js:3607:103)
at _next (C:\Users\computer\Desktop\news-web\.next\server\pages\index.js:3609:194)
at processTicksAndRejections (node:internal/process/task_queues:96:5)
I'm trying to scrape data from atp ranking site:
https://www.atptour.com/en/rankings/singles
So I tried to do a GET on this URL, but I have this error code: "403 Forbidden" with Postman. I don't have this error code on other websites like https://www.worldometers.info/coronavirus/.
ATP do not have API, so I want to scrape the website but it seems scraping is not allowed on this website, do you have a solution ?
Thanks
error code
The problem is that I don't know the headers to be included in the request.
Sometimes it works in Postman but i still get the error in my express JS application:
config: {
adapter: [Function: httpAdapter],
transformRequest: { '0': [Function: transformRequest] },
transformResponse: { '0': [Function: transformResponse] },
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/0.18.1'
},
method: 'get',
url: 'https://www.atptour.com/en/rankings/singles',
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype] {
socket: [Function (anonymous)],
abort: [Function (anonymous)],
aborted: [Function (anonymous)],
error: [Function (anonymous)],
timeout: [Function (anonymous)],
prefinish: [Function: requestOnPrefinish]
},
_eventsCount: 6,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: TLSSocket {
_tlsOptions: [Object],
_secureEstablished: true,
_securePending: false,
_newSessionPending: false,
_controlReleased: true,
secureConnecting: false,
_SNICallback: null,
servername: 'www.atptour.com',
alpnProtocol: false,
authorized: true,
authorizationError: null,
encrypted: true,
_events: [Object: null prototype],
_eventsCount: 10,
connecting: false,
_hadError: false,
_parent: null,
_host: 'www.atptour.com',
_readableState: [ReadableState],
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: undefined,
_server: null,
ssl: [TLSWrap],
_requestCert: true,
_rejectUnauthorized: true,
parser: null,
_httpMessage: [Circular *1],
[Symbol(res)]: [TLSWrap],
[Symbol(verified)]: true,
[Symbol(pendingSession)]: null,
[Symbol(async_id_symbol)]: 13,
[Symbol(kHandle)]: [TLSWrap],
[Symbol(kSetNoDelay)]: false,
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0,
[Symbol(connect-options)]: [Object],
[Symbol(RequestTimeout)]: undefined
},
_header: 'GET /en/rankings/singles HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.18.1\r\n' +
'Host: www.atptour.com\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object],
requests: {},
sockets: [Object],
freeSockets: {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'fifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
maxCachedSessions: 100,
_sessionCache: [Object],
[Symbol(kCapture)]: false
},
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/en/rankings/singles',
_ended: true,
res: IncomingMessage {
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 3,
_maxListeners: undefined,
socket: [TLSSocket],
httpVersionMajor: 1,
httpVersionMinor: 1,
httpVersion: '1.1',
complete: true,
headers: [Object],
rawHeaders: [Array],
trailers: {},
rawTrailers: [],
aborted: false,
upgrade: false,
url: '',
method: null,
statusCode: 403,
statusMessage: 'Forbidden',
client: [TLSSocket],
_consuming: true,
_dumped: false,
req: [Circular *1],
responseUrl: 'https://www.atptour.com/en/rankings/singles',
redirects: [],
[Symbol(kCapture)]: false,
[Symbol(RequestTimeout)]: undefined
},
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'www.atptour.com',
protocol: 'https:',
_redirectable: Writable {
_writableState: [WritableState],
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
_options: [Object],
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: [Circular *1],
_currentUrl: 'https://www.atptour.com/en/rankings/singles',
[Symbol(kCapture)]: false
},
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype] {
accept: [Array],
'user-agent': [Array],
host: [Array]
}
},
response: {
status: 403,
statusText: 'Forbidden',
headers: {
date: 'Fri, 21 May 2021 18:27:42 GMT',
'content-type': 'text/html; charset=UTF-8',
'transfer-encoding': 'chunked',
connection: 'close',
'cf-chl-bypass': '1',
'permissions-policy': 'accelerometer=(),autoplay=(),camera=(),clipboard-read=(),clipboard-write=(),fullscreen=(),geolocation=(),gyroscope=(),hid=(),interest-cohort=(),magnetometer=(),microphone=(),payment=(),publickey-credentials-get=(),screen-wake-lock=(),serial=(),sync-xhr=(),usb=()',
'cache-control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0',
expires: 'Thu, 01 Jan 1970 00:00:01 GMT',
'x-frame-options': 'SAMEORIGIN',
'cf-request-id': '0a31c7fa700000ee0f36b38000000001',
'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
'set-cookie': [Array],
server: 'cloudflare',
'cf-ray': '652fdc3d7c82ee0f-CDG'
},
config: {
adapter: [Function: httpAdapter],
transformRequest: [Object],
transformResponse: [Object],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers: [Object],
method: 'get',
url: 'https://www.atptour.com/en/rankings/singles',
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype],
_eventsCount: 6,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [TLSSocket],
_header: 'GET /en/rankings/singles HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.18.1\r\n' +
'Host: www.atptour.com\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/en/rankings/singles',
_ended: true,
res: [IncomingMessage],
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'www.atptour.com',
protocol: 'https:',
_redirectable: [Writable],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
I've encountered read:ECONNRESET error while I was trying to some data through axios post.
A configuration for Axios is
const http = require('http');
const https = require('https');
const axios = require('axios').create({
//keepAlive pools and reuses TCP connections, so it's faster
httpAgent: new http.Agent({ keepAlive: true }),
httpsAgent: new https.Agent({ keepAlive: true }),
});
function to send post request is
axios.post(`${HOST}/sendEmail/`, qs.stringify(params))
.then(() => {
if (conversation.managerUnreadCount) {
console.log('[todo] conversation.managerUnreadCount ', conversation.managerUnreadCount);
}
})
.catch((err) => {
console.error('err in axios', err, url);
});
Error is below..
{ Error: read ECONNRESET
at exports._errnoException (util.js:1020:11)
at TLSWrap.onread (net.js:580:26)
code: 'ECONNRESET',
errno: 'ECONNRESET',
syscall: 'read',
config:
{ adapter: [Function: httpAdapter],
transformRequest: { '0': [Function: transformRequest] },
transformResponse: { '0': [Function: transformResponse] },
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers:
{ Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'axios/0.16.2',
'Content-Length': 737 },
httpAgent:
Agent {
domain: null,
_events: [Object],
_eventsCount: 1,
_maxListeners: undefined,
defaultPort: 80,
protocol: 'http:',
options: [Object],
requests: {},
sockets: {},
freeSockets: {},
keepAliveMsecs: 1000,
keepAlive: true,
maxSockets: Infinity,
maxFreeSockets: 256 },
httpsAgent:
Agent {
domain: null,
_events: [Object],
_eventsCount: 1,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object],
requests: {},
sockets: [Object],
freeSockets: [Object],
keepAliveMsecs: 1000,
keepAlive: true,
maxSockets: Infinity,
maxFreeSockets: 256,
maxCachedSessions: 100,
_sessionCache: [Object] },
method: 'post',
url: 'https://us-central1-customurl.cloudfunctions.net/sendEmail/',
data: 'fromEmail=testerA11%40mailinator.com&fromEmailName=tester%20A11&replyTo=testerA11%40msg.dev.hostthere.email&replyToName=tester%20A11&toEmail=hostthere.auth.test%40gmail.com&subject=test%20...*(custom data string here)*' },
request:
Writable {
_writableState:
WritableState {
objectMode: false,
highWaterMark: 16384,
needDrain: false,
ending: false,
ended: false,
finished: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function],
writecb: null,
writelen: 0,
bufferedRequest: null,
lastBufferedRequest: null,
pendingcb: 0,
prefinished: false,
errorEmitted: false,
bufferedRequestCount: 0,
corkedRequestsFree: [Object] },
writable: true,
domain: null,
_events:
{ response: [Function: handleResponse],
error: [Function: handleRequestError] },
_eventsCount: 2,
_maxListeners: undefined,
_options:
{ protocol: 'https:',
maxRedirects: 21,
maxBodyLength: 10485760,
hostname: 'us-central1-customurl.cloudfunctions.net',
port: null,
path: '/sendEmail/',
method: 'post',
headers: [Object],
agent: [Object],
auth: undefined,
nativeProtocols: [Object],
pathname: '/sendEmail/' },
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 737,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function],
_currentRequest:
ClientRequest {
domain: null,
_events: [Object],
_eventsCount: 6,
_maxListeners: undefined,
output: [],
outputEncodings: [],
outputCallbacks: [],
outputSize: 0,
writable: true,
_last: false,
upgrading: false,
chunkedEncoding: false,
shouldKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedHeader: [Object],
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
socket: [Object],
connection: [Object],
_header: 'POST /sendEmail/ HTTP/1.1\r\nAccept: application/json, text/plain, */*\r\nContent-Type: application/x-www-form-urlencoded\r\nUser-Agent: axios/0.16.2\r\nContent-Length: 737\r\nHost: us-central1-customurl.cloudfunctions.net\r\nConnection: keep-alive\r\n\r\n',
_headers: [Object],
_headerNames: [Object],
_onPendingData: null,
agent: [Object],
... }
And in my system many different post requests are made at same time.
(not same url with above)
I thought if I added keep-alive options where axios created, but it did not help.
Any other ideas to resolve this error?
Thank you for reading.
I had the same problem. Turns out I was running script from a local machine that had intermittent Internet connection. Fixing the connection solved the issue.
check error log at api side.
In my case, it was error from api side which was throwing exception and its leading to this error.
I want to extends query method of model, this is what i wrote:
store.query.expose('todos', 'forGroup', function(group) {
return this.where('group').equals(group);
});
but when i run the app, console says "Cannot call method expose of undefined", which means store.query is undefined.Why is that?
the store variable is:
var store = derby.createStore({
db: liveDbMongo(mongoUrl + '?auto_reconnect', {safe: true})
, redis: redis
});
And i also output the store using console.log:
{ domain: null,
_events: { client: [Function] },
_maxListeners: 10,
racer:
{ get: [Function],
_plugins: [ [Function: derbyServer] ],
run: [Function: run],
createApp: [Function: createApp],
createStatic: [Function: createStatic] },
modelOptions: undefined,
shareClient:
{ options: { db: [Object], redis: [Object] },
preValidate: undefined,
validate: undefined,
backend:
{ snapshotDb: [Object],
getOps: [Function],
publish: [Function],
submit: [Function],
_subscribeChannels: [Function],
subscribe: [Function],
fetch: [Function],
bulkFetchCached: [Function],
fetchAndSubscribe: [Function],
queryFetch: [Function],
query: [Function],
collection: [Function],
destroy: [Function] },
extensions: { '': [] },
docFilters: [],
opFilters: [] },
logger: undefined }
and console.log(store.query) outputs undefined.
The Access Control Api was changed in the 0.5 version of Racer, which is mentioned here http://blog.derbyjs.com/2013/06/04/migrating-from-derby-0-dot-3-to-0-dot-5/
I have not played around that much with authentication, but a pointer is to look through the Auth example on GitHub - https://github.com/codeparty/derby-examples/tree/master/auth.