When I run Protractor, it seems like I get terminal printouts of every single GET request it makes.
Test Suites & Specs:
1) App
[Sun Apr 30 2017 18:43:12 GMT+0200 (Vest-Europa (sommertid))] "GET /" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:12 GMT+0200 (Vest-Europa (sommertid))] "GET /polyfills.dll.js" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:12 GMT+0200 (Vest-Europa (sommertid))] "GET /vendor.dll.js" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:12 GMT+0200 (Vest-Europa (sommertid))] "GET /polyfills.bundle.js" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:12 GMT+0200 (Vest-Europa (sommertid))] "GET /vendor.bundle.js" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:12 GMT+0200 (Vest-Europa (sommertid))] "GET /main.bundle.js" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /0.chunk.js""Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /1.chunk.js""Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /assets/img/angularclass-avatar.png" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /5.chunk.js""Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /6.chunk.js""Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /assets/icon/favicon-16x16.png" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /assets/icon/favicon-96x96.png" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /assets/icon/favicon-32x32.png" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /assets/icon/android-icon-192x192.png" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
√ should have a title
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
[Sun Apr 30 2017 18:43:13 GMT+0200 (Vest-Europa (sommertid))] "GET /polyfills.dll.js" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"
...
How can I hide these?
I have added various Jasmine reporters, and would like to only see the results from them. As it is, all these resource requests make the console a mess.
Edit:
My protractor.conf.js (from Angular 2 Webpack starter with some irrelevant reporter additions)
require('ts-node/register');
var helpers = require('./helpers');
exports.config = {
baseUrl: 'http://localhost:3000/',
// use `npm run e2e`
specs: [
helpers.root('src/**/**.e2e.ts'),
helpers.root('src/**/*.e2e.ts')
],
exclude: [],
framework: 'jasmine2',
allScriptsTimeout: 110000,
jasmineNodeOpts: {
showTiming: true,
showColors: true,
isVerbose: false,
includeStackTrace: false,
defaultTimeoutInterval: 400000,
print: function() {}
},
directConnect: true,
capabilities: {
'browserName': 'chrome',
'chromeOptions': {
'args': ['show-fps-counter=true']
}
},
onPrepare: function() {
browser.ignoreSynchronization = true;
const reporters = require('jasmine-reporters');
const Jasmine2HtmlReporter = require('protractor-jasmine2-html-reporter');
const JasmineConsoleReporter = require('jasmine-console-reporter');
jasmine.getEnv().addReporter(new reporters.JUnitXmlReporter('reports', true, true));
jasmine.getEnv().addReporter(
new JasmineConsoleReporter({
colors: 1,
cleanStack: 1,
verbosity: 4,
listStyle: 'indent',
activity: false
})
);
jasmine.getEnv().addReporter(
new Jasmine2HtmlReporter({
savePath: 'target/screenshots',
takeScreenshots: true,
takeScreenshotsOnlyOnFailures: true
})
);
},
useAllAngular2AppRoots: true
};
Started from npm scripts
"e2e": "npm-run-all -p -r server:prod:ci protractor",
"server:prod:ci": "http-server dist -p 3000 -c-1 --cors",
"protractor": "protractor",
Related
I am debugging http requests to our server and decided to try Dio dart package. After some trials (with no difference in results from standard http packages), I decided to stop using the Dio package.
I though happen to notice extraneous requests from random location (traced back to China telecom). Considering we are only trying to setup the server, and the requests started showing up only after I used Dio in my flutter app - Is DIO snooping on my server?
Seen on Server
X-Forwarded-Protocol: https
X-Real-Ip: 183.136.225.35
Host: 0.0.0.0:5002
Connection: close
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE
Accept: */*
Referer: ******
Accept-Encoding: gzip
2022-10-06 15:06:06,768 [DEBUG] root:
X-Forwarded-Protocol: https
X-Real-Ip: 45.79.204.46
Host: 0.0.0.0:5002
Connection: close
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36
Accept: */*
Referer: *****
Accept-Encoding: gzip
Traceroute on IP
4 142 ms 141 ms 153 ms 116.119.68.60
5 140 ms 138 ms 140 ms be6391.rcr21.b015591-1.lon13.atlas.cogentco.com [149.14.224.161]
6 139 ms 139 ms 139 ms be2053.ccr41.lon13.atlas.cogentco.com [130.117.2.65]
7 144 ms 142 ms 142 ms 154.54.61.158
8 191 ms 190 ms 190 ms chinatelecom.demarc.cogentco.com [149.14.81.226]
9 299 ms * 299 ms 202.97.13.18
10 * 316 ms * 202.97.90.30
11 * 317 ms * 202.97.24.141
12 * * * Request timed out.
13 317 ms 308 ms 320 ms 220.191.200.166
14 334 ms 354 ms * 115.233.128.133
15 * * * Request timed out.
16 * * * Request timed out.
17 * * * Request timed out.
18 * * * Request timed out.
19 * * * Request timed out.
20 * * * Request timed out.
21 325 ms 325 ms 333 ms 183.136.225.35
I am trying to parse a log file with millions of records. It contains host name, timestamp, status code etc. After successfully parsing host and status code and url, when I am trying to parse timestamp, I am getting an error. following is my code:
lines=sc.textFile(filepath)
df_log= lines.map(lambda x: Row(header=x)).toDF()
timestamp_pattern= r'\[\d{2}\/\w{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2}\s\S+\d{4}]'
df2=df_log.select(regexp_extract(col('header'),timestamp_pattern,1).alias("timestamp"))
everything is working fine till here. after this when I am trying to df2.show(10). I am getting following error:
Py4JJavaErrorTraceback (most recent call last) <ipython-input-112-5a86d13b2926> in <module>()
----> 1 df2.show(1)
/opt/cloudera/parcels/SPARK2/lib/spark2/python/pyspark/sql/dataframe.pyc in show(self, n, truncate)
316 """
317 if isinstance(truncate, bool) and truncate:
--> 318 print(self._jdf.showString(n, 20))
319 else:
320 print(self._jdf.showString(n, int(truncate)))
/opt/cloudera/parcels/SPARK2/lib/spark2/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in __call__(self, *args) 1131 answer = self.gateway_client.send_command(command) 1132 return_value
= get_return_value(
-> 1133 answer, self.gateway_client, self.target_id, self.name) 1134 1135 for temp_arg in temp_args:
/opt/cloudera/parcels/SPARK2/lib/spark2/python/pyspark/sql/utils.pyc in deco(*a, **kw)
61 def deco(*a, **kw):
62 try:
---> 63 return f(*a, **kw)
64 except py4j.protocol.Py4JJavaError as e:
65 s = e.java_exception.toString()
/opt/cloudera/parcels/SPARK2/lib/spark2/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
317 raise Py4JJavaError(
318 "An error occurred while calling {0}{1}{2}.\n".
--> 319 format(target_id, ".", name), value)
320 else:
321 raise Py4JError(
Py4JJavaError: An error occurred while calling o965.showString. : org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 2.0 failed 4 times, most recent failure: Lost task 0.3 in stage 2.0 (TID 5, ip-20-0-31-210.ec2.internal, executor 2): java.lang.IndexOutOfBoundsException: No group 1 at java.util.regex.Matcher.group(Matcher.java:538) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:377) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:231) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:225) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:826) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:826) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at org.apache.spark.scheduler.Task.run(Task.scala:99) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1430) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1418) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1417) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1417) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:797) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:797) at scala.Option.foreach(Option.scala:257) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:797) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1645) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1600) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1589) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:623) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1930) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1943) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1956) at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:333) at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38) at org.apache.spark.sql.Dataset$$anonfun$org$apache$spark$sql$Dataset$$execute$1$1.apply(Dataset.scala:2378) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57) at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2772) at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$execute$1(Dataset.scala:2377) at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collect(Dataset.scala:2384) at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2120) at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2119) at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2802) at org.apache.spark.sql.Dataset.head(Dataset.scala:2119) at org.apache.spark.sql.Dataset.take(Dataset.scala:2334) at org.apache.spark.sql.Dataset.showString(Dataset.scala:248) at sun.reflect.GeneratedMethodAccessor71.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:280) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:214) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.IndexOutOfBoundsException: No group 1 at java.util.regex.Matcher.group(Matcher.java:538) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:377) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:231) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:225) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:826) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:826) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at org.apache.spark.scheduler.Task.run(Task.scala:99) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more
Please help me with this.
following is the excerpt of the records in the log file for referance:
109.169.248.247 - - [12/Dec/2015:18:25:11 +0100] GET /administrator/ HTTP/1.1 200 4263 - Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
109.169.248.247 - - [12/Dec/2015:18:25:11 +0100] GET /administrator/ HTTP/1.1 200 4263 - Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
109.169.248.247 - - [12/Dec/2015:18:25:11 +0100] GET /administrator/ HTTP/1.1 200 4263 - Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
109.169.248.247 - - [12/Dec/2015:18:25:11 +0100] POST /administrator/index.php HTTP/1.1 200 4494 http://almhuette-raith.at/administrator/ Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
46.72.177.4 - - [12/Dec/2015:18:31:08 +0100] GET /administrator/ HTTP/1.1 200 4263 - Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
46.72.177.4 - - [12/Dec/2015:18:31:08 +0100] POST /administrator/index.php HTTP/1.1 200 4494 http://almhuette-raith.at/administrator/ Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
83.167.113.100 - - [12/Dec/2015:18:31:25 +0100] GET /administrator/ HTTP/1.1 200 4263 - Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
83.167.113.100 - - [12/Dec/2015:18:31:25 +0100] POST /administrator/index.php HTTP/1.1 200 4494 http://almhuette-raith.at/administrator/ Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
95.29.198.15 - - [12/Dec/2015:18:32:10 +0100] GET /administrator/ HTTP/1.1 200 4263 - Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
95.29.198.15 - - [12/Dec/2015:18:32:11 +0100] POST /administrator/index.php HTTP/1.1 200 4494 http://almhuette-raith.at/administrator/ Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
109.184.11.34 - - [12/Dec/2015:18:32:56 +0100] GET /administrator/ HTTP/1.1 200 4263 - Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
109.184.11.34 - - [12/Dec/2015:18:32:56 +0100] POST /administrator/index.php HTTP/1.1 200 4494 http://almhuette-raith.at/administrator/ Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0 -
`
change last line to -
timestamp_pattern= r'\[\d{2}\/\w{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2}\s\S+\d{4}]'
df2=df_log.select(regexp_extract(col('header'),timestamp_pattern,0).alias("timestamp"))
note: I changed the groupId from 1 to 0 since there is no group set in your timestamp_pattern
In the following code sample, I start a proxy server using HTTP::Proxy and attempt to use it to request an HTTPS URL, but the proxy server either doesn't actually make the request, or doesn't return the response. However, if I make the URL use HTTP (not secure), the request succeeds. I've installed both IO::Socket::SSL and LWP::UserAgent::https (yay secret deps!), but am still unable to get HTTPS requests to go through the proxy. How can I get HTTP::Proxy to work with HTTPS URLs?
Here's my code:
#!/usr/bin/env perl
use strict;
use warnings;
use Data::Printer;
use HTTP::Proxy ':log';
use Mojo::UserAgent ();
my $URL = 'https://www.yahoo.com';
my $PROXY_PORT = 8667;
my $pid = fork();
if ($pid) { # I am the parent
print "Press ^c to kill proxy server...\n";
my $proxy = HTTP::Proxy->new( port => $PROXY_PORT );
$proxy->logmask(ALL);
$proxy->via(q{});
$proxy->x_forwarded_for(0);
$proxy->start;
waitpid $pid, 0;
}
elsif ($pid == 0) { # I am the child
sleep 3; # Allow the proxy server to start
my $ua = Mojo::UserAgent->new;
$ua->proxy
->http("http://127.0.0.1:$PROXY_PORT")
->https("http://127.0.0.1:$PROXY_PORT");
my $tx = $ua->get($URL);
if ($tx->error) {
p $tx->error;
}
else {
print "Success!\n";
}
}
else {
die 'Unknown result after forking';
}
Saving the above script as testcase-so.pl and running it:
$ MOJO_CLIENT_DEBUG=1 ./testcase-so.pl
Press ^c to kill proxy server...
-- Blocking request (https://www.yahoo.com)
-- Connect c66a92739c09c76fa24029e8079808c7 (https://www.yahoo.com:443)
-- Client >>> Server (https://www.yahoo.com)
CONNECT www.yahoo.com:443 HTTP/1.1\x0d
User-Agent: Mojolicious (Perl)\x0d
Content-Length: 0\x0d
Host: www.yahoo.com\x0d
Accept-Encoding: gzip\x0d
\x0d
-- Client >>> Server (https://www.yahoo.com)
[Tue Oct 9 12:02:54 2018] (12348) PROCESS: Forked child process 12352
[Tue Oct 9 12:02:54 2018] (12352) SOCKET: New connection from 127.0.0.1:45312
[Tue Oct 9 12:02:54 2018] (12352) REQUEST: CONNECT www.yahoo.com:443
[Tue Oct 9 12:02:54 2018] (12352) REQUEST: Accept-Encoding: gzip
[Tue Oct 9 12:02:54 2018] (12352) REQUEST: Host: www.yahoo.com
[Tue Oct 9 12:02:54 2018] (12352) REQUEST: User-Agent: Mojolicious (Perl)
[Tue Oct 9 12:02:54 2018] (12352) REQUEST: Content-Length: 0
[Tue Oct 9 12:02:54 2018] (12352) RESPONSE: 200 OK
[Tue Oct 9 12:02:54 2018] (12352) RESPONSE: Date: Tue, 09 Oct 2018 12:02:54 GMT
[Tue Oct 9 12:02:54 2018] (12352) RESPONSE: Transfer-Encoding: chunked
[Tue Oct 9 12:02:54 2018] (12352) RESPONSE: Server: HTTP::Proxy/0.304
-- Client <<< Server (https://www.yahoo.com)
HTTP/1.1 200 OK\x0d
Date: Tue, 09 Oct 2018 12:02:54 GMT\x0d
Transfer-Encoding: chunked\x0d
Server: HTTP::Proxy/0.304\x0d
\x0d
[Tue Oct 9 12:03:14 2018] (12352) CONNECT: Connection closed by the client
[Tue Oct 9 12:03:14 2018] (12352) PROCESS: Served 1 requests
[Tue Oct 9 12:03:14 2018] (12352) CONNECT: End of CONNECT proxyfication
\ {
message "Proxy connection failed"
}
[Tue Oct 9 12:03:15 2018] (12348) PROCESS: Reaped child process 12349
[Tue Oct 9 12:03:15 2018] (12348) PROCESS: 1 remaining kids: 12352
[Tue Oct 9 12:03:15 2018] (12348) PROCESS: Reaped child process 12352
[Tue Oct 9 12:03:15 2018] (12348) PROCESS: 0 remaining kids:
^C[Tue Oct 9 12:04:04 2018] (12348) STATUS: Processed 2 connection(s)
$
And with the $URL switched to not use https:
$ MOJO_CLIENT_DEBUG=1 ./testcase-so.pl
Press ^c to kill proxy server...
-- Blocking request (http://www.yahoo.com)
-- Connect f792ee97a0362ab493575d8116e69e59 (http://127.0.0.1:8667)
-- Client >>> Server (http://www.yahoo.com)
GET http://www.yahoo.com HTTP/1.1\x0d
Accept-Encoding: gzip\x0d
Content-Length: 0\x0d
Host: www.yahoo.com\x0d
User-Agent: Mojolicious (Perl)\x0d
\x0d
[Tue Oct 9 12:09:38 2018] (12656) PROCESS: Forked child process 12659
-- Client >>> Server (http://www.yahoo.com)
[Tue Oct 9 12:09:38 2018] (12659) SOCKET: New connection from 127.0.0.1:58288
[Tue Oct 9 12:09:38 2018] (12659) REQUEST: GET http://www.yahoo.com
[Tue Oct 9 12:09:38 2018] (12659) REQUEST: Accept-Encoding: gzip
[Tue Oct 9 12:09:38 2018] (12659) REQUEST: Host: www.yahoo.com
[Tue Oct 9 12:09:38 2018] (12659) REQUEST: User-Agent: Mojolicious (Perl)
[Tue Oct 9 12:09:38 2018] (12659) REQUEST: Content-Length: 0
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: 301 Moved Permanently
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Cache-Control: no-store, no-cache
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Date: Tue, 09 Oct 2018 14:10:01 GMT
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Transfer-Encoding: chunked
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Via: http/1.1 media-router-fp1006.prod.media.bf1.yahoo.com (ApacheTrafficServer [c s f ])
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Location: https://www.yahoo.com/
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Server: ATS
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Content-Language: en
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Content-Length: 8
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Content-Type: text/html
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: Content-Security-Policy: sandbox allow-forms allow-same-origin allow-scripts allow-popups allow-popups-to-escape-sandbox allow-presentation; report-uri https://csp.yahoo.com/beacon/csp?src=ats&site=frontpage®ion=US&lang=en-US&device=desktop&yrid=&partner=;
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: X-Frame-Options: SAMEORIGIN
[Tue Oct 9 12:09:38 2018] (12659) RESPONSE: X-XSS-Protection: 1; report="https://csp.yahoo.com/beacon/csp?src=fp-hpkp-www"
-- Client <<< Server (http://www.yahoo.com)
HTTP/1.1 301 Moved Permanently\x0d
Cache-Control: no-store, no-cache\x0d
Date: Tue, 09 Oct 2018 14:10:01 GMT\x0d
Transfer-Encoding: chunked\x0d
Via: http/1.1 media-router-fp1006.prod.media.bf1.yahoo.com (ApacheTrafficServer [c s f ])\x0d
Location: https://www.yahoo.com/\x0d
Server: ATS\x0d
Content-Language: en\x0d
Content-Length: 8\x0d
Content-Type: text/html\x0d
Content-Security-Policy: sandbox allow-forms allow-same-origin allow-scripts allow-popups allow-popups-to-escape-sandbox allow-presentation; report-uri https://csp.yahoo.com/beacon/csp?src=ats&site=frontpage®ion=US&lang=en-US&device=desktop&yrid=&partner=;\x0d
X-Frame-Options: SAMEORIGIN\x0d
X-XSS-Protection: 1; report="https://csp.yahoo.com/beacon/csp?src=fp-hpkp-www"\x0d
\x0d
-- Client <<< Server (http://www.yahoo.com)
8\x0d
redirect\x0d
0\x0d
\x0d
Success!
[Tue Oct 9 12:09:38 2018] (12659) SOCKET: Getting request failed: Client closed
[Tue Oct 9 12:09:39 2018] (12656) PROCESS: Reaped child process 12657
[Tue Oct 9 12:09:39 2018] (12656) PROCESS: 1 remaining kids: 12659
[Tue Oct 9 12:09:39 2018] (12656) PROCESS: Reaped child process 12659
[Tue Oct 9 12:09:39 2018] (12656) PROCESS: 0 remaining kids:
^C[Tue Oct 9 12:09:45 2018] (12656) STATUS: Processed 2 connection(s)
$
There is a bug in HTTP::Proxy in that it returns the wrong response to a CONNECT request:
-- Client <<< Server (https://www.yahoo.com)
HTTP/1.1 200 OK\x0d
Date: Tue, 09 Oct 2018 12:02:54 GMT\x0d
Transfer-Encoding: chunked\x0d
Server: HTTP::Proxy/0.304\x0d
\x0d
The response to a CONNECT request can have no body which means that it should not have a HTTP header announcing a body like Transfer-Encoding: chunked does. This bug happens with all clients which do a CONNECT request using HTTP/1.1. If the CONNECT is instead done with HTTP/1.0 the problem vanishes since Transfer-Encoding: chunked is not defined with HTTP/1.0 yet and thus HTTP::Proxy does not send it.
The same problem happens when trying to use curl with HTTP::Proxy, thus this is not a problem solely of Mojo::UserAgent. I`ve made a patch to HTTP::Proxy to respond properly. See this pull request for the details and for the (small) diff you need to apply.
trying to when change /client/BigBlueButton.html portion of the URL according to - http://docs.bigbluebutton.org/support/faq.html#how-do-i-change-the-client-bigbluebutton-html-portion-of-the-url,
but getting -
404 Not Found
nginx/1.4.6 (Ubuntu)
my /etc/bigbluebutton/nginx/client.nginx:
#location /client/BigBlueButton.html {
# root /home/firstuser/dev/bigbluebutton/bigbluebutton-client;
# index index.html index.htm;
# expires 1m;
#}
# BigBlueButton Flash client.
location /client {
root /home/firstuser/dev/bigbluebutton/bigbluebutton-client;
index index.html index.htm;
}
my /etc/bigbluebutton/nginx/rewrite.nginx:
location /client/BigBlueButton.html {
rewrite ^ /conference permanent;
}
location /conference {
alias /var/www/bigbluebutton/client;
index BigBlueButton.html;
expires 1m;
}
$sudo /etc/init.d/nginx restart
/var/log/nginx/bigbluebutton.access.log
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET / HTTP/1.1" 200 2852 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET /css/bijou.min.css HTTP/1.1" 200 2753 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET /css/style.css HTTP/1.1" 200 2918 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET /css/font-awesome.min.css HTTP/1.1" 200 20766 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET /images/jimtalk-logo.png HTTP/1.1" 200 10251 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET /images/bbb-setup-audio.jpg HTTP/1.1" 200 18876 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET /images/bbb-viewer-overview.jpg HTTP/1.1" 200 21929 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:57 -0400] "GET /images/bbb-presenter-overview.jpg HTTP/1.1" 200 18309 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:37:58 -0400] "GET /fonts/fontawesome-webfont.woff?v=4.1.0 HTTP/1.1" 200 83760 "http://M_IP_Ad/css/font-awesome.min.css" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
M_IP_Ad - - [16/Aug/2015:12:38:00 -0400] "POST /bigbluebutton/api/create?name=Demo+Meeting&meetingID=Demo+Meeting&voiceBridge=72274&attendeePW=ap&moderatorPW=mp&record=false&checksum=a93ab8433532c633ab2467afc0d91e0eb1dc4e88 HTTP/1.1" 200 488 "-" "Java/1.7.0_79"
33.126.263.65 - - [16/Aug/2015:12:38:00 -0400] "GET /demo/demo1.jsp?username=%D7%99%D7%A2%D7%99%D7%A2%D7%99&action=create HTTP/1.1" 200 1003 "http://M_IP_Ad/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:38:01 -0400] "GET /bigbluebutton/api/join?meetingID=Demo+Meeting&fullName=%D7%99%D7%A2%D7%99%D7%A2%D7%99&password=mp&checksum=3438ed39a50723be59798038f86fcba0af30b325 HTTP/1.1" 302 0 "http://M_IP_Ad/demo/demo1.jsp?username=%D7%99%D7%A2%D7%99%D7%A2%D7%99&action=create" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:38:02 -0400] "GET /client/BigBlueButton.html HTTP/1.1" 301 193 "http://M_IP_Ad/demo/demo1.jsp?username=%D7%99%D7%A2%D7%99%D7%A2%D7%99&action=create" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:38:02 -0400] "GET /conference HTTP/1.1" 301 193 "http://M_IP_Ad/demo/demo1.jsp?username=%D7%99%D7%A2%D7%99%D7%A2%D7%99&action=create" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
33.126.263.65 - - [16/Aug/2015:12:38:02 -0400] "GET /conference/ HTTP/1.1" 403 208 "http://M_IP_Ad/demo/demo1.jsp?username=%D7%99%D7%A2%D7%99%D7%A2%D7%99&action=create" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"
Thank you
The nginx read all the files with nginx extension (*.nginx). I guess the problem is that nginx is not able to handle same locations.
How can I delete symbols, whitespaces, characters, words everything between two characters in a line?
My 5-line file is:
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; InfoPath.1)" 120.94.30.12 264 556 -
"Skype for Macintosh" 120.94.30.9 1038 482 -
-129.94.30.4 217 309 -
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; InfoPath.1)" 120.94.30.8 1197 747 -
"¢¢HttpClient" 120.94.30.12 594 231 -
I want to delete everything comes in between " and " (including the " characters) so that the required output should be:
120.94.30.12 264 556 -
120.94.30.9 1038 482 -
-120.94.30.4 217 309 -
120.94.30.8 1197 747 -
120.94.30.12 594 231 -
You mean like this?
sed 's/"[^"]*"//' file
echo '"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; InfoPath.1)" 120.94.30.12 264 556 -' |\
sed -e 's/".*"\(.*\)/\1/g'