jq -r --arg summary "$summary" '.fields |= .+ {"summary":$summary}' $CREATE_JSON > $TEMP1_JSON
jq -r --arg component "$component" '.fields.components += [{"name":$component}]' $TEMP1_JSON > $TEMP2_JSON
jq '.fields.labels = ["Android","'$build_number'","'$feature'","'$branch'","'$system'"]' $TEMP2_JSON > $SUMMARY_JSON
test_execution_id=$(curl -u$xray_token -XPOST --data #$SUMMARY_JSON -H "Content-Type: application/json" $tracker_url/rest/api/2/issue
Related
For testing one of my application I need some huge data in parquet format. which I don't have.I have written a shell script which is performing slow.I wanted go with spark.How can I generate random data using spark scala.
Each filed has to be in specific range.Id should be in between (1-10),date(any date from 2010-2018),start and end time should be any.
My shell scipt.
!/bin/bash
if [ $# -eq 2 ]; then
LIMIT=40 # to generate 40KB file
for((i=0;i<$2;i++))
{
FILE_NAME="$1$i.csv"
echo "id,date,start_time,end_time,distance,amount,persons,longitude,latitude" >> "$FILE_NAME"
while [ $(du -k $FILE_NAME | cut -f 1) -le $LIMIT ]
do
start_time=`date -d "$(date +%H:%M:%S) + $(shuf -i 0-24 -n 1) hours $(shuf -i 0-60 -n 1) minutes $(shuf -i 0-60 -n 1) seconds" +'%H:%M:%S'`
echo "`shuf -i 1-10 -n 1`,`date -d "2011-01-01 + $(shuf -i 1-2557 -n 1) days" +'%m-%d-%Y'`,$start_time,`date -d "$start_time + $(shuf -i 1-6 -n 1) hours $(shuf -i 0-60 -n 1) minutes $(shuf -i 0-60 -n 1) seconds" +'%H:%M:%S'`,`shuf -i 1-60 -n 1`,`shuf -i 100-1500 -n 1`,`shuf -i 1-6 -n 1`,`shuf -i 10-99 -n 1`.`shuf -i 100000-999999 -n 1`,`shuf -i 10-99 -n 1`.`shuf -i 100000-999999 -n 1`" >> "$FILE_NAME"
done
}
else
printf "Usage: sh GenerateCSV.sh <filename without extension> <No of files to generate> \nThe files will be generated with .csv extension\n"
fi
I want data to be like this,which should be parquet format.
2,20-10-2010,23:18:10,23:40:40
How can I do it in spark.
You can try this option.
Below are the Unix Timestamp values for the dates mentioned.
val ss = SparkSession.builder().appName("local").master("local[*]").getOrCreate()
ss.sqlContext.sql("SELECT unix_timestamp ('2010-01-01', 'yyyy-MM-dd')") // 1262284200
ss.sqlContext.sql("SELECT unix_timestamp ('2018-12-31', 'yyyy-MM-dd')") // 1546194600
You can code in this below way for generating random number between the above numbers.
val r = new scala.util.Random
val x = 1262284200 + r.nextInt((1546194600-1262284200))
You can code in this below way for generating random date between the dates by using above generated value x
ss.sqlContext.sql(s"SELECT FROM_UNIXTIME($x)")
how can the below curl code from the command prompt curl be represented in the perl WWW:Curl ?
curl https://api.stripe.com/v1/charges \
-u sk_test_a02zSeLS9cMPlJvu2GkWgSDB: \
-d amount=1000 \
-d currency=sgd \
-d description="Example charge" \
-d source=tok_2s0QJK6exWUdbSGZb4SpAKep
perl
use WWW::Curl::Easy;
my $curl = WWW::Curl::Easy->new;
$curl->setopt(CURLOPT_HEADER,1);
$curl->setopt(CURLOPT_URL, 'https://api.stripe.com/v1/charges');
# A filehandle, reference to a scalar or reference to a typeglob can be used here.
my $response_body;
$curl->setopt(CURLOPT_WRITEDATA,\$response_body);
# Starts the actual request
my $retcode = $curl->perform;
as you can see in the examples here,
use WWW::Curl::Form;
my $curlf = WWW::Curl::Form->new;
$curlf->formadd("amount", "1000");
$curlf->formadd("currency", "sgd");
$curlf->formadd("description", "Example charge");
$curlf->formadd("source", "tok_2s0QJK6exWUdbSGZb4SpAKep");
$curl->setopt(CURLOPT_HTTPPOST, $curlf);
$curl->setopt(CURLOPT_USERPWD,"sk_test_a02zSeLS9cMPlJvu2GkWgSDB:");
# pwd
/Apache/httpd-2.4.23
# ./configure --prefix=/Apache/apache2/ --with-ssl=/usr/local/include/openssl --with-included-apr --with-pcre=/usr/local/share/pcre
Configure succeeded with latest openssl 1.0.1t.
Make fails:
# make clean
# make
/Apache/httpd-2.4.23/srclib/apr/libtool --silent --mode=compile gcc -std=gnu99 -g -O2 -DSOLARIS2=10 -D_POSIX_PTHREAD_SEMANTICS -D_REENTRANT -D_LARGEFILE64_SOURCE -I. -I/Apache/httpd-2.4.23/os/unix -I/Apache/httpd-2.4.23/include -I/Apache/httpd-2.4.23/srclib/apr/include -I/Apache/httpd-2.4.23/srclib/apr-util/include -I/usr/local/share/pcre/include -I/Apache/httpd-2.4.23/modules/aaa -I/Apache/httpd-2.4.23/modules/cache -I/Apache/httpd-2.4.23/modules/core -I/Apache/httpd-2.4.23/modules/database -I/Apache/httpd-2.4.23/modules/filters -I/Apache/httpd-2.4.23/modules/ldap -I/Apache/httpd-2.4.23/modules/loggers -I/Apache/httpd-2.4.23/modules/lua -I/Apache/httpd-2.4.23/modules/proxy -I/Apache/httpd-2.4.23/modules/session -I/Apache/httpd-2.4.23/modules/ssl -I/Apache/httpd-2.4.23/modules/test -I/Apache/httpd-2.4.23/server -I/Apache/httpd-2.4.23/modules/arch/unix -I/Apache/httpd-2.4.23/modules/dav/main -I/Apache/httpd-2.4.23/modules/generators -I/Apache/httpd-2.4.23/modules/mappers -prefer-non-pic -static -c vhost.c && touch vhost.lo
make: Fatal error: Don't know how to make target `/Apache/httpd-2.4.23/server/test_char.h'
Current working directory /Apache/httpd-2.4.23/server
*** Error code 1
The following command caused the error:
otarget=`echo all-recursive|sed s/-recursive//`; \
list=' mpm'; \
for i in $list; do \
if test -d "$i"; then \
target="$otarget"; \
echo "Making $target in $i"; \
if test "$i" = "."; then \
made_local=yes; \
target="local-$target"; \
fi; \
(cd $i && make $target) || exit 1; \
fi; \
done; \
if test "$otarget" = "all" && test -z 'delete-exports libmain.la export_vars.h httpd.exp'; then \
made_local=yes; \
fi; \
if test "$made_local" != "yes"; then \
make "local-$otarget" || exit 1; \
fi
make: Fatal error: Command failed for target `all-recursive'
Current working directory /Apache/httpd-2.4.23/server
*** Error code 1
The following command caused the error:
otarget=`echo all-recursive|sed s/-recursive//`; \
list=' srclib os server modules support'; \
for i in $list; do \
if test -d "$i"; then \
target="$otarget"; \
echo "Making $target in $i"; \
if test "$i" = "."; then \
made_local=yes; \
target="local-$target"; \
fi; \
(cd $i && make $target) || exit 1; \
fi; \
done; \
if test "$otarget" = "all" && test -z 'httpd shared-build '; then \
made_local=yes; \
fi; \
if test "$made_local" != "yes"; then \
make "local-$otarget" || exit 1; \
fi
make: Fatal error: Command failed for target `all-recursive'
Please help me to sort out this problem.
Often I need to analyze large logs in console.
I use the following command to colorize important keywords:
echo "string1\nerror\nsuccess\nstring2\nfail" | perl -p -e 's/(success)/\e[1;32;10m$&\e[0m/g;' -e 's/(error|fail)/\e[0;31;10m$&\e[0m/g'
It will colorize "success" with green, and error messages with red and keeps others lines unchanged (as they contain some useful info).
But in some cases I need to colorize values after some marker, but not marker itself, i.e. in these lines
Marker1: value1
Marker2: value2
need to highlight only value1 and value2 by known markers.
I'm looking for a way to modify my current oneliner to add this function
Also I tried the following solution, which I like less
#!/bin/bash
default=$(tput op)
red=$(tput setaf 1 || tput AF 1)
green=$(tput setaf 2 || tput AF 2)
sed -u -r "s/(Marker1: )(.+)$/\1${red}\2${default}/
s/(Marker2: )(.+)$/\1${green}\2${default}/" "${#}"`
But it has some problem with buffering, so it's ok for some constant file, but log which is continuosly running is not displayed at all
UPDATE:
Found a solution with help of some perl guru.
echo -e "string1\nerror\nsuccess\nstring2\nfail\nMaker1: value1\nMaker2: value2" | \
perl -p \
-e 's/(success)/\e[32m$&\e[0m/g;' \
-e 's/(error|fail)/\e[31m$&\e[0m/g;' \
-e 's/(Maker1:) (.*)/$1 \e[36m$2\e[0m/m;' \
-e 's/(Maker2:) (.*)/$1 \e[01;34m$2\e[0m/m;'
echo -e "string1\nerror\nsuccess\nstring2\nfail\nMaker1: value1\nMaker2: value2" | \
perl -p \
-e 's/(success)/\e[32m$&\e[0m/g;' \
-e 's/(error|fail)/\e[31m$&\e[0m/g;' \
-e 's/(Maker1:) (.*)/$1 \e[36m$2\e[0m/m;' \
-e 's/(Maker2:) (.*)/$1 \e[01;34m$2\e[0m/m;'
#!/bin/bash
default=$(tput op)
red=$(tput setaf 1 || tput AF 1)
green=$(tput setaf 2 || tput AF 2)
#default='e[0m'
#red='e[0;31;10m'
#green='e[1;32;10m'
# automaticaly use passed argument file if any or stdin if not
sed -u -r \
"/success/ s//${green}&${default}/
/error|fail/ s//${red}&${default}/
/^Marker1:/ {s//\1${red}/;s/$/${default}/;}
/^Marker2:/ {s//\1${green}/;s/$/${default}/;}" \
$( [ ${##} -gt 0 ] && echo ${#} )
For a one line:
remove other line thans sed one
replace newline in sed by ;
use directly the terminal code in place of variable
remove the last line if you pipe or use specific file instead
I need to get job duration via curl command.
I can get access to it
curl -s $JENKINS_JOB_URL/146/api/json?tree=duration
{"duration":14182142}
How convert the 14182142 to "3 hr 56 min" ? ( via ksh, or in the curl command )
Thanks
You could always use some python:
python -c"import datetime;print datetime.timedelta(milliseconds=14182142);"
3:56:22.142000
Is awk acceptible?
curl -s $JENKINS_JOB_URL/146/api/json?tree=duration | awk -F : '{ print $2 }' | awk -F } '{ printf "%i hr %i min", ($1 / 3600000), ($1 % 3600000)/60000 }'
If anyone knows of a nice way to get the number out of the string and into awk, I can remove the nasty double-invocation of awk.