Convert an iteration to recursion - scala

I have an iteration logic that returns a query like this :
val baseQuery = s"select agg_id from quality.QUALITY_AGGREGATOR where job_id = 123 and level ="
val reprocessDate ="2018-10-24"
for( i <- 0 to level){
var currLevelSubQuery=""
if (i==0 ){
currLevelSubQuery= baseQuery + s"$i and agg_value >= '${reprocessDate}'"
}
else{
currLevelSubQuery= baseQuery + s"$i and parent_agg_id in ( $prevLevelSubQuery )"
}
prevLevelSubQuery= currLevelSubQuery
finalQuery = finalQuery + currLevelSubQuery + (if (i<level) " union " else "")
}
It returns a query of this nature for level = 2.
SELECT agg_id
FROM quality.quality_aggregator
WHERE job_id = 123
AND level = 0
AND agg_value >= '2018-10-24'
UNION
SELECT agg_id
FROM quality.quality_aggregator
WHERE job_id = 123
AND level = 1
AND parent_agg_id IN (SELECT agg_id
FROM quality.quality_aggregator
WHERE job_id = 123
AND level = 0
AND agg_value >= '2018-10-24')
UNION
SELECT agg_id
FROM quality.quality_aggregator
WHERE job_id = 123
AND level = 2
AND parent_agg_id IN (SELECT agg_id
FROM quality.quality_aggregator
WHERE job_id = 123
AND level = 1
AND parent_agg_id IN
(SELECT agg_id
FROM quality.quality_aggregator
WHERE job_id = 123
AND level = 0
AND agg_value >=
'2018-10-24'))
I am trying to convert it into a recursive logic like this
val baseQuery = s"select agg_id from quality.QUALITY_AGGREGATOR where job_id = 123 and level ="
val finalQuery = getAggIdSQLGenerator(2,"2018-10-24")
def getAggIdSQLGenerator(level : Int, reprocessDate:String):String={
if (level == 0)
( baseQuery + s"$level and agg_value >= '${reprocessDate}'")
else
{
val subQuery=getAggIdSQLGenerator(level-1 ,reprocessDate)
baseQuery + s"$level and parent_agg_id in (" +subQuery +") union "+ subQuery
}
}
But this is not yielding correct results. What am I missing?

This isn't recursive but I think it's a smaller, cleaner, implementation of what you're after.
val baseQuery = ".... level="
val levelRE = "level=(\\d+)".r.unanchored
val reprocessDate ="2018-10-24"
val av = s" and agg_value >= '${reprocessDate}'"
val pai = " and parent_agg_id in "
val itrs = 3
val query = List.iterate(s"${baseQuery}0$av", itrs) { prevStr =>
val level = prevStr match {
case levelRE(n) => n.toInt + 1
case _ => 0
}
s"$baseQuery$level$pai($prevStr)"
}.mkString(" union ")

The issue is you have added union on every query and you have concatenated subquery 2 times.
This gives your expected output.
val baseQuery = s"select agg_id from quality.QUALITY_AGGREGATOR where job_id = 123 and level ="
val finalQuery = getAggIdSQLGenerator(2, "2018-10-24")
def getAggIdSQLGenerator(level: Int, reprocessDate: String): String = {
if (level == 0) {
baseQuery + s" $level and agg_value >= '${reprocessDate}'"
} else {
val subQuery = getAggIdSQLGenerator(level - 1, reprocessDate)
baseQuery + s" $level and parent_agg_id in (" + subQuery + ")"
}
}
println("UNION " + finalQuery)

Related

Functioning postgres query not working in JPA

I've been trying to write a working postgres query in JPA, but I'm getting errors in Intellij and when I run Spring it fails with a generic validation exception.
This is my query in postgres
SELECT
( "header" :: json ) ->> 'company' AS "Company",
LOWER(( "header" :: json ) ->> 'user') AS "User",
MAX ( ( "header" :: json ) ->> 'version' ) AS "Version",
MIN ( date_res ) AS "First CN",
MAX ( date_res ) AS "Last CN",
SUM ( CASE WHEN url = '/url-1' AND status = 200 THEN 1 ELSE 0 END ) AS "Survey",
SUM ( CASE WHEN url = '/url-2' AND status = 200 THEN 1 ELSE 0 END ) AS "Work",
SUM ( CASE WHEN url = '/url-3' AND status = 200 THEN 1 ELSE 0 END ) AS "Home",
SUM ( CASE WHEN url = '/url-4' AND status = 200 THEN 1 ELSE 0 END ) AS "DeploySurvey",
SUM ( CASE WHEN url = '/url-5' AND status = 200 THEN 1 ELSE 0 END ) AS "DeployWork"
FROM
"public"."seg_ws_log_res"
INNER JOIN "seg_company" ON ("seg_company"."companyName" = ( "header" :: json ) ->> 'company')
WHERE
"service" = 'service'
AND NOT ( "header" :: json ) ->> 'company' IS NULL
AND NOT ( "header" :: json ) ->> 'user' IS NULL
AND NOT ( "header" :: json ) ->> 'company' = ''
AND NOT UPPER(( "header" :: json ) ->> 'user') IN ('USER_1', 'USER_2')
AND NOT ( "header" :: json ) ->> 'version' = 'Dev'
AND NOT UPPER(( "header" :: json ) ->> 'company') IN ('text_1','text_2','text_3')
GROUP BY
"Company",
"User"
ORDER BY
"Company",
"User";
This is my #Query code
#Query(value = "SELECT " +
"cast(log.header as json) ->> 'company' AS \"Company\"," +
"LOWER(cast(log.header as json) ->> 'user') AS \"User\"," +
"MAX ( cast(log.header as json) ->> 'version' ) AS \"Version\"," +
"MIN (log.dateRes) AS \"First CN\"," +
"MAX ( log.dateRes ) AS \"Last CN\"," +
"SUM ( CASE WHEN log.url = '/url-1' AND log.status = 200 THEN 1 ELSE 0 END ) AS \"Survey\"," +
"SUM ( CASE WHEN log.url = '/url-2' AND log.status = 200 THEN 1 ELSE 0 END ) AS \"Work\"," +
"SUM ( CASE WHEN log.url = '/url-3' AND log.status = 200 THEN 1 ELSE 0 END ) AS \"Home\"," +
"SUM ( CASE WHEN log.url = '/url-4' AND log.status = 200 THEN 1 ELSE 0 END ) AS \"DeploySurvey\"," +
"SUM ( CASE WHEN log.url = '/url-5' AND log.status = 200 THEN 1 ELSE 0 END ) AS \"DeployWork\"" +
"FROM" +
" SegWsLogResEntity log" +
"INNER JOIN SegCompanyEntity c ON (c.companyName = ( cast(log.header as json)) ->> 'company') " +
"WHERE" +
" log.service = 'service' " +
" AND NOT cast(log.header as json) ->> 'company' IS NULL " +
" AND NOT cast(log.header as json) ->> 'user' IS NULL " +
" AND NOT cast(log.header as json) ->> 'company' = '' " +
" AND NOT UPPER(cast(log.header as json) ->> 'user') IN ('USER_1', 'USER_2') " +
" AND NOT cast(log.header as json) ->> 'version' = 'Dev'" +
" AND NOT UPPER(cast(log.header as json) ->> 'company') IN ('text_1','text_2','text_3')" +
"GROUP BY" +
"\"Company\"," +
"\"User\"" +
"ORDER BY" +
"\"Company\"," +
"\"User\"")
In this line:
cast(log.header ->> 'company' as json) AS \"Company\"
in the operator --> the IDE indicates:
<expression> expected, got '>'
LOWER(cast(log.header as json) ->> 'user') AS \"User\"
In this line:
"MIN (log.dateRes) AS \"First CN\","
The IDE indicates:
identifier expected, got '"First CN"'
the same for the each identifier until the from
And in this line
"INNER JOIN SegCompanyEntity c ON (c.companyName = ( \"header\" :: json ) ->> 'company') " +
After the ON the IDE indicates
'(', , FUNCTION or identifier expected, got '('
I have gone through many posts about JPA and native queries but none seem to be similar to my problem.
The query you define in the jpa-#Query annotations is by default a JPQL query. JPQL doesn't understand Postgres (or any other DB) specific features e.g. Postgres' Json(b) type.
There is a flag nativeQuery in the #Query annotation. You can set this to true -- then your query won't be interpreted as JPQL but as native SQL. Obviously, this way, the query will only work with postgres.
#Query(
nativeQuery = true,
value = "SELECT..."
)

How to read a csv file and assign values to the variable in spark scala

I am trying to achieve the below.
I have csv file with the below content
Sno,ColumnName,ColumnValue
——————————-———————————
1,svEmp,(Case when col1 = 1 then 2 else 1 end)
2,svCol,(Case when col2 = 2 then 3 else 6 end)
I have a requirement where I need to read the csv file in spark scala and assign the columnValue to ColumnName(create staging variables which can be later used in spark.sql statements) as below
val svEmp = "(Case when col1 = 1 then 2 else 1 end)"
//and so on
spark.sql("select $svEmp as CalCol from tableName")
How is this achievable?
Any help would be appreciated.
You can read the CSV that contains select expression into dataframe then aggregate to concatenate the columns ColumnValue and ColumnName, and finally collect the generated select statement into string variable :
val df = spark.read.option("header", "true").csv(path)
val selectStm = df.agg(
concat_ws(
", ",
collect_list(concat(col("ColumnValue"), lit(" as "), col("ColumnName")))
)
).first.getString(0)
//selectStm: String = (Case when col1 = 1 then 2 else 1 end) as svEmp, (Case when col2 = 2 then 3 else 6 end) as svCol
val sqlQuery = s"select $selectStm from tableName"
//sqlQuery: String = select (Case when col1 = 1 then 2 else 1 end) as svEmp, (Case when col2 = 2 then 3 else 6 end) as svCol from tableName
val df2 = spark.sql(sqlQuery)

Where condition in Slick

How to achieve the equivalent in Slick?
select * from table1 where col1 = 1 AND (col2 = 2 or col3 = 3)
This doesn't work:
val action = table.filter(_.col1 === 1 && (_.col2 === 2 || _.col3 === 3)).result
You cannot use the short hand in this case. Try this:
val action = table.filter( x => x.col1 == 1 && (x.col2 == 2 || x.col3 == 3)).result

PostgreSQL 9.3: REPLACE string

I have a string to replace with the expected form.
Input: I have the following string.
'A,B,C,D,E,F,G,H,I,J,K,L'
And I want to replace the above string into the following format:
'x.A = z.A ,
x.B = z.B ,
x.C = z.C ,
x.D = z.D ,
x.E = z.E ,
x.F = z.F ,
.........
.........
x.L = z.L'
My try:
SELECT 'x.'||REPLACE('A,B,C,D,E,F,G,H,I,J,K,L',',',' = z.')
SELECT 'x.' || col || '=z.' || col
FROM (
SELECT unnest(regexp_split_to_array('A,B,C,D,E,F,G,H,I,J,K,L', ',')) col
) t
You can use string_agg and FORMAT:
SELECT string_agg(FORMAT('x.%s = z.%s', t,t) , ',')
FROM (SELECT unnest(regexp_split_to_array('A,B,C,D,E,F,G,H,I,J,K,L', ',')) AS t
) AS sub;
You can use as delimeter E',\r\n' to get carriage return:
SELECT string_agg(FORMAT('x.%s = z.%s', t,t) , E',\r\n')
FROM (SELECT unnest(regexp_split_to_array('A,B,C,D,E,F,G,H,I,J,K,L', ',')) AS t)
AS sub
Output:
x.A = z.A,
x.B = z.B,
x.C = z.C,
x.D = z.D,
x.E = z.E,
x.F = z.F,
x.G = z.G,
x.H = z.H,
x.I = z.I,
x.J = z.J,
x.K = z.K,
x.L = z.L

How can I create a datatable of my 3 select SQL statements

I have a database with one table and I have 3 select SQL statement.
Those select items have different conditions. How can I merge the answer of this 3 SQL command?
I don't want to merge them, rows by rows to a data table. Is there any other way?
Some thing like this..
OleDbCommand cmd = new OleDbCommand("select top "+ cont0 +" * from (select * from db where tablenumber=0) order by ID ASC", mycon);
OleDbDataAdapter adapt=new OleDbDataAdapter(cmd);
DataTable dt = new DataTable();
DataTable dttemp = new DataTable();
adapt = new OleDbDataAdapter(cmd);
adapt.Fill(dt);
cmd = new OleDbCommand("select top "+ cont1 +" * from (select * from db where tablenumber=1) order by ID ASC", mycon);
adapt = new OleDbDataAdapter(cmd);
adapt.Fill(dttemp);
foreach (DataRow row in dttemp.Rows)
{
dt.Rows.Add(row.ItemArray);
}
if (cont2 != 0)
{
cmd = new OleDbCommand("select top " + cont2 + " * from (select * from db where tablenumber=2) order by ID ASC", mycon);
adapt = new OleDbDataAdapter(cmd);
dttemp = new DataTable();
adapt.Fill(dttemp);
foreach (DataRow row in dttemp.Rows)
{
dt.Rows.Add(row.ItemArray);
}
}
if (cont3 != 0)
{
cmd = new OleDbCommand("select top " + cont3 + " * from (select * from db where tablenumber=3) order by ID ASC", mycon);
adapt = new OleDbDataAdapter(cmd);
dttemp = new DataTable();
adapt.Fill(dttemp);
foreach (DataRow row in dttemp.Rows)
{
dt.Rows.Add(row.ItemArray);
}
}
if (cont4 != 0)
{
cmd = new OleDbCommand("select top " + cont4 + " * from (select * from db where tablenumber=4) order by ID ASC", mycon);
adapt = new OleDbDataAdapter(cmd);
dttemp = new DataTable();
adapt.Fill(dttemp);
foreach (DataRow row in dttemp.Rows)
{
dt.Rows.Add(row.ItemArray);
}
}
if (cont5 != 0)
{
cmd = new OleDbCommand("select top " + cont5 + " * from (select * from db where tablenumber=5) order by ID ASC", mycon);
adapt = new OleDbDataAdapter(cmd);
dttemp = new DataTable();
adapt.Fill(dttemp);
foreach (DataRow row in dttemp.Rows)
{
dt.Rows.Add(row.ItemArray);
}
}
You can merge them simply like this:
"select * from
(select *, row_number() over(partition by field order by id) as rn
from table1 where field in(0, 5, 9)) t
where rn <= " + cont
EDIT:
string command = "select * from ( select top "+ cont0 +" * from db where tablenumber=0 order by ID) t union all
select * from ( select top "+ cont1 +" * from db where tablenumber=1 order by ID) t";
if (cont2 != 0)
comand += " union all select * from ( select top " + cont2 + " * from db where tablenumber=2 order by ID) t";
if (cont3 != 0)
comand += " union all select * from ( select top " + cont3 + " * from db where tablenumber=3 order by ID) t";
....
OleDbCommand cmd = new OleDbCommand(command, mycon);
OleDbDataAdapter adapt=new OleDbDataAdapter(cmd);
DataTable dt = new DataTable();
....