postgres generate sql statement with dynamic fields - postgresql

im building a package to generate HTML code that later will be sent by email.
the data query has be dynamic as it will have all days of the month.
in the end query should be something like
select day_20230101, day_20230102, ...,day_20230131,avg_cap, rank1,... from v_html_report;
and then execute in the FOR to generate the HTML. The problem I'm facing is that the list of the columns is being seen as an entire string. when executing the loop the result_rec.day_20230101 doesnt exist but exist the result_rec.column_list.
is there a way to execute for example the query from a string in the FOR or the query seen this as fields and not a single field?
Below my Procedure:
create or replace procedure proc_html_gen ()
language plpgsql
as $$
DECLARE
days_month int;
column_list text ;
result_rec record;
html_header text ;
html_result text ;
html_days text ;
html_sql text ;
begin
SELECT
date_part(
'days',
((to_date((select value from parameters rtp where id='html_report_month' and id2='execute_month')||'01','YYYYMMDD')) + interval '1 month - 1 day')
) into days_month;
update html_gen rtp
set aux1=days_month
where html_type='html_report';
SELECT to_char(generate_series(01,days_month ),'00') into column_list;
select string_agg(concat('days_',days_list),',') from (
SELECT concat(
(select value from parameters rtp where id='html_report' and id2='execute_month')
,trim(to_char(generate_series(01,days_month),'00'))) as days_list
) a into column_list;
update html_gen rtp
set aux2=column_list
where html_type='html_report';
html_sql := ' select ' || column_list || ', avg_cap,rank1, rank2, rank3, rank4, rank5, rank6, rank7, rank8, rank9, rank10 from v_monthly_polygon_peak_cap_table' ;
FOR result_rec IN execute html_sql
LOOP
select string_agg(days_list,'') into html_days
from (
select concat('<td>',concat(1,trim(to_char(generate_series(01,31),'00'))),'</td>') as days_list
) a ;
html_result := concat(html_result,'<tr>',html_days,'<td> ' || result_rec.avg_cap || '</td></tr>');
END LOOP;
commit;
end;$$
update the query with the solution
now im facing a new problem. how to get the field name dynamically?
like below im getting an error. days_ doesnt exist. i need to append the number of the day in the end of the variable.
html_result := result_rec.days_|| trim(to_char(generate_series(01,31),'00')))
Best Regards,
Macieira

Related

Using copy command within format in a stored procedure using Postgres plpgsql

I am trying to create a stored procedure in Postgres for generating CSV files containing the average values of some tags/variables between the last 15 minutes. But when I try to implement the 'where' condition for selecting the tag values in the last 15 minutes using Execute 'format()' method, it gives me syntax errors as output. Since the actual code with multiple tags is too complicated, I am having a hard time fixing the syntax of this code. Can you guys please take a look at this? I am doing this in Postgres SQL using PLPGSQL language. Here I am able to add my file name dynamically, but conditions in the select query are generating problems.
CREATE OR REPLACE PROCEDURE public.csv_test0002()
LANGUAGE plpgsql
AS $$
declare sDate varchar;
declare sFileName varchar;
declare DEST varchar;
DECLARE sFromTime varchar(150);
DECLARE sToTime varchar(150);
begin
sDate := to_char(current_timestamp, 'YYYY-MM-DD_HH24_MI_SS');
DEST := concat('Filename_',sDate);
sFromTime := TO_CHAR(current_timestamp - interval '15 minutes', 'YYYY-MM-DD HH24:MI:SS');
sToTime := TO_CHAR(current_timestamp, 'YYYY-MM-DD HH24:MI:SS');
EXECUTE format(
'COPY'
'(select avg(ptvalue) as ptvalue from pro_opc where ptname = ''OPC_SR'' and pttime between ',sFromTime,' and ',sToTime,')'
'to %L CSV HEADER',
'D:\' || DEST || '.csv'
);
end $$;
The problematic part is the sFromTime and sTotime in the select query. Postgres is not recognizing the value inside them. Can you help me correct the syntax of the format() function here so that I can pass the timestamp condition dynamically?!!
Output image of current code when the stored procedure is called
Try this using double-dollar quoting format for better readability.
Keeping your query -
execute format
(
$$
COPY
(
select avg(ptvalue) as ptvalue,
from pro_opc
where ptname = 'OPC_SR'
and pttime between %L and %L
)
to %L CSV HEADER
$$,
sFromTime,
sToTime,
'D:\' || DEST || '.csv'
);
But better remove sFromTime and sToTime completely and use this equivalent query.
execute format
(
$$
COPY
(
select avg(ptvalue) as ptvalue,
current_timestamp as local_time
from pro_opc
where ptname = 'OPC_SR'
and pttime between current_timestamp - interval '15 minutes'
and current_timestamp
)
to %L CSV HEADER
$$,
'D:\' || DEST || '.csv'
);

Cannot Get Dynamic Exec of SP to Return INOUT Param

Using PostgreSQL 13.2, wherein a stored procedure (the Requestor) is given a name of a list of stored procedures to run (the job group). All sp's executed this way are coded to write a log record as their last task. I have chosen to pull that 'append log' code from all of the sp's, and instead send back the log record (always a single record) using an INOUT rowtype param, but have run into trouble. In my example below, the requestor sp will load the records returned from the sp's it calls into a temp table shaped like the permanent log table.
That permanent table looks like this:
create table public.job_log (
log_id integer,
event_id integer,
job_id integer,
rows_affected integer);
Any one of the jobs that is executed by the requestor sp might look like this one:
CREATE OR REPLACE procedure public.get_log_rcd(
inout p_log_rcd public.job_log)
LANGUAGE 'plpgsql'
as
$BODY$
declare
v_log_id integer = 40;
v_event_id integer = 698;
v_job_id integer = 45;
v_rows_affected integer = 60;
begin
select
v_log_id
, v_event_id
, v_job_id
, v_rows_affected
into
p_log_rcd.log_id,
p_log_rcd.event_id,
p_log_rcd.job_id,
p_log_rcd.rows_affected;
end;
$BODY$
This sample sp doesn't do anything--it's purpose here is only to simulate initialize of the log parameters to return to caller.
Again, the requestor sp that's going to run jobs like the one above creates a temp table with the same structure as the permanent log:
drop table if exists tmp_log_cache;
create temp table tmp_log_cache as table public.job_log with no data;
If the requestor sp didn't have to do dynamic SQL, it would look something like this block here:
do
$$
declare
big_local public.job_log;
begin
call public.get_log_rcd( big_local );
insert into tmp_log_cache (
log_id
, event_id
, job_id
, rows_affected )
values (
big_local.log_id
, big_local.event_id
, big_local.job_id
, big_local.rows_affected);
end;
$$;
Doing a
select * from tmp_log_cache;
Returns a row containing the 4 column values expected, all is well. But, dynamic execution is required. And, as I'm sure most folks here know, the following dog don't hunt:
do
$$
declare
big_local public.job_log;
v_query_text varchar;
v_job_name varchar = 'public.get_log_rcd';
begin
select 'call ' || v_job_name || '( $1 );'
into v_query_text;
execute v_query_text using big_local::public.job_log;
insert into tmp_log_cache (
log_id
, event_id
, job_id
, rows_affected )
values (
big_local.log_id
, big_local.event_id
, big_local.job_id
, big_local.rows_affected);
end;
$$;
The above dynamic statement executes without error, but the insert statement only has NULL values to work with--a row is inserted, all nulls. Any suggestions warmly welcomed. The sp's that comprise the various job groups could probably have been implemented as functions, although in all cases their primary tasks are to massage, normalize, cleanse telemetry data, not to spit anything out, per se.
Hmm, the documentation states that "parameter symbols (...) only work in SELECT, INSERT, UPDATE, and DELETE commands.", so this probably isn't possible using parameters.
But as a workaround you can build a dynamic DO and include a variable to get the values and the INSERT in there.
DO
$o$
DECLARE
v_query_text varchar;
v_job_name varchar := format('%I.%I',
'public',
'get_log_rcd');
BEGIN
v_query_text := concat('DO ',
'$i$ ',
'DECLARE ',
' big_local public.job_log; ',
'BEGIN ',
' CALL ', v_job_name, '(big_local); ',
' INSERT INTO tmp_log_cache ',
' (log_id, ',
' event_id, ',
' job_id, ',
' rows_affected) ',
' VALUES (big_local.log_id, ',
' big_local.event_id, ',
' big_local.job_id, '
' big_local.rows_affected); ',
'END; ',
'$i$; ');
EXECUTE v_query_text;
END;
$o$;
db<>fiddle
Thanks--I would not have considered the ability to execute a 'do' using execute. It just would not have occurred to me. Well, here's my solution: flip to functions.
With the understanding that my 'Requestor' is only given sp's to run because that's what we had to do with SQL Server and it was reflex, I did the 1-line change needed to flip my example sp above to a function:
CREATE OR REPLACE function public.get_log_rcdf(
inout p_log_rcd public.job_log)
returns public.job_log
LANGUAGE 'plpgsql'
as
$BODY$
declare
v_log_id integer = 40;
v_event_id integer = 698;
v_job_id integer = 45;
v_rows_affected integer = 60;
begin
select
v_log_id
, v_event_id
, v_job_id
, v_rows_affected
into
p_log_rcd.log_id,
p_log_rcd.event_id,
p_log_rcd.job_id,
p_log_rcd.rows_affected;
end;
$BODY$
In fact, the change to a function required the addition of a RETURNS line. Done. Then, the dynamic call was tweaked to a SELECT and the execute modified with an INTO:
do
$$
declare
big_local public.job_log;
v_query_text varchar;
v_job_name varchar = 'public.get_log_rcdf';
begin
select 'select * from ' || v_job_name || '( $1 );'
into v_query_text;
raise info 'SQL text is: %', v_query_text;
execute v_query_text into big_local using big_local;
insert into tmp_log_cache (
log_id
, event_id
, job_id
, rows_affected )
values (
big_local.log_id
, big_local.event_id
, big_local.job_id
, big_local.rows_affected);
end;
$$;
and the process now works exactly as desired. I tidy up my handling of the dynamic function name as illustrated in the first answer, and I think we're done here.

Dynamically add a column with multiple values to any table using a PL/pgSQL function

I would like to use a function/procedure to add to a 'template' table an additional column (e.g. period name) with multiple values, and do a cartesian product on the rows, so my 'template' is duplicated with the different values provided for the new column.
E.g. Add a period column with 2 values to my template_country_channel table:
SELECT *
FROM unnest(ARRAY['P1', 'P2']) AS prd(period)
, template_country_channel
ORDER BY period DESC
, sort_cnty
, sort_chan;
/*
-- this is equivalent to:
(
SELECT 'P2'::text AS period
, *
FROM template_country_channel
) UNION ALL (
SELECT 'P1'::text AS period
, *
FROM template_country_channel
)
--
*/
This query is working fine, but I was wondering if I could turn that into a PL/pgSQL function/procedure, providing the new column values to be added, the column to add the extra column to (and optionally specify the order by conditions).
What I would like to do is:
SELECT *
FROM template_with_periods(
'template_country_channel' -- table name
, ARRAY['P1', 'P2'] -- values for the new column to be added
, 'period DESC, sort_cnty, sort_chan' -- ORDER BY string (optional)
);
and have the same result as the 1st query.
So I created a function like:
CREATE OR REPLACE FUNCTION template_with_periods(template regclass, periods text[], order_by text)
RETURNS SETOF RECORD
AS $BODY$
BEGIN
RETURN QUERY EXECUTE 'SELECT * FROM unnest($2) AS prd(period), $1 ORDER BY $3' USING template, periods, order_by ;
END;
$BODY$
LANGUAGE 'plpgsql'
;
But when I run:
SELECT *
FROM template_with_periods('template_country_channel', ARRAY['P1', 'P2'], 'period DESC, sort_cnty, sort_chan');
I have the error ERROR: 42601: a column definition list is required for functions returning “record”
After some googling, it seems that I need to define the list of columns and types to perform the RETURN QUERY (as the error message precisely states).
Unfortunately, the whole idea is to use the function with many 'template' tables, so columns name & type lists is not fixed.
Is there any other approach to try?
Or is the only way to make it work is to have within the function, a way to get list of columns' names and types of the template table?
I did this with refcursor if You want output columns list completely dynamic:
CREATE OR REPLACE FUNCTION is_record_exists(tablename character varying, columns character varying[], keepcolumns character varying[] DEFAULT NULL::character varying[])
RETURNS SETOF refcursor AS
$BODY$
DECLARE
ref refcursor;
keepColumnsList text;
columnsList text;
valuesList text;
existQuery text;
keepQuery text;
BEGIN
IF keepcolumns IS NOT NULL AND array_length(keepColumns, 1) > 0 THEN
keepColumnsList := array_to_string(keepColumns, ', ');
ELSE
keepColumnsList := 'COUNT(*)';
END IF;
columnsList := (SELECT array_to_string(array_agg(name || ' = ' || value), ' OR ') FROM
(SELECT unnest(columns[1:1]) AS name, unnest(columns[2:2]) AS value) pair);
existQuery := 'SELECT ' || keepColumnsList || ' FROM ' || tableName || ' WHERE ' || columnsList;
RAISE NOTICE 'Exist query: %', existQuery;
OPEN ref FOR EXECUTE
existQuery;
RETURN next ref;
END;$BODY$
LANGUAGE plpgsql;
Then need to call FETCH ALL IN to get results. Detailed syntax here or there: https://stackoverflow.com/a/12483222/630169. Seems it is the only way for now. Hope something will be changed in PostgreSQL 11 with PROCEDURES.

DB2. Select all columns in a table that contains a value

I would like to find all column names in a table that contains a value in any given record.
I.e All columns that contains a string in the record value.
'%ABC%' or '%QAW%' or '%IGH%'
If possible give me all the tables and columns in a DB schema, so I do not have to query ever table manually
2016-06-15
So I got a little further, I can now get all the values from each column in each row in each table. Now I need to see if that value ( v_value ) exist in a list of airport codes. i.e ['LAS','LAX','BIL']
I have all the airports in a table that I want to read into and array.
I am having trouble with creating that array and getting the data into it.
Here is what I have so far.
Look at the TODO's
CREATE OR REPLACE PROCEDURE "CMSDB"."TEST1"
()
LANGUAGE SQL
SPECIFIC SQL3
P1: BEGIN
DECLARE v_tabschema VARCHAR(255);
DECLARE v_tabname VARCHAR(255);
DECLARE v_colname VARCHAR(255);
DECLARE v_airport VARCHAR(255);
DECLARE v_stmt VARCHAR(3000);
DECLARE V_SQL VARCHAR(3000);
DECLARE v_value VARCHAR(255);
DECLARE SQLSTATE CHAR(5) DEFAULT '00000';
DECLARE v_stmt2 STATEMENT;
DECLARE v_value_cursor CURSOR FOR v_stmt2;
DECLARE v_airport_cursor CURSOR FOR select IDX from CMSDB.AIRPORTS;
DECLARE syscat_cursor CURSOR FOR select trim(tabschema), tabname, colname from cmsdb.syscat.columns where tabname = 'ACCTGROUP' and tabschema = 'CMSDB' and TYPENAME = 'VARCHAR' and colname not in ('CHGDATE','CHGPAGE','CHGPROG','CHGTYPE','CHGUSER','CREATEDATETIME','CREATEDBYID','REC_ID');
open v_airport_cursor;
FETCH FROM v_airport_cursor INTO v_airport;
WHILE (SQLSTATE = '00000') DO
call DBMS_OUTPUT.PUT_LINE(v_airport);
-- TODO Add each value to a list, arryalist that can be used to check if the v_value is in the list.
FETCH FROM v_airport_cursor INTO v_airport;
END WHILE;
close v_airport_cursor;
OPEN syscat_cursor;
FETCH FROM syscat_cursor INTO v_tabschema, v_tabname, v_colname;
WHILE (SQLSTATE = '00000') DO
--call DBMS_OUTPUT.PUT_LINE(v_tabschema || ' ' || v_tabname || ' ' || v_colname);
SET v_stmt = 'select ' || v_colname || ' from ' || v_tabschema || '.' || v_tabname;
--call DBMS_OUTPUT.PUT_LINE(v_stmt);
PREPARE v_stmt2 FROM v_stmt;
OPEN v_value_cursor;
FETCH FROM v_value_cursor INTO v_value;
WHILE (SQLSTATE = '00000') DO
-- TODO
--IF ( airportList contains v_value) THEN
--call DBMS_OUTPUT.PUT_LINE(v_value);
--END IF;
FETCH FROM v_value_cursor INTO v_value;
END WHILE;
CLOSE v_value_cursor;
FETCH FROM syscat_cursor INTO v_tabschema, v_tabname, v_colname;
END WHILE;
close syscat_cursor;
END P1
You can use sysibm.syscolumns:
select colname
from sysibm.syscolumns
where tbname = 'XX' and
(name like %ABC%' or name like '%QAW%' or name like '%IGH%');
You'll need to create a cursor over SYSTABLES that returns all the tables in the system. Then have another cursor that returns all the column names in a given table. Once you have those, you can build a dynamic statement that checks all the columns in a given table for the value you are looking for. Fetch the next table name and do it all over again.
Obviously, if you can narrow down your search to a particular schema or even limit the search to tables/columns with a particular naming pattern; you'd be better off.
Another technique, depending on your platform and version of DB2. You might be able to do some sort of a bulk export to a set of text files. Then use a tool that will serach the contents of those text files.

PL/SQL SELECT INTO alternative

I have few SELECT statements that are made by joining multiple tables.
Each select is returning single row but from 10 to 20 fields in that row. What is the easiest way to store that data for later use?
I would like to avoid creating 50 variables and writing select into statements, using cursors and loop for single row is not the smartest idea from what i read around.
Is there a good way to do this?
Stupid example so you can get general idea
SELECT t1.field1
, t1.field2
, t1.field3
, t1.field4
, t2.field5
, t2.field6
, t2.field7
, t3.field8
FROM table1 t1
JOIN table2 t2 ON something
JOIN table3 t3 ON something
Sorry for errors in my english and thanks in advance
Create a view from your select statement. Thereafter can reference a record by using a single variable of type <viewname>%ROWTYPE.
Another option would be to wrap the select in an implicit cursor loop:
DECLARE
strvar VARCHAR2(400); -- demo purpose only
BEGIN
-- ...
FOR i IN (
-- ... here goesyour select statement ...
) LOOP
strvar := i.field1 || i.field2; -- ... whatever
END LOOP;
-- ...
END;
-- ...
Still another option is the declaration of a record type and a record variable:
DECLARE
TYPE tRec IS RECORD (
field1 table1.field1%TYPE
, field2 table1.field2%TYPE
, field3 table1.field3%TYPE
, field4 table1.field4%TYPE
, field5 table2.field5%TYPE
, field6 table2.field6%TYPE
, field7 table2.field7%TYPE
, field8 table3.field8%TYPE
)
r tRec;
BEGIN
-- ...
SELECT --...
INTO r
FROM --...
;
-- ...
END;
-- ...
Every time you make a select you will have a cursor - there's no escape from that.
Views and explicit cursors are a way to reuse select statements. Which one is better option depends on the case.
rowtype-attribute is handy way to create records automatically based on tables/views/cursors. I think this is the closest to your requirement what PL/SQL can offer.
create or replace package so51 is
cursor cursor1 is -- an example single row query
select
dual.* -- all table columns included
,'Y' as str
,rownum as num
,sysdate as date_
from dual
;
function get_data return cursor1%rowtype;
end;
/
show errors
create or replace package body so51 is
-- function never changes when cursor1 changes
function get_data return cursor1%rowtype is
v_data cursor1%rowtype;
begin
open cursor1;
fetch cursor1 into v_data;
close cursor1;
return v_data;
end;
end;
/
show errors
declare
v_data constant so51.cursor1%rowtype := so51.get_data;
begin
-- use only the data you need
dbms_output.put_line('v_data.dummy = ' || v_data.dummy);
dbms_output.put_line('v_data.str = ' || v_data.str);
dbms_output.put_line('v_data.num = ' || v_data.num);
dbms_output.put_line('v_data.date_ = ' || v_data.date_);
end;
/
Example run
SQL> #so51.sql
Package created.
No errors.
Package body created.
No errors.
v_data.dummy = X
v_data.str = Y
v_data.num = 1
v_data.date_ = 2015-11-23 09:42:02
PL/SQL procedure successfully completed.
SQL>