I am running a do loop that has four queries that can be run independently of each other inside two doubly nested FOR LOOPs. I have marked the functions PARALLEL SAFE, but they still won't execute in parallel. Are the EXCEPTION blocks causing any problems? I have these settings in postgresql.conf:
#max_worker_processes = 8 # (change requires restart)
max_parallel_maintenance_workers = 4 # taken from max_parallel_workers
max_parallel_workers_per_gather = 4 # taken from max_parallel_workers
#parallel_leader_participation = on
#max_parallel_workers = 8 # maximum number of max_worker_processes that
Here's the code. I'm starting to believe the four create materialized view statements have to be executed sequentially, but I don't know for sure why.
do $code$
declare
f text;
qs text;
oid int;
tbl text;
key text;
pair cursor(key text) for select tablename from pg_tables where tablename like key order by tablename;
begin
FOR y in 2010..2022 LOOP
select substring('''%' || y || '%''',2,6) into key;
FOR f in pair(key)
LOOP
select substring(f::text, 2, 6) into tbl;
raise info 'f: %, tbl: %', f, tbl;
qs = format('create materialized view %I_2010_candlestick1m tablespace forex_view as select * from candlestick(''%I_2010'', 1, ''minute'', ''-infinity'', ''infinity'') with data;', tbl, tbl );
begin
raise info 'executing: %', qs;
execute qs;
exception when others then
raise notice '% %', SQLERRM, SQLSTATE;
end;
qs = format('create materialized view %I_2010_candlestick5m tablespace forex_view as select * from candlestick(''%I_2010'', 5, ''minute'', ''-infinity'', ''infinity'') with data;', tbl, tbl );
begin
raise info 'executing: %', qs;
execute qs;
exception when others then
raise notice '% %', SQLERRM, SQLSTATE;
end;
qs = format('create materialized view %I_2010_candlestick15m tablespace forex_view as select * from candlestick(''%I_2010'', 15, ''minute'', ''-infinity'', ''infinity'') with data;', tbl, tbl );
begin
raise info 'executing: %', qs;
execute qs;
exception when others then
raise notice '% %', SQLERRM, SQLSTATE;
end;
qs = format('create materialized view %I_2010_candlestick1day tablespace forex_view as select * from candlestick(''%I_2010'', 1, ''day'', ''-infinity'', ''infinity'') with data;', tbl, tbl );
begin
raise info 'executing: %', qs;
execute qs;
exception when others then
raise notice '% %', SQLERRM, SQLSTATE;
end;
COMMIT;
END LOOP;
END LOOP;
end $code$
language 'plpgsql';
Related
I am getting folowwing error when execute my stored procedure. Does anyone knows how can i solve this issue? The table mentioned in error is always the same and it's the first one from the top of the list from my schema.
This is my error:
relation "t_ser_d2ert" does not exist 42P01
This is my stored procedure:
CREATE OR REPLACE PROCEDURE schem.myprocedure()
LANGUAGE plpgsql
AS $procedure$
declare
sessionPid numeric = pg_backend_pid();
BEGIN
begin
raise notice 'schem.myprocedure()';
insert into myschema.t_tables (tablename, sizemb, rowcounts, created)
SELECT
table_name as tablename,
(pg_relation_size(quote_ident(table_name)) / 1024 /1024) sizemb,
reltuples as rowcounts, NOW() FROM information_schema.tables ist
left join pg_class pc on pc.relname = ist.table_name
WHERE table_schema = 'myschema';
exception
when others then
raise notice ' >> Error for schem.myprocedure()>> % %', sqlerrm, sqlstate;
end ;
END;
$procedure$
;
Here's below my procedure. Inside u find a loop. Target is if any of single loop execution will throw exception i don't want to stop the process and want to continue and commit each loop query which was executed succesfully. Therefore i placed exception catch inside loop. As u can see i also got a commit at the end and some begin/end blocks. My question is whether did i correctly or maybe i should put additional commit inside begin/end inside the loop (just after execute mysql;)? Thank you in advance.
CREATE OR REPLACE PROCEDURE myProc()
LANGUAGE plpgsql
AS $procedure$
declare
mysql text;
tb_name text;
myTables CURSOR for
SELECT table_name
FROM information_schema.tables
WHERE table_type='BASE TABLE'
AND table_schema='dist';
begin
begin
call DoSomeJob();
for tb in myTables loop
tb_name := tb;
begin
mysql := format('delete from %I where somecol=2', tb_name);
execute mysql;
exception
when others then
raise notice '% %', SQLERRM, SQLSTATE;
end ;
end loop;
call doOtherJob();
exception
when others then
raise notice 'The transaction is in an uncommittable state. '
'Transaction was rolled back';
raise notice '%: %', SQLSTATE, sqlerrm;
end ;
commit;
end;
$procedure$
;
UPDATE:
CREATE OR REPLACE PROCEDURE myProc()
LANGUAGE plpgsql
AS $procedure$
declare
mysql text;
tb_name text;
myTables CURSOR for
SELECT table_name
FROM information_schema.tables
WHERE table_type='BASE TABLE'
AND table_schema='dist';
begin
begin
call DoSomeJob();
exception
when others then
raise notice 'The transaction is in an uncommittable state. '
'Transaction was rolled back';
raise notice '%: %', SQLSTATE, sqlerrm;
end;
RAISE EXCEPTION 'ERROR test';
for tb in myTables loop
tb_name := tb;
begin
mysql := format('delete from %I where somecol=2', tb_name);
execute mysql;
exception
when others then
raise notice '% %', SQLERRM, SQLSTATE;
end ;
end loop;
begin
call doOtherJob();
exception
when others then
raise notice 'The transaction is in an uncommittable state. '
'Transaction was rolled back';
raise notice '%: %', SQLSTATE, sqlerrm;
end;
commit;
end;
$procedure$;
I explain you how to works PostgreSQL transactions in procedures.
Firstly
begin is a - "begin transaction";
end is a - "commit transaction"
And when you using transaction in another transaction (sub transactions) when your first level transaction rollbacked then all sub transactions also will be rollbacked.
For example:
begin --block call2
begin --block call1
call1;
exception
when others then
raise notice 'error call1'
end;
call2;
exception
when others then
raise notice 'error call2'
end;
Here when call2 failed then call1 also be failed. Because block call1 is in the block call2
And on your procedure - when doOtherJob() will be fail then all your inserted data will fail to. For solving this problem you can write your procedure as below:
CREATE OR REPLACE PROCEDURE myProc()
LANGUAGE plpgsql
AS $procedure$
declare
mysql text;
tb_name text;
myTables CURSOR for
SELECT table_name
FROM information_schema.tables
WHERE table_type='BASE TABLE'
AND table_schema='dist';
begin
call DoSomeJob();
for tb in myTables loop
tb_name := tb;
begin
mysql := format('delete from %I where somecol=2', tb_name);
execute mysql;
exception
when others then
raise notice '% %', SQLERRM, SQLSTATE;
end ;
end loop;
begin
call doOtherJob();
exception
when others then
raise notice 'The transaction is in an uncommittable state. '
'Transaction was rolled back';
raise notice '%: %', SQLSTATE, sqlerrm;
end;
commit;
end;
$procedure$;
You can set for block DoSomeJob() to the same logic.
When using procedures in PostgreSQL, especially about transactions, you need to know and understand some details well. I will now explain to you the important details and you will understand how it works.
begin - this is start transaction (always)
exception
end - this is not always commit-transaction, because if exception is created then end will be rollback-transaction if else end will be commit-transaction
And remember that: each block end affects only its own block begin
At the same time, pay attention to this:
During rollback process, all other begin-exception-end blocks which is under between the begin block and the exception block will be rollback.
Please execute this block and view result: select * from table test1
DO
$body$
begin
CREATE TABLE test1 (
id int4 NOT NULL,
CONSTRAINT test1_pk PRIMARY KEY (id)
);
begin -- block 1
insert into test1 (id) values (1);
exception
when others then
raise notice 'exception ';
end;
begin -- block 2
insert into test1 (id) values (error);
exception
when others then
raise notice 'exception ';
end;
begin -- block 3
insert into test1 (id) values (3);
exception
when others then
raise notice 'exception ';
end;
begin --block 4
begin -- block 5
insert into test1 (id) values (5);
exception
when others then
raise notice 'exception ';
end;
insert into test1 (id) values (error);
exception
when others then
raise notice 'exception ';
end;
END;
$body$
LANGUAGE 'plpgsql';
block 1 - success execute (success insert 1) - commit
block 2 - execute with error (no inserted record) - rollback
block 3 - execute success (success insert 3) - commit
block 5 - execute success (success insert 5) - commit
block 4 - execute with error (no inserted record) - rollback
Since block 5 is inside block 4 so block 5 already rollbacked
Result: select * from test1
1
3
The following sql file runs without error; however it does no display any output.
The file is executed with: pslq -d database -f filename.
How can this be modified to display output?
do $$
declare tn varchar ;
begin
for tn in
select tablename from pg_tables where tableowner not like 'postgres%'
loop
EXECUTE format('SELECT count(*) from %I;', tn);
end loop;
end;
$$;
The problem is that the result of a dynamic query is discarded unless you use SELECT ... INTO:
DO $$
DECLARE
tn text;
total bigint;
BEGIN
FOR tn IN
SELECT tablename FROM pg_tables WHERE tableowner NOT LIKE 'postgres%'
LOOP
EXECUTE format('SELECT count(*) from %I;', tn) INTO total;
RAISE NOTICE 'Table: % rows: %', tn, total;
END LOOP;
END;
$$;
I have this piece of code :
DO
$$
DECLARE v_RowCountInt Int;
BEGIN
execute ('insert into vcc.toto select * from vcc.segment s limit 50
on conflict (id_segment,pds) do update
set id_segment = excluded.id_segment
,pds = excluded.pds');
GET DIAGNOSTICS v_RowCountInt = ROW_COUNT;
RAISE NOTICE 'Returned % rows', v_RowCountInt;
END;
$$
The RAISE NOTICE return 50 rows which is what i want.
But if i add this to my code
DO
$$
DECLARE v_RowCountInt Int;
DECLARE query_plan varchar;
BEGIN
execute ('explain (analyze, COSTS, VERBOSE, BUFFERS, format json)
insert into vcc.toto select * from vcc.segment s limit 50
on conflict (id_segment,pds) do update
set id_segment = excluded.id_segment
,pds = excluded.pds')
into query_plan;
GET DIAGNOSTICS v_RowCountInt = ROW_COUNT;
RAISE NOTICE 'Returned % rows', v_RowCountInt;
END;
$$
This RAISE 1 row only (because of into query_plan i think)
How can i get 50 rows instead while keeping the "INTO query_plan" variable ?
Thanks
I have faced a strange problem,I am using postgresql and I want to store result of a query into a variable. this is my code in function
select "GRIDS"."IDcluster" into clusterIDrow1 from "GRIDS" where "IDraster"=arow."IDraster";
RAISE DEBUG 'arow.IDraster is %',arow."IDraster";
RAISE DEBUG 'clusterIDrow1 is %',clusterIDrow1;
this gives me
DEBUG: arow.IDraster is 1
DEBUG: clusterIDrow1 is 0
but it must return clusterIDrow1 =44 since this query returns 44
select "GRIDS"."IDcluster" from "GRIDS" where "IDraster"=1;
can you please help me find out my mistake?
this is all of my function
CREATE OR REPLACE FUNCTION public.similarity_cal(c double precision,t double precision)
RETURNS text AS
$func$
DECLARE
num integer=1;
algabra raster;
width integer;
height integer;
sumOfsimilarity integer=1;
arow record;
secrow record;
clusterIDrow1 double precision;
clusterIDrow2 integer;
result text;
BEGIN
set client_min_messages to 'debug';
UPDATE public."GRIDS" SET "IDcluster"=0;
FOR arow IN
SELECT *
FROM "rasters"
LOOP
FOR secrow IN
SELECT *
FROM "rasters"
LOOP
SELECT ST_width(arow.rast),ST_height(arow.rast) into width,height As pvc ;
RAISE DEBUG 'first loop';
RAISE DEBUG ' width %', width;
RAISE DEBUG ' height %', height;
CONTINUE WHEN arow.rid=secrow.rid;
RAISE DEBUG 'start run algabra';
SELECT
ST_MapAlgebra(
arow.rast, 1,
secrow.rast, 1,
'(abs([rast2.val] - [rast1.val]))'
) into algabra AS algabraTable;
RAISE DEBUG 'end run algabra';
RAISE DEBUG 'start run count';
SELECT sum((pvc).count) into sumOfsimilarity
FROM (SELECT ST_ValueCount(algabra) As pvc) As foo where (pvc).value<c;
RAISE DEBUG 'end run count, value is %',sumOfsimilarity;
RAISE DEBUG 'Similarity condition is %',width*height*t;
IF (sumOfsimilarity>=width*height*t) THEN
RAISE DEBUG 'similarity is true';
select "GRIDS"."IDcluster" into clusterIDrow1 from "GRIDS" where "IDraster"=arow."IDraster";
RAISE DEBUG 'arow.IDraster is %',arow."IDraster";
RAISE DEBUG 'clusterIDrow1 is %',clusterIDrow1;
IF (clusterIDrow1<>0) THEN
select "IDcluster" into clusterIDrow2 from "GRIDS" where "IDraster"=secrow."IDraster";
RAISE DEBUG 'clusterIDrow2 is %',clusterIDrow2;
IF (clusterIDrow2=0) THEN
RAISE DEBUG 'update grids %',secrow."IDraster";
UPDATE public."GRIDS"
SET "IDcluster"=clusterIDrow1 WHERE "IDraster"=secrow."IDraster";
END IF;
ELSE
UPDATE public."GRIDS"
SET "IDcluster"=num WHERE "IDraster"=arow."IDraster";
num=num+1;
END IF;
END IF;
END LOOP;
END LOOP;
END
$func$ LANGUAGE plpgsql;