How to insert before update (Custom audit) - postgresql

I would like execute insert before I update some rows, insert would contain information about which table is updated, which column is updated, what query is executed etc..
And I wrote something like this:
DO $$
DECLARE _new_product_id bigint := 1;
_update_by_product_id bigint:= 2;
-- BEGIN TRANSACTION
BEGIN
DECLARE product_sql_query text := 'UPDATE products SET product_id =' + _new_product_id + 'WHERE product_id =' + _update_by_product_id + ';'
INSERT INTO products_audit (ordinal_number, table_name, column_name, action_applied, query_executed, value_before_update, value_after_update, created_date )
VALUES (1, 'products', 'product_id', 'UPDATE', users_query, 1, 1, NOW());
-- END TRANSACTION
COMMIT;
END;
$$
But I'm receiving an syntax error which says:
ERROR: syntax error at or near "INSERT" LINE 9: INSERT INTO
products_audit (ordinal_number, table...
What's wrong here, I guess product_sql_query value is not correct?

The DECLARE block needs to go before the first BEGIN of the PL/pgSQL - which is not the same as a BEGIN TRANSACTION. You can't control transactions in an anonymous PL/pgSQL block.
It's unclear to me what you intend with the product_sql_query variable, but removing all the syntax errors, the block should look like this:
DO $$
DECLARE
_new_product_id bigint := 1;
_update_by_product_id bigint:= 2;
product_sql_query text;
BEGIN
product_sql_query := 'UPDATE products SET product_id = ' || _new_product_id::text ||' WHERE product_id = ' || _update_by_product_id::text;
INSERT INTO products_audit
(ordinal_number, table_name, column_name, action_applied, query_executed, value_before_update, value_after_update, created_date)
VALUES
(1, 'products', 'product_id', 'UPDATE', users_query, 1, 1, NOW());
END;
$$

Related

How to use dynamic column names in an UPDATE statement in a function?

I want to update all the text columns by fetching the column names on runtime within the loop.
drop function if exists aa.clean_data();
create or replace function aa.clean_data()
returns void
language plpgsql
as
$$
declare
col_name varchar(50);
begin
for col_name in select column_name from information_schema.columns where table_schema = 'aa' and table_name = 'test' and data_type in ('character varying', 'character','text', '"char"', 'name') loop
update autopipe.test set col_name = initcap(col_name); -- case correction
update autopipe.test set col_name = REGEXP_REPLACE(col_name, '[^a-zA-Z\d\s:/\//]', '', 'g'); -- non alpha-numeric characters
update autopipe.test set col_name = REGEXP_REPLACE(col_name, '\s+$', '', 'g'); -- Trim padding
end loop;
end;
$$;
All the update statements above gives error below as it is working on runtime and can not figure out the value in "col_name"
SQL Error [42703]: ERROR: column "col_name" of relation "test" does not exist
Where: PL/pgSQL function autopipe.clean_data() line 7 at SQL statement
The question below seems to be near to what my problem is but I don't know how to use it in my case
How to use dynamic column names in an UPDATE or SELECT statement in a function?
Some dynamic SQL will help. The select and update queries are taken from the question w/o change.
drop function if exists aa.clean_data();
create or replace function aa.clean_data() returns void language plpgsql as
$$
declare
UPDATE_A constant text := $dynsql$ UPDATE autopipe.test SET %I = initcap(%I) $dynsql$;
UPDATE_B constant text := $dynsql$ UPDATE autopipe.test SET %I = REGEXP_REPLACE(%I, '[^a-zA-Z\d\s:/\//]', '', 'g') $dynsql$;
UPDATE_C constant text := $dynsql$ UPDATE autopipe.test SET %I = REGEXP_REPLACE(%I, '\s+$', '', 'g') $dynsql$;
col_name text;
begin
for col_name in select column_name from information_schema.columns where table_schema = 'aa' and table_name = 'test' and data_type in ('character varying', 'character','text', '"char"', 'name') loop
execute format(UPDATE_A, col_name, col_name);
execute format(UPDATE_B, col_name, col_name);
execute format(UPDATE_C, col_name, col_name);
end loop;
end;
$$;
It might be worth running a VACCUM on the target table after such a massive UPDATE.

How to use a Function Parameter in a Cursor that's incorporated with Dynamic SQL in Postgres Functions?

Created this Postgres Function which is working fine, but the actual requirement is to pass the input parameter in the function to the Cursor which uses the dynamic SQL as follows,
The below is the Function
CREATE OR REPLACE FUNCTION ssp2_pcat.find_shift_dates (date_to_find date)
RETURNS void
LANGUAGE 'plpgsql'
COST 100
VOLATILE
AS $BODY$
DECLARE
C1 CURSOR FOR
SELECT TABLE_NAME, 'SELECT COUNT(*) FROM ' || TABLE_NAME || ' WHERE ' ||
COLUMN_NAME || ' = '||
'CASE WHEN ' || COLUMN_NAME || ' LIKE ' || '''%START%'''||' THEN
date_to_find ELSE date_to_find-1 END;' SQL_TEXT
FROM (
SELECT TABLE_NAME, COLUMN_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME IN (SELECT TABLE_NAME FROM RESET_DATES WHERE RESET_IT =
'Y') AND
UPPER(DATA_TYPE) = 'DATE'
AND (COLUMN_NAME LIKE '%START%' OR COLUMN_NAME LIKE '%END%')
AND (COLUMN_NAME NOT LIKE '%TEST%'
AND COLUMN_NAME NOT LIKE '%PCAT%'
AND COLUMN_NAME NOT LIKE '%ORDER%'
AND COLUMN_NAME NOT LIKE '%SEASON%'
AND COLUMN_NAME NOT LIKE '%_AT')
ORDER BY 1, 2) A;
END_COUNT INTEGER := 0;
START_COUNT INTEGER := 0;
TABLENAME VARCHAR(32) := 'ALFU';
l_start TIMESTAMP;
l_end TIMESTAMP;
Time_Taken VARCHAR(20);
BEGIN
l_start := clock_timestamp();
DELETE FROM SHIFT_DATES_COUNT;
FOR I IN C1 LOOP
IF I.TABLE_NAME <> TABLENAME THEN
INSERT INTO SHIFT_DATES_COUNT VALUES (TABLENAME, START_COUNT,
END_COUNT, current_timestamp::timestamp(0));
TABLENAME := I.TABLE_NAME;
END_COUNT := 0;
START_COUNT := 0;
END IF;
IF STRPOS(I.SQL_TEXT, 'END') > 0 THEN
EXECUTE I.SQL_TEXT INTO END_COUNT;
RAISE NOTICE '% ', ('END: ' || I.SQL_TEXT);
ELSE
EXECUTE I.SQL_TEXT INTO START_COUNT;
RAISE NOTICE '% ', ('START: ' || I.SQL_TEXT);
END IF;
END LOOP;
INSERT INTO SHIFT_DATES_COUNT VALUES (TABLENAME, START_COUNT, END_COUNT,
current_timestamp::timestamp(0));
RAISE NOTICE '% ', ('INSERT INTO SHIFT_DATES_COUNT Done...');
l_end := clock_timestamp();
Time_Taken := (l_end-l_start);
RAISE NOTICE '% ', ('FIND_SHIFT_DATES Took: ' || Time_Taken );
END;
$BODY$;
Please let me know how can I use the date_to_find input parameter in the Dynamic SQL in the Cursor in the above Function.
You can use unbound cursor, clause fetch to get data from cursor, and exit when not found to finish, like:
CREATE OR REPLACE FUNCTION example (p_name text) RETURNS void LANGUAGE 'plpgsql' AS $$
DECLARE
C1 refcursor;
res record;
BEGIN
OPEN c1 FOR EXECUTE 'SELECT * FROM pg_database WHERE datname like ''%'||p_name||'%''';
LOOP
FETCH c1 INTO res;
EXIT WHEN not found;
raise notice 'value datname: %',res.datname;
END LOOP;
CLOSE c1;
RETURN;
END; $$;
--in my case
select example ('test')
NOTICE: value datname: test
NOTICE: value datname: test_msmov
NOTICE: value datname: test_resources
NOTICE: value datname: test_load_table
NOTICE: value datname: test_resources2
Total query runtime: 63 msec
1 row retrieved.
You can use EXECUTE clause for open cursor, see the documentation of PostgreSQL
https://www.postgresql.org/docs/10/plpgsql-cursors.html#PLPGSQL-CURSOR-OPENING
Example:
OPEN curs1 FOR EXECUTE format('SELECT * FROM %I WHERE col1 = $1',tabname) USING keyvalue;

Having an ordinal column with no gaps

I want to have an ordinal column in which values always start from 1 and have no gaps. I have devised a solution with triggers, but I'd like to know if there is a better or more elegant way.
BEFORE INSERT trigger renumbers the rows that come after the inserted value. If value is not provided or too high, it is set to row count + 1. Similarly, AFTER DELETE trigger renumbers the rows that come after the deleted value. Both triggers lock rows before changing the value.
CREATE OR REPLACE FUNCTION ids_insert() RETURNS trigger AS $BODY$
DECLARE
_lock_sql text;
_id bigint;
BEGIN
IF TG_OP = 'INSERT' THEN
IF NEW.id < 1 THEN
RAISE EXCEPTION 'ID must be greater than zero.';
END IF;
EXECUTE format('SELECT COUNT(*) + 1 FROM %I', TG_TABLE_NAME)
INTO _id;
IF NEW.id IS NULL OR NEW.id > _id THEN
NEW.id := _id;
ELSE
_lock_sql := format(
'SELECT id FROM %I '
'WHERE id >= %s '
'ORDER BY id DESC '
'FOR UPDATE', TG_TABLE_NAME, NEW.id
);
FOR _id IN EXECUTE _lock_sql LOOP
EXECUTE format('UPDATE %I SET id = id + 1 WHERE id = %s', TG_TABLE_NAME, _id);
END LOOP;
END IF;
ELSE
IF NEW.id != OLD.id THEN
RAISE EXCEPTION 'Changing the ID directly is not allowed.';
END IF;
END IF;
RETURN NEW;
END;
$BODY$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION ids_delete() RETURNS trigger AS $BODY$
DECLARE
_lock_sql text;
_id bigint;
BEGIN
_lock_sql := format(
'SELECT id FROM %I '
'WHERE id > %s '
'ORDER BY id '
'FOR UPDATE', TG_TABLE_NAME, OLD.id
);
FOR _id IN EXECUTE _lock_sql LOOP
EXECUTE format('UPDATE %I SET id = id - 1 WHERE id = %s', TG_TABLE_NAME, _id);
END LOOP;
RETURN OLD;
END;
$BODY$ LANGUAGE plpgsql;
CREATE TABLE test (
id bigint PRIMARY KEY,
...
)
CREATE TRIGGER test_insert BEFORE INSERT OR UPDATE OF id ON test
FOR EACH ROW WHEN (pg_trigger_depth() < 1) EXECUTE PROCEDURE ids_insert();
CREATE TRIGGER test_delete AFTER DELETE ON test
FOR EACH ROW EXECUTE PROCEDURE ids_delete();

Query string argument of EXECUTE is null

EDIT
It seems my issue is when this select statement returns null (which is the case I'm trying to handle - when it returns null, I want my new value to be -999). How can I go about doing this if it errors out whenever a null is found?
ORIGINAL
I have read every other SO post I could find regarding this error, but none of which seemed to address the root of my issue.
The error is pretty straightforward - one of my arguments within my EXECUTE statement is null. Great. However, I print out each of the values that make up my EXECUTE statement right before it gets called, and I can clearly see that none of the values are null.
Code:
CREATE FUNCTION inform_icrm_prob_flow_query(tablename text, location_id int,
product_date_str text, lead_time_start int,
lead_time_end int, first_member_id int,
last_member_id int, dest_file text)
RETURNS void AS $$
DECLARE
count int;
product_date TIMESTAMPTZ;
interval_lead_time_start text;
interval_lead_time_end text;
curr_value double precision;
query text;
BEGIN
product_date := product_date_str::TIMESTAMPTZ;
count := first_member_id;
curr_value := 0;
interval_lead_time_start := ''''|| product_date ||'''::timestamptz +
interval '''||lead_time_start||' hours''';
interval_lead_time_end := ''''|| product_date ||'''::timestamptz +
interval '''||lead_time_end||' hours'' -
interval ''6 hours''';
--create our temporary table and populate it's date column
EXECUTE 'CREATE TEMPORARY TABLE temp_table_icrm_prob_flow AS
SELECT * FROM generate_series('||interval_lead_time_start || ',' ||
interval_lead_time_end || ', ''6 hours'')
AS date_valid';
LOOP
EXIT WHEN count > last_member_id;
IF NOT EXISTS(
SELECT 'date_valid'
FROM information_schema.columns
WHERE table_name='temp_table_icrm_prob_flow'
and column_name='value'||count||'')
THEN
EXECUTE 'ALTER TABLE temp_table_icrm_prob_flow ADD COLUMN value' || count
|| ' double precision DEFAULT -999';
END IF;
raise notice 'tablename: %', tablename;
raise notice 'location_id: %', location_id;
raise notice 'product_date: %', product_date;
raise notice 'count: %', count;
query := 'SELECT value FROM '|| tablename ||'
INNER JOIN temp_table_icrm_prob_flow
ON (temp_table_icrm_prob_flow.date_valid = '|| tablename ||'.date_valid)
WHERE '|| tablename ||'.id_location = '|| location_id ||'
AND '|| tablename ||'.date_product = '''|| product_date ||'''
AND '|| tablename ||'.id_member = '|| count ||'';
EXECUTE query INTO curr_value;
EXECUTE 'UPDATE temp_table_icrm_prob_flow
SET value'|| count ||' = COALESCE('|| curr_value ||', -999)';
count := count + 1;
END LOOP;
EXECUTE 'ALTER TABLE temp_table_icrm_prob_flow DROP COLUMN date_valid';
EXECUTE 'COPY temp_table_icrm_prob_flow TO '''||dest_file||''' DELIMITER '','' CSV';
EXECUTE 'DROP TABLE temp_table_icrm_prob_flow';
END;
$$ LANGUAGE plpgsql;
Output:
NOTICE: tablename: inform_tseries_data_basin_proc_fcst_prob_flow
NOTICE: location_id: 38
NOTICE: product_date: 2015-02-05 12:00:00+00
NOTICE: count: 1
ERROR: query string argument of EXECUTE is null
CONTEXT: PL/pgSQL function inform_icrm_prob_flow_query(text,integer,text,integer,integer,integer,integer,text) line 38 at EXECUTE
If none of the variables I am passing in are null, and the only other thing referenced is a temp table that I know exists, what could be causing this error?
Note: when changing my query to:
query := 'SELECT value FROM '|| tablename ||' WHERE '|| tablename ||'.id_location = '|| location_id ||' AND '|| tablename ||'.date_product = '''|| product_date ||''' AND '|| tablename ||'.id_member = '|| count ||' AND temp_table_icrm_prob_flow.date_va lid = '|| tablename ||'.date_valid';
I get the following error:
NOTICE: tablename: inform_tseries_data_basin_proc_fcst_prob_flow
NOTICE: location_id: 38
NOTICE: product_date: 2015-02-05 12:00:00+00
NOTICE: count: 1
ERROR: missing FROM-clause entry for table "temp_table_icrm_prob_flow"
LINE 1: ..._data_basin_proc_fcst_prob_flow.id_member = 1 AND temp_table...
^
QUERY: SELECT value FROM inform_tseries_data_basin_proc_fcst_prob_flow WHERE inform_tseries_data_basin_proc_fcst_prob_flow.id_location = 38 AND inform_tseries_data_basin_proc_fcst_prob_flow.date_product = '2015-02-05 12:00:00+00' AND inform_tseries_data_basin_proc_fcst_prob_flow.id_member = 1 AND temp_table_icrm_prob_flow.date_valid = inform_tseries_data_basin_proc_fcst_prob_flow.date_valid
CONTEXT: PL/pgSQL function inform_icrm_prob_flow_query(text,integer,text,integer,integer,integer,integer,text) line 35 at EXECUTE
Sorry for small offtopic. Your code is pretty unreadable (and SQL injecttion vulnerable). There are some techniques, that you can use:
Use clause USING of EXECUTE statement for usual parameters.
DO $$
DECLARE
tablename text := 'mytab';
from_date date := CURRENT_DATE;
BEGIN
EXECUTE 'INSERT INTO ' || quote_ident(tablename) || ' VALUES($1)'
USING from_date;
END
$$;
This code will be safe (due using quote_ident function), little bit faster (due using binary value of from_date variable - removed multiple string<->date conversions and little bit more readable (because string expression is shorter).
Use function format. The building query string will be shorter and more readable (table aliases helps too):
query := format('
SELECT value
FROM %I _dtn
INNER JOIN temp_table_icrm_prob_flow t ON t.date_valid = _dtn.date_valid
WHERE _dtn.id_location = $1
AND _dtn.date_product = $2
AND _dtd.id_member = $3'
, tablename);
EXECUTE query INTO curr_value USING location_id, product_date, count;
Using variables named like important SQL keywords and identifier is wrong idea - names count, values are wrong.
The error message is clean - you are using the identifier temp_table_icrm_prob_flow.date_valid, but the table temp_table_icrm_prob_flow is not mentioned in query. The query missing JOIN part.

ERROR: missing FROM-clause entry for table "new"

I have a parent table layer_1_ and a number of child tables layer_1_points, layer_1_linestrings etc. which contain some geometry data. Each child table has its own geometry constraint. So, for example, layer_1_points has this constraint:
CONSTRAINT enforce_geotype_geom_geom CHECK (geometrytype(geom) = 'POINT'::text)
Whereas layer_1_linestrings table has this constraint:
CONSTRAINT enforce_geotype_geom_geom CHECK (geometrytype(geom) = 'LINESTRING'::text)
Many other layer tables have similar names: layer_2_, layer_3_, ..., layer_N_. And all of them have their own child tables. What I want to achive is that when a user inserts to a parent table (layer_N_), then this insert statement should be forwarded to a particular child table (layer_N_points etc.). So, for example, when I do:
INSERT INTO layer_1_ (geom) VALUES(ST_GeomFromText('POINT(0 0)', 3857))
I should actually insert to layer_1_points, because geom type is POINT. To achive all this I created this trigger function and the trigger itself:
CREATE OR REPLACE FUNCTION trigger_layer_insert()
RETURNS trigger AS
$$
DECLARE
var_geomtype text;
table_name text;
layer_id text := (TG_ARGV[0])::text;
BEGIN
var_geomtype := geometrytype(NEW.geom);
IF var_geomtype = 'POINT' THEN
table_name := (SELECT concat ('layer_', layer_id, '_points'));
ELSIF var_geomtype = 'MULTIPOINT' THEN
table_name := (SELECT concat ('layer_', layer_id, '_multipoints'));
ELSIF var_geomtype = 'LINESTRING' THEN
table_name := (SELECT concat ('layer_', layer_id, '_linestrings'));
ELSIF var_geomtype = 'MULTILINESTRING' THEN
table_name := (SELECT concat ('layer_', layer_id, '_multilinestrings'));
ELSIF var_geomtype = 'POLYGON' THEN
table_name := (SELECT concat ('layer_', layer_id, '_polygons'));
ELSIF var_geomtype = 'MULTIPOLYGON' THEN
table_name := (SELECT concat ('layer_', layer_id, '_multipolygons'));
END IF;
EXECUTE '
INSERT INTO ' || table_name || '
SELECT * FROM (SELECT NEW.*) AS t
';
RETURN NULL;
END;
$$
LANGUAGE 'plpgsql' VOLATILE;
CREATE TRIGGER trigger_layer_1_ BEFORE INSERT
ON layer_1_ FOR EACH ROW
EXECUTE PROCEDURE trigger_layer_insert(1);
However, when I do actual insert like:
INSERT INTO layer_1_ (geom) VALUES(ST_GeomFromText('POINT(0 0)', 3857))
I get an error message:
ERROR: missing FROM-clause entry for table "new"
LINE 3: SELECT * FROM (SELECT NEW.*) AS t
^
QUERY:
INSERT INTO layer_1_points
SELECT * FROM (SELECT NEW.*) AS t
So, what is wrong with SELECT NEW.* and how can I fix it? Thanks!
EDIT
I also tried this:
EXECUTE '
INSERT INTO ' || table_name || '
SELECT * FROM (SELECT NEW.*) AS t
' USING NEW;
But it has no effect.
When you execute something using PLPGSQL statement EXECUTE it runs in the different context so local variables is not visible there. To pass variable(s) the EXECUTE '<SQL script>' USING <variables list>; form is used:
EXECUTE 'insert into table(field1, field2) values ($1, $2)' USING var1, var2;
So the statement should be:
EXECUTE 'INSERT INTO ' || table_name || ' SELECT * FROM SELECT $1.*) AS t'
USING NEW;
But much more secure is using format function:
execute format('INSERT INTO %I SELECT * FROM SELECT $1.*) AS t', table_name)