Having an ordinal column with no gaps - postgresql

I want to have an ordinal column in which values always start from 1 and have no gaps. I have devised a solution with triggers, but I'd like to know if there is a better or more elegant way.
BEFORE INSERT trigger renumbers the rows that come after the inserted value. If value is not provided or too high, it is set to row count + 1. Similarly, AFTER DELETE trigger renumbers the rows that come after the deleted value. Both triggers lock rows before changing the value.
CREATE OR REPLACE FUNCTION ids_insert() RETURNS trigger AS $BODY$
DECLARE
_lock_sql text;
_id bigint;
BEGIN
IF TG_OP = 'INSERT' THEN
IF NEW.id < 1 THEN
RAISE EXCEPTION 'ID must be greater than zero.';
END IF;
EXECUTE format('SELECT COUNT(*) + 1 FROM %I', TG_TABLE_NAME)
INTO _id;
IF NEW.id IS NULL OR NEW.id > _id THEN
NEW.id := _id;
ELSE
_lock_sql := format(
'SELECT id FROM %I '
'WHERE id >= %s '
'ORDER BY id DESC '
'FOR UPDATE', TG_TABLE_NAME, NEW.id
);
FOR _id IN EXECUTE _lock_sql LOOP
EXECUTE format('UPDATE %I SET id = id + 1 WHERE id = %s', TG_TABLE_NAME, _id);
END LOOP;
END IF;
ELSE
IF NEW.id != OLD.id THEN
RAISE EXCEPTION 'Changing the ID directly is not allowed.';
END IF;
END IF;
RETURN NEW;
END;
$BODY$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION ids_delete() RETURNS trigger AS $BODY$
DECLARE
_lock_sql text;
_id bigint;
BEGIN
_lock_sql := format(
'SELECT id FROM %I '
'WHERE id > %s '
'ORDER BY id '
'FOR UPDATE', TG_TABLE_NAME, OLD.id
);
FOR _id IN EXECUTE _lock_sql LOOP
EXECUTE format('UPDATE %I SET id = id - 1 WHERE id = %s', TG_TABLE_NAME, _id);
END LOOP;
RETURN OLD;
END;
$BODY$ LANGUAGE plpgsql;
CREATE TABLE test (
id bigint PRIMARY KEY,
...
)
CREATE TRIGGER test_insert BEFORE INSERT OR UPDATE OF id ON test
FOR EACH ROW WHEN (pg_trigger_depth() < 1) EXECUTE PROCEDURE ids_insert();
CREATE TRIGGER test_delete AFTER DELETE ON test
FOR EACH ROW EXECUTE PROCEDURE ids_delete();

Related

Save dynamic query to variable in postgres stored procedure

I have the following postgres stored procedure:
CREATE OR REPLACE PROCEDURE
schema.MyProcedure()
AS $$
DECLARE
RowCount int;
BEGIN
SELECT cnt INTO RowCount
FROM (
SELECT COUNT(*) AS cnt
FROM MySchema.MyTable
) AS sub;
RAISE NOTICE 'RowCount: %', RowCount;
END;
$$
LANGUAGE plpgsql;
which "prints" out the row count of the static table MySchema.MyTable. How can it make it so I pass the Table and Schema name as an input.
eg:
CREATE OR REPLACE PROCEDURE
schema.MyProcedure(MySchema_In varchar, MyTable_In varchar)
AS $$
DECLARE
RowCount int;
BEGIN
SELECT cnt INTO RowCount
FROM (
SELECT COUNT(*) AS cnt
FROM || **MySchema_In** || . || **MyTable_In** ||
) AS sub;
RAISE NOTICE 'RowCount: %', RowCount;
END;
$$
LANGUAGE plpgsql;
You should use format() instead of concatenating the strings with || and then EXECUTE ... INTO to get the query's result, e.g.
CREATE OR REPLACE PROCEDURE MyProcedure(MySchema_In varchar, MyTable_In varchar)
AS $$
DECLARE RowCount int;
BEGIN
EXECUTE FORMAT('SELECT count(*) FROM %I.%I',$1,$2) INTO RowCount;
RAISE NOTICE 'RowCount: %', RowCount;
END;
$$
LANGUAGE plpgsql;

Using NEW.* inside EXECUTE regarding psql

I checked all related questions on SO but none helped in my case.
I have 2 loops(outside for the tables and inside for the columns). Tables are represented by 'r', and columns by 'm'. While being inside the 'm' loop which is supposed to send column values to the to-be-created trigger function. When I try to use 'NEW.m' (with trying many different formatting attempts) compiler always gives error.
Can you kindly advice on it please? Br
FOR r IN SELECT table_name FROM information_schema.tables LOOP
FOR m IN SELECT column_name FROM information_schema.columns WHERE (table_name = r.table_name ) LOOP
function_name := 'dictionary_functions_foreach_trigger';
EXECUTE format('CREATE OR REPLACE FUNCTION %s()
RETURNS trigger AS
$BODY$
DECLARE
BEGIN
IF NEW.m IS NOT NULL AND NEW.m IN (SELECT key FROM tableX.tableX_key)
THEN RETURN NEW;
END IF;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION %s()
OWNER TO mydb;',function_name, function_name);
EXECUTE 'CREATE TRIGGER ' || function_name || ' BEFORE INSERT OR UPDATE ON ' || belonging_to_schema || '.' || r.table_name || ' FOR EACH ROW EXECUTE PROCEDURE ' || function_name || '();';
----Trigger Functions after edit-
EXECUTE format(
'CREATE OR REPLACE FUNCTION %s()
RETURNS trigger AS
$BODY$
DECLARE
insideIs text := %s ;
BEGIN
FOR %s IN 0..(TG_NARGS-1) LOOP
IF %I= TG_ARGV[%s]
THEN insideIs := %s ;
END IF;
END LOOP;
IF NEW.%I IS NOT NULL AND (insideIs =%s) AND NEW.%I IN (SELECT key FROM tableX.tableX_key)
THEN RETURN NEW;
ELSE RETURN OLD;
END IF;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION %s()
OWNER TO mydb;' , function_name, 'notInside', 'i' , m.column_name, 'i' , 'ok', m.column_name, 'ok', m.column_name ,function_name);
You need to use another placeholder for the column name, they way you have written it, the column name "m" is hardcoded in the function.
You also don't really need the outer loop, as the table_name is also available in information_schema.columns.
Your trigger would also fail with a runtime error if the condition is not true as you don't have a return in that case. If you want to abort the statement, use return null;
You should also use format() for the create trigger statement.
FOR m IN SELECT table_schema, table_name, column_name
FROM information_schema.columns
WHERE table_name in (...)
LOOP
function_name := 'dictionary_functions_foreach_trigger';
EXECUTE format('CREATE OR REPLACE FUNCTION %I()
RETURNS trigger AS
$BODY$
DECLARE
BEGIN
IF NEW.%I IS NOT NULL AND NEW.%I IN (SELECT key FROM tableX.tableX_key) THEN
RETURN NEW;
END IF;
RETURN null; --<< you need some kind of return here!
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION %s()
OWNER TO mydb;', function_name, m.column_name, m.column_name, function_name, function_name);
EXECUTE format('CREATE TRIGGER %I BEFORE INSERT OR UPDATE ON %I.%I FOR EACH ROW EXECUTE PROCEDURE %I()',
function_name, m.table_schema, m.table_name, function_name);
END LOOP;
Online example

PostgreSQL log trigger optimalization

I spent a lot of time trying to optimize our pgsql log trigger which started to be a problem. I did huge progress (from 18min to 2.5min by inserting 3M rows) but I would like to know if some pgSql masters will be able to do it even better.
CREATE OR REPLACE FUNCTION table_log_trig()
RETURNS trigger AS
$BODY$
DECLARE
col TEXT; -- Single column name to save
newVal TEXT; -- New value for column
oldVal TEXT; -- Old value for column
colLimit TEXT[]; -- Columns that should be logged
BEGIN
IF TG_ARGV[0] IS NOT NULL THEN
-- Trigger specifies columns to log
SELECT array_agg(unnest)
FROM unnest(string_to_array(TG_ARGV[0], ','))
INTO colLimit;
ELSE
-- Trigger with no params. Log all columns
SELECT array_agg(json_object_keys)
FROM json_object_keys(row_to_json(NEW))
WHERE json_object_keys NOT IN ('id', 'created_at', 'updated_at') -- Exceptions
INTO colLimit;
END IF;
-- Loop over columns that should be saved in log
FOREACH col IN ARRAY colLimit
LOOP
-- INSERT & UPDATE
EXECUTE 'SELECT ($1).' || col || '::text' INTO newVal USING NEW;
-- UPDATE
IF TG_OP = 'UPDATE' THEN
EXECUTE 'SELECT ($1).' || col || '::text' INTO oldVal USING OLD;
END iF;
-- Add only new or changed data
IF
newVal != oldVal OR
(oldVal IS NULL AND newVal IS NOT NULL) OR
(oldVal IS NOT NULL AND newVal IS NULL)
THEN
INSERT INTO tab_logs (record_id, field_name, old_value, new_value, created_at, created_by, action)
VALUES (NEW.id, col, oldVal, newVal, NOW(), 999, 'O');
END IF;
END LOOP;
RETURN NEW;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
row_to_json() returns both column names and values; you may as well make use of these values, rather than extracting them later via dynamic SQL.
I haven't thoroughly tested this, let alone benchmarked it, but here's the gist of it:
CREATE OR REPLACE FUNCTION table_log_trig() RETURNS trigger AS
$$
DECLARE
OldJson JSONB = NULL;
BEGIN
IF TG_OP <> 'INSERT' THEN
OldJson := to_jsonb(old);
END IF;
INSERT INTO tab_logs (record_id, field_name, old_value, new_value, created_at, created_by, action)
SELECT new.id, key, OldValues.value, NewValues.value, now(), 999, 'O'
FROM jsonb_each(to_jsonb(new)) NewValues
LEFT JOIN jsonb_each(OldJson) OldValues USING (key)
WHERE
(
(TG_ARGV[0] IS NULL AND key NOT IN ('id', 'created_at', 'updated_at')) OR
(TG_ARGV[0] IS NOT NULL AND key = ANY(string_to_array(TG_ARGV[0], ',')))
) AND
OldValues.value::text IS DISTINCT FROM NewValues.value::text;
RETURN NULL;
END
$$
LANGUAGE plpgsql VOLATILE;

PostgreSQL update trigger Comparing Hstore values

I am creating trigger in PostgresSQL. On update I would like to compare all of the values in a Hstore column and update changes in my mirror table. I managed to get names of my columns in variable k but I am not able to get values using it from NEW and OLD.
CREATE OR REPLACE FUNCTION function_replication() RETURNS TRIGGER AS
$BODY$
DECLARE
k text;
BEGIN
FOR k IN SELECT key FROM EACH(hstore(NEW)) LOOP
IF NEW.k != OLD.k THEN
EXECUTE 'UPDATE ' || TG_TABLE_NAME || '_2' || 'SET ' || k || '=' || new.k || ' WHERE ID=$1.ID;' USING OLD;
END IF;
END LOOP;
RETURN NEW;
END;
$BODY$
language plpgsql;
You should operate on hstore representations of the records new and old. Also, use the format() function for better control and readibility.
create or replace function function_replication()
returns trigger as
$body$
declare
newh hstore = hstore(new);
oldh hstore = hstore(old);
key text;
begin
foreach key in array akeys(newh) loop
if newh->key != oldh->key then
execute format(
'update %s_2 set %s = %L where id = %s',
tg_table_name, key, newh->key, oldh->'id');
end if;
end loop;
return new;
end;
$body$
language plpgsql;
Another version - with minimalistic numbers of updates - in partially functional design (where it is possible).
This trigger should be AFTER trigger, to be ensured correct behave.
CREATE OR REPLACE FUNCTION function_replication()
RETURNS trigger AS $$
DECLARE
newh hstore;
oldh hstore;
update_vec text[];
pair text[];
BEGIN
IF new IS DISTINCT FROM old THEN
IF new.id <> old.id THEN
RAISE EXCEPTION 'id should be immutable';
END IF;
newh := hstore(new); oldh := hstore(old); update_vec := '{}';
FOREACH pair SLICE 1 IN ARRAY hstore_to_matrix(newh - oldh)
LOOP
update_vec := update_vec || format('%I = %L', pair[1], pair[2]);
END LOOP;
EXECUTE
format('UPDATE %I SET %s WHERE id = $1',
tg_table_name || '_2',
array_to_string(update_vec, ', '))
USING old.id;
END IF;
RETURN NEW; -- the value is not important in AFTER trg
END;
$$ LANGUAGE plpgsql;
CREATE TABLE foo(id int PRIMARY KEY, a int, b int);
CREATE TABLE foo_2(LIKE foo INCLUDING ALL);
CREATE TRIGGER xxx AFTER UPDATE ON foo
FOR EACH ROW EXECUTE PROCEDURE function_replication();
INSERT INTO foo VALUES(1, NULL, NULL);
INSERT INTO foo VALUES(2, 1,1);
INSERT INTO foo_2 VALUES(1, NULL, NULL);
INSERT INTO foo_2 VALUES(2, 1,1);
UPDATE foo SET a = 20, b = 30 WHERE id = 1;
UPDATE foo SET a = NULL WHERE id = 1;
This code is little bit more complex, but all what should be escaped is escaped and reduce number of executed UPDATE commands. UPDATE is full SQL command and the overhead of full SQL commands should be significantly higher than code that reduce number of full SQL commands.

DRY postgresql 9.4 trigger functions

I've written 3 functions to log transactions to designated tables:
CREATE OR REPLACE FUNCTION log_sites() RETURNS TRIGGER AS $body$
DECLARE
target_row sites%ROWTYPE;
BEGIN
IF (TG_OP = 'DELETE') THEN
-- No NEW row
target_row = OLD;
ELSE
target_row = NEW;
END IF;
INSERT INTO sites_history (transaction_type,
transaction_time,
site_id,
address,
name,
shared_key)
VALUES (TG_OP,
NOW(),
target_row.site_id,
target_row.address,
target_row.name,
target_row.shared_key);
RETURN target_row;
END;
$body$
LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION log_licenses() RETURNS TRIGGER AS $body$
DECLARE
target_row licenses%ROWTYPE;
BEGIN
IF (TG_OP = 'DELETE') THEN
target_row = OLD;
ELSE
target_row = NEW;
END IF;
INSERT INTO licenses_history (transaction_type,
transaction_time,
license_id,
start_date,
expiration_date,
site_id)
VALUES (TG_OP,
NOW(),
target_row.license_id,
target_row.start_date,
target_row.expiration_date,
target_row.site_id);
RETURN target_row;
END;
$body$
LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION log_clients() RETURNS TRIGGER AS $body$
DECLARE
target_row clients%ROWTYPE;
BEGIN
IF (TG_OP = 'DELETE') THEN
target_row = OLD;
ELSE
target_row = NEW;
END IF;
INSERT INTO clients_history (transaction_type,
transaction_time,
mac_address,
hardware,
license_id,
site_id)
VALUES (TG_OP,
NOW(),
target_row.mac_address,
target_row.hardware,
target_row.license_id,
target_row.site_id);
RETURN target_row;
END;
$body$
LANGUAGE plpgsql;
This results in a big ugly block of PL/pgSQL, which nobody at my job is especially familiar with, myself included. A coworker suggested it'd be nice to consolidate/DRYify all this stuff, but for the life of me, I'm not sure how, especially considering each table needs a separate trigger, and the way triggers pass data to their functions. Any suggestions?
ETA:
1) Here are the triggers:
CREATE TRIGGER sites_log
AFTER INSERT OR UPDATE OR DELETE
ON sites
FOR EACH ROW EXECUTE PROCEDURE log_transactions();
CREATE TRIGGER licenses_log
AFTER INSERT OR UPDATE OR DELETE
ON licenses
FOR EACH ROW EXECUTE PROCEDURE log_transactions();
CREATE TRIGGER clients_log
AFTER INSERT OR UPDATE OR DELETE
ON clients
FOR EACH ROW EXECUTE PROCEDURE log_transactions();
Here's what I've got now, after quite a bit of messing around:
CREATE OR REPLACE FUNCTION log_transactions() RETURNS TRIGGER LANGUAGE plpgsql AS $body$
DECLARE
target_row RECORD;
target_cols text[];
col_name RECORD;
col_name_str text;
right_now timestamp without time zone;
q_str text;
BEGIN
right_now := now();
target_cols := '{}';
FOR col_name IN SELECT column_name::text FROM information_schema.columns WHERE table_name = TG_TABLE_NAME AND table_schema = TG_TABLE_SCHEMA LOOP
col_name_str := col_name.column_name::text;
target_cols = ARRAY_APPEND(target_cols, col_name_str);
END LOOP;
RAISE NOTICE 'target_cols: %', target_cols;
IF (TG_OP = 'DELETE') THEN
target_row := OLD;
ELSE
target_row := NEW;
END IF;
RAISE NOTICE 'target_row: %', target_row;
EXECUTE format('INSERT INTO %I_history (transaction_time, transaction_type) VALUES (%L, %L)', TG_TABLE_NAME, right_now, TG_OP);
q_str := format('UPDATE %I_history SET (%s) = ', TG_TABLE_NAME, array_to_string(target_cols, ', ')) || '$1' || format(' WHERE transaction_type = %L AND transaction_time = %L', TG_OP, right_now);
EXECUTE q_str USING target_row;
RETURN target_row;
END;
$body$;
This doesn't work either, and it's spiraling out of control, complexity-wise.
Personally, I use a set of home-grown functions Below for auditing any table I want. All I do is run audit.enable() on any table I want to audit and it's stored in the general tables I have here. Yes, it's not exactly what you're doing, but it's the most "DRY" think there is really, I do it once and never again-- ever.
CREATE TABLE audit.audit_log
(
audit_log_seq serial primary key,
schema_name text NOT NULL,
table_name text NOT NULL,
db_username text,
user_seq bigint,
logged_ip inet,
log_timestamp TIMESTAMP NOT NULL DEFAULT (now()),
action TEXT NOT NULL CHECK (action IN ('I','D','U')),
comment varchar(500),
old_data hstore,
new_data hstore,
query text
) WITH
(
fillfactor=100
);
CREATE INDEX audit_log_schema_table_idx ON audit.audit_log(schema_name,table_name);
CREATE INDEX audit_log_timestamp_utc_idx ON audit.audit_log(log_timestamp);
CREATE INDEX audit_log_uname on audit.audit_log(user_seq);
-- generic function for all tables
CREATE OR REPLACE FUNCTION audit.log_func() RETURNS TRIGGER AS $body$
DECLARE
v_old_data hstore;
v_new_data hstore;
v_query text;
v_comment varchar;
-- v_old_data TEXT;
-- v_new_data TEXT;
BEGIN
v_query=current_query();
IF (TG_OP = 'UPDATE') THEN
v_old_data := hstore(OLD.*);
v_new_data := hstore(NEW.*);
v_comment=v_new_data -> 'audit_comment';
ELSIF (TG_OP = 'DELETE') THEN
v_old_data := hstore(OLD.*);
ELSIF (TG_OP = 'INSERT') THEN
v_new_data := hstore(NEW.*);
v_comment=v_new_data -> 'audit_comment';
ELSE
RAISE WARNING '[audit.log_func] - Other action occurred: %, at %',TG_OP,now();
RETURN NULL;
END IF;
INSERT INTO audit.audit_log (schema_name,table_name,db_username,user_seq,logged_ip,action,old_data,new_data,query, comment)
VALUES (TG_TABLE_SCHEMA::TEXT,
TG_TABLE_NAME::TEXT,
session_user::TEXT,
coalesce(current_setting('mvc.user_seq'),'0')::bigint, --current user
inet_client_addr(),
substring(TG_OP,1,1),
v_old_data,
v_new_data,
v_query,
v_comment);
IF (TG_OP = 'DELETE') THEN
RETURN OLD;
END IF;
RETURN NEW;
EXCEPTION
WHEN data_exception THEN
RAISE WARNING '[audit.log_func] - UDF ERROR [DATA EXCEPTION] - SQLSTATE: %, SQLERRM: %',SQLSTATE,SQLERRM;
RETURN NULL;
WHEN unique_violation THEN
RAISE WARNING '[audit.log_func] - UDF ERROR [UNIQUE] - SQLSTATE: %, SQLERRM: %',SQLSTATE,SQLERRM;
RETURN NULL;
--WHEN OTHERS THEN
-- RAISE WARNING '[audit.log_func] - UDF ERROR [OTHER] - SQLSTATE: %, SQLERRM: %',SQLSTATE,SQLERRM;
-- RETURN NULL;
END;
$body$ language plpgsql security definer;
CREATE OR REPLACE FUNCTION audit.enable(p_table_name text,p_schema_name text DEFAULT 'dallas') RETURNS VOID as $body$
DECLARE
BEGIN
EXECUTE 'create trigger trg_audit_'||p_table_name||' BEFORE INSERT OR UPDATE OR DELETE ON '||p_schema_name||'.'||p_table_name|| ' FOR EACH ROW EXECUTE PROCEDURE audit.log_func()';
exception when duplicate_object then null;
END;
$body$ language plpgsql security definer;