Postgres - Track changes - postgresql

I want to track the history changes from my table, for that I created one history table, right now is saving old and new values, but I want to save in one column the fk from my regular table.
This is my history table:
CREATE TABLE IF NOT EXISTS sales.history (
history_id integer PRIMARY KEY NOT NULL,
mytable_id integer,
table_name varchar(30),
new_val json,
old_val json,
operation varchar(10),
CONSTRAINT "fk sales.mytable_id to history.mytable_id" FOREIGN KEY (mytable_id) REFERENCES sales.sales (mytable_id)
) TABLESPACE pg_default;
I created a trigger to track update,insert and delete
CREATE TRIGGER t_history BEFORE INSERT OR UPDATE OR DELETE ON sales.sales
FOR EACH ROW EXECUTE PROCEDURE core.func_store_history_changes();
Now my function
CREATE OR REPLACE FUNCTION core.func_store_history_changes() RETURNS trigger AS $$
BEGIN
IF TG_OP = 'INSERT'
THEN
INSERT INTO sales.history (tabname, schemaname, operation, new_val)
VALUES (TG_RELNAME, TG_TABLE_SCHEMA, TG_OP, row_to_json(NEW));
RETURN NEW;
ELSIF TG_OP = 'UPDATE'
THEN
INSERT INTO sales.history (tabname, schemaname, operation, new_val, old_val)
VALUES (TG_RELNAME, TG_TABLE_SCHEMA, TG_OP,
row_to_json(NEW), row_to_json(OLD));
RETURN NEW;
ELSIF TG_OP = 'DELETE'
THEN
INSERT INTO sales.history (tabname, schemaname, operation, old_val)
VALUES (TG_RELNAME, TG_TABLE_SCHEMA, TG_OP, row_to_json(OLD));
RETURN OLD;
END IF;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
As you can see my history table has mytable_id which is a foreign key and primary key from sales table, how i can get that value and insert into my history table?
Regards

For you specific problem, I guess that you could use NEW.mytable_id and use trigger AFTER INSERT ...
Here is a example, I've wrote only the first if (INSERT).
CREATE TABLE IF NOT EXISTS sales.history (
history_id serial PRIMARY KEY NOT NULL,
mytable_id integer,
table_name varchar(30),
new_val json,
old_val json,
operation varchar(10),
CONSTRAINT "fk sales.mytable_id to history.mytable_id" FOREIGN KEY (mytable_id) REFERENCES sales.sales (mytable_id)
) TABLESPACE pg_default;
CREATE TRIGGER t_history AFTER INSERT OR UPDATE OR DELETE ON sales.sales
FOR EACH ROW EXECUTE PROCEDURE core.func_store_history_changes();
CREATE OR REPLACE FUNCTION core.func_store_history_changes() RETURNS trigger AS $$
BEGIN
IF TG_OP = 'INSERT'
THEN
INSERT INTO sales.history (table_name, operation,mytable_id, new_val)
VALUES (TG_RELNAME, TG_OP, NEW.mytable_id, row_to_json(NEW));
RETURN NEW;
ELSIF TG_OP = 'UPDATE'
THEN
INSERT INTO sales.history (tabname, schemaname, operation, new_val, old_val)
VALUES (TG_RELNAME, TG_TABLE_SCHEMA, TG_OP,
row_to_json(NEW), row_to_json(OLD));
RETURN NEW;
ELSIF TG_OP = 'DELETE'
THEN
INSERT INTO sales.history (tabname, schemaname, operation, old_val)
VALUES (TG_RELNAME, TG_TABLE_SCHEMA, TG_OP, row_to_json(OLD));
RETURN OLD;
END IF;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
Result:
# insert into sales.sales (value) values ('2.3');
INSERT 0 1
# select * from sales.sales;
mytable_id | value
------------+-------
5 | 2.30
(2 rows)
# select * from sales.history;
history_id | mytable_id | table_name | new_val | old_val | operation
------------+------------+------------+-------------------------------+---------+-----------
3 | 5 | sales | {"mytable_id":5,"value":2.30} | | INSERT
(2 rows)
But, maybe, you could try to use this generic trigger for any table.
https://github.com/2ndQuadrant/audit-trigger

Related

How insert count of a table in another table attribute by trigger postgresql

I have created two tables "post" and "node" and I want to assign the sum of the entities of the "node" table in the attribute "nb_noeud" of the "post" table by trigger. But, the code below does not work and I think I missed something.
My code is as follows:
CREATE TABLE noeud
(
id_noeud serial NOT NULL,
code_noeud varchar(10) NULL,
type_noeud t_noeud NULL,
phase t_phase NULL
x_32632 bigint NULL,
y_32632 bigint NULL,
geom_noeud geometry(point) NULL,
obs text NULL
)
;
CREATE TABLE poste
(
id_pt serial NOT NULL,
code_pt varchar(8) NULL,
nom_pt varchar(50) NULL,
nb_noeud smallint NULL,
geom_pt geometry(polygon) NULL,
surf_pt numeric(15,2) NULL,
obs text NULL
)
;
CREATE OR REPLACE FUNCTION recap_noeud() RETURNS TRIGGER
language plpgsql AS
$$
DECLARE
som_noeud smallint;
BEGIN
IF (TG_OP = 'INSERT') THEN
SELECT COUNT(*) INTO som_noeud FROM noeud;
UPDATE poste set NEW.nb_noeud = som_noeud;
RETURN NEW;
ELSIF (TG_OP = 'DELETE') THEN
SELECT COUNT(*) INTO som_noeud FROM noeud;
UPDATE poste set NEW.nb_noeud = som_noeud;
RETURN NEW;
ELSIF (TG_OP = 'UPDATE') THEN
RETURN NULL;
ELSE
RAISE WARNING 'Other action occurred: %, at %', TG_OP, now();
RETURN NULL;
END IF;
END;
$$
;
DROP TRIGGER IF EXISTS trig_recap_noeud ON noeud;
CREATE TRIGGER trig_recap_noeud AFTER INSERT OR UPDATE OR DELETE ON noeud FOR EACH ROW EXECUTE PROCEDURE recap_noeud();
Replace DELETE and INSERT clauses with
IF TG_OP = 'INSERT' OR TG_OP = 'DELETE' THEN
SELECT COUNT(*) INTO som_noeud FROM noeud;
UPDATE poste set nb_noeud = som_noeud;
RETURN NULL;
Best regards, Bjarni

trigger in postgresql old value and new value

how to write trigger in postgresql which maintain old value, new value and table name.
I have 5 tables and each tables is different data structure i want to maintain audit details in single table with old value new value and table name.old value and new value contain multiple columns in json format with column name and value.
example
audit_details
----------------------------------------------------------------
date_time|table_name|old_data|new_data|user|primary_key_of_table
----------------------------------------------------------------
I wrote sample trigger function for you. You can make additional changes yourself.
CREATE TABLE test.log_table_data (
id serial not null,
schema_name varchar(100) NOT NULL,
table_name varchar(100) NOT NULL,
action_date timestamp NOT NULL DEFAULT now(),
action_type varchar(10) NOT NULL,
table_id int4 NOT NULL,
old_data jsonb NULL,
new_data jsonb NULL,
CONSTRAINT log_table_data_pk PRIMARY KEY (id)
);
create or replace function test.register_as_log()
returns trigger
language plpgsql
security definer
as $function$
declare
v_old_data json;
v_new_data json;
shemaname varchar;
tablename varchar;
begin
shemaname = tg_table_schema;
tablename = tg_table_name;
/* json_strip_nulls - removes null values */
if (tg_op = 'UPDATE') then
v_old_data = (select json_strip_nulls(row_to_json(old.*)));
v_new_data = (select json_strip_nulls(row_to_json(new.*)));
insert into test.log_table_data (schema_name, table_name, action_type, table_id, old_data, new_data)
values (shemaname, tablename, 'update', old.id, v_old_data, v_new_data);
return new;
end if;
if (tg_op = 'DELETE') then
v_old_data = (select json_strip_nulls(row_to_json(old.*)));
insert into test.log_table_data (schema_name, table_name, action_type, table_id, old_data, new_data)
values (shemaname, tablename, 'delete', old.id, v_old_data, null);
return old;
end if;
end;
$function$
;

PL/pgSQL: How to use IF NEW.<variable_column_name> <> OLD.<variable_column_name>

I am pretty new to PL/pgSQL programming. I have a requirement of audit logging updated columns in my table
Table
create table sample_table(name varchar(15),city varchar(15),age int,mail varchar(20) primary key);
Audit table
create table sample_table__audits_dynamicols(mail varchar(20), columnchanged varchar(10), oldvalue varchar(10), changed_on timestamp(6) NOT NULL)
Trigger Function
CREATE FUNCTION public.log_sample_table_allchanges() RETURNS trigger AS $BODY$DECLARE
_colname text;
_tablename varchar(15) := 'sample_table';
_schema varchar(15) := 'public';
_changed_on time := now();
BEGIN
FOR _colname IN SELECT column_name FROM information_schema.Columns WHERE table_schema = _schema AND table_name = _tablename LOOP
IF NEW._colname <> OLD._colname THEN
INSERT INTO sample_table__audits_dynamicols(mail,columnchanged, oldvalue ,changed_on)
VALUES(OLD.mail,_colname,OLD.:_colname,_changed_on);
END IF;
END LOOP;
RETURN NEW;
END$BODY$
LANGUAGE plpgsql VOLATILE NOT LEAKPROOF;
Trigger
create TRIGGER log_sample_table_allchanges
BEFORE UPDATE
ON SAMPLE_TABLE
FOR EACH ROW
EXECUTE PROCEDURE log_sample_table_allchanges();
Requirement: Whenever a column value is changed i want to log it as
(mail, columnname, columnvalue, date)
E.g:
insert into sample_table (name, mail, city, age) values('kanta','mk#foo.com','hyd',23);
insert into sample_table (name, mail, city, age) values('kmk','mk#gmail.com','hyd',23);
So when i update like the following
update sample_table set age=24 where mail='mk#foo.com';
update sample_table set city='bza' where mail='mk#gmail.com'
I want audit table to record like
(mk#foo.com,age,23, timestamp)
(mk#gmail.com, city, hyd, timestamp)
Right now I am facing issue with column comparison in my Trigger function. Please help me rectifying my Trigger function to meet my requirement.
You may use EXECUTE to get the values of columns dynamically and do the comparison.
CREATE OR REPLACE FUNCTION public.log_sample_table_allchanges() RETURNS trigger AS
$BODY$
DECLARE
_colname text;
_tablename varchar(15) := 'sample_table';
_schema varchar(15) := 'public';
_changed_on timestamp := now();
_old_val text;
_new_val text;
BEGIN
FOR _colname IN SELECT column_name FROM information_schema.Columns WHERE table_schema = _schema AND table_name = _tablename
LOOP
EXECUTE 'SELECT $1.' || _colname || ', $2.' || _colname
USING OLD,NEW
INTO _old_val, _new_val; --get the old and new values for the column.
IF _new_val <> _old_val THEN
INSERT INTO sample_table__audits_dynamicols(mail,columnchanged, oldvalue ,changed_on)
VALUES(OLD.mail,_colname,_old_val,_changed_on);
END IF;
END LOOP;
RETURN NEW;
END$BODY$
LANGUAGE plpgsql VOLATILE NOT LEAKPROOF;
I'm not sure why you have defined mail as a PRIMARY KEY in the audits table, it will cause unique constraint violation if the same mail gets updated twice.

Issue with dynamic sql

Hi every one I have multiple spatial tables That I want to control, so that I created a table where I will store the name of the operation applied on my layers tables(insert,update or delete), operation time and the team who did it, number of spatial tables created.
My script table
CREATE TABLE public.monitoring_table
(
operation character(1) COLLATE pg_catalog."default",
operat_ime timestamp without time zone,
userid text COLLATE pg_catalog."default",
dc_team text COLLATE pg_catalog."default",
number_pts_created integer,
id integer NOT NULL DEFAULT nextval('monitoring_table_id_seq'::regclass),
CONSTRAINT monitoring_table_pkey PRIMARY KEY (id)
)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
ALTER TABLE public.monitoring_table
OWNER to postgres;
after that I stored all the teams that I have on my table :
insert into monitoring_table (dc_team) values ('abdoulhassan');
insert into monitoring_table (dc_team) values ('abdoulei');
insert into monitoring_table (dc_team) values ('danis');
insert into monitoring_table (dc_team) values ('david');
insert into monitoring_table (dc_team) values ('joseph');
To calculate the number of spatial tables created, I executed this function :
My counting function :
DROP FUNCTION get_dc_team_counting();
CREATE OR REPLACE FUNCTION get_dc_team_counting()
RETURNS bigint AS
$func$
DECLARE
dc_team text;
_tbl_pattern text;
_schema text = 'sige';
_tb_name information_schema.tables.table_name%TYPE;
_tc bigint;
BEGIN
FOR _tb_name IN
SELECT table_name
FROM information_schema.tables
WHERE table_schema = _schema
AND table_name ~ _tbl_pattern
LOOP
EXECUTE format('SELECT count(*) FROM %I.%I where id= 583', _schema, _tb_name)
INTO _tc;
return _tc;
END LOOP;
END
$func$ LANGUAGE plpgsql
To get the team I executed this function :
CREATE OR REPLACE FUNCTION get_team()
RETURNS text AS -- or whatever you want to return
$func$
DECLARE
dc_team text;
_tbl_pattern text;
_schema text = 'public';
_tb_name information_schema.tables.table_name%TYPE; -- currently varchar
_tc text;
BEGIN
FOR _tb_name IN
SELECT table_name
FROM information_schema.tables
WHERE table_schema = _schema
AND table_name ~ _tbl_pattern -- see below!
LOOP
EXECUTE format('SELECT dc_team FROM %I.%I where id = 26', _schema, _tb_name)
INTO _tc;
return _tc;
END LOOP;
END
$func$ LANGUAGE plpgsql;
In my where clause I want to get dynamically the value of the ID. I don't want to give it manually in the function. I don't see how to do it.
Now I created a trigger function to be able to update my table if a row was inserted, updated or deleted. I did it this way :
CREATE OR REPLACE FUNCTION process_monitoring() RETURNS TRIGGER AS $monitoring$
BEGIN
IF (TG_OP = 'DELETE') THEN
update monitoring_table set operation = 'D', operat_ime = now(), userid = user ,dc_team = OLD.dc_team, number_pts_created = get_dc_team_counting() where dc_team = get_team();
RETURN OLD;
ELSIF (TG_OP = 'UPDATE') THEN
update monitoring_table set operation = 'U',operat_ime = now(),userid = user , dc_team = NEW.dc_team, number_pts_created = get_dc_team_counting() where dc_team = get_team();
RETURN NEW;
ELSIF (TG_OP = 'INSERT') THEN
update monitoring_table set operation = 'I', operat_ime = now(), userid = user, dc_team = NEW.dc_team, number_pts_created = get_dc_team_counting() where dc_team = get_team();
RETURN NEW;
END IF;
RETURN NULL;
END;
$monitoring$ LANGUAGE plpgsql;
CREATE TRIGGER monitoring
AFTER INSERT OR UPDATE OR DELETE ON sige.valve
FOR EACH ROW EXECUTE PROCEDURE process_monitoring();
In my previous functions I used known values of ids in the where clauses, but when I try to insert or update a value on the concerned table I get this error :
the control attempted it's end without return
CONTEXT: fonction PL/pgsql get_team()
instruction SQL « update monitoring_table set operation = 'U',operat_ime = now(),userid = user , dc_team = NEW.dc_team, number_pts_created = get_dc_team_counting() where dc_team = get_team() »
fonction PL/pgsql process_monitoring(), ligne 10 à instruction SQL
If you have any idea about the origin of the error and how to get dynamically a value of a field and put it in the where clause tell me please.
I'm a newbie and I'm struggling, any assistance would be warmly appreciated.
I'm using PostgreSQL.

Audit trigger (Statement Level) storing identifier of inserted/updated/deleted rows

I have the following small MVWE for a basic micro-auditing system which works fine but lacks a functionality:
DROP TABLE IF EXISTS audit CASCADE;
CREATE TABLE audit(
Id BIGSERIAL NOT NULL
,TimeValue TIMESTAMP NOT NULL
,RoleName NAME NOT NULL
,Operation NAME NOT NULL
,SchemaName NAME NOT NULL
,TableName NAME NOT NULL
,Identifiers BIGINT[]
---
,PRIMARY KEY(Id)
);
-- Audit Trigger:
DROP FUNCTION IF EXISTS audit_trigger() CASCADE;
CREATE OR REPLACE FUNCTION audit_trigger()
RETURNS TRIGGER AS
$BODY$
BEGIN
INSERT INTO audit(TimeValue, RoleName, Operation, SchemaName, TableName) VALUES
(now()::TIMESTAMP, current_user, TG_OP, TG_TABLE_SCHEMA, TG_RELNAME);
RETURN NULL;
END;
$BODY$
LANGUAGE plpgsql SECURITY DEFINER;
-- Channels:
DROP TABLE IF EXISTS channels CASCADE;
CREATE TABLE channels(
Id INTEGER NOT NULL
,UserKey TEXT NOT NULL
,Active BOOLEAN NOT NULL DEFAULT(TRUE)
---
,PRIMARY KEY(Id)
,UNIQUE(UserKey)
);
CREATE TRIGGER channel_audit_trigger BEFORE INSERT OR UPDATE OR DELETE ON channels
FOR EACH STATEMENT EXECUTE PROCEDURE audit_trigger();
-- Perform some operations:
INSERT INTO channels(
SELECT C.Id, 'Channel-' || C.Id
FROM generate_series(1, 300, 10) AS C(Id)
);
DELETE FROM channels WHERE id < 10;
UPDATE channels
SET UserKey = 'wild channel'
WHERE id = 21;
I would like to add into the last column of audit table, identifiers of rows that have been inserted/updated/deleted in channels.
I have used STATEMENT level because I just need to collects identifiers in a array. But I do not find how to access DML statistics. Conversely at the ROW level I must handle OLD and NEW cases and I cannot succeed in aggregate all touched identifier.
How can I proceed in order to fill the last column of audit table with touched identifiers?
Update
Finally I reached my goal, but this solution might not be scalable and may have some unwanted drawbacks (I am open to any constructive feedback and advice).
Basically, how I have solved my problem:
Log at ROW level BEFORE DML is performed into a table audit_rowlevel;
Aggregate the freshly added content of audit_rowlevel into audit_statementlevel at STATEMENT level AFTER DML is performed;
Minimal Working Example is now:
DROP TABLE IF EXISTS audit_rowlevel CASCADE;
CREATE TABLE audit_rowlevel(
Id BIGSERIAL NOT NULL
,Aggregated BOOLEAN NOT NULL DEFAULT(FALSE)
,TimeValue TIMESTAMP NOT NULL
-- https://www.postgresql.org/docs/current/static/functions-info.html
,RoleName NAME NOT NULL
,ClientIP INET NOT NULL
,ClientPid INTEGER NOT NULL
-- https://www.postgresql.org/docs/current/static/plpgsql-trigger.html
,Operation TEXT NOT NULL
,SchemaName NAME NOT NULL
,TableName NAME NOT NULL
,RowId BIGINT NOT NULL
-- https://www.postgresql.org/docs/current/static/functions-json.html
,OldValue JSONB
,NewValue JSONB
---
,PRIMARY KEY(Id)
);
-- Row Level Trigger:
DROP FUNCTION IF EXISTS audit_rowlevel_trigger() CASCADE;
CREATE OR REPLACE FUNCTION audit_rowlevel_trigger()
RETURNS TRIGGER AS
$BODY$
DECLARE
history BOOLEAN := (TG_NARGS > 0) AND (TG_ARGV[0]::BOOLEAN);
rowid BIGINT;
oldvalue JSONB;
newvalue JSONB;
BEGIN
-- Handle NEW:
IF TG_OP = ANY('{INSERT,UPDATE}') THEN
IF history THEN
newvalue := to_jsonb(NEW);
END IF;
rowid := NEW.Id::BIGINT;
END IF;
-- Handle OLD:
IF TG_OP = ANY('{UPDATE,DELETE}') THEN
IF history THEN
oldvalue := to_jsonb(OLD);
END IF;
rowid := OLD.Id::BIGINT;
END IF;
-- INSERT:
INSERT INTO audit_rowlevel(TimeValue, RoleName, ClientIP, ClientPID, Operation, SchemaName, TableName, RowId, NewValue, OldValue) VALUES
(now()::TIMESTAMP, current_user, inet_client_addr(), pg_backend_pid(), TG_OP, TG_TABLE_SCHEMA, TG_RELNAME, RowId, NewValue, OldValue);
-- RETURN:
IF TG_OP = ANY('{INSERT,UPDATE}') THEN
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
RETURN OLD;
ELSE
RETURN NULL;
END IF;
END;
$BODY$
LANGUAGE plpgsql SECURITY DEFINER;
-- Statement Level Trigger:
DROP TABLE IF EXISTS audit_statementlevel CASCADE;
CREATE TABLE audit_statementlevel(
Id BIGSERIAL NOT NULL
,TimeValue TIMESTAMP NOT NULL
,RoleName NAME NOT NULL
,ClientIP INET NOT NULL
,ClientPid INTEGER NOT NULL
,Operation TEXT NOT NULL
,SchemaName NAME NOT NULL
,TableName NAME NOT NULL
,RowCount BIGINT NOT NULL
,RowIds BIGINT[] NOT NULL
,AuditIds BIGINT[] NOT NULL
---
,PRIMARY KEY(Id)
);
-- Row Level Trigger:
DROP FUNCTION IF EXISTS audit_statementlevel_trigger() CASCADE;
CREATE OR REPLACE FUNCTION audit_statementlevel_trigger()
RETURNS TRIGGER AS
$BODY$
DECLARE
rowcount BIGINT;
BEGIN
WITH
A AS (
SELECT
TimeValue, RoleName, ClientIP, ClientPid, Operation, SchemaName, TableName
,COUNT(*)
,array_agg(RowId)
,array_agg(Id)
FROM
audit_rowlevel
WHERE
NOT Aggregated
GROUP BY
TimeValue, RoleName, ClientIP, ClientPid, Operation, SchemaName, TableName
ORDER BY
TimeValue
),
B AS (
INSERT INTO audit_statementlevel(TimeValue, RoleName, ClientIP, ClientPid, Operation, SchemaName, TableName, RowCount, RowIds, AuditIds)
(SELECT * FROM A)
RETURNING AuditIds
),
C AS (
SELECT array_agg(DISTINCT T.id) AS Ids FROM B, unnest(B.AuditIds) AS T(id)
)
UPDATE
audit_rowlevel
SET
Aggregated = TRUE
FROM
C
WHERE
Id = ANY(C.Ids);
RETURN NULL;
END;
$BODY$
LANGUAGE plpgsql SECURITY DEFINER;
-- Channels:
DROP TABLE IF EXISTS channels CASCADE;
CREATE TABLE channels(
Id INTEGER NOT NULL
,UserKey TEXT NOT NULL
,Active BOOLEAN NOT NULL DEFAULT(TRUE)
---
,PRIMARY KEY(Id)
,UNIQUE(UserKey)
);
CREATE TRIGGER channel_audit_rowlevel_trigger BEFORE INSERT OR UPDATE OR DELETE ON channels
FOR EACH ROW EXECUTE PROCEDURE audit_rowlevel_trigger(TRUE);
CREATE TRIGGER channel_audit_statementlevel_trigger AFTER INSERT OR UPDATE OR DELETE ON channels
FOR EACH STATEMENT EXECUTE PROCEDURE audit_statementlevel_trigger();
-- Perform some operations:
INSERT INTO channels(
SELECT C.Id, 'Channel-' || C.Id
FROM generate_series(1, 300, 10) AS C(Id)
);
DELETE FROM channels WHERE id < 10;
UPDATE channels
SET UserKey = 'wild channel'
WHERE id = 21;
I am interested knowing if this solution looks right to professional developers.
Am I going into the good direction or is this solution evil?