Postgres 9.3 x64 bit ,I am using a history table with user log.huge data insertion occurring on every second till now the data is is around 90 millions and still counting
I partitioned the table using date column for every month and I used a trigger where for each insert it will check weather for the particular month table is there or not, if there it will directly insert else it will try to create and insert the data. but the trigger taking soo much time to execute is there any way to improvise the performance
CREATE OR REPLACE FUNCTION history_master_part_trigger()
RETURNS TRIGGER AS $$
DECLARE
yr int;
mnth int;
tbname character varying;
sdate character varying;
edate character varying;
cnt int;
Begin
raise notice '%','something';
EXECUTE 'select EXTRACT(YEAR FROM DATE '''||NEW.hm_date||''')' INTO yr;
EXECUTE 'select EXTRACT(month FROM DATE '''||NEW.hm_date||''')' INTO mnth;
tbname := 'history_master_part_y'||yr||'m'||mnth;
sdate :=yr||'-'||mnth||'-01';
IF mnth = 12 THEN
edate :=yr+1||'-'||'01-01';
ELSE
edate :=yr||'-'||mnth+1||'-01';
END IF;
--raise notice 'table-----', tbname;
raise notice '%', sdate;
raise notice '%', edate;
raise notice '%',yr;
If(SELECT EXISTS (
SELECT 1
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public'
AND c.relname = tbname
AND c.relkind = 'r' -- only tables(?)
)) THEN
--INSERT INTO ||tbname|| VALUES ||NEW'.*)';
raise notice '%','inserting into'||tbname;
EXECUTE format('INSERT INTO ' || tbname || ' SELECT ($1).*')
USING NEW;
ELSE
EXECUTE 'CREATE TABLE '|| tbname||'(
CHECK ( hm_date >= DATE '''||sdate||''' AND hm_date < DATE '''||edate||''' )
) INHERITS (history_master_part)';
EXECUTE format('INSERT INTO ' || tbname || ' SELECT ($1).*')
USING NEW;
--RAISE EXCEPTION 'Caution Caution Caution Amit is getting angry';
END IF;
Return NULL;
End;
$$
LANGUAGE 'plpgsql'
Related
I am trying to call this function
CREATE or replace FUNCTION get_column_names(param text)
RETURNS text AS $get_column_names$
DECLARE
return_value text;
x record;
y int;
begin
return_value := '';
y := 0;
for x in SELECT *
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = param
ORDER BY ordinal_position
loop
if (y = 0)
then
return_value = x.column_name;
else
return_value := return_value ||','|| x.column_name;
end if;
Y = Y+1;
end loop;
return return_value;
END;
$get_column_names$ LANGUAGE plpgsql;
Which this function works here
select get_column_names('users');
Results
first_name,middle_name,last_name,gender,locale,auth_user_id,identity_id,active,date_created,provisioned,user_id,timezone,last_seen
However when I use it in another function I get an error.
Here is the other function I am calling it from.
CREATE OR REPLACE FUNCTION process_users_audit() RETURNS TRIGGER AS $users_audit$
BEGIN
--
-- Create a row in users_audit to reflect the operation performed on users
--
IF (TG_OP = 'DELETE') THEN
INSERT INTO users_audit (user_audit_id, stamp, operation, db_user, get_column_names(TG_TABLE_NAME)) values (uuid_generate_v4(), now(), 'D', user, OLD.*);
RETURN OLD;
ELSIF (TG_OP = 'UPDATE') THEN
INSERT INTO users_audit (user_audit_id, stamp, operation, db_user, get_column_names(TG_TABLE_NAME)) values (uuid_generate_v4(), now(), 'U', user, NEW.*);
RETURN NEW;
ELSIF (TG_OP = 'INSERT') THEN
INSERT INTO users_audit (user_audit_id, stamp, operation, db_user, get_column_names(TG_TABLE_NAME)) values (uuid_generate_v4(), now(), 'I', user, NEW.*);
RETURN NEW;
END IF;
RETURN NULL; -- result is ignored since this is an AFTER trigger
END;
$users_audit$ LANGUAGE plpgsql;
The error I am receiving is this (same as title).
SQL Error [42601]: ERROR: syntax error at or near "("
Position: 406
UPDATE
#Ed Brook's answer worked except for had to change how NEW was being used. Will remove this update once he has updated his answer but here is what worked.
CREATE OR REPLACE FUNCTION process_audit_table() RETURNS TRIGGER AS $audit_table$
BEGIN
--
-- Create a row in the requesting table's audit to reflect the operation performed on users,
-- make use of the special variable TG_OP to work out the operation.
--
IF (TG_OP = 'DELETE') then
EXECUTE 'INSERT INTO ' || concat(TG_TABLE_NAME, '_audit') || ' (' || get_primary_key_name(concat(TG_TABLE_NAME, '_audit')) || ', stamp, operation, db_user, ' || get_column_names(TG_TABLE_NAME) || ') values (uuid_generate_v4(), now(), ''D'', user, $1.*);'
USING OLD;
ELSIF (TG_OP = 'UPDATE') then
EXECUTE 'INSERT INTO ' || concat(TG_TABLE_NAME, '_audit') || ' (' || get_primary_key_name(concat(TG_TABLE_NAME, '_audit')) || ', stamp, operation, db_user, ' || get_column_names(TG_TABLE_NAME) || ') values (uuid_generate_v4(), now(), ''U'', user, $1.*);'
USING NEW;
ELSIF (TG_OP = 'INSERT') then
EXECUTE 'INSERT INTO ' || concat(TG_TABLE_NAME, '_audit') || ' (' || get_primary_key_name(concat(TG_TABLE_NAME, '_audit')) || ', stamp, operation, db_user, ' || get_column_names(TG_TABLE_NAME) || ') values (uuid_generate_v4(), now(), ''I'', user, $1.*);'
USING NEW;
END IF;
RETURN NULL; -- result is ignored since this is an AFTER trigger
END;
$audit_table$ LANGUAGE plpgsql;
Also added some extra bits in there like dynamically getting the audit table name and then get_primary_key_name below is that function.
CREATE OR REPLACE FUNCTION get_primary_key_name(table_name text)
RETURNS text AS $primary_key_name$
DECLARE
return_value text;
BEGIN
SELECT pg_attribute.attname INTO return_value
FROM pg_index, pg_class, pg_attribute, pg_namespace
WHERE pg_class.oid = table_name::regclass
AND indrelid = pg_class.oid
AND nspname = 'public'
AND pg_class.relnamespace = pg_namespace.oid
AND pg_attribute.attrelid = pg_class.oid
AND pg_attribute.attnum = any(pg_index.indkey)
AND indisprimary;
RETURN return_value;
END
$primary_key_name$ LANGUAGE plpgsql;
You may need to use EXECUTE, as I don't think you can have function calls in the column list?
ref: https://www.postgresql.org/docs/current/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN
So, something like this?
EXECUTE 'INSERT INTO users_audit (user_audit_id, stamp, operation, db_user, ' || get_column_names(TG_TABLE_NAME) || ') values (uuid_generate_v4(), now(), ''U'', user, NEW.*)';
Created this Postgres Function which is working fine, but the actual requirement is to pass the input parameter in the function to the Cursor which uses the dynamic SQL as follows,
The below is the Function
CREATE OR REPLACE FUNCTION ssp2_pcat.find_shift_dates (date_to_find date)
RETURNS void
LANGUAGE 'plpgsql'
COST 100
VOLATILE
AS $BODY$
DECLARE
C1 CURSOR FOR
SELECT TABLE_NAME, 'SELECT COUNT(*) FROM ' || TABLE_NAME || ' WHERE ' ||
COLUMN_NAME || ' = '||
'CASE WHEN ' || COLUMN_NAME || ' LIKE ' || '''%START%'''||' THEN
date_to_find ELSE date_to_find-1 END;' SQL_TEXT
FROM (
SELECT TABLE_NAME, COLUMN_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME IN (SELECT TABLE_NAME FROM RESET_DATES WHERE RESET_IT =
'Y') AND
UPPER(DATA_TYPE) = 'DATE'
AND (COLUMN_NAME LIKE '%START%' OR COLUMN_NAME LIKE '%END%')
AND (COLUMN_NAME NOT LIKE '%TEST%'
AND COLUMN_NAME NOT LIKE '%PCAT%'
AND COLUMN_NAME NOT LIKE '%ORDER%'
AND COLUMN_NAME NOT LIKE '%SEASON%'
AND COLUMN_NAME NOT LIKE '%_AT')
ORDER BY 1, 2) A;
END_COUNT INTEGER := 0;
START_COUNT INTEGER := 0;
TABLENAME VARCHAR(32) := 'ALFU';
l_start TIMESTAMP;
l_end TIMESTAMP;
Time_Taken VARCHAR(20);
BEGIN
l_start := clock_timestamp();
DELETE FROM SHIFT_DATES_COUNT;
FOR I IN C1 LOOP
IF I.TABLE_NAME <> TABLENAME THEN
INSERT INTO SHIFT_DATES_COUNT VALUES (TABLENAME, START_COUNT,
END_COUNT, current_timestamp::timestamp(0));
TABLENAME := I.TABLE_NAME;
END_COUNT := 0;
START_COUNT := 0;
END IF;
IF STRPOS(I.SQL_TEXT, 'END') > 0 THEN
EXECUTE I.SQL_TEXT INTO END_COUNT;
RAISE NOTICE '% ', ('END: ' || I.SQL_TEXT);
ELSE
EXECUTE I.SQL_TEXT INTO START_COUNT;
RAISE NOTICE '% ', ('START: ' || I.SQL_TEXT);
END IF;
END LOOP;
INSERT INTO SHIFT_DATES_COUNT VALUES (TABLENAME, START_COUNT, END_COUNT,
current_timestamp::timestamp(0));
RAISE NOTICE '% ', ('INSERT INTO SHIFT_DATES_COUNT Done...');
l_end := clock_timestamp();
Time_Taken := (l_end-l_start);
RAISE NOTICE '% ', ('FIND_SHIFT_DATES Took: ' || Time_Taken );
END;
$BODY$;
Please let me know how can I use the date_to_find input parameter in the Dynamic SQL in the Cursor in the above Function.
You can use unbound cursor, clause fetch to get data from cursor, and exit when not found to finish, like:
CREATE OR REPLACE FUNCTION example (p_name text) RETURNS void LANGUAGE 'plpgsql' AS $$
DECLARE
C1 refcursor;
res record;
BEGIN
OPEN c1 FOR EXECUTE 'SELECT * FROM pg_database WHERE datname like ''%'||p_name||'%''';
LOOP
FETCH c1 INTO res;
EXIT WHEN not found;
raise notice 'value datname: %',res.datname;
END LOOP;
CLOSE c1;
RETURN;
END; $$;
--in my case
select example ('test')
NOTICE: value datname: test
NOTICE: value datname: test_msmov
NOTICE: value datname: test_resources
NOTICE: value datname: test_load_table
NOTICE: value datname: test_resources2
Total query runtime: 63 msec
1 row retrieved.
You can use EXECUTE clause for open cursor, see the documentation of PostgreSQL
https://www.postgresql.org/docs/10/plpgsql-cursors.html#PLPGSQL-CURSOR-OPENING
Example:
OPEN curs1 FOR EXECUTE format('SELECT * FROM %I WHERE col1 = $1',tabname) USING keyvalue;
I created a table partition that will create a table if it is not yet existing the table names are on a monthly basis. I need this function to return the inserted ID but I'm getting this error of column "partition" does not exist it seems that my schema(partition) is considered column in this code
CREATE OR REPLACE FUNCTION partition.itinerary_partition_function()
RETURNS TRIGGER AS
$BODY$
DECLARE
reflowId bigint;
_tablename text;
_startyear text;
_startmonth text;
_fulltablename text;
BEGIN
--Takes the current inbound "time" value and determines when midnight is for the given date
_startyear := to_char(now(), 'YYYY');
_startmonth := to_char(now(), 'MM');
_tablename := 'itinerary_'||_startyear || '_' || _startmonth;
_fulltablename := 'partition.' || _tablename;
-- Check if the partition needed for the current record exists
PERFORM 1
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
AND c.relname = _tablename
AND n.nspname = 'partition';
-- If the partition needed does not yet exist, then we create it:
-- Note that || is string concatenation (joining two strings to make one)
IF NOT FOUND THEN
EXECUTE 'CREATE TABLE partition.' || quote_ident(_tablename) || '()INHERITS (partition.itinerary)';
-- Table permissions are not inherited from the parent.
-- If permissions change on the master be sure to change them on the child also.
EXECUTE 'ALTER TABLE partition.' || quote_ident(_tablename) || ' OWNER TO postgres';
-- Indexes are defined per child, so we assign a default index that uses the partition columns
EXECUTE 'CREATE INDEX ' || quote_ident(_tablename||'_indx1') || ' ON partition.' || quote_ident(_tablename) || ' (id)';
END IF;
BEGIN
EXECUTE format('INSERT INTO %I SELECT $1.*', "partition." || _tablename)
USING NEW;
RETURN NEW;
END;
END;
$BODY$
LANGUAGE plpgsql;
After this code I am calling it in another insert function
CREATE OR REPLACE FUNCTION partition.insert_data(username text,jsonData jsonb) RETURNS bigint AS
$$
DECLARE reflowId bigint;
BEGIN
INSERT INTO reflow_partition.itinerary(username, data)
VALUES (username, jsonData) RETURNING id;
END;
$$
LANGUAGE plpgsql;
Try changing this:
EXECUTE format('INSERT INTO %I SELECT $1.*', "partition." || _tablename)
to this:
EXECUTE format('INSERT INTO %1$I.%2$I SELECT $1.*', 'partition', _tablename)
CREATE OR REPLACE FUNCTION event_partition()
RETURNS trigger AS
$BODY$
DECLARE
_tbl text := to_char(NEW.the_date, 'YYYY-MM-DD');
BEGIN
***IF NOT EXISTS (
SELECT 1
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' -- your schema
AND c.relname = _tbl
AND c.relkind = 'r') THEN***
EXECUTE format('CREATE TABLE %I (CHECK (the_date >= %L AND
the_date < %L)) INHERITS (events)'
, _tbl
, to_char(NEW.the_date, 'YYYY-MM-DD')
, to_char(NEW.the_date + 1, 'YYYY-MM-DD')
);
END IF;
EXECUTE 'INSERT INTO ' || quote_ident(_tbl) || ' VALUES ($1.*)'
USING NEW;
RETURN NULL;
END
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION event_partition() SET search_path=public;
It's querying the database's internal table list (pg_class) to see if the table _tbl already exists in the public schema, and creating it if it doesn't.
You should be able to remove the check, and just use a CREATE TABLE IF NOT EXISTS... statement instead.
I got a syntax error while creating a procedure in postgresql.Here I attached my code.I got a error syntax error near "Continue"
create function patient_form_values() RETURNS void AS
$$ begin
DECLARE columnName varchar(200) ;
DECLARE done boolean default true;
DECLARE CONTINUE handler for not found set done = false;
DECLARE cur1 cursor for select distinct COLUMN_NAME from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME = 'currentdiagnosis';
open cur1;
read_loop : loop
fetch from cur1 into columnName;
if done then leave read_loop;
end if;
set #insertValues := concat('INSERT INTO patient_form_temp(patient_id, form_template_id, creator_id, created_date)
SELECT c.patient_id as patient_id, 41 AS form_template_id, 2 AS creator_id, c.created_date AS created_date
FROM currentdiagnosis c
WHERE c.', columnName,' IS NOT NULL GROUP BY c.patient_id, c.created_date');
select #insertValues;
prepare stmt from #insertValues;
execute stmt;
end loop;
close cur1;
end ;
$$ LANGUAGE plpgsql
You are trying to use a MySQL (or other DB?) function in PostgreSQL. There is no concept of CONTINUE HANDLER in PostgreSQL, so you have to convert the function into PostgreSQL format.
drop FUNCTION if exists migratePartnerAdvertiser();
CREATE OR REPLACE FUNCTION migratePartnerAdvertiser() RETURNS int4 AS '
DECLARE r RECORD;
BEGIN
FOR r IN select distinct COLUMN_NAME from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME = ''currentdiagnosis'' and table_schema=''public'' LOOP
EXECUTE concat(''INSERT INTO patient_form_temp(patient_id, form_template_id, creator_id, created_date) SELECT c.patient_id as patient_id, 41 AS form_template_id, 2 AS creator_id, c.reg_date AS created_date FROM currentdiagnosis c WHERE c.'' , r.column_name , '' IS NOT NULL GROUP BY c.patient_id, c.reg_date'');
END LOOP;
return 1;
END;
' LANGUAGE plpgsql;