I would like to create N partition tables for the last N days. I have created a table like the following
create table metrics.my_table (
id bigserial NOT NULL primary key,
...
logdate date NOT NULL
) PARTITION BY LIST (logdate);
Then I have the following function to create those tables:
CREATE OR REPLACE function metrics.create_my_partitions(init_date numeric default 30, current_date_parameter timestamp default current_date)
returns void as $$
DECLARE
partition_date TEXT;
partition_name TEXT;
begin
for cnt in 0..init_date loop
partition_date := to_char((current_date_parameter - (cnt * interval '1 day')),'YYYY-MM-DD');
raise notice 'cnt: %', cnt;
raise notice 'partition_date: %', partition_date;
partition_name := 'my_table_' || partition_date;
raise notice 'partition_name: %', partition_name;
EXECUTE format('CREATE table if not exists metrics.%I PARTITION OF metrics.my_table for VALUES IN ($1)', partition_name) using partition_date;
end loop;
END
$$
LANGUAGE plpgsql;
select metrics.create_my_partitions(30, current_date);
But it throws the following error in the EXECUTE format line:
SQL Error [42P02]: ERROR: there is no parameter $1
Any idea on how to create those tables?
The EXECUTE ... USING ... option only works for data values in DML commands (SELECT,INSERT, etc.). Since CREATE TABLE is a DDL command, use a parameter in format():
execute format(
'create table if not exists metrics.%I partition of metrics.my_table for values in (%L)',
partition_name, partition_date);
Related
--Existing Data
create table tbl_Master
(
col_Date date not null,
id int not null,
value int not null
);
insert into tbl_Master values('2021-01-14',10,21);
insert into tbl_Master values('2020-09-30',11,22);
insert into tbl_Master values('2021-11-28',12,23);
--alter table name
alter table tbl_Master rename to tbl_Master_old;
--Create Master table again with Partition by range
create table tbl_Master(
col_Date date not null,
id int not null,
value int not null
) partition by range (col_Date);
--Function:
create or replace function fn_create_partition(col_Date date) returns void
as
$body$
declare v_startDate date := date_trunc('month', col_Date)::date;
declare v_EndDate date := (v_startDate + interval '1 month')::date;
declare tableName text := 'tbl_Master_Part_' || to_char(col_Date, 'YYYYmm');
begin
if to_regclass(tableName) is null then
execute format('create table %I partition of tbl_Master for values from (%L) to (%L)', tableName, v_startDate, v_EndDate);
end if;
end;
$body$
language plpgsql;
--Create Partition tables for existing data
do
$$
declare rec record;
begin
for rec in select distinct date_trunc('month', col_Date)::date yearmonth from tbl_Master_old
loop
perform fn_create_partition(rec.yearmonth);
end loop;
end
$$;
--Insert backup data
insert into tbl_Master (col_Date, id, value) select * from tbl_Master_old;
--When I insert record
insert into tbl_Master values('2021-02-22',12,22);
Getting an error:
ERROR: no partition of relation "tbl_master" found for row DETAIL: Partition key of the failing row contains (col_date) = (2021-02-22).
So I have created rule for this:
--Rule
create or replace rule rule_fn_create_partition as on insert
to tbl_Master
do instead
(
select fn_create_partition(NEW.col_Date);
insert into tbl_Master values(New.*)
);
--When I insert record
insert into tbl_Master values('2021-02-22',12,22);
Getting an error:
ERROR: infinite recursion detected in rules for relation "tbl_master"
The rule contains an INSERT, so that is an infinite recursion.
You cannot have PostgreSQL automatically create partitions as you insert data. It just doesn't work. Triggers won't work either. The problem is that that would require modifying the underlying table while the insert is in progress.
I wrote a function that should automatically create new partitions for the table. I created a trigger, but when the trigger fires and the function is called, nothing happens, just an error appears:
ERROR: query string argument of EXECUTE is null
Function code:
CREATE FUNCTION public.auto_part()
RETURNS trigger
LANGUAGE 'plpgsql'
COST 100
VOLATILE NOT LEAKPROOF
AS $BODY$DECLARE
partition_date TEXT;
partition TEXT;
startdate TEXT;
enddate TEXT;
query TEXT;
BEGIN
partition_date := TO_CHAR(new.created_on,'YYYY-MM');
startdate := partition_date || '-01';
enddate := to_char(to_timestamp('YYYY-MM',partition_date) + '1 MONTH'::interval,'YYYY-MM') || '-01';
partition := TG_TABLE_NAME || '_' || partition_date;
IF NOT EXISTS(SELECT relname FROM pg_class WHERE relname=partition) THEN
RAISE NOTICE 'TABLE %',TG_TABLE_NAME;
RAISE NOTICE 'ID %',new.id;
RAISE NOTICE 'NAME %',new.username;
RAISE NOTICE 'CREATED_ON %',new.created_on;
RAISE NOTICE 'PARTITION_DATE %',partition_date;
RAISE NOTICE 'STARTDATE %',startdate;
RAISE NOTICE 'A partition has been created %',partition;
EXECUTE 'CREATE TABLE ' || partition || ' PARTITION OF ' || TG_TABLE_NAME || ' FOR VALUES FROM ('|| startdate || ') TO (' || enddate || ');';
--RAISE NOTICE query;
--EXECUTE query;
RETURN 1;
END IF;
END
$BODY$;
Conclusion "RAISE NOTICE":
NOTICE: TABLE users_sec
NOTICE: ID <NULL>
NOTICE: NAME <NULL>
NOTICE: CREATED_ON <NULL>
NOTICE: PARTITION_DATE <NULL>
NOTICE: STARTDATE <NULL>
NOTICE: A partition has been created <NULL>
ERROR: query string argument of EXECUTE is null
CONTEXT: PL/pgSQL function auto_part() line 22 at EXECUTE
SQL state: 22004
Trigger code:
CREATE TRIGGER auto_part_trigger
BEFORE INSERT OR UPDATE
ON public.users_sec
FOR EACH STATEMENT
EXECUTE PROCEDURE public.auto_part();
Insert example:
INSERT INTO users_sec(
username, password, created_on, last_logged_on)
VALUES (
'qwerty',
random_string( 20 ),
'2021-03-23',
'2021-03-24'
);
Table creation code:
CREATE TABLE public.users_sec
(
id integer NOT NULL DEFAULT nextval('users_sec_id_seq'::regclass) ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
username text COLLATE pg_catalog."default" NOT NULL,
password text COLLATE pg_catalog."default",
created_on timestamp with time zone NOT NULL,
last_logged_on timestamp with time zone NOT NULL,
CONSTRAINT users_sec_pkey PRIMARY KEY (id, created_on)
) PARTITION BY RANGE (created_on)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
If you want NEW to contain the row about to be inserted, you have to use a FOR EACH ROW level trigger.
Since you cannot have a BEFORE trigger FOR EACH ROW on a partitioned table, that's kind of a catch 22 situiation.
A possible way our may be this:
Create a DEFAULT partition. All rows that don't match an existing partition will be inserted there.
Define a BEFORE trigger FOR EACH ROW on the default partition. The trigger creates a new partition as appropriate and inserts the row into that partition. The trigger function uses RETURN NULL to avoid inserting anything in the default partition itself.
This way, the default partition remains empty. Moreover, you only have to pay the overhead for a trigger for rows that don't go in any of the existing partitions!
I'm trying to auto partition my oltpsales table using a function below and a trigger below but then I try to perform in insert on the table I get error code below. I have referenced a few threads below and suggestions are welcomed.
INSERT with dynamic table name in trigger function
Postgres format string using array
ERROR: too few arguments for format() CONTEXT: PL/pgSQL function
testoltpsales_insert_function() line 17 at EXECUTE
CREATE TRIGGER testoltpsales_insert_trg
BEFORE INSERT ON myschema."testoltpsales"
FOR EACH ROW EXECUTE PROCEDURE testoltpsales_insert_function();
CREATE OR REPLACE FUNCTION testoltpsales_insert_function()
RETURNS TRIGGER AS $$
DECLARE
partition_date TEXT;
partition_name TEXT;
start_of_month TEXT;
end_of_next_month TEXT;
BEGIN
partition_date := to_char(NEW."CreateDateTime",'YYYY_MM');
partition_name := 'testoltpsaless_' || partition_date;
start_of_month := to_char((NEW."CreateDateTime"),'YYYY-MM') || '-01';
end_of_next_month := to_char((NEW."CreateDateTime" + interval '1 month'),'YYYY-MM') || '-01';
IF NOT EXISTS
(SELECT 1
FROM information_schema.tables
WHERE table_name = partition_name)
THEN
EXECUTE format(E'CREATE TABLE %I (CHECK ( date_trunc(\'day\', %I.CreateDateTime) >= ''%s'' AND date_trunc(\'day\', %I.CreateDateTime) < ''%s'')) INHERITS (myschema."Testoltpsaless")',
VARIADIC ARRAY [partition_name, start_of_month,end_of_next_month]);
RAISE NOTICE 'A partition has been created %', partition_name;
-- EXECUTE format('GRANT SELECT ON TABLE %I TO readonly', partition_name); -- use this if you use role based permission
END IF;
EXECUTE format('INSERT INTO %I ("OwnerId","DaddyTable","SaleId","RunId","CreateDateTime","SalesetId","Result","Score","NumberOfMatches" ) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)', partition_name)
USING NEW."OwnerId",NEW."DaddyTable",NEW."SaleId",NEW."RunId",NEW."CreateDateTime",NEW."SalesetId",NEW."Result",NEW."Score",NEW."NumberOfMatches";
RETURN NULL;
END
$$
LANGUAGE plpgsql;
EXECUTE format(E'CREATE TABLE %I (CHECK ( date_trunc(\'day\', %I.CreateDateTime) >= ''%s'' AND date_trunc(\'day\',
%I.CreateDateTime) < ''%s'')) INHERITS (myschema."Testoltpsaless")',
VARIADIC ARRAY [partition_name, start_of_month,end_of_next_month]); ```
There are five format specifiers in your format string but you're passing it only three arguments. Unless you're using positional formatting e.g. %1$I, you must supply the same number of args, as they are used sequentially.
https://www.postgresql.org/docs/current/functions-string.html#FUNCTIONS-STRING-FORMAT
I'm trying to create simple table with partitions.
this is my command:
CREATE TABLE measurement (
city_id int not null,
logdate date not null,
peaktemp int,
unitsales int
) PARTITION BY RANGE (logdate);
this is the error I got:
SQL Error [42601]: ERROR: syntax error at or near "PARTITION"
Unable to understand with is the problem..
I am using PostgreSQL 9.6.3
"Declarative table partitioning", that is partitioning as a first-class feature of the DBMS with its own syntax, was added in PostgreSQL 10.
In earlier versions, you can achieve the same effect with a bit more effort using "table inheritance". There is a page in the manual describing how to do this manually, summarised as:
Create the "master" table, from which all of the partitions will inherit.
Create several "child" tables that each inherit from the master table.
Add table constraints to the partition tables to define the allowed key values in each partition.
For each partition, create an index on the key column(s), as well as any other indexes you might want.
Optionally, define a trigger or rule to redirect data inserted into the master table to the appropriate partition.
Ensure that the constraint_exclusion configuration parameter is not disabled in postgresql.conf. If it is, queries will not be optimized as desired.
To make this easier, if you can't upgrade to version 10, you can use an extension such as pg_partman which gives you additional functions for setting up and managing partition sets.
Here is the example of automatically creating monthly partition for 9.6 version, may be compatible with other versions:
`----------Function to create partitions by system------------------
CREATE OR REPLACE FUNCTION schema.insert_function()
RETURNS TRIGGER
AS $$
DECLARE
partition_date TEXT;
partition_schema TEXT;
partition_name TEXT;
start_of_month TEXT;
end_of_month TEXT;
BEGIN
partition_date := to_char(NEW.created_dtm,'YYYY_MM');
partition_schema := 'temp';
partition_name := 'patition_name_' || partition_date;
start_of_month := to_char((NEW.created_dtm),'YYYY-MM'|| '-01');
end_of_month := to_char((NEW.created_dtm + interval '1 month'),'YYYY-MM'|| '-01');
IF NOT EXISTS
(SELECT 1
FROM information_schema.tables
WHERE table_name = partition_name
AND table_schema = partition_schema)
THEN
EXECUTE 'CREATE TABLE '|| partition_schema ||' . '|| partition_name ||' (check (created_dtm >= ''' || start_of_month || ''' and created_dtm < ''' || end_of_month || ''' ), ' || 'LIKE master_schema.master_table INCLUDING ALL) INHERITS (master_schema.master_table)';
EXECUTE format('ALTER TABLE %I.%I OWNER TO role1', partition_schema, partition_name);
EXECUTE format('GRANT SELECT ON TABLE %I.%I TO read_only_role', partition_schema, partition_name);
EXECUTE format('GRANT INSERT, SELECT, UPDATE, DELETE ON TABLE %I.%I TO read_write_role', partition_schema, partition_name);
EXECUTE format('CREATE INDEX ON %I.%I(column1, column2, column3)', partition_schema, partition_name);
EXECUTE format('CREATE UNIQUE INDEX ON %I.%I(column4)', partition_schema, partition_name);
........
RAISE NOTICE 'A partition has been created %.%', partition_schema, partition_name;
RAISE NOTICE 'All necessary indices are created on %.%', partition_schema, partition_name;
END IF;
EXECUTE format('INSERT INTO %I.%I VALUES($1.*)', partition_schema, partition_name) using NEW;
RETURN NULL;
END
$$
LANGUAGE plpgsql;
ALTER FUNCTION schema.insert_function()
OWNER TO super_user;
-----------------------------------------Trigger on master table--------------
CREATE TRIGGER insert_trigger
BEFORE INSERT
ON master_schema.master_table
FOR EACH ROW
EXECUTE PROCEDURE schema.insert_function();`
I'm trying to implement table partitioning with dynamic table creation using BEFORE INSERT trigger to create new tables and indexes when necesarry using following solution:
create table mylog (
mylog_id serial not null primary key,
ts timestamp(0) not null default now(),
data text not null
);
CREATE OR REPLACE FUNCTION mylog_insert() RETURNS trigger AS
$BODY$
DECLARE
_name text;
_from timestamp(0);
_to timestamp(0);
BEGIN
SELECT into _name 'mylog_'||replace(substring(date_trunc('day', new.ts)::text from 0 for 11), '-', '');
IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name=_name) then
SELECT into _from date_trunc('day', new.ts)::timestamp(0);
SELECT into _to _from + INTERVAL '1 day';
EXECUTE 'CREATE TABLE '||_name||' () INHERITS (mylog)';
EXECUTE 'ALTER TABLE '||_name||' ADD CONSTRAINT ts_check CHECK (ts >= '||quote_literal(_from)||' AND ts < '||quote_literal(_to)||')';
EXECUTE 'CREATE INDEX '||_name||'_ts_idx on '||_name||'(ts)';
END IF;
EXECUTE 'INSERT INTO '||_name||' (ts, data) VALUES ($1, $2)' USING
new.ts, new.data;
RETURN null;
END;
$BODY$
LANGUAGE plpgsql;
CREATE TRIGGER mylog_insert
BEFORE INSERT
ON mylog
FOR EACH ROW
EXECUTE PROCEDURE mylog_insert();
Everything works as expected but each day when concurrent INSERT statements are being fired for the first time that day, one of them fails trying to "create table that already exists". I suspect that this is caused by the triggers being fired concurrently and both trying to create new table and only one can succeed.
I could be using CREATE TABLE IF NOT EXIST but I cannot detect the outcome so I cannot reliably create constraints and indexes.
What can I do to avoid such problem? Is there any way to signal the fact that the table has been already created to other concurrent triggers? Or maybe there is a way of knowing if CREATE TABLE IF NOT EXISTS created new table or not?
What I do is create a pgAgent job to run every day and create 3 months of tables ahead of time.
CREATE OR REPLACE FUNCTION avl_db.create_alltables()
RETURNS numeric AS
$BODY$
DECLARE
rec record;
BEGIN
FOR rec IN
SELECT date_trunc('day', i::timestamp without time zone) as table_day
FROM generate_series(now()::date,
now()::date + '3 MONTH'::interval,
'1 DAY'::interval) as i
LOOP
PERFORM avl_db.create_table (rec.table_day);
END LOOP;
PERFORM avl_db.avl_partition(now()::date,
now()::date + '3 MONTH'::interval);
RETURN 0;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION avl_db.create_alltables()
OWNER TO postgres;
create_table is very similar to your CREATE TABLE code
avl_partition update the BEFORE INSERT TRIGGER but I saw you do that part with dynamic query. Will have to check again that.
Also I see you are doing inherit, but you are missing a very important CONSTRAINT
CONSTRAINT route_sources_20170601_event_time_check CHECK (
event_time >= '2017-06-01 00:00:00'::timestamp without time zone
AND event_time < '2017-06-02 00:00:00'::timestamp without time zone
)
This improve the query a lot when doing a search for event_time because doesn't have to check every table.
See how doesn't check all tables for the month:
Eventually I wrapped CREATE TABLE in BEGIN...EXCEPTION block that catches duplicate_table exception - this did the trick, but creating the tables upfront in a cronjob is much better approach performance-wise.
CREATE OR REPLACE FUNCTION mylog_insert() RETURNS trigger AS
$BODY$
DECLARE
_name text;
_from timestamp(0);
_to timestamp(0);
BEGIN
SELECT into _name 'mylog_'||replace(substring(date_trunc('day', new.ts)::text from 0 for 11), '-', '');
IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name=_name) then
SELECT into _from date_trunc('day', new.ts)::timestamp(0);
SELECT into _to _from + INTERVAL '1 day';
BEGIN
EXECUTE 'CREATE TABLE '||_name||' () INHERITS (mylog)';
EXECUTE 'ALTER TABLE '||_name||' ADD CONSTRAINT ts_check CHECK (ts >= '||quote_literal(_from)||' AND ts < '||quote_literal(_to)||')';
EXECUTE 'CREATE INDEX '||_name||'_ts_idx on '||_name||'(ts)';
EXCEPTION WHEN duplicate_table THEN
RAISE NOTICE 'table exists -- ignoring';
END;
END IF;
EXECUTE 'INSERT INTO '||_name||' (ts, data) VALUES ($1, $2)' USING
new.ts, new.data;
RETURN null;
END;
$BODY$
LANGUAGE plpgsql;