UPDATE
Adding create table statement to replicate the issue:
CREATE TABLE table1
(
id serial NOT NULL,
"timestamp" timestamp with time zone,
dir integer,
created timestamp without time zone,
deleted boolean DEFAULT false,
CONSTRAINT pk_table1_id PRIMARY KEY (id)
);
CREATE TABLE table2
(
id serial NOT NULL,
"timestamp" timestamp with time zone,
dir integer,
created timestamp without time zone,
deleted boolean DEFAULT false,
CONSTRAINT pk_table2_id PRIMARY KEY (id)
);
insert into table1 (timestamp, dir, created) values('2015-01-01 17:52:00', 3, now()), ('2015-01-02 17:52:00', 1, now()), ('2015-01-03 17:52:00', 1, now());
insert into table2
select * from table1
insert into table2 (timestamp, dir, created) values('2015-01-04 17:52:00', 3, now());
ERROR: duplicate key value violates unique constraint "pk_table2_id"
DETAIL: Key (id)=(1) already exists.
I am trying to reset the serial counter of a newly created table in postgres using the following:
SELECT setval('table2_id_seq', (SELECT MAX(id) FROM table2));
setval returns 3.
But counter is not being set as my next insert statement fails with the following error:
insert into table2 (timestamp, dir, created) values('2015-01-04 17:52:00', 3, now());
ERROR: duplicate key value violates unique constraint "pk_table2_id"
DETAIL: Key (id)=(2) already exists.
select version();
PostgreSQL 9.4.5 on x86_64-apple-darwin14.5.0, compiled by Apple LLVM version 7.0.0 (clang-700.0.72), 64-bit
Related
I have tables master_bill and master_bill_order_leg and order_leg.
MasterBill and OrderLeg have many to many relationship.
This is the quick documentation from IntelliJ for the DB tables.
Table master_bill:
create table master_bill
(
id bigint not null
primary key,
mb_no bigint not null,
created_by text not null,
updated_by text not null,
created_at timestamp with time zone default now() not null,
updated_at timestamp with time zone default now() not null
);
Table master_bill_order_leg:
create table master_bill_order_leg
(
mb_id bigint
references master_bill
on delete cascade,
order_leg_id bigint
references order_leg
on delete cascade,
constraint master_bill_order_mb_id_order_leg_id_key
unique (mb_id, order_leg_id)
);
Table order_leg:
create table order_leg
(
id bigserial
primary key,
created_by text not null,
updated_by text not null,
created_at timestamp with time zone default now() not null,
updated_at timestamp with time zone default now() not null,
constraint order_leg_unique_c
unique (flight_id, flight_date, departure_iata, arrival_iata)
);
I have set the foreign keys and they look like this:
Table master_bill_order_leg
master_bill_order_leg_mb_id_fkey (mb_id) -> master_bill(id)
master_bill_order_leg_order_leg_id_fkey (order_leg_id) -> order_leg(id)
I thought that if I delete a row from the parent master_bill table that all relevant rows from the child table master_bill_order_leg would be deleted too because of foreign keys that are set with on delete cascade. So, I have tried with deleting a row from master_bill table:
DELETE
FROM master_bill
WHERE id = :mbId
But, then I get an error:
org.postgresql.util.PSQLException: ERROR: update or delete on table
"master_bill" violates foreign key constraint
"master_bill_order_leg_mb_id_fkey" on table
"master_bill_order_leg" Detail: Key (id)=(1076) is
still referenced from table "master_bill_order_leg".
Why do I get this error, what am I doing wrong here?
If I run this query directly from the query console then a row is deleted from the table master_bill and a child table master_bill_order_leg.
I have an issue when trying to modify and existing PostgreSQL (version 13.3) table to support partitioning it gets stuck when inserting the new data from the old table because the inserted timestamp in some cases may not be unique, so it fails on execution.
The partition forces me to create the primary to be the range (timestamp) value. You can see the new table definition below:
CREATE TABLE "UserFavorites_master" (
"Id" int4 NOT NULL GENERATED BY DEFAULT AS IDENTITY,
"UserId" int4 NOT NULL,
"CardId" int4 NOT NULL,
"CreationDate" timestamp NOT NULL,
CONSTRAINT "PK_UserFavorites_CreationDate" PRIMARY KEY ("CreationDate")
) partition by range ("CreationDate");
The original table didn't have a constraint on timestamp to either be unique or a primary key nor would we particularly want that but that seems to be a requirement of partitioning. Looking for alternatives or good ideas to solve the issue.
You can see the full code below:
alter table "UserFavorites" rename to "UserFavorites_old";
CREATE TABLE "UserFavorites_master" (
"Id" int4 NOT NULL GENERATED BY DEFAULT AS IDENTITY,
"UserId" int4 NOT NULL,
"CardId" int4 NOT NULL,
"CreationDate" timestamp NOT NULL,
CONSTRAINT "PK_UserFavorites_CreationDate" PRIMARY KEY ("CreationDate")
) partition by range ("CreationDate");
-- Frome Reference: https://stackoverflow.com/a/53600145/1190540
create or replace function createPartitionIfNotExists(forDate timestamp) returns void
as $body$
declare yearStart date := date_trunc('year', forDate);
declare yearEndExclusive date := yearStart + interval '1 year';
declare tableName text := 'UserFavorites_Partition_' || to_char(forDate, 'YYYY');
begin
if to_regclass(tableName) is null then
execute format('create table %I partition of "UserFavorites_master" for values from (%L) to (%L)', tableName, yearStart, yearEndExclusive);
-- Unfortunatelly Postgres forces us to define index for each table individually:
--execute format('create unique index on %I (%I)', tableName, 'UserId'::text);
end if;
end;
$body$ language plpgsql;
do
$$
declare rec record;
begin
loop
for rec in 2015..2030 loop
-- ... and create a partition for them
perform createPartitionIfNotExists(to_date(rec::varchar,'yyyy'));
end loop;
end
$$;
create or replace view "UserFavorites" as select * from "UserFavorites_master";
insert into "UserFavorites" ("Id", "UserId", "CardId", "CreationDate") select * from "UserFavorites_old";
It fails on the Last line with the following error:
SQL Error [23505]: ERROR: duplicate key value violates unique constraint "UserFavorites_Partition_2020_pkey"
Detail: Key ("CreationDate")=(2020-11-02 09:38:54.997) already exists.
ERROR: duplicate key value violates unique constraint "UserFavorites_Partition_2020_pkey"
Detail: Key ("CreationDate")=(2020-11-02 09:38:54.997) already exists.
ERROR: duplicate key value violates unique constraint "UserFavorites_Partition_2020_pkey"
Detail: Key ("CreationDate")=(2020-11-02 09:38:54.997) already exists.
No, partitioning doesn't force you to create a primary key. Just omit that line, and your example should work.
However, you definitely always should have a primary key on your tables. Otherwise, you can end up with identical rows, which is a major headache in a relational database. You might have to clean up your data.
#Laurenz Albe is correct, it seems I also have the ability to specify multiple keys though it may affect performance as referenced here Multiple Keys Performance, even indexing the creation date of the partition seemed to make the performance worse.
You can see a reference to multiple keys below, you mileage may vary.
CREATE TABLE "UserFavorites_master" (
"Id" int4 NOT NULL GENERATED BY DEFAULT AS IDENTITY,
"UserId" int4 NOT NULL,
"CardId" int4 NOT NULL,
"CreationDate" timestamp NOT NULL,
CONSTRAINT "PK_UserFavorites" PRIMARY KEY ("Id", "CreationDate")
) partition by range ("CreationDate");
#Error description:
It's possible to create a table that has a foreign key into a hypertable provided the foreign key is defined when the table is created
#To Reproduce, there are next tables:
CREATE TABLE ids (
measurement_id int DEFAULT 0,
description text DEFAULT 0,
m_id bigserial NOT NULL,
service_id int DEFAULT NULL,
time bigint NOT NULL DEFAULT cast((EXTRACT(EPOCH FROM now() AT TIME ZONE 'UTC') * 1000) as bigint),
user_id int DEFAULT NULL,
end_time DOUBLE PRECISION DEFAULT 0,
start_time int NOT NULL DEFAULT 0
);
CREATE INDEX ON ids (time DESC, user_id);
CREATE INDEX ON ids (time DESC, service_id);
SELECT create_hypertable('ids', 'start_time', chunk_time_interval => 604800016);
---------
CREATE TABLE IF NOT EXISTS metrics (
id bigserial NOT NULL,
duration real DEFAULT NULL,
metric integer DEFAULT 0,
m_id bigint NOT NULL,
time bigint NOT NULL DEFAULT 0
);
ALTER TABLE metrics ADD PRIMARY KEY (time, m_id);
CREATE INDEX ON metrics (time DESC);
CREATE INDEX ON metrics (time DESC, measurement );
CREATE INDEX ON metrics (time DESC, m_id );
grant all privileges on ids, metrics to your_db_user;
SELECT create_hypertable('metrics', 'time' , chunk_time_interval => 604800016);
SELECT table_catalog, table_schema, table_name, privilege_type FROM information_schema.table_privileges WHERE grantee = 'your_db_user';
---------
DROP TABLE IF EXISTS resource;
CREATE TABLE resource(
id int NOT NULL,
cpu text DEFAULT 0,
storing text DEFAULT 0,
memory text DEFAULT 0
);
ALTER TABLE resource ADD PRIMARY KEY (id);
CREATE SEQUENCE resource_id_seq
INCREMENT 1
MINVALUE 1
MAXVALUE 2147483647
START 1
CACHE 1;
ALTER TABLE resource_id_seq
OWNER TO your_db_user;
ALTER TABLE resource ALTER COLUMN id SET DEFAULT nextval('resource_id_seq'::regclass);
---------
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
DROP TABLE IF EXISTS ns;
CREATE TABLE ns(
id bigint NOT NULL,
uuid uuid NOT NULL DEFAULT uuid_generate_v4 (),
availability double precision,
faultTolerance boolean,
activated boolean,
UNIQUE (id, uuid),
PRIMARY KEY(id),
CONSTRAINT fk_resource
FOREIGN KEY(id)
REFERENCES resource(id)
ON DELETE CASCADE
);
CREATE SEQUENCE ns_id_seq
INCREMENT 1
MINVALUE 1
MAXVALUE 9223372036854775807
START 1
CACHE 1;
ALTER TABLE ns_id_seq
OWNER TO your_db_user;
ALTER TABLE ns ALTER COLUMN id SET DEFAULT nextval('ns_id_seq'::regclass);
---------
DROP TABLE IF EXISTS authentication;
CREATE TABLE authentication(
id integer NOT NULL,
username character varying(255) NOT NULL,
password character varying(255) NOT NULL,
host character varying(255) NOT NULL,
port character varying(10) NOT NULL,
PRIMARY KEY(id)
);
CREATE SEQUENCE auth_id_seq
INCREMENT 1
MINVALUE 1
MAXVALUE 2147483647
START 1
CACHE 1;
ALTER TABLE auth_id_seq
OWNER TO your_db_user;
ALTER TABLE authentication ALTER COLUMN id SET DEFAULT nextval('auth_id_seq'::regclass);
---------
DROP TABLE IF EXISTS job;
CREATE TABLE job(
id int NOT NULL,
interval integer NOT NULL,
auth_id integer REFERENCES authentication (id),
ns_id integer REFERENCES ns (id),
UNIQUE (auth_id, ns_id),
PRIMARY KEY(id)
);
ALTER TABLE job
ADD CONSTRAINT fk_auth_id
FOREIGN KEY (id) REFERENCES authentication (id)
ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED;
ALTER TABLE job
ADD CONSTRAINT fk_ns_id
FOREIGN KEY (id) REFERENCES ns (id)
ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED;
CREATE SEQUENCE job_id_seq
INCREMENT 1
MINVALUE 1
MAXVALUE 2147483647
START 1
CACHE 1;
ALTER TABLE job_id_seq
OWNER TO your_db_user;
ALTER TABLE job ALTER COLUMN id SET DEFAULT nextval('job_id_seq'::regclass);
---------
DROP TABLE IF EXISTS job_metric;
CREATE TABLE job_metric (
id int NOT NULL,
j_id int NOT NULL REFERENCES job (id),
mj_id bigint NOT NULL,
jm_time bigint NOT NULL
);
CREATE INDEX ON job_metric (jm_time DESC);
CREATE INDEX ON job_metric (jm_time DESC, id);
CREATE INDEX ON job_metric (jm_time DESC, mj_id);
ALTER TABLE job_metric ADD PRIMARY KEY (jm_time, id);
grant all privileges on job_metric to your_db_user;
SELECT create_hypertable('job_metric', 'jm_time' , chunk_time_interval => 604800016);
CREATE SEQUENCE mjob_metric_id_seq
INCREMENT 1
MINVALUE 1
MAXVALUE 2147483647
START 1
CACHE 1;
ALTER TABLE mjob_metric_id_seq
OWNER TO your_db_user;
ALTER TABLE job_metric ALTER COLUMN id SET DEFAULT nextval('mjob_metric_id_seq'::regclass);
---------
After creating the tables, I have used the solution proposed by #Laurenz in a database with PostgreSQL 12.6 using the extension of timescaledb 1.7.5 as follows:
#To fill the table with the appropriate values:
UPDATE job_metric AS jm_point
SET jm_time = qm.time
FROM metrics AS qm
WHERE qm.m_id = jm_point.mj_id;
#Then set it NOT NULL:
ALTER TABLE job_metric ALTER jm_time SET NOT NULL;
#To define your foreign key:
ALTER TABLE job_metric
ADD FOREIGN KEY (mj_id, jm_time)
REFERENCES metrics (time, m_id) MATCH FULL;
#Response of the last reference table to enable foreign key: Query returned successfully in 40 msec.
Expected behavior:
The idea is to use the table job_metric in an even many-to-many relationship to access the information of job and metrics tables.
Actual behavior and error:
Tables are created and FKs were created but cannot be used when data is inserted at job_metric as is detailed in the following:
INSERT INTO job_metric (j_id, mj_id, jm_time)
VALUES(13, 185063, 1621957192266);
ERROR: foreign keys to hypertables are not supported CONTEXT: SQL
statement " ALTER TABLE _timescaledb_internal._hyper_5_5_chunk ADD
CONSTRAINT "5_13_job_metric_j_id_mj_id_jm_time_fkey" FOREIGN KEY
(j_id, mj_id, jm_time) REFERENCES qmetrics("time", m_id) MATCH FULL "
PL/pgSQL function
_timescaledb_internal.chunk_constraint_add_table_constraint(_timescaledb_catalog.chunk_constraint)
line 42 at EXECUTE SQL state: 0A000
***According to https://docs.timescale.com/timescaledb/latest/overview/limitations/##distributed-hypertable-limitations, it looks like the above error is part of the hypertable limitations:
Foreign key constraints referencing a hypertable are not supported.
#Request:
Given the above information and errors, does anyone know any solution at the DB level to establish the relationships (many-to-many or one-to-many) using timescaledb extension and mainly hypertables?
Actually, I have obtained similar of above error when I had attempted to create many-to-many relation among the tables metrics and job_metric using the Django Rest Framework:
class Job_Metrics(models.Model):
job = models.OneToOneField(Job, on_delete=models.CASCADE)
metrics = models.ManyToManyField(Metrics)
time = models.IntegerField(default=0)
Running the application metrics pointing out directly metrics_db:
$ python3 manage.py migrate metrics --database=metrics_db
Operations to perform: Apply all migrations: metrics Running migrations: Applying
metrics.0002_job...Traceback (most recent call last): File
"/var/myproject/myprojectenv/lib/python3.8/site-packages/django/db/backends/utils.py",
line 84, in _execute return self.cursor.execute(sql, params)
psycopg2.errors.FeatureNotSupported: foreign keys to hypertables are
not supported
If someone knows a solution or has an idea to deal with the above error at the REST API level, please could you share your idea with the aim to access data associated tables (metrics and jobs) and modify them together when is required to delete e.g., a job_metric. So far, using hypertables amendments of timescaledb extension seems to be not a viable solution.
Using Postgres, what I would like to achieve is to be able to have many different instrument types, with corresponding [TYPE].instrument tables, which all have a unique ID in the table, but also reference a unique ID in the instrument.master table. I have the following:
create schema instrument
CREATE TABLE instrument.type (
id smallserial NOT NULL,
name text not null,
code text not null,
CONSTRAINT pk_instrument_type PRIMARY KEY (id)
);
ALTER TABLE instrument.type ADD CONSTRAINT unq_instrument_type_code UNIQUE(code);
ALTER TABLE instrument.type ADD CONSTRAINT unq_instrument_type_name UNIQUE(name);
insert into instrument.type (name, code) values ('futures', 'f');
CREATE TABLE instrument.master (
id serial NOT NULL,
type smallint not null references instrument.type (id),
timestamp timestamp with time zone not null,
CONSTRAINT pk_instrument_master PRIMARY KEY (id)
);
CREATE TABLE futures.definition (
id smallserial NOT NULL,
code text not null,
CONSTRAINT pk_futures_definition PRIMARY KEY (id)
);
ALTER TABLE futures.definition ADD CONSTRAINT unq_futures_definition_code UNIQUE(code);
insert into futures.definition (code) values ('ED');
CREATE TABLE futures.instrument (
id smallserial NOT NULL,
master serial not null references instrument.master (id),
definition smallint not null references futures.definition (id),
month smallint not null,
year smallint not null,
CONSTRAINT pk_futures_instrument PRIMARY KEY (id),
check (month >= 1),
check (month <= 12),
check (year >= 1900)
);
ALTER TABLE futures.instrument ADD CONSTRAINT unq_futures_instrument UNIQUE(definition, month, year);
CREATE OR REPLACE FUNCTION trigger_master_futures()
RETURNS trigger AS
$BODY$
BEGIN
insert into instrument.master (type, timestamp)
select id, current_timestamp from instrument.type where code = 'f';
NEW.master := currval('instrument.master_id_seq');
RETURN NEW;
END;
$BODY$
LANGUAGE 'plpgsql' VOLATILE;
create trigger trg_futures_instrument before insert on futures.instrument
for each row
execute procedure trigger_master_futures();
I then test with:
insert into futures.instrument (definition, month, year)
select id, 3, 2015 from futures.definition where code = 'ED';
Everything works almost as I would like it to. The only issue is that somehow, instrument.master.id ends up being one more than futures.instrument.master. I am not sure what I need to do to achieve the behavior I want, which is that whenever an entry is inserted into futures.instrument, an entry should be inserted into instrument.master, and the id entry of the latter should be inserted into the master entry of the former. I actually think it should have failed since the foreign key relationship is violated somehow.
As it turns out, everything was correct. The issue was that in futures.instrument, the type of the master column is serial, and it should have been int.
i have a table:
CREATE TABLE my_table
(
id integer NOT NULL DEFAULT nextval('seq_my_table_id'::regclass),
fk_id1 integer NOT NULL,
fk_id2 smallint NOT NULL,
name character varying(255) NOT NULL,
description text,
currency_name character varying(3) NOT NULL,
created timestamp with time zone NOT NULL DEFAULT now(),
updated timestamp with time zone NOT NULL DEFAULT now(),
CONSTRAINT "PK_my_table_id" PRIMARY KEY (id ),
CONSTRAINT "FK_my_table_fk_id1" FOREIGN KEY (fk_id1)
REFERENCES my_table2 (id) MATCH SIMPLE
ON UPDATE NO ACTION ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED,
CONSTRAINT "FK_my_table_fk_id2" FOREIGN KEY (fk_id2)
REFERENCES my_table3 (id) MATCH SIMPLE
ON UPDATE NO ACTION ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED
)
WITH (
OIDS=FALSE,
autovacuum_enabled=true,
autovacuum_vacuum_threshold=50,
autovacuum_vacuum_scale_factor=0.2,
autovacuum_analyze_threshold=50,
autovacuum_analyze_scale_factor=0.1,
autovacuum_vacuum_cost_delay=20,
autovacuum_vacuum_cost_limit=200,
autovacuum_freeze_min_age=50000000,
autovacuum_freeze_max_age=200000000,
autovacuum_freeze_table_age=150000000
);
ALTER TABLE my_table
OWNER TO postgres;
CREATE INDEX my_table_fk_id1
ON my_table
USING btree
(fk_id1 );
CREATE INDEX my_table_fk_id2
ON my_table
USING btree
(fk_id2 );
tables records count
select count(id) from my_table; --24061
select count(id) from my_table2; --24061
select count(id) from my_table3; --123
execution time
select * from my_table -- ~17sec
vacuum/analyze - no effect
description - length ~ 4000 chars in each row
postgres.conf - standart settings
Version: 9.1
select all fields except description reduce execution time to ~1,5 sec
How to icrease select speed with description ?
upd
--explain analyze select * from my_table
"Seq Scan on my_table (cost=0.00..3425.79 rows=24079 width=1015) (actual time=0.019..17.238 rows=24079 loops=1)"
"Total runtime: 18.649 ms"
The question is how to make this fast. The issue is not on the server since it takes 18ms there. The simple solution is to select fewer columns so that there is less to transfer over the network. My guess is that you have long descriptions on some. Leave that column off your select and try again.