CTE query performance improvement (postgres 9.6)

CTE query performance improvement (postgres 9.6) - postgresql

I have got a query below that does what needed, but super slow. It returns in a given period of time (for example below between 2017-05-01 00:00:00 and 2017-05-01 01:00:00, but could be anything between 1 second and several days) first and last records of a given intervals (15 sec as example, but could be anything from 1 second to several days).
It is unbearably slow. For example for interval on 1 second for a period 2017-05-01 00:00:00 to 2017-05-01 00:00:01 it runs 6 sec on my i7 7700HQ. for interval for 1 for period 2017-05-01 00:00:00 to 2017-05-01 00:00:05 I never saw the result! The database has about 60 millions row now. in production it will be 1 billion with approx 50 millions added every months.
QUERY:
WITH ranges as (
SELECT dd as start_range,
dd + '15 seconds'::interval as end_range,
ROW_NUMBER() over () as grp
FROM generate_series
( '2017-05-01 00:00:00'::timestamp
, '2017-05-01 01:00:00'::timestamp
, '15 seconds'::interval) dd
), create_grp as (
SELECT r.grp, r.start_range, r.end_range, p.*
FROM prices p
JOIN ranges r
ON p.dt >= r.start_range
AND p.dt < r.end_range
WHERE instrument='EURGBP'
), minmax as (
SELECT row_number() over (partition by grp
order by dt asc) as rn1,
row_number() over (partition by grp
order by dt desc) as rn2,
create_grp.*
FROM create_grp
)
SELECT *,
CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
WHEN rn1 = 1 THEN 'first'
WHEN rn2 = 1 THEN 'last'
END as row_position
FROM minmax
WHERE
1 IN (rn1, rn2)
ORDER BY dt
;
SCHEMA:
CREATE TABLE public.prices
(
uid uuid NOT NULL DEFAULT uuid_generate_v4(),
instrument character varying COLLATE pg_catalog."default" NOT NULL,
bid double precision NOT NULL,
ask double precision NOT NULL,
dt timestamp without time zone NOT NULL DEFAULT now(),
CONSTRAINT prices_pkey PRIMARY KEY (uid)
)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
INDEXES:
CREATE INDEX idx_dt_instrument
ON public.prices USING btree
(dt, instrument COLLATE pg_catalog."default")
TABLESPACE pg_default;
CREATE INDEX idx_dt_instrument_bid_ask
ON public.prices USING btree
(dt, instrument COLLATE pg_catalog."default", bid, ask)
TABLESPACE pg_default;
CREATE INDEX idx_instrument
ON public.prices USING btree
(instrument COLLATE pg_catalog."default")
TABLESPACE pg_default;
EXAMPLE dataset for 5 seconds staring from 2017-05-01 00:00:00:
"uid","instrument","bid","ask","dt"
"4ecaa607-3733-4aba-9093-abc8f59e1638","EURGBP","0.84331","0.8434","2017-05-01 00:00:00.031"
"d1a41847-4945-4cf4-a45f-781db977ae07","GBPJPY","143.949005","143.970993","2017-05-01 00:00:00.031"
"34972c12-899b-404a-b0de-bae0fd3f6733","GBPJPY","143.947998","143.970993","2017-05-01 00:00:00.056"
"384b8246-3eac-4826-b6d2-d6caaa364f81","GBPUSD","1.29311","1.29323","2017-05-01 00:00:00.066"
"d879b04d-a4ed-452e-9208-7dfff672e860","GBPJPY","143.947006","143.970993","2017-05-01 00:00:00.067"
"a9e735ec-30d9-4c9a-a28e-5e5553273372","GBPJPY","143.945999","143.970993","2017-05-01 00:00:00.079"
"ee40ee5f-d8ac-41ce-9f39-ae50ef15d02d","GBPJPY","143.947006","143.964005","2017-05-01 00:00:00.091"
"7b605c3b-121f-46c2-a3e4-297d187f0a28","GBPJPY","143.947006","143.968994","2017-05-01 00:00:00.115"
"ccb307b0-7fa3-4354-8707-1426eded49e8","GBPJPY","143.942001","143.968994","2017-05-01 00:00:00.205"
"206c339d-bc36-469d-82d1-c7ae74002f44","EURGBP","0.84332","0.8434","2017-05-01 00:00:00.206"
"bc581318-91c7-4c80-85e0-e06f7236b277","GBPJPY","143.944","143.968994","2017-05-01 00:00:00.206"
"1fabf850-9045-4beb-81ae-bfa3adada62e","GBPUSD","1.29311","1.29324","2017-05-01 00:00:00.208"
"06d40f9a-a47e-466d-aebf-97a0154bdc74","GBPJPY","143.942001","143.968994","2017-05-01 00:00:00.209"
"b3b7fac9-340f-4e3b-8946-7bdecc383191","GBPUSD","1.29311","1.29327","2017-05-01 00:00:00.211"
"b5c28955-b40f-446f-9f43-9d5e9f145c1b","EURGBP","0.84331","0.8434","2017-05-01 00:00:00.212"
"192a40d6-8001-42ea-9430-96e800f2d4e8","GBPJPY","143.942993","143.968994","2017-05-01 00:00:00.212"
"e98dbba7-8231-4fa3-926b-291eb22b0f87","GBPJPY","143.944","143.968994","2017-05-01 00:00:00.215"
"7c6e952d-d01f-4dac-a1df-6b32168246fd","GBPUSD","1.29311","1.29326","2017-05-01 00:00:00.216"
"c86ba29f-3edb-4147-ba99-d9dfe594b0ff","GBPUSD","1.29312","1.29327","2017-05-01 00:00:00.243"
"35ca131e-b714-462d-827f-b5bc593aa3e6","GBPJPY","143.942993","143.968994","2017-05-01 00:00:00.262"
"91fc6fc0-7af9-4036-8e0e-29d4a3bb3e43","EURGBP","0.8433","0.8434","2017-05-01 00:00:00.283"
"e71946e0-1859-461a-b3eb-0539584ac4dc","GBPJPY","143.944","143.968994","2017-05-01 00:00:00.296"
"7321eea8-2610-408b-8dbf-4087f01e8c6e","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.377"
"f146716d-cadf-4e2f-9f17-6e7e8c5f2175","GBPUSD","1.29312","1.29327","2017-05-01 00:00:00.38"
"b3d81295-8cd5-44e7-879c-f15476ffac21","GBPUSD","1.29312","1.29326","2017-05-01 00:00:00.391"
"c037e27b-a8f4-4ec3-8472-b0f72a58fd33","EURGBP","0.8433","0.8434","2017-05-01 00:00:00.413"
"dba0f8f5-f218-49ea-8ebf-132f3ecf8910","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.443"
"a08449e3-44aa-4fed-b8e9-bf1a6bfc35c5","EURGBP","0.8433","0.84339","2017-05-01 00:00:00.585"
"a0b7ba20-653f-46db-93e9-d1edd8972dba","GBPUSD","1.29312","1.29326","2017-05-01 00:00:00.588"
"c2855ce8-8c5b-4de7-a92b-186d928e8f31","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.591"
"01b19a70-3ce7-44c7-9abd-9945321fdfd0","EURGBP","0.8433","0.84339","2017-05-01 00:00:00.621"
"4518aa9d-1f76-428e-ace4-7dcffeb6aa22","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.796"
"e4d4bac4-dd02-4da3-b231-20bfa6424412","EURGBP","0.8433","0.84339","2017-05-01 00:00:00.907"
"e48d3721-3157-4033-bd4f-09baae0f989c","GBPUSD","1.29312","1.29326","2017-05-01 00:00:00.909"
"64a33b2d-c756-4a0a-823e-075143ae7263","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.913"
"e477c47d-efd1-44dd-8058-cac17e08bc5d","GBPUSD","1.29314","1.29327","2017-05-01 00:00:00.914"
"cf6d5341-f7fd-47bc-89f6-a5448f78fb99","EURGBP","0.84329","0.84339","2017-05-01 00:00:00.943"
"4caa8bb9-094e-48ca-8a9a-7b2dd23a4fb4","GBPJPY","143.947006","143.968994","2017-05-01 00:00:00.967"
"274b7f51-3b07-430d-bfc5-a0b1ce62a750","GBPUSD","1.29312","1.29327","2017-05-01 00:00:00.975"
"2e3c2cd6-2525-46b3-86c5-b88e48bb0138","GBPJPY","143.947998","143.968994","2017-05-01 00:00:01.076"
"bcb12b90-2795-4789-bfa2-8e9a494eaeb1","GBPUSD","1.29312","1.29326","2017-05-01 00:00:01.076"
"d63d6037-fa81-47cc-bd62-4655850c0f80","GBPUSD","1.29312","1.29327","2017-05-01 00:00:01.077"
"6dbf8d8e-37c8-4537-80b5-c9219f4356b1","EURGBP","0.8433","0.84339","2017-05-01 00:00:01.079"
"7e3b6eaf-22e1-4f87-a7c1-226d3ee76146","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.08"
"63e451c5-b8c6-4b57-ac9e-2171bc1dfbfa","GBPJPY","143.947998","143.968994","2017-05-01 00:00:01.121"
"866316e7-90a2-4c80-9a38-b3a062837415","GBPJPY","143.949005","143.968994","2017-05-01 00:00:01.143"
"fb11e963-cd36-4cfc-89b3-1bba3b8595b5","GBPUSD","1.29312","1.29327","2017-05-01 00:00:01.156"
"ad57e34f-5cbe-4b79-8579-b2c77c83b50b","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.249"
"46b1840e-e424-41c2-8c71-691b201183ab","GBPJPY","143.949005","143.968994","2017-05-01 00:00:01.259"
"c5fa5e09-46df-4ea4-8640-9cf18e1f8fcc","GBPUSD","1.29312","1.29327","2017-05-01 00:00:01.265"
"926d092d-601e-43ec-b398-3300b0345cfd","GBPJPY","143.949997","143.968994","2017-05-01 00:00:01.267"
"4ddfbc84-20a1-4281-86c2-f0a4e77d0152","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.305"
"e75af139-51a2-4a0a-acc8-e0adfbe5472a","GBPUSD","1.29313","1.29327","2017-05-01 00:00:01.346"
"408bca2d-2d57-471b-b6a7-819a3257741f","GBPJPY","143.951004","143.968994","2017-05-01 00:00:01.348"
"53c6f444-bd76-4a28-a370-7af98d5fa9ec","GBPUSD","1.29313","1.29326","2017-05-01 00:00:01.359"
"cd002ef4-925e-469b-8f2f-848cadd5943f","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.443"
"13587e44-b694-4ce9-a626-e592d28c507d","EURGBP","0.8433","0.84339","2017-05-01 00:00:01.45"
"aabbacaf-3f09-4313-b992-e8b8d91df7ad","GBPJPY","143.951004","143.968994","2017-05-01 00:00:01.461"
"6b2111ef-c285-4482-b93c-238f27522ca3","GBPUSD","1.29313","1.29326","2017-05-01 00:00:01.477"
"19507f29-149c-4fdf-a1fa-aac312ab8479","EURGBP","0.8433","0.84338","2017-05-01 00:00:01.506"
"916cc759-a536-449d-b825-8203ebea9bf8","GBPJPY","143.949997","143.968994","2017-05-01 00:00:01.649"
"1fdd2b35-fd44-4dbb-81df-b98514af7004","GBPUSD","1.29313","1.29326","2017-05-01 00:00:01.649"
"3e9fc214-3cc6-4991-9eb3-2f5a5e557e96","GBPUSD","1.29312","1.29326","2017-05-01 00:00:01.65"
"5da9a29b-0d8a-42b1-98b6-f89dd2893c77","EURGBP","0.8433","0.84338","2017-05-01 00:00:01.651"
"b87f85e3-fba5-4556-8ca5-52625d978d53","EURGBP","0.84329","0.84338","2017-05-01 00:00:01.652"
"75b7624c-b90c-40d4-a7fc-09df7da9f659","GBPJPY","143.949997","143.968994","2017-05-01 00:00:01.715"
"b076534f-b893-4b55-9d9a-e3bac6e77ab2","GBPUSD","1.29312","1.29325","2017-05-01 00:00:01.732"
"6464da85-9eb5-4548-bd5e-3331a69f2121","EURGBP","0.84329","0.84338","2017-05-01 00:00:01.817"
"f9937464-e36a-4c57-a212-2f32943307d3","EURGBP","0.8433","0.84338","2017-05-01 00:00:01.83"
"1c7848e5-c101-4a50-87fe-1f5e980dbb95","GBPJPY","143.949005","143.968994","2017-05-01 00:00:01.83"
"30ad5bfc-f968-4b49-9b22-80c12e801f37","GBPUSD","1.29312","1.29325","2017-05-01 00:00:01.847"
"94eb4b51-4901-4e56-873a-05f93ef35e64","EURGBP","0.8433","0.84338","2017-05-01 00:00:02.007"
"97ef7e05-0a2f-4d70-80d1-1d3c748ea70f","GBPJPY","143.949005","143.968994","2017-05-01 00:00:02.007"
"af9274d9-02d6-43fc-9336-c3f512e4bd78","GBPUSD","1.29312","1.29325","2017-05-01 00:00:02.008"
"78c12ba8-992a-469e-bdda-bef004596aa1","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.062"
"4e1e31a9-bfdc-4f56-b8e1-0c7e83852fe0","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.071"
"e4f1c459-f75b-4005-8dd1-00c572ded4f9","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.203"
"2c8b38ce-50de-43e7-ba63-b6d1d7a7f76c","GBPJPY","143.947998","143.968994","2017-05-01 00:00:02.244"
"438f4439-a259-421f-8f45-a0dc88d80e1d","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.257"
"f12b7e18-84f0-4686-9588-0ecfab8f8bf8","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.301"
"f335ee32-cd5e-4cd2-aabe-06c14985778f","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.394"
"3c55786e-f9b8-424b-9c32-0b4b432a553f","EURGBP","0.8433","0.84338","2017-05-01 00:00:02.404"
"53c3e013-b016-41bb-a3ed-48158640831a","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.417"
"a944e2b6-3700-4ecd-994d-cb5ab97cf5ad","EURGBP","0.8433","0.84339","2017-05-01 00:00:02.508"
"ab32781d-1502-4047-b320-c3f4cda389fd","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.605"
"24759950-e095-4bb7-be3d-fda725423589","EURGBP","0.84329","0.84339","2017-05-01 00:00:02.625"
"cd7940d9-8a30-4a12-9dc9-a2c710cbb982","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.649"
"233e0a3d-eeae-403f-b512-2add055d6735","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.761"
"58c7c33c-75b2-45bd-a50c-615d9eaaec07","EURGBP","0.84329","0.84339","2017-05-01 00:00:02.763"
"675b2847-b0f9-44f1-ab75-d6ddafbddfd5","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.766"
"d2946714-e84c-4a36-9819-0cca4f2dd197","EURGBP","0.8433","0.84339","2017-05-01 00:00:02.81"
"5322418b-4c45-4771-b591-cc8fea167424","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.832"
"9a1c572e-28b9-4d16-b196-ccef3e48f602","EURGBP","0.84331","0.84339","2017-05-01 00:00:02.929"
"a4377169-5400-4dd6-81f0-8b455d178b65","EURGBP","0.8433","0.84339","2017-05-01 00:00:02.951"
"37cdddb0-db5e-4574-bf4d-45fb8d52c6be","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.951"
"1258c886-b797-43ed-9886-323daac83dde","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.999"
"453406c3-5902-4dab-b39c-2e0b85739767","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.005"
"95699a6d-8412-4f2f-9dd4-86a1d3eed57d","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.129"
"2268fcc4-104e-434b-8237-bce765bda084","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.131"
"ba771b59-b04c-4683-8ceb-8e9b8451bcc7","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.17"
"00541afa-79a6-4d09-8c6e-4d59e8d15bb2","GBPJPY","143.947006","143.968994","2017-05-01 00:00:03.216"
"b4dd6e15-0e25-4da1-b210-021b91e4245a","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.323"
"79b11513-97c5-4146-a4c8-7daf7d8ffe6f","GBPJPY","143.947006","143.966995","2017-05-01 00:00:03.324"
"c5283502-eb09-45d3-b92b-d5625de9e9fc","GBPJPY","143.947006","143.966003","2017-05-01 00:00:03.411"
"67715855-9171-4ae7-b205-5ba017d328f2","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.501"
"00b7647e-7d53-407d-9704-78a8712ac580","GBPJPY","143.947006","143.966003","2017-05-01 00:00:03.525"
"b41b5483-5fb2-4c57-9892-0d02e5e6a823","GBPJPY","143.947998","143.966003","2017-05-01 00:00:03.549"
"881712b2-ff9a-4065-ad6a-9caee03283e3","GBPJPY","143.947998","143.964996","2017-05-01 00:00:03.56"
"7248bc13-5a9f-4fba-97a1-fe041e975c38","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.581"
"0379b4bb-7691-48e8-a668-2388f6e5d510","GBPJPY","143.947998","143.964996","2017-05-01 00:00:03.672"
"2715fe9a-eb6f-445d-b7d0-850293bb5b2e","GBPJPY","143.947006","143.964996","2017-05-01 00:00:03.698"
"52ce7685-0964-46e4-ab8f-282e68bdb73d","GBPJPY","143.947006","143.964005","2017-05-01 00:00:03.711"
"6b31dee4-1aec-4d10-9b37-0c58bae7e449","GBPJPY","143.947006","143.964005","2017-05-01 00:00:03.848"
"7e61571a-ea8b-44cb-9481-ae31ba8b86c4","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.868"
"89e316cb-e01d-4f01-9dcd-a380e94ed4e8","GBPJPY","143.947006","143.964005","2017-05-01 00:00:03.943"
"966318e4-a6b5-4c1b-82e1-fbbf0029ce02","GBPUSD","1.29312","1.29325","2017-05-01 00:00:04.096"
"8fc7b075-7f5e-40f5-9879-55604f77a3c5","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.227"
"23ab8b66-337e-4c4e-b73a-f31101716565","GBPJPY","143.947006","143.964005","2017-05-01 00:00:04.227"
"eb7469be-8a40-4498-af85-7b2c42630149","GBPJPY","143.947006","143.964005","2017-05-01 00:00:04.288"
"f945a821-9790-4760-beea-1b54bb1c2a85","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.406"
"978c6d4c-22f9-4218-b447-940a3d3c436e","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.505"
"2ace7d77-b228-491c-9cf0-610b8e48cdf7","GBPJPY","143.947006","143.966003","2017-05-01 00:00:04.786"
"0f89d5dd-52f2-44e9-aa54-cbb423cd7416","GBPJPY","143.947006","143.964996","2017-05-01 00:00:04.787"
"db7748a8-317b-4c62-a1f2-d6679430b343","GBPUSD","1.29312","1.29325","2017-05-01 00:00:04.895"
"ae29728e-773b-42a9-9f1e-199ba996731a","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.986"
"19ffa8e4-63c5-4f47-90d5-771dd5a839ba","GBPJPY","143.947006","143.962997","2017-05-01 00:00:04.986"
The slowdown is caused by " WHERE instrument='EURGBP' ". As soon as I remove it the query flies. However, I must have filtering by 'instrument'.
EXPLAIN OUTPUT:
Sort (cost=2356660289.95..2356711257.16 rows=20386886 width=144)
Sort Key: minmax.dt
CTE ranges
-> WindowAgg (cost=0.00..25.00 rows=1000 width=24)
-> Function Scan on generate_series dd (cost=0.00..10.00 rows=1000 width=8)
CTE create_grp
-> Nested Loop (cost=344299.48..460976348.77 rows=2043798111 width=71)
Join Filter: ((p.dt >= r.start_range) AND (p.dt < r.end_range))
-> Bitmap Heap Scan on prices p (cost=344299.48..1121763.77 rows=18394183 width=47)
Recheck Cond: ((instrument)::text = 'EURGBP'::text)
-> Bitmap Index Scan on idx_instrument (cost=0.00..339700.94 rows=18394183 width=0)
Index Cond: ((instrument)::text = 'EURGBP'::text)
-> CTE Scan on ranges r (cost=0.00..20.00 rows=1000 width=24)
CTE minmax
-> WindowAgg (cost=1796644092.85..1837520055.07 rows=2043798111 width=112)
-> Sort (cost=1796644092.85..1801753588.13 rows=2043798111 width=104)
Sort Key: create_grp.grp, create_grp.dt DESC
-> WindowAgg (cost=880857947.67..921733909.89 rows=2043798111 width=104)
-> Sort (cost=880857947.67..885967442.95 rows=2043798111 width=96)
Sort Key: create_grp.grp, create_grp.dt
-> CTE Scan on create_grp (cost=0.00..40875962.22 rows=2043798111 width=96)
-> CTE Scan on minmax (cost=0.00..51298821.64 rows=20386886 width=144)
Filter: ((1 = rn1) OR (1 = rn2))
Any suggestions to optimize it are very very welcome.

CREATE TEMP TABLE ranges
( start_range timestamp NOT NULL
, end_range timestamp NOT NULL
, grp INTEGER NOT NULL UNIQUE
, PRIMARY KEY (start_range)
);
INSERT INTO ranges(start_range, end_range, grp)
SELECT dd as start_range,
dd + '1 seconds'::interval as end_range,
ROW_NUMBER() over () as grp
FROM generate_series
( '2017-05-01 00:00:00'::timestamp
, '2017-05-01 01:00:00'::timestamp
--, '15 seconds'::interval) dd
, '1 seconds'::interval) dd
;
VACUUM ANALYZE ranges;
-- EXPLAIN ANALYZE
SELECT *,
CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
WHEN rn1 = 1 THEN 'first'
WHEN rn2 = 1 THEN 'last'
END as row_position
FROM (
SELECT r.grp, p.uid, p.instrument, r.start_range , p.dt AS dt
, p.bid, p.ask
, row_number() over (partition by r.grp order by p.dt asc) as rn1
, row_number() over (partition by r.grp order by p.dt desc) as rn2
FROM prices p
JOIN ranges r ON p.dt >= r.start_range AND p.dt < r.end_range
WHERE p.instrument='EURGBP'
) zzzz
WHERE (rn1=1 OR rn2=1)
-- ORDER BY instrument,grp, dt
;
And, you don't actually need the row number, since you only want the beginning/end of a window:
-- EXPLAIN ANALYZE
SELECT *,
CASE WHEN (prev IS NULL AND next IS NULL) THEN 'first and last'
WHEN prev IS NULL THEN 'first'
WHEN next IS NULL THEN 'last'
END as row_position
FROM (
SELECT r.grp, p.uid, p.instrument, r.start_range , p.dt AS dt, p.bid, p.ask
, lag(uid) over (www) as prev
, lead(uid) over (www) as next
FROM prices p
JOIN ranges r ON p.dt >= r.start_range AND p.dt < r.end_range
WHERE p.instrument='EURGBP'
WINDOW www AS (partition by r.grp order by p.dt )
-- ORDER BY p.instrument,r.grp, p.dt
) qqqq
WHERE (prev IS NULL OR next IS NULL)
-- ORDER BY instrument,grp, dt
;

Thanks to all for help and hints! Apparently if I move WHERE clause from create_gpr to minimax, the query become extremely fast. I have no explanation to it whatsoever. But it works. Here it is:
WITH ranges as (
SELECT dd as start_range,
dd + '1 seconds'::interval as end_range,
ROW_NUMBER() over () as grp
FROM generate_series
( '2017-05-01 00:00:00'::timestamp
, '2017-05-01 00:01:00'::timestamp
, '1 seconds'::interval) dd
), create_grp as (
SELECT r.grp, r.start_range, r.end_range, p.*
FROM prices p
JOIN ranges r
ON p.dt >= r.start_range
AND p.dt < r.end_range
-- WHERE need to be moved out of here, which has no indexes as being temporary
-- WHERE instrument='EURGBP'
), minmax as (
SELECT row_number() over (partition by grp
order by dt asc) as rn1,
row_number() over (partition by grp
order by dt desc) as rn2,
create_grp.*
FROM create_grp
-- Here WHERE goes! It does use index here for some reason. And the query flies.
WHERE instrument='EURGBP'
)
SELECT *,
CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
WHEN rn1 = 1 THEN 'first'
WHEN rn2 = 1 THEN 'last'
END as row_position
FROM minmax
WHERE
1 IN (rn1, rn2)
ORDER BY dt
;
This query (for 1 minute range) runs in 84 msec, the old one never finished to run (I gave up after 10 minutes). The old query took 10 sec to run for the range of 1 sec.
Thanks to everyone for help!

Related

Optimise query: count latest win/lose streak for all teams

I'm not an expert in data-warehousing nor analytics, so I give birth to a monster-query that I'd like to optimise (if possible).
The problem is: I need to display the stagings table for a given tournament. This table should display team id, total score, team position (i.e. bucket for same-score teams), order and latest (!) win-lose strike.
I.e. for the following sequence (most recent first) WWLLLWLD I should get 2W
W = win, L = lose, D = draft
Schema
create table matches (
id integer primary key,
stage_id integer not null,
scheduled_at timestamp not null,
winner_id integer null,
status text not null default 'finished' -- just to give extra context
);
create table teams (
id integer primary key
);
create table match_teams (
match_id integer,
team_id integer,
constraint fk_mt_m foreign key (match_id) references matches(id),
constraint fk_mt_t foreign key (team_id) references teams(id)
);
insert into teams(id) values(1),(2);
insert into matches(id, stage_id, scheduled_at, winner_id) values
(1, 1, now() - interval '1 day', 1),
(2, 1, now() - interval '2 days', 1),
(3, 1, now() - interval '3 days', 2),
(4, 1, now() - interval '4 days', 1),
(5, 1, now() - interval '5 days', null);
insert into match_teams(match_id, team_id) values
(1, 1),
(1, 2),
(2, 1),
(2, 2),
(3, 1),
(3, 2),
(4, 1),
(4, 2),
(5, 1),
(5, 2);
Query itself:
with v_mto as (
SELECT
m.id,
m."stage_id",
mt."team_id",
m."scheduled_at",
(
case
when m."winner_id" IS NULL then 0
when (m."winner_id" = mt."team_id") then 1
else -1
end
) win
FROM matches m
INNER JOIN match_teams mt ON m.id = mt."match_id"
WHERE m.status = 'finished'
ORDER BY "stage_id", "team_id", "scheduled_at" desc
),
v_lag as (
select
"stage_id",
"team_id",
win,
lag(win, 1, win) over (partition by "stage_id", "team_id" order by "scheduled_at" desc ) lag_win,
first_value(win) over (partition by "stage_id", "team_id" order by "scheduled_at" desc ) first_win
from v_mto
)
select
"stage_id",
"team_id",
v_lag.win,
count(1)
from v_lag
where v_lag.win = v_lag.lag_win and v_lag.win = v_lag.first_win
group by 1, 2, 3
-- This is the query for the final table (on a screenshot)
-- with team_scores as (
-- select
-- m."tournamentStageId",
-- "teamId",
-- sum(
-- -- each win gives 3 score, each draft gives 1 score
-- coalesce((m."winner_id" = mt."team_id")::integer, 0) * 3
-- +
-- (m."winner_id" IS NULL)::int
-- ) as score
-- from matches m
-- inner join match_teams mt on m.id = mt."matchId"
-- where m.status = 1
-- group by m."tournamentStageId", "teamId")
-- select
-- "tournamentStageId",
-- "teamId",
-- t.name,
-- score,
-- dense_rank() over (partition by "tournamentStageId" order by score desc) rank,
-- row_number() over (partition by "tournamentStageId" order by t.name) position
-- -- total number of wins/losses/drafts to be added (the "score" column from the screenshot)
-- from team_scores ts
-- inner join teams t on t.id = ts."teamId"
-- order by "tournamentStageId", rank, position
I've created a sandbox for those who is brave enough to get a deep dive into the task: https://www.db-fiddle.com/f/6jsFFnxQMKwNQWznR3VXHC/2
Also, I've already crafted the part that creates a list of teams together with scores and points, so the attached query will be used as a joined one or sub-select.
Query plan on the real database and query (some indexes, probably, are missing, but that's ok for this moment):
GroupAggregate (cost=24862.28..29423.68 rows=3 width=24)
" Group Key: v_lag.""computerGameId"", v_lag.""tournamentStageId"", v_lag.""teamId"", v_lag.win"
-> Incremental Sort (cost=24862.28..29423.61 rows=3 width=16)
" Sort Key: v_lag.""computerGameId"", v_lag.""tournamentStageId"", v_lag.""teamId"", v_lag.win"
" Presorted Key: v_lag.""computerGameId"", v_lag.""tournamentStageId"", v_lag.""teamId"""
-> Subquery Scan on v_lag (cost=22581.67..29423.47 rows=3 width=16)
Filter: ((v_lag.win = v_lag.lag_win) AND (v_lag.lag_win = v_lag.first_win))
-> WindowAgg (cost=22581.67..27468.67 rows=130320 width=32)
-> Subquery Scan on v_mto (cost=22581.67..24210.67 rows=130320 width=24)
-> Sort (cost=22581.67..22907.47 rows=130320 width=28)
" Sort Key: m.""computerGameId"", m.""tournamentStageId"", mt.""teamId"", m.""scheduledAt"" DESC"
-> Hash Join (cost=3863.39..8391.38 rows=130320 width=28)
" Hash Cond: (mt.""matchId"" = m.id)"
-> Seq Scan on match_teams mt (cost=0.00..2382.81 rows=137281 width=8)
-> Hash (cost=2658.10..2658.10 rows=65623 width=24)
-> Seq Scan on matches m (cost=0.00..2658.10 rows=65623 width=24)
Filter: (status = 1)
Thanks everyone for help and suggestions!
The final result:
P.S. it is possible to convert the first query (v_mto) as materialised view or de-normalise win into the match_teams table, as this piece will be used in different queries to build match/game stats.

So, the original query is wrong - gives incorrect result for the standings results.
I've moved to row_number math to solve this task.
The final query (with scores) looks like this:
create materialized view vm_tournament_stage_standings as
with v_mto as (SELECT m.id,
m."computerGameId",
m."tournamentStageId",
mt."teamId",
m."scheduledAt",
(
case
when m."winnerId" IS NULL then 'D'
when m."winnerId" = mt."teamId" then 'W'
else 'L'
end
) win
FROM matches m
INNER JOIN match_teams mt ON
m.id = mt."matchId"
WHERE m.status = 1),
v_streaks as (select "computerGameId",
"tournamentStageId",
"teamId",
row_number()
over grp_ord_matches
- row_number()
over (partition by "computerGameId", "tournamentStageId", "teamId", win order by "scheduledAt" desc ) streak_index,
win
from v_mto
window grp_ord_matches as (partition by "computerGameId", "tournamentStageId", "teamId" order by "scheduledAt" desc)),
v_streak as (select "computerGameId",
"tournamentStageId",
"teamId",
count(1) || win as streak
from v_streaks
where streak_index = 0
group by "computerGameId", "tournamentStageId", "teamId", "win"),
team_scores as (select m."tournamentStageId",
"teamId",
sum((m."winnerId" = mt."teamId")::int) as wins,
sum((m."winnerId" is null)::int) draws,
sum((m."winnerId" <> mt."teamId")::int) loses,
sum(
coalesce((m."winnerId" = mt."teamId")::integer, 0) * 3
+
(m."winnerId" IS NULL)::int
) as score
from matches m
inner join match_teams mt on m.id = mt."matchId"
where m.status = 1
group by m."tournamentStageId", "teamId")
select ts."tournamentStageId",
ts."teamId",
score,
wins,
draws,
loses,
vs.streak as streak,
dense_rank() over (partition by ts."tournamentStageId" order by score desc) rank,
row_number() over (partition by ts."tournamentStageId" order by t.name) position
from team_scores ts
inner join teams t on t.id = ts."teamId"
inner join v_streak vs on vs."teamId" = t.id and vs."tournamentStageId" = ts."tournamentStageId"
order by "tournamentStageId", rank, position

How to group by when using a modulo

For each company, I want to sum the revenue for the 4 most recent quarters, then the 4 subsequent ones, and so on (see screenshot attached for details). How can I do that?
SQL query and result - 1st attempt (failed)
https://i.stack.imgur.com/wWhhb.png
SELECT
ticker,
period_end_date,
revenue,
1+ ((rn - 1) % 4) AS test
FROM (
SELECT
ticker,
period_end_date,
revenue,
ROW_NUMBER() OVER (PARTITION BY ticker ORDER BY period_end_date DESC) rn
FROM "ANALYTICS"."vQUARTERLY_MASTER_MATERIALIZED"
--WHERE ticker = 'ACN'
ORDER BY ticker
) q
EDIT: the following code meets my needs. The 'revenue' is summed using the most recent quarter and the 3 quarters thereafter.
SELECT
ticker,
period_end_date,
SUM(revenue) OVER (PARTITION BY ticker ORDER BY period_end_date DESC ROWS BETWEEN CURRENT ROW AND 3 FOLLOWING) AS total_revenue
FROM "ANALYTICS"."vQUARTERLY_MASTER_MATERIALIZED"
--WHERE ticker = 'ACN'
ORDER BY ticker

You can try this :
SELECT ticker
, period_end_date
, total_revenue
FROM (
SELECT ticker
, period_end_date
, SUM(revenue) OVER (PARTITION BY ticker ORDER BY period_end_date DESC ROWS BETWEEN CURRENT ROW AND 3 FOLLOWING) AS total_revenue
, max(period_end_date) OVER (PARTITION BY ticker) AS period_end_date_max
FROM "ANALYTICS"."vQUARTERLY_MASTER_MATERIALIZED"
--WHERE ticker = 'ACN
) q
WHERE EXTRACT(MONTH FROM period_end_date) = EXTRACT(MONTH FROM period_end_date_max)
ORDER BY ticker, period_end_date ASC

How to optimize join and sub-query in pgSQL

I've 2 tables in PostgreSQL, Schema ==> sts
product_price_log contains columns product_name (character varying), K (numeric), sale_date (date)
product_transaction contains columns product_name (character varying), price (numeric), sale_date (date)
I want to get json data for sale_date = '2018-01-15' and '2018-01-01'.
Product price should be from product_price_log table and sale data from product_transaction table for summarized report.
this my query:
(SELECT
json_agg(j.*)
from(
select
a.sale_date, a.product_name, sum(a.quantity) as quantity, sum( a.quantity * b.price) as total_sale
from(
select
sale_date, product_name, quantity
from
sts.product_transaction p
where
sale_date = '2018-01-15' or sale_date = '2018-01-01' and quantity > 0
)a
inner join (
select * from(
select product_name, price, sale_date from sts.product_price_log where sale_date <= '2018-01-15' order by sale_date desc limit 1
union all
select product_name, price, sale_date from sts.product_price_log where sale_date <= '2018-01-01' order by sale_date desc limit 1
)a order by sale_date
)r on (a.product_name = r.product_name and a.sale_date = r.sale_date)
group by
sale_date,
product_name
)j)
;
QUERY PLAN
Aggregate (cost=230444.51..230444.52 rows=1 width=28)
-> Subquery Scan on j (cost=230290.98..230444.50 rows=1 width=28)
-> GroupAggregate (cost=230290.98..230444.49 rows=1 width=20)
How to I reduce time cost or optimize this query?

Bug in SQL query finding First and Last records per interval within a selected period of time in postgres

Postgres 9.6.
I had been suggested a solution which seems to work, but with all the beauty of the query, it randomly misses First or Last records. There is an example with 1 second interval for simplicity. I'm struggling to find the bugfix. Please help!
SCHEMA:
CREATE TABLE public.prices
(
uid uuid NOT NULL DEFAULT uuid_generate_v4(),
instrument character varying COLLATE pg_catalog."default" NOT NULL,
bid double precision NOT NULL,
ask double precision NOT NULL,
dt timestamp without time zone NOT NULL DEFAULT now(),
CONSTRAINT prices_pkey PRIMARY KEY (uid)
)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
INDEXES:
CREATE INDEX idx_dt_instrument
ON public.prices USING btree
(dt, instrument COLLATE pg_catalog."default")
TABLESPACE pg_default;
CREATE INDEX idx_dt_instrument_bid_ask
ON public.prices USING btree
(dt, instrument COLLATE pg_catalog."default", bid, ask)
TABLESPACE pg_default;
CREATE INDEX idx_instrument
ON public.prices USING btree
(instrument COLLATE pg_catalog."default")
TABLESPACE pg_default;
QUERY:
WITH ranges as (
SELECT dd as start_range,
dd + '1 seconds'::interval as end_range,
ROW_NUMBER() over () as grp
FROM generate_series
( '2017-05-01 00:01:00'::timestamp
, '2017-05-01 00:01:01'::timestamp
, '1 seconds'::interval) dd
), create_grp as (
SELECT r.grp, r.start_range, r.end_range, p.*
FROM prices p
JOIN ranges r
ON p.dt >= r.start_range
AND p.dt < r.end_range
), minmax as (
SELECT row_number() over (partition by grp
order by dt asc) as rn1,
row_number() over (partition by grp
order by dt desc) as rn2,
create_grp.*
FROM create_grp
)
SELECT *,
CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
WHEN rn1 = 1 THEN 'first'
WHEN rn2 = 1 THEN 'last'
END as row_position
FROM minmax
WHERE
1 IN (rn1, rn2)
AND instrument='EURGBP'
ORDER BY dt
;
ORIGINAL DATASET:
"uid","instrument","bid","ask","dt"
"62d857cd-4830-4146-9516-dfff57abf9de","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.057"
"da4b37f8-673e-4708-95e9-2c316b46d703","GBPUSD","1.29334","1.29344","2017-05-01 00:01:00.057"
"fceb5de9-bf68-450b-91bf-b97d1af77587","GBPUSD","1.29334","1.29345","2017-05-01 00:01:00.058"
"35b0113b-209d-4195-b27d-f1539055b7de","GBPJPY","143.964005","143.977997","2017-05-01 00:01:00.059"
"df347488-d5e0-4e53-b02f-88e3d63246d2","GBPUSD","1.29334","1.29346","2017-05-01 00:01:00.059"
"83e5eb95-4e02-4c5d-917f-770eb232c54b","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.096"
"d4ed4f64-c292-4172-b7c9-91ee2e7e1b7b","GBPJPY","143.964005","143.979996","2017-05-01 00:01:00.132"
"d9c86d19-5f90-4afe-b29a-a3823366d700","GBPJPY","143.964005","143.981003","2017-05-01 00:01:00.143"
"830b0b6b-a520-462f-9a6d-9f0b18581ba7","GBPUSD","1.29334","1.29345","2017-05-01 00:01:00.172"
"9f4a0c3b-d902-4d17-8e3d-f4867e58ed9f","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.29"
"84c303ea-777a-40c4-9d26-b0767698bda7","GBPJPY","143.964005","143.981003","2017-05-01 00:01:00.294"
"1a79250d-4c3d-47ad-9fbd-6ad208afb1fc","GBPJPY","143.964005","143.988998","2017-05-01 00:01:00.297"
"d3467ea7-0406-46c1-b95a-26fd63b7a28f","GBPUSD","1.29334","1.29345","2017-05-01 00:01:00.298"
"5da4f08d-c4f8-41d3-bc55-4a0d706b872a","GBPJPY","143.964005","143.992004","2017-05-01 00:01:00.312"
"c163ff17-c7ff-4197-8858-8ebc87f1045a","GBPJPY","143.968994","143.992004","2017-05-01 00:01:00.324"
"089ee425-324b-4111-98a1-4cf8dba883c3","GBPJPY","143.968994","143.994003","2017-05-01 00:01:00.347"
"f78a9d85-485e-42ee-8d71-666238cebb02","GBPJPY","143.964005","143.994003","2017-05-01 00:01:00.357"
"c318fdb7-3503-4072-ba74-281780ac2a91","GBPUSD","1.29334","1.29344","2017-05-01 00:01:00.37"
"c5a9a7b0-ee83-4d44-ba48-5f8aa82aa641","GBPJPY","143.964005","143.996002","2017-05-01 00:01:00.382"
"6c0811d3-4129-4e52-93fc-fd9f949e6459","GBPJPY","143.964996","143.996002","2017-05-01 00:01:00.484"
"eb8691da-2480-4853-b49f-ba9ee9d86086","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.485"
"0ceea454-e079-4c25-b58b-635e7c2f2e5e","GBPUSD","1.29334","1.29342","2017-05-01 00:01:00.486"
"156aeaa6-4266-4414-bf5c-400a58929a67","GBPJPY","143.968002","143.983994","2017-05-01 00:01:00.487"
"2dc08833-8a11-4dde-907e-b4af9f795e75","GBPJPY","143.964996","143.992004","2017-05-01 00:01:00.505"
"9685ee23-3365-4db5-84da-041f1d26c6cb","EURGBP","0.8432","0.8433","2017-05-01 00:01:00.507"
"54beeda6-c693-4362-8f96-7c93fd36c4f5","GBPUSD","1.2933","1.29343","2017-05-01 00:01:00.508"
"1b09afe4-5970-43c9-9101-81be1be9c032","EURGBP","0.84321","0.8433","2017-05-01 00:01:00.519"
"40aea241-4ffe-4321-9314-b53ea5fd04d4","GBPJPY","143.964996","143.994995","2017-05-01 00:01:00.532"
"e43c2214-300b-45eb-91a7-eb4411addb06","GBPUSD","1.2933","1.2934","2017-05-01 00:01:00.54"
"fe2eaeb9-94d5-4d53-95ba-53be2864e10c","GBPJPY","143.964005","143.994995","2017-05-01 00:01:00.543"
"eacc0e09-4712-4d7c-9974-1ff4f8884a60","EURGBP","0.84322","0.84332","2017-05-01 00:01:00.553"
"7b9fda5e-dba0-4523-9f01-e7636de5ef78","GBPJPY","143.962006","143.992004","2017-05-01 00:01:00.553"
"37f5d872-1e21-4e92-bb7d-76c17e9e0278","EURGBP","0.84321","0.84332","2017-05-01 00:01:00.579"
"c7553b67-cf59-439e-ad39-ca1374d7fd81","GBPJPY","143.962006","143.983994","2017-05-01 00:01:00.587"
"e7addcce-55b0-4d7a-ae64-177964a6c698","GBPJPY","143.962006","143.985992","2017-05-01 00:01:00.669"
"57d8aa45-eeee-4809-91c5-9e0ff5495351","GBPUSD","1.29327","1.29341","2017-05-01 00:01:00.669"
"1c9b66f9-6558-44dd-94b7-892689ed6a8e","EURGBP","0.84321","0.84333","2017-05-01 00:01:00.67"
"f94c87b3-8dd8-4e97-8cf4-84af7e921c14","GBPJPY","143.957001","143.985992","2017-05-01 00:01:00.67"
"43fd091c-be15-476b-9e76-b9c71062ba92","GBPUSD","1.29325","1.29341","2017-05-01 00:01:00.671"
"90931fdf-5507-44cb-8b25-2e06a634caad","EURGBP","0.84323","0.84333","2017-05-01 00:01:00.672"
"a3e57590-e673-406d-9bd9-a8ac739fac37","GBPUSD","1.29325","1.29338","2017-05-01 00:01:00.679"
"4d2cbbec-06e3-450c-857d-97bb9a4b4271","EURGBP","0.84323","0.84333","2017-05-01 00:01:00.833"
"4a036cce-e34a-4ec2-9bb0-5760465cd6fc","GBPJPY","143.957001","143.985992","2017-05-01 00:01:00.833"
"3ddfddb4-7c5d-4dea-92ee-411bbba4f699","GBPUSD","1.29325","1.29339","2017-05-01 00:01:00.834"
"2a0334b9-f6cd-4bd9-8517-7bd7cfb83415","GBPJPY","143.957001","143.983994","2017-05-01 00:01:00.836"
"bf9293a1-d52e-4354-9583-c3c02178f985","EURGBP","0.84323","0.84333","2017-05-01 00:01:00.888"
"4ce95ac5-2de0-4123-8916-4dc4ab2659ea","GBPUSD","1.29325","1.29339","2017-05-01 00:01:00.891"
"377f5831-931b-4870-8b08-c6a7f3b6ad2e","GBPJPY","143.957001","143.983994","2017-05-01 00:01:01.014"
"96977442-9f60-4c02-87e9-c4ffa685e324","EURGBP","0.84323","0.84333","2017-05-01 00:01:01.017"
"509a3f68-7126-45ea-9f89-99d92c71697a","GBPUSD","1.29325","1.29339","2017-05-01 00:01:01.017"
"14e4d804-b419-43fb-9189-44a1fbe7a4e4","GBPUSD","1.29325","1.29338","2017-05-01 00:01:01.02"
"34dd5dac-d0e1-49e5-92cf-6d9d9de14e49","GBPUSD","1.29325","1.29339","2017-05-01 00:01:01.052"
"91cdab81-8a50-4db7-8292-a90f4c8c42cb","GBPJPY","143.957001","143.983994","2017-05-01 00:01:01.065"
"e8b9b67a-3708-4463-83d9-39b0286af14e","EURGBP","0.84323","0.84333","2017-05-01 00:01:01.193"
"abbb50d6-e764-48e6-9172-98dbfe04f961","EURGBP","0.84324","0.84333","2017-05-01 00:01:01.195"
"d4e07afc-c0b6-45b5-9004-e378116ac8e0","GBPUSD","1.29325","1.29339","2017-05-01 00:01:01.195"
"706004b0-dc76-4dfd-ba78-f8fcf455a86b","GBPJPY","143.957001","143.983994","2017-05-01 00:01:01.196"
"a1a5c7c9-8579-4477-ba55-f3bb6f9888c7","EURGBP","0.84324","0.84334","2017-05-01 00:01:01.231"
"5e29a0b9-e47c-4547-b9ab-30e80c50bb96","GBPUSD","1.29325","1.29339","2017-05-01 00:01:01.284"
"7fe3cea7-e2dc-4c82-b425-8cb4514a0742","GBPJPY","143.957001","143.983994","2017-05-01 00:01:01.297"
"91db0d7e-422a-4236-90f9-dbd1ca569c14","EURGBP","0.84324","0.84334","2017-05-01 00:01:01.38"
"dfeaf1d3-adea-4476-8717-3724ffc253f4","GBPUSD","1.29325","1.29339","2017-05-01 00:01:01.403"
"378a0924-510f-41b6-aa01-0508c22f1745","GBPUSD","1.29325","1.29338","2017-05-01 00:01:01.449"
"763237fc-85ba-4263-9ab6-4348b08c5cfc","EURGBP","0.84324","0.84334","2017-05-01 00:01:01.461"
"85a77007-c176-4ce0-9500-0d3919552fe5","EURGBP","0.84325","0.84334","2017-05-01 00:01:01.573"
"bdf1c190-d517-4b15-8b20-b01936e8e7ba","GBPJPY","143.957001","143.983994","2017-05-01 00:01:01.574"
"a80c4a57-8c3f-42f4-a6c1-54ab8f9fa13e","EURGBP","0.84327","0.84334","2017-05-01 00:01:01.576"
"7c811f46-3772-434e-b466-ee830c62c61e","GBPUSD","1.29325","1.29338","2017-05-01 00:01:01.576"
"ec7e93ad-af3e-4d64-9c7d-5881ed6444d8","GBPUSD","1.29325","1.29339","2017-05-01 00:01:01.593"
"65f794fb-278f-40b1-9974-142b0fc03125","EURGBP","0.84327","0.84338","2017-05-01 00:01:01.649"
"14e496ed-c732-444d-ada5-ef4e1f7434ba","GBPJPY","143.957001","143.983994","2017-05-01 00:01:01.649"
"3d28c092-3d42-4399-b774-c927c2f2283c","GBPUSD","1.29323","1.29339","2017-05-01 00:01:01.65"
"eb0bdae8-b67a-4262-ad87-0db109b53434","GBPJPY","143.955002","143.983994","2017-05-01 00:01:01.672"
"cde342f0-5bd3-469b-9f98-6053f9ea153e","EURGBP","0.84327","0.84338","2017-05-01 00:01:01.777"
"637bec3a-c65e-49ca-8381-ab5c7df29580","GBPUSD","1.29323","1.29339","2017-05-01 00:01:01.78"
"60af62a4-524b-4ad3-a4a8-1db39a8bccf5","EURGBP","0.84327","0.84339","2017-05-01 00:01:01.783"
"8ef15da4-7836-4186-b30b-9c4eb1e1aa3f","GBPJPY","143.955002","143.983994","2017-05-01 00:01:01.796"
"7ae9a50f-35b7-4a09-a392-1459abf12122","EURGBP","0.84327","0.8434","2017-05-01 00:01:01.832"
"00c63fdc-ddef-4dde-9f44-69d7482882e0","EURGBP","0.84328","0.8434","2017-05-01 00:01:01.846"
"e6791f56-e321-416b-bef5-6d45f89feee0","GBPUSD","1.29323","1.29338","2017-05-01 00:01:01.946"
"7d31abc9-5ea3-4d8b-a68e-5f3c8cc6addd","GBPJPY","143.955002","143.983994","2017-05-01 00:01:01.95"
"ca6ac100-8fd8-46e7-afc7-9966526a5c14","EURGBP","0.84329","0.8434","2017-05-01 00:01:01.96"
"5b3e48ca-b9e8-4d18-86eb-a326e33db188","GBPUSD","1.29323","1.29338","2017-05-01 00:01:01.982"
"5299b60d-654f-411a-8eab-34ed6bc61fb3","EURGBP","0.84329","0.8434","2017-05-01 00:01:02.148"
"48dc504c-ba88-4665-a2e5-5f584529f19d","GBPUSD","1.29323","1.29338","2017-05-01 00:01:02.149"
"cafacce6-7ec1-4386-9323-6bad8d07ca50","GBPJPY","143.955002","143.983994","2017-05-01 00:01:02.15"
"959efc0d-6116-4c50-9a02-d3483c116732","GBPUSD","1.29324","1.29339","2017-05-01 00:01:02.151"
"097e7e31-8cb4-4f3e-9fb5-71569c5be2b7","EURGBP","0.84329","0.8434","2017-05-01 00:01:02.187"
"18a62956-93fc-401a-9f89-8f94a3e3f463","GBPJPY","143.955002","143.983994","2017-05-01 00:01:02.258"
"fba787c1-db9b-4af7-80e1-8823209d30ba","GBPUSD","1.29324","1.29339","2017-05-01 00:01:02.259"
"c59d01fb-67fc-4b0d-b11a-13790ac512f5","GBPUSD","1.29325","1.29339","2017-05-01 00:01:02.356"
"13f9d5f0-f933-4633-9b79-5eaec4975997","EURGBP","0.84329","0.8434","2017-05-01 00:01:02.357"
"e23dcfdc-1845-4c6c-b78a-c80562079853","GBPJPY","143.955002","143.983994","2017-05-01 00:01:02.371"
"afdf5a3a-ca4c-4b92-bbb2-f83391aaf603","GBPUSD","1.29325","1.29339","2017-05-01 00:01:02.399"
"67d58e76-34ae-4c06-b53b-dab360500f22","EURGBP","0.84329","0.8434","2017-05-01 00:01:02.418"
"e9b1e743-2f3e-4d3e-8d01-14d7ec968efc","GBPUSD","1.29325","1.29339","2017-05-01 00:01:02.582"
"76bf9147-a9b7-4db8-bd09-4d2cd3635ace","GBPUSD","1.29325","1.29339","2017-05-01 00:01:02.796"
"4b8d5562-7bef-410d-9d83-ca5b3590aa0d","EURGBP","0.84329","0.8434","2017-05-01 00:01:02.797"
"77849b8e-527f-47aa-967c-f4f161305537","GBPJPY","143.955002","143.983994","2017-05-01 00:01:02.832"
"3840acd1-c790-4302-b61d-35f76f9feb3f","EURGBP","0.84329","0.8434","2017-05-01 00:01:02.878"
"b2786f7b-9f2c-40ae-af02-31bba7ad8ab0","GBPUSD","1.29325","1.29339","2017-05-01 00:01:02.987"
"0b99d3f6-1146-46ac-ba16-bc3b1f8fdf45","EURGBP","0.84329","0.8434","2017-05-01 00:01:02.995"
"54cecdbd-8cee-4f77-bcc5-9fc46bd3b0e9","GBPJPY","143.955002","143.983994","2017-05-01 00:01:03.067"
"23c82fb7-b264-421f-b7a7-60c5f6e04065","GBPUSD","1.29325","1.29339","2017-05-01 00:01:03.09"
"564fdf6d-3a5c-46e3-8b16-51a78c08a382","EURGBP","0.84328","0.8434","2017-05-01 00:01:03.091"
"df67f32a-3783-4c23-88cd-ef809dc0766b","GBPJPY","143.955002","143.983994","2017-05-01 00:01:03.186"
"ddb57e22-eb1f-4ba6-b442-c1bb60d3543f","EURGBP","0.84328","0.8434","2017-05-01 00:01:03.208"
"8e6e2053-4aa2-4e75-8740-ea89202f727a","GBPUSD","1.29325","1.29339","2017-05-01 00:01:03.208"
"8b1ff49d-57bd-4bfd-8305-38e76ed24672","GBPUSD","1.29324","1.29339","2017-05-01 00:01:03.287"
"a65d1000-0603-4131-95b8-30c9e9c63df4","GBPJPY","143.955002","143.983994","2017-05-01 00:01:03.301"
"ebc03747-c019-4aa2-9fb0-36f7177546d0","EURGBP","0.84328","0.8434","2017-05-01 00:01:03.389"
"abb5f2b9-869d-48d0-ab69-9d312018fdf9","GBPUSD","1.29324","1.29339","2017-05-01 00:01:03.401"
"b8a26b37-8863-497c-aafd-9ba7281e2e40","GBPJPY","143.955002","143.983994","2017-05-01 00:01:03.417"
"52fccae5-7af7-4e9d-8fb3-f9f0d8b25241","EURGBP","0.84328","0.8434","2017-05-01 00:01:03.44"
"78690be1-773b-4872-a0ab-6f77c5a0a5e7","GBPUSD","1.29324","1.29339","2017-05-01 00:01:03.587"
"b3d6fd81-4c5a-49f4-896b-8b4790aa2a73","GBPJPY","143.955002","143.985001","2017-05-01 00:01:03.672"
"c9ae32fe-33d1-4486-bf79-3d23aaa42e59","GBPUSD","1.29324","1.29339","2017-05-01 00:01:03.792"
"0c005686-0b70-4d40-9dfb-913c2b4f0a01","EURGBP","0.84328","0.8434","2017-05-01 00:01:03.793"
"5b8789ff-2a66-46e8-aca8-ba3f24b91cd3","GBPUSD","1.29324","1.29339","2017-05-01 00:01:03.854"
"8ee7180a-df32-4411-ab76-795148532b2f","GBPUSD","1.29324","1.29339","2017-05-01 00:01:03.995"
"4ef53c94-3228-42a4-a1a7-932e11df1b28","EURGBP","0.84328","0.8434","2017-05-01 00:01:04.023"
"7016a0c0-297f-4299-b85b-c1cd27e7c992","GBPJPY","143.955002","143.985001","2017-05-01 00:01:04.023"
"474efc11-599d-456a-9e14-aeb906c8da8a","GBPJPY","143.955002","143.985001","2017-05-01 00:01:04.195"
"d6011bd8-4a16-40dd-9648-f401cf7d4bcb","GBPUSD","1.29324","1.29339","2017-05-01 00:01:04.199"
"1c8e0210-4198-4cb2-9758-73d9c68c5681","EURGBP","0.84328","0.8434","2017-05-01 00:01:04.25"
"98f975ae-b410-4d00-812d-2c0f289f9ac6","GBPJPY","143.955994","143.985001","2017-05-01 00:01:04.273"
"75808e7b-be90-4284-b491-b577928e0a69","GBPUSD","1.29324","1.29339","2017-05-01 00:01:04.382"
"dc10ab53-1e6b-4bd2-a77e-be3fd8bf8a42","EURGBP","0.84328","0.8434","2017-05-01 00:01:04.383"
"e40ab9d5-8e2e-49e5-8748-5b3c43024a6d","GBPJPY","143.955994","143.985992","2017-05-01 00:01:04.383"
"8ee41b6c-e56b-4aa0-9e79-6204cc4785a0","GBPUSD","1.29324","1.29339","2017-05-01 00:01:04.656"
Just as extra info with row numbers in groups. It is obviously missing rn2=1 for the first second:
"rn1","rn2","grp","start_range","end_range","uid","instrument","bid","ask","dt","row_position"
"1","46","1","2017-05-01 00:01:00","2017-05-01 00:01:01","62d857cd-4830-4146-9516-dfff57abf9de","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.057","first"
"6","42","1","2017-05-01 00:01:00","2017-05-01 00:01:01","83e5eb95-4e02-4c5d-917f-770eb232c54b","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.096",""
"10","38","1","2017-05-01 00:01:00","2017-05-01 00:01:01","9f4a0c3b-d902-4d17-8e3d-f4867e58ed9f","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.29",""
"21","27","1","2017-05-01 00:01:00","2017-05-01 00:01:01","eb8691da-2480-4853-b49f-ba9ee9d86086","EURGBP","0.84319","0.8433","2017-05-01 00:01:00.485",""
"25","23","1","2017-05-01 00:01:00","2017-05-01 00:01:01","9685ee23-3365-4db5-84da-041f1d26c6cb","EURGBP","0.8432","0.8433","2017-05-01 00:01:00.507",""
"27","21","1","2017-05-01 00:01:00","2017-05-01 00:01:01","1b09afe4-5970-43c9-9101-81be1be9c032","EURGBP","0.84321","0.8433","2017-05-01 00:01:00.519",""
"31","17","1","2017-05-01 00:01:00","2017-05-01 00:01:01","eacc0e09-4712-4d7c-9974-1ff4f8884a60","EURGBP","0.84322","0.84332","2017-05-01 00:01:00.553",""
"33","15","1","2017-05-01 00:01:00","2017-05-01 00:01:01","37f5d872-1e21-4e92-bb7d-76c17e9e0278","EURGBP","0.84321","0.84332","2017-05-01 00:01:00.579",""
"37","11","1","2017-05-01 00:01:00","2017-05-01 00:01:01","1c9b66f9-6558-44dd-94b7-892689ed6a8e","EURGBP","0.84321","0.84333","2017-05-01 00:01:00.67",""
"40","8","1","2017-05-01 00:01:00","2017-05-01 00:01:01","90931fdf-5507-44cb-8b25-2e06a634caad","EURGBP","0.84323","0.84333","2017-05-01 00:01:00.672",""
"43","5","1","2017-05-01 00:01:00","2017-05-01 00:01:01","4d2cbbec-06e3-450c-857d-97bb9a4b4271","EURGBP","0.84323","0.84333","2017-05-01 00:01:00.833",""
"46","2","1","2017-05-01 00:01:00","2017-05-01 00:01:01","bf9293a1-d52e-4354-9583-c3c02178f985","EURGBP","0.84323","0.84333","2017-05-01 00:01:00.888",""
"2","35","2","2017-05-01 00:01:01","2017-05-01 00:01:02","96977442-9f60-4c02-87e9-c4ffa685e324","EURGBP","0.84323","0.84333","2017-05-01 00:01:01.017",""
"7","30","2","2017-05-01 00:01:01","2017-05-01 00:01:02","e8b9b67a-3708-4463-83d9-39b0286af14e","EURGBP","0.84323","0.84333","2017-05-01 00:01:01.193",""
"9","29","2","2017-05-01 00:01:01","2017-05-01 00:01:02","abbb50d6-e764-48e6-9172-98dbfe04f961","EURGBP","0.84324","0.84333","2017-05-01 00:01:01.195",""
"11","26","2","2017-05-01 00:01:01","2017-05-01 00:01:02","a1a5c7c9-8579-4477-ba55-f3bb6f9888c7","EURGBP","0.84324","0.84334","2017-05-01 00:01:01.231",""
"14","23","2","2017-05-01 00:01:01","2017-05-01 00:01:02","91db0d7e-422a-4236-90f9-dbd1ca569c14","EURGBP","0.84324","0.84334","2017-05-01 00:01:01.38",""
"17","20","2","2017-05-01 00:01:01","2017-05-01 00:01:02","763237fc-85ba-4263-9ab6-4348b08c5cfc","EURGBP","0.84324","0.84334","2017-05-01 00:01:01.461",""
"18","19","2","2017-05-01 00:01:01","2017-05-01 00:01:02","85a77007-c176-4ce0-9500-0d3919552fe5","EURGBP","0.84325","0.84334","2017-05-01 00:01:01.573",""
"20","17","2","2017-05-01 00:01:01","2017-05-01 00:01:02","a80c4a57-8c3f-42f4-a6c1-54ab8f9fa13e","EURGBP","0.84327","0.84334","2017-05-01 00:01:01.576",""
"23","14","2","2017-05-01 00:01:01","2017-05-01 00:01:02","65f794fb-278f-40b1-9974-142b0fc03125","EURGBP","0.84327","0.84338","2017-05-01 00:01:01.649",""
"27","10","2","2017-05-01 00:01:01","2017-05-01 00:01:02","cde342f0-5bd3-469b-9f98-6053f9ea153e","EURGBP","0.84327","0.84338","2017-05-01 00:01:01.777",""
"29","8","2","2017-05-01 00:01:01","2017-05-01 00:01:02","60af62a4-524b-4ad3-a4a8-1db39a8bccf5","EURGBP","0.84327","0.84339","2017-05-01 00:01:01.783",""
"31","6","2","2017-05-01 00:01:01","2017-05-01 00:01:02","7ae9a50f-35b7-4a09-a392-1459abf12122","EURGBP","0.84327","0.8434","2017-05-01 00:01:01.832",""
"32","5","2","2017-05-01 00:01:01","2017-05-01 00:01:02","00c63fdc-ddef-4dde-9f44-69d7482882e0","EURGBP","0.84328","0.8434","2017-05-01 00:01:01.846",""
"35","2","2","2017-05-01 00:01:01","2017-05-01 00:01:02","ca6ac100-8fd8-46e7-afc7-9966526a5c14","EURGBP","0.84329","0.8434","2017-05-01 00:01:01.96",""
Is it some kind of Postgres bug?
Any ideas how to fix it are very welcome!

Similar query without the CTE and the calendar-seconds:
SELECT *,
CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
WHEN rn1 = 1 THEN 'first'
WHEN rn2 = 1 THEN 'last'
END as row_position
FROM (
SELECT p.uid, p.instrument, date_trunc('second', p.dt) AS the_second , p.dt AS dt, p.bid, p.ask
, row_number() over (partition by date_trunc('second', p.dt) order by dt asc) as rn1
, row_number() over (partition by date_trunc('second', p.dt) order by dt desc) as rn2
FROM prices p
)minmax
WHERE instrument='EURGBP'
AND (rn1=1 OR rn2=1)
ORDER BY dt
;

This query below works. The problem was in WHERE clause location . It must be within groups, otherwise row numbers become impacted by other instruments.
WITH ranges as (
SELECT dd as start_range,
dd + '1 seconds'::interval as end_range,
ROW_NUMBER() over () as grp
FROM generate_series
( '2017-05-01 00:01:00'::timestamp
, '2017-05-01 00:01:00'::timestamp
, '1 seconds'::interval) dd
), create_grp as (
SELECT r.grp, r.start_range, r.end_range, p.*
FROM prices p
JOIN ranges r
ON p.dt >= r.start_range
AND p.dt < r.end_range
-- WHERE moves here from the end of the query
WHERE instrument='EURGBP'
), minmax as (
SELECT row_number() over (partition by grp
order by dt asc) as rn1,
row_number() over (partition by grp
order by dt desc) as rn2,
create_grp.*
FROM create_grp
)
SELECT *,
CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
WHEN rn1 = 1 THEN 'first'
WHEN rn2 = 1 THEN 'last'
END as row_position
FROM minmax
WHERE
1 IN (rn1, rn2)
-- AND instrument='EURGBP'
ORDER BY dt
;
The the problem solved, but it raises new one: such change to query make is to run very slow! despite of the index on instruments. I'm raising new question about it and linking it here.

TSQL get overlapping periods from datetime ranges

I have a table with date range an i need the sum of overlapping periods (in hours) between its rows.
This is a schema example:
create table period (
id int,
starttime datetime,
endtime datetime,
type varchar(64)
);
insert into period values (1,'2013-04-07 8:00','2013-04-07 13:00','Work');
insert into period values (2,'2013-04-07 14:00','2013-04-07 17:00','Work');
insert into period values (3,'2013-04-08 8:00','2013-04-08 13:00','Work');
insert into period values (4,'2013-04-08 14:00','2013-04-08 17:00','Work');
insert into period values (5,'2013-04-07 10:00','2013-04-07 11:00','Holyday'); /* 1h overlapping with 1*/
insert into period values (6,'2013-04-08 10:00','2013-04-08 20:00','Transfer'); /* 6h overlapping with 3 and 4*/
insert into period values (7,'2013-04-08 11:00','2013-04-08 12:00','Test'); /* 1h overlapping with 3 and 6*/
And its fiddle: http://sqlfiddle.com/#!6/9ca31/10
I expect a sum of 8h overlapping hours:
1h (id 5 over id 1)
6h (id 6 over id 3 and 4)
1h (id 7 over id 3 and 6)
I check this: select overlapping datetime events with SQL but seems to not do what I need.
Thank you.

select sum(datediff(hh, case when t2.starttime > t1.starttime then t2.starttime else t1.starttime end,
case when t2.endtime > t1.endtime then t1.endtime else t2.endtime end))
from period t1
join period t2 on t1.id < t2.id
where t2.endtime > t1.starttime and t2.starttime < t1.endtime;
Updated to handle several overlaps:
select sum(datediff(hh, start, fin))
from (select distinct
case when t2.starttime > t1.starttime then t2.starttime else t1.starttime end as start,
case when t2.endtime > t1.endtime then t1.endtime else t2.endtime end as fin
from period t1
join period t2 on t1.id < t2.id
where t2.endtime > t1.starttime and t2.starttime < t1.endtime
) as overlaps;

I have some "dirty" solution. Hope this helps :)
with src as (
select
convert(varchar, starttime, 112) [start_date]
, cast(left(convert(varchar, starttime, 108), 2) as int) [start_time]
, convert(varchar, endtime, 112) [end_date]
, cast(left(convert(varchar, endtime, 108), 2) as int) [end_time]
, id
from [period]),
[gr] as (
select
row_number() over(order by s1.[start_date], s1.[start_time], s1.[end_time], s2.[start_time], s2.[end_time]) [no]
, s1.[start_date] [date]
, s1.[start_time] [t1]
, s1.[end_time] [t2]
, s2.[start_time] [t3]
, s2.[end_time] [t4]
from src s1
join src s2 on s1.[start_date] = s2.[start_date]
and s1.[end_date] = s2.[end_date]
and (s1.[start_time] between s2.[start_time] and s2.[end_time] or s1.[end_time] between s2.[start_time] and s2.[end_time])
and s1.id != s2.id),
[raw] as (
select [no], [date], [t1] [h] from [gr] union all
select [no], [date], [t2] from [gr] union all
select [no], [date], [t3] from [gr] union all
select [no], [date], [t4] from [gr]),
[max_min] as (
select [no], [date], max(h) [max_h], min(h) [min_h]
from [raw]
group by [no], [date]
),
[result] as (
select [raw].*
from [raw]
left join [max_min] on [raw].[no] = [max_min].[no]
and ([raw].h = [max_min].[max_h] or [raw].h = [max_min].[min_h])
where [max_min].[no] is null),
[final] as (
select distinct r1.[date], r1.h [start_h], r2.h [end_h], abs(r1.h - r2.h) [dif]
from [result] r1
join [result] r2 on r1.[no] = r2.[no]
where abs(r1.h - r2.h) > 0
and r1.h > r2.h)
select sum(dif) [overlapping hours] from [final]
SQLFiddle

We Keep Coding

iphone swift flutter scala powershell matlab mongodb postgresql perl eclipse

CTE query performance improvement (postgres 9.6) - postgresql

Related

Optimise query: count latest win/lose streak for all teams

How to group by when using a modulo

How to optimize join and sub-query in pgSQL

Bug in SQL query finding First and Last records per interval within a selected period of time in postgres

TSQL get overlapping periods from datetime ranges

Categories

Resources