I have next data:
Here I already calculated total for conf_id. But want also calculate total for whole partition. eg:
Calculate total suma by agreement for each its order (not goods at order which are with slightly different rounding)
How to sum 737.38 and 1238.3? eg. take only one number among group
(I can not sum( item_suma ), because it will return 1975.67. Notice round for conf_suma as intermediate step)
UPD
Full query. Here I want to calculate rounded suma for each group. Then I need to calculate total suma for those groups
SELECT app_period( '2021-02-01', '2021-03-01' );
WITH
target_date AS ( SELECT '2021-02-01'::timestamptz ),
target_order as (
SELECT
tstzrange( '2021-01-01', '2021-02-01') as bill_range,
o.*
FROM ( SELECT * FROM "order_bt" WHERE sys_period #> sys_time() ) o
WHERE FALSE
OR o.agreement_id = 3385 and o.period_id = 10
),
USAGE AS ( SELECT
ocd.*,
o.agreement_id as agreement_id,
o.id AS order_id,
(dense_rank() over (PARTITION BY o.agreement_id ORDER BY o.id )) as zzzz_id,
(dense_rank() over (PARTITION BY o.agreement_id, o.id ORDER BY (ocd.ic).consumed_period )) as conf_id,
sum( ocd.item_suma ) OVER( PARTITION BY (ocd.o).agreement_id ) AS agreement_suma2,
(sum( ocd.item_suma ) OVER( PARTITION BY (ocd.o).agreement_id, (ocd.o).id, (ocd.ic).consumed_period )) AS x_suma,
(sum( ocd.item_cost ) OVER( PARTITION BY (ocd.o).agreement_id, (ocd.o).id, (ocd.ic).consumed_period )) AS x_cost,
(sum( ocd.item_suma ) OVER( PARTITION BY (ocd.o).agreement_id, (ocd.o).id, (ocd.ic).consumed_period ))::numeric( 10, 2) AS conf_suma,
(sum( ocd.item_cost ) OVER( PARTITION BY (ocd.o).agreement_id, (ocd.o).id, (ocd.ic).consumed_period ))::numeric( 10, 2) AS conf_cost,
max((ocd.ic).consumed) OVER( PARTITION BY (ocd.o).agreement_id, (ocd.o).id, (ocd.ic).consumed_period ) AS consumed,
(sum( ocd.item_suma ) OVER( PARTITION BY (ocd.o).agreement_id, (ocd.o).id )) AS order_suma2
FROM target_order o
LEFT JOIN order_cost_details( o.bill_range ) ocd
ON (ocd.o).id = o.id AND (ocd.ic).consumed_period && o.app_period
)
SELECT
*,
(conf_suma/6) ::numeric( 10, 2 ) as group_nds,
(SELECT sum(x) from (SELECT sum( DISTINCT conf_suma ) AS x FROM usage sub_u WHERE sub_u.agreement_id = usage.agreement_id GROUP BY agreement_id, order_id) t) as total_suma,
(SELECT sum(x) from (SELECT (sum( DISTINCT conf_suma ) /6)::numeric( 10, 2 ) AS x FROM usage sub_u WHERE sub_u.agreement_id = usage.agreement_id GROUP BY agreement_id, order_id) t) as total_nds
FROM USAGE
WINDOW w AS ( PARTITION BY usage.agreement_id ROWS CURRENT ROW EXCLUDE TIES)
ORDER BY
order_id,
conf_id
My old question
I found solution. See dbfiddle.
To run window function for distinct values I should get first value from each peer. To complete this I
aggregate IDs of rows for this peer
lag this aggregation by one
Mark rows that are not aggregated yet (this is first row at peer) as _distinct
sum( ) FILTER ( WHERE _distinct ) over ( ... )
Voila. You get sum over DISTINCT values at target PARTITION
which are not implemented yet by PostgreSQL
with data as (
select * from (values
( 1, 1, 1, 1.0049 ), (2, 1,1,1.0049), ( 3, 1,1,1.0049 ) ,
( 4, 1, 2, 1.0049 ), (5, 1,2,1.0057),
( 6, 2, 1, 1.53 ), ( 7,2,1,2.18), ( 8,2,2,3.48 )
) t (id, agreement_id, order_id, suma)
),
intermediate as (select
*,
sum( suma ) over ( partition by agreement_id, order_id ) as fract_order_suma,
sum( suma ) over ( partition by agreement_id ) as fract_agreement_total,
(sum( suma::numeric(10,2) ) over ( partition by agreement_id, order_id )) as wrong_order_suma,
(sum( suma ) over ( partition by agreement_id, order_id ))::numeric( 10, 2) as order_suma,
(sum( suma ) over ( partition by agreement_id ))::numeric( 10, 2) as wrong_agreement_total,
id as xid,
array_agg( id ) over ( partition by agreement_id, order_id ) as agg
from data),
distinc as (select *,
lag( agg ) over ( partition by agreement_id ) as prev,
id = any (lag( agg ) over ()) is not true as _distinct, -- allow to match first ID from next peer
order_suma as xorder_suma, -- repeat column to easily visually compare with _distinct
(SELECT sum(x) from (SELECT sum( DISTINCT order_suma ) AS x FROM intermediate sub_q WHERE sub_q.agreement_id = intermediate.agreement_id GROUP BY agreement_id, order_id) t) as correct_total_suma
from intermediate
)
select
*,
sum( order_suma ) filter ( where _distinct ) over ( partition by agreement_id ) as also_correct_total_suma
from distinc
better approach dbfiddle:
Assign row_number at each order: row_number() over (partition by agreement_id, order_id ) as nrow
Take only first suma: filter nrow = 1
with data as (
select * from (values
( 1, 1, 1, 1.0049 ), (2, 1,1,1.0049), ( 3, 1,1,1.0049 ) ,
( 4, 1, 2, 1.0049 ), (5, 1,2,1.0057),
( 6, 2, 1, 1.53 ), ( 7,2,1,2.18), ( 8,2,2,3.48 )
) t (id, agreement_id, order_id, suma)
),
intermediate as (select
*,
row_number() over (partition by agreement_id, order_id ) as nrow,
(sum( suma ) over ( partition by agreement_id, order_id ))::numeric( 10, 2) as order_suma,
from data)
select
*,
sum( order_suma ) filter (where nrow = 1) over (partition by agreement_id)
from intermediate```
Related
I am currently using the UNION ALL workaround below to calculate old_eps_tfq regression slopes of each ticker based off its corresponding rownum value (see WHERE rownum < x). I am interested to know what the old_eps_tfq is when rownum < 4 then increment 4 by 1 to find out what old_eps_tfq is when rownum < 5, and so on (there are ~20 rownum)
Could I use PL/pgSQL for this?
SELECT * FROM(
WITH regression_slope AS(
SELECT
ROW_NUMBER() OVER ( PARTITION BY ticker ORDER BY earnings_growths_ped) AS rownum,
*
FROM "ANALYTICS"."vEARNINGS_GROWTHS"
--WHERE ticker = 'ACN'
ORDER BY ticker )
SELECT
ticker,
current_period_end_date,
max(earnings_growths_ped) AS max_earnings_growths_ped,
--max(rownum) AS max_rownum,
round(regr_slope(old_eps_tfq, rownum)::numeric, 2) AS slope,
round(regr_intercept(old_eps_tfq, rownum)::numeric, 2) AS y_intercept,
round(regr_r2(old_eps_tfq, rownum)::numeric, 3) AS r_squared
FROM regression_slope
WHERE rownum < 4
GROUP BY ticker, current_period_end_date
ORDER BY ticker asc ) q
UNION ALL
SELECT * FROM(
WITH regression_slope AS(
SELECT
ROW_NUMBER() OVER ( PARTITION BY ticker ORDER BY earnings_growths_ped) AS rownum,
*
FROM "ANALYTICS"."vEARNINGS_GROWTHS"
--WHERE ticker = 'ACN'
ORDER BY ticker )
SELECT
ticker,
current_period_end_date,
max(earnings_growths_ped) AS max_earnings_growths_ped,
--max(rownum) AS max_rownum,
round(regr_slope(old_eps_tfq, rownum)::numeric, 2) AS slope,
round(regr_intercept(old_eps_tfq, rownum)::numeric, 2) AS y_intercept,
round(regr_r2(old_eps_tfq, rownum)::numeric, 3) AS r_squared
FROM regression_slope
WHERE rownum < 5
GROUP BY ticker, current_period_end_date
ORDER BY ticker asc ) q
Here is my table
The top query SELECT * FROM (...) q sounds like useless.
Then you can try this :
WITH regression_slope AS(
SELECT
ROW_NUMBER() OVER ( PARTITION BY ticker ORDER BY earnings_growths_ped) AS rownum,
*
FROM "ANALYTICS"."vEARNINGS_GROWTHS"
--WHERE ticker = 'ACN'
ORDER BY ticker )
SELECT
max,
ticker,
current_period_end_date,
max(earnings_growths_ped) AS max_earnings_growths_ped,
--max(rownum) AS max_rownum,
round(regr_slope(old_eps_tfq, rownum)::numeric, 2) AS slope,
round(regr_intercept(old_eps_tfq, rownum)::numeric, 2) AS y_intercept,
round(regr_r2(old_eps_tfq, rownum)::numeric, 3) AS r_squared
FROM regression_slope
INNER JOIN generate_series(4, 24) AS max -- the range 4 to 24 can be adjusted to the need
ON rownum < max
GROUP BY max, ticker, current_period_end_date
ORDER BY max asc, ticker asc
When I run query and filter by agreement_id it is slow,
but when I filter by an alias id it is fast. (Look at the end of the query)
Why using same field when filtering cause different execution time?
Links to explain analyze:
slow1, slow2
fast1, fast2
Difference start at #20: Where different indexes are used:
Index Cond: (o.sys_period #> sys_time()) VS Index Cond: (o.agreement_id = 38)
PS. It would be nice if I can contact to developer of this feature (I have one more similar problem)
UPD I did some experiments. when I remove window functions from my query it works fast in any case. So why window function stop index usage in some cases? How to escape/workaround that?
dbfiddle with minimal test case
Server version is v13.1
Full query:
WITH gconf AS
-- https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-SELECT
NOT MATERIALIZED -- force it to be merged into the parent query
-- it gives a net savings because each usage of the WITH query needs only a small part of the WITH query's full output.
( SELECT
ocd.*,
tstzrange( '2021-05-01', '2021-05-01', '[]') AS acc_period,
(o).agreement_id AS id, -- Required to passthrough WINDOW FUNCTION
(o).id AS order_id,
(ic).consumed_period AS consumed_period,
dense_rank() OVER ( PARTITION BY (o).agreement_id, (o).id ORDER BY (ic).consumed_period ) AS nconf,
row_number() OVER ( wconf ORDER BY (c).sort_order NULLS LAST ) AS nitem,
(sum( ocd.item_cost ) OVER wconf)::numeric( 10, 2) AS conf_cost,
max((ocd.ic).consumed) OVER wconf AS consumed,
CASE WHEN true
THEN (sum( ocd.item_suma ) OVER wconf)::numeric( 10, 2 )
ELSE (sum( ocd.item_cost ) OVER wconf)::numeric( 10, 2 )
END AS conf_suma
FROM order_cost_details( tstzrange( '2021-05-01', '2021-05-01', '[]') ) ocd
WHERE true OR (ocd.ic).consumed_period #> lower( tstzrange( '2021-05-01', '2021-05-01', '[]') )
WINDOW wconf AS ( PARTITION BY (o).agreement_id, (o).id, (ic).consumed_period )
),
gorder AS (
SELECT *,
(conf_suma/6)::numeric( 10, 2 ) as conf_nds,
sum( conf_suma ) FILTER (WHERE nitem = 1) OVER worder AS order_suma
FROM gconf
WINDOW worder AS ( PARTITION BY gconf.id, (o).id )
-- TODO: Ask PG developers: Why changing to (o).agreement_id slows down query?
-- WINDOW worder AS ( PARTITION BY (o).agreement_id, (o).id )
)
SELECT
u.id, consumed_period, nconf, nitem,
(c).id as item_id,
COALESCE( (c).sort_order, pd.sort_order ) as item_order,
COALESCE( st.display, st.name, rt.display, rt.name ) as item_name,
COALESCE( item_qty, (c).amount/rt.unit ) as item_qty,
COALESCE( (p).label, rt.label ) as measure,
item_price, item_cost, item_suma,
conf_cost, consumed, conf_suma, conf_nds, order_suma,
(order_suma/6)::numeric( 10, 2 ) as order_nds,
sum( conf_suma ) FILTER (WHERE nitem = 1 ) OVER wagreement AS total_suma,
sum( (order_suma/6)::numeric( 10, 2 ) ) FILTER (WHERE nitem = 1 AND nconf = 1) OVER wagreement AS total_nds,
pkg.id as package_id,
pkg.link_1c_id as package_1c_id,
COALESCE( pkg.display, pkg.name ) as package,
acc_period
FROM gorder u
LEFT JOIN resource_type rt ON rt.id = (c).resource_type_id
LEFT JOIN service_type st ON st.id = (c).service_type_id
LEFT JOIN package pkg ON pkg.id = (o).package_id
LEFT JOIN package_detail pd ON pd.package_id = (o).package_id
AND pd.resource_type_id IS NOT DISTINCT FROM (c).resource_type_id
AND pd.service_type_id IS NOT DISTINCT FROM (c).service_type_id
-- WHERE (o).agreement_id = 38 -- slow
WHERE u.id = 38 -- fast
WINDOW wagreement AS ( PARTITION BY (o).agreement_id )
As problem workaround we can additionally SELECT an alias for column used at PARTITION BY expression. Then PG apply optimization and use index.
The answer to the question could be: PG does not apply optimization if composite type is used. Notice as it works:
PARTITION | FILTER | IS USED?
------------------------------
ALIAS | ORIG | NO
ALIAS | ALIAS | YES
ORIG | ALIAS | NO
ORIG | ORIG | NO
See this dbfiddle
create table agreement ( ag_id int, name text, cost numeric(10,2) );
create index ag_idx on agreement (ag_id);
insert into agreement (ag_id, name, cost) values ( 1, '333', 22 ),
(1,'333', 33), (1, '333', 7), (2, '555', 18 ), (2, '555', 2), (3, '777', 4);
select * from agreement;
create function initial ()
returns table( agreement_id int, ag agreement ) language sql stable AS $$
select ag_id, t from agreement t;
$$;
select * from initial() t;
explain( analyze, costs, buffers, verbose ) with totals_by_ag as (
select
*,
sum( (t.ag).cost ) over ( partition by agreement_id ) as total
from initial() t
)
select * from totals_by_ag t
where (t.ag).ag_id = 1; -- index is NOT USED
explain( analyze, costs, buffers, verbose ) with totals_by_ag as (
select
*,
sum( (t.ag).cost ) over ( partition by agreement_id ) as total
from initial() t
)
select * from totals_by_ag t
where agreement_id = 1; -- index is used when alias for column is used
explain( analyze, costs, buffers, verbose ) with totals_by_ag as (
select
*,
sum( (t.ag).cost ) over ( partition by (t.ag).ag_id ) as total --renamed
from initial() t
)
select * from totals_by_ag t
where agreement_id = 1; -- index is NOT USED because grouping by original column
explain( analyze, costs, buffers, verbose ) with totals_by_ag as (
select
*,
sum( (t.ag).cost ) over ( partition by (t.ag).ag_id ) as total --renamed
from initial() t
)
select * from totals_by_ag t
where (t.ag).ag_id = 1; -- index is NOT USED even if at both cases original column
PostgreSQL 11
How to pass o.create_date value into INNER JOIN? I need Max ID before o.create_date
SELECT o.id,
o.create_date date,
sum(oi.quantity) qty,
sum(oi.quantity * sp.price) total
FROM ax_order o
LEFT JOIN ax_order_invenotry oi on o.id = oi.order_id
LEFT JOIN ax_inventory i on i.id = oi.inventory_id
LEFT JOIN ax_suppliers s on s.id = o.supplier_id
INNER JOIN ax_supplier_price sp ON (sp.inventory_id = oi.inventory_id and sp.supplier_id = o.supplier_id)
INNER JOIN
(
SELECT inventory_id,
max(id) id
FROM ax_supplier_price
WHERE create_date <= o.create_date
GROUP BY inventory_id
) lsp ON (sp.id = lsp.id)
WHERE o.store_id = 13
AND o.supplier_id = 35
GROUP BY o.id, o.create_date
ORDER BY o.id
You could use the LATERAL join mechanism to make it work:
WITH ax_order AS (
SELECT *
FROM (VALUES (1, '2000-1-1'::date, 1, 1)) as x(id, create_date, store_id, supplier_id)
), ax_order_inventory AS (
SELECT *
FROM (VALUES (1, 2, 4)) as x(order_id, inventory_id, quantity)
), ax_supplier_price AS (
SELECT *
FROM (VALUES (1, 2, 1, 10, '1999-12-31'::date)) as x(id, inventory_id, supplier_id, price, create_date)
)
SELECT o.id,
o.create_date date,
sum(oi.quantity) qty,
sum(oi.quantity * sp.price) total
FROM ax_order o
LEFT JOIN ax_order_inventory oi on o.id = oi.order_id
INNER JOIN ax_supplier_price sp ON (sp.inventory_id = oi.inventory_id and sp.supplier_id = o.supplier_id)
INNER JOIN LATERAL
(
SELECT inventory_id,
max(lsp.id) id
FROM ax_supplier_price lsp
WHERE sp.create_date <= o.create_date
GROUP BY inventory_id
) lsp ON sp.id = lsp.id
GROUP BY o.id, o.create_date
ORDER BY o.id
I deleted some JOINs that were not strictly necessary and mocked your data as well as I could see. Note, however, that you could also use a WHERE clause to find it - which should be more efficient:
WITH ax_order AS (
SELECT *
FROM (VALUES (1, '2000-1-1'::date, 1, 1)) as x(id, create_date, store_id, supplier_id)
),
ax_order_inventory AS (
SELECT *
FROM (VALUES (1, 2, 4)) as x(order_id, inventory_id, quantity)
),
ax_supplier_price AS (
SELECT *
FROM (VALUES (1, 2, 1, 10, '1999-12-31'::date)) as x(id, inventory_id, supplier_id, price, create_date)
)
SELECT o.id,
o.create_date date,
sum(oi.quantity) qty,
sum(oi.quantity * sp.price) total
FROM ax_order o
LEFT JOIN ax_order_inventory oi on o.id = oi.order_id
INNER JOIN ax_supplier_price sp
ON (sp.inventory_id = oi.inventory_id and sp.supplier_id = o.supplier_id)
WHERE sp.id =
(
-- NOTE: no GROUP BY necessary!
SELECT max(lsp.id) id
FROM ax_supplier_price lsp
WHERE sp.create_date <= o.create_date
AND lsp.inventory_id = sp.inventory_id
)
GROUP BY o.id, o.create_date
ORDER BY o.id
This question already has answers here:
How to `sum( DISTINCT <column> ) OVER ()` using window function?
(2 answers)
Closed 1 year ago.
Example data:
id | docn | item | suma
---------------------
1 33 x | 10
1 33 y | 20
2 37 a | 10
2 37 b | 20
2 37 c | 30
To group results I can write:
SELECT sum( suma ),
(ocd.o).*
FROM order_cost_details() ocd
where (ocd.o).id IN ( 6154, 10805 )
GROUP BY ocd.o
But in a place with a group I want to select last_value for each group. Next does not work:
SELECT sum( suma ),
(ocd.o).*,
last_value( ocd.c ) OVER (PARTITION BY ocd.o )
FROM order_cost_details() ocd
where (ocd.o).id IN ( 6154, 10805 )
GROUP BY ocd.o
SQL Error [42803]: ERROR: column "ocd.c" must appear in the GROUP BY clause or be used in an aggregate function
I rewrite my query like next:
SELECT DISTINCT sum( suma ) OVER ( PARTITION BY ocd.o ),
(ocd.o).*,
last_value( ocd.c ) OVER (PARTITION BY ocd.o )
FROM order_cost_details() ocd
where (ocd.o).id IN ( 6154, 10805 )
Results seems expected:
with correct last_value:
But I am not sure is this correct to use DISTINCT instead of GROUP BY here?
last_value() often does not work as expected Window Functions: last_value(ORDER BY ... ASC) same as last_value(ORDER BY ... DESC)
To get the last value of a partition, a more valid way is getting the first value of the descending order:
SELECT
first_value(my_column) OVER (PARTITION BY partitioned_column ORDER BY order_column DESC)
FROM
...
You can use a subselect:
SELECT sum(suma),
(o).*,
last_c
FROM (SELECT suma,
ocd.o
last_value(ocd.c)
OVER (PARTITION BY ocd.o
ORDER BY some_col)
AS last_c
FROM order_cost_details() ocd
where (ocd.o).id IN (6154, 10805)
) AS q
GROUP BY o, last_c;
From the IRC
RhodiumToad: using DISTINCT ON is almost always a mistake (do bear in mind it's completely non-standard)
The basic rule of thumb is that you use GROUP BY when you want to reduce the number of output rows, and window functions when you want to keep the number of rows the same
Nothing stops you doing a total over the orders using a window function after the group by
Thus I rewrite my query to look like:
SELECT *,
sum( t.group_suma ) OVER( PARTITION BY (t.o).id ) AS total_suma
FROM (
SELECT
sum( ocd.item_cost ) AS group_cost,
sum( ocd.item_suma ) AS group_suma,
max( (ocd.ic).consumed ) AS consumed,
ocd.o
FROM order_cost_details() ocd
where (ocd.o).id IN ( 6154, 10805 )
GROUP BY ocd.o, (ocd.ic).consumed_period
) t
I have a table which has 3 columns: Product, Date, Status
I want to rank in this manner:
for each product order by Date, and Rank if Status = FALSE then 0, if it's TRUE then start ranking by 1, continue ranking by the same value if previous Status is TRUE.
In this ordered set if FALSE comes assign to it 0, and for the next coming TRUE status for same product assign x+1 (x here is previous rank value for status TRUE).
I hope picture makes it more clear
This code uses SS2008R2 features which do not include LEAD/LAG. A better solution is certainly possible with more modern versions of SQL Server.
-- Sample data.
declare #Samples as Table ( Product VarChar(10), ProductDate Date,
ProductStatus Bit, DesiredRank Int );
insert into #Samples values
( 'a', '20160525', 0, 0 ), ( 'a', '20160526', 1, 1 ), ( 'a', '20160529', 1, 1 ),
( 'a', '20160601', 1, 1 ), ( 'a', '20160603', 0, 0 ), ( 'a', '20160604', 0, 0 ),
( 'a', '20160611', 1, 2 ), ( 'a', '20160612', 0, 0 ), ( 'a', '20160613', 1, 3 ),
( 'b', '20160521', 1, 1 ), ( 'b', '20160522', 0, 0 ), ( 'b', '20160525', 1, 2 );
select * from #Samples;
-- Query to rank data as requested.
with WithRN as (
select Product, ProductDate, ProductStatus, DesiredRank,
Row_Number() over ( partition by Product order by ProductDate ) as RN
from #Samples
),
RCTE as (
select *, Cast( ProductStatus as Int ) as C
from WithRN
where RN = 1
union all
select WRN.*, C + Cast( 1 - R.ProductStatus as Int ) * Cast( WRN.ProductStatus as Int )
from RCTE as R inner join
WithRN as WRN on WRN.Product = R.Product and WRN.RN = R.RN + 1 )
select Product, ProductDate, ProductStatus, DesiredRank,
C * ProductStatus as CalculatedRank
from RCTE
order by Product, ProductDate;
Note that the sample data was extracted from an image using a Mark I Eyeball. Had the OP taken heed of advice here it would have been somewhat easier.
Tip: Using column names that don't happen to match data types and keywords makes life somewhat simpler.
Try this query,
SELECT a.Product ,
a.Date ,
a.Status ,
CASE WHEN a.Status = 'FALSE' THEN 0
ELSE 1
END [Rank]
FROM ( SELECT Product ,
Date ,
Status ,
ROW_NUMBER() OVER ( PARTITION BY Product ORDER BY DATE, Status ) RNK
FROM TableProduct
) a
ORDER BY Product, a.RNK