Cumulative count on history table with deleted attributes - postgresql

I've got a history table of updates to records, and I want to calculate cumulative totals where values may be added or deleted to the set. (ie the cumulative total for one month may be less than the previous).
For example, here's a table with the history of updates to tags for a person record. (id is the id of the person record).
I want to count how many people had the "established" tag in any given month, accounting for when it was added or removed in a prior month.
+----+------------------------+---------------------+
| id | tags | created_at |
+----+------------------------+---------------------+
| 1 | ["vip", "established"] | 2017-01-01 00:00:00 |
| 2 | ["established"] | 2017-01-01 00:00:00 |
| 3 | ["established"] | 2017-02-01 00:00:00 |
| 1 | ["vip"] | 2017-03-01 00:00:00 |
| 4 | ["established"] | 2017-05-01 00:00:00 |
+----+------------------------+---------------------+
With some help from these posts, I've gotten this far:
SELECT
item_month,
sum(count(distinct(id))) OVER (ORDER BY item_month)
FROM (
SELECT
to_char("created_at", 'yyyy-mm') as item_month,
id
FROM person_history
WHERE tags ? 'established'
) t1
GROUP BY item_month;
Which gives me:
month count
2017-01 2
2017-02 3
2017-05 4 <--- should be 3
And it's also missing an entry for 2017-03 which should be 2.
(An entry for 2017-04 would be nice too, but the UI could always infer it from the previous month if need be)

Here is step-by-step tutorial, you could try to collapse all those CTEs:
with
-- Example data
person_history(id, tags, created_at) as (values
(1, '["vip", "est"]'::jsonb, '2017-01-01'::timestamp),
(2, '["est"]', '2017-01-01'), -- Note that Person 2 changed its tags several times per month
(2, '["vip"]', '2017-01-02'),
(2, '["vip", "est"]', '2017-01-03'),
(3, '["est"]', '2017-02-01'),
(1, '["vip"]', '2017-03-01'),
(4, '["est"]', '2017-05-01')),
-- Get the last tags for each person per month
monthly as (
select distinct on (id, date_trunc('month', created_at))
id,
date_trunc('month', created_at) as month,
tags,
created_at
from person_history
order by 1, 2, created_at desc),
-- Retrieve tags from previous month
monthly_prev as (
select
*,
coalesce((lag(tags) over (partition by id order by month)), '[]') as prev_tags
from monthly),
-- Calculate delta: if "est" was added then 1, removed then -1, nothing heppens then 0
monthly_delta as (
select
*,
case
when tags ? 'est' and not prev_tags ? 'est' then 1
when not tags ? 'est' and prev_tags ? 'est' then -1
else 0
end as delta
from monthly_prev),
-- Sum all deltas for each month
monthly_total as (
select month, sum(delta) as total
from monthly_delta
group by month)
-- Finally calculate cumulative sum
select *, sum(total) over (order by month) from monthly_total
order by month;
Result:
┌─────────────────────┬───────┬─────┐
│ month │ total │ sum │
├─────────────────────┼───────┼─────┤
│ 2017-01-01 00:00:00 │ 2 │ 2 │
│ 2017-02-01 00:00:00 │ 1 │ 3 │
│ 2017-03-01 00:00:00 │ -1 │ 2 │
│ 2017-05-01 00:00:00 │ 1 │ 3 │
└─────────────────────┴───────┴─────┘

Related

Using unnest to join in Postgres

Appreciate this is a simple use case but having difficulty doing a join in Postgres using an array.
I have two tables:
table: shares
id | likes_id_array timestamp share_site
-----------------+-----------------+----------+-----------
12345_6789 | [xxx, yyy , zzz]| date1 | fb
abcde_wxyz | [vbd, fka, fhx] | date2 | tw
table: likes
likes_id | name | location
--------+-------+----------+-----
xxx | aaaa | nice
fpg | bbbb | dfpb
yyy | mmmm | place
dhf | cccc | fiwk
zzz | dddd | here
desired - a result set based on shares.id = 12345_6789:
likes_id | name | location | timestamp
--------+-------+----------+------------+-----------
xxx | aaaa | nice | date1
yyy | mmmm | place | date1
zzz | dddd | here | date1
the first step is using unnest() for the likes_id_array:
SELECT unnest(likes_id_array) as i FROM shares
WHERE id = '12345_6789'
but I can't figure out how to join the results set this produces, with the likes table on likes_id. Any help would be much appreciated!
You can create a CTE with your query with the likes identifiers, and then make a regular inner join with the table of likes
with like_ids as (
select
unnest(likes_id_array) as like_id
from shares
where id = '12345_6789'
)
select
likes_id,
name,
location
from likes
inner join like_ids
on likes.likes_id = like_ids.like_id
Demo
You can use ANY:
SELECT a.*, b.timestamp FROM likes a JOIN shares b ON a.likes_id = ANY(b.likes_id_array) WHERE id = '12345_6789';
You could do this with subqueries or a CTE, but the easiest way is to call the unnest function not in the SELECT clause but as a table expression in the FROM clause:
SELECT likes.*, shares.timestamp
FROM shares, unnest(likes_id_array) as arr(likes_id)
JOIN likes USING (likes_id)
WHERE shares.id = '12345_6789'
You can use jsonb_array_elements_text with a (implicit) lateral join:
SELECT
likes.likes_id,
likes.name,
likes.location,
shares.timestamp
FROM
shares,
jsonb_array_elements_text(shares.likes_id_array) AS share_likes(id),
likes
WHERE
likes.likes_id = share_likes.id AND
shares.id = '12345_6789';
Output:
┌──────────┬──────┬──────────┬─────────────────────┐
│ likes_id │ name │ location │ timestamp │
├──────────┼──────┼──────────┼─────────────────────┤
│ xxx │ aaaa │ nice │ 2022-10-12 11:32:39 │
│ yyy │ mmmm │ place │ 2022-10-12 11:32:39 │
│ zzz │ dddd │ here │ 2022-10-12 11:32:39 │
└──────────┴──────┴──────────┴─────────────────────┘
(3 rows)
Or if you want to make the lateral join explicit (notice the addition of the LATERAL keyword):
SELECT
likes.likes_id,
likes.name,
likes.location,
shares.timestamp
FROM
shares,
LATERAL jsonb_array_elements_text(shares.likes_id_array) AS share_likes(id),
likes
WHERE
likes.likes_id = share_likes.id AND
shares.id = '12345_6789';

Using Timescale to find the latest value per interval

I have timeseries data that has up to millisecond accuracy. Some of these timestamps can coincide on the exact time which can therefore be sorted out by a database id column to figure out which is the latest.
I am trying to use Timescale to get the latest values per second.
Here is an example of the data I'm looking at
time db_id value
2020-01-01 08:39:23.293 | 4460 | 136.01 |
2020-01-01 08:39:23.393 | 4461 | 197.95 |
2020-01-01 08:40:38.973 | 4462 | 57.95 |
2020-01-01 08:43:01.223 | 4463 | 156 |
2020-01-01 08:43:26.577 | 4464 | 253.43 |
2020-01-01 08:43:26.577 | 4465 | 53.68 |
2020-01-01 08:43:26.577 | 4466 | 160.00 |
When obtaining latest price per second, my results should look like this
time value
2020-01-01 08:39:23 | 197.95 |
2020-01-01 08:39:24 | 197.95 |
.
.
.
2020-01-01 08:40:37 | 197.95 |
2020-01-01 08:40:38 | 57.95 |
2020-01-01 08:40:39 | 57.95 |
.
.
.
2020-01-01 08:43:25 | 57.95 |
2020-01-01 08:43:26 | 160.00 |
2020-01-01 08:43:27 | 160.00 |
.
.
.
I've successfully obtained the latest results per second using the Timescale time_bucket
SELECT last(value, db_id), time_bucket('1 seconds', time) AS per_second FROM timeseries GROUP BY per_second ORDER BY per_second DESC;
but it leaves holes in the time column.
time value
2020-01-01 08:39:23 | 197.95 |
2020-01-01 08:40:38 | 57.95 |
2020-01-01 08:43:26 | 160.00 |
The solution I thought up of is creating a database with per second timestamps and null values, migrating data from the previous resulting table and then replacing the null values with last occurring value but it seems like a lot of intermediary steps.
I'd like to know if there is a better approach to this issue of finding the "latest value" per second, minute, hour etc. I originally tried solving the issue with python as it seemed like a simple issue but it took up a lot of computing time.
Found a nice working solution to my problem.
It involves four main steps:
getting latest values
select
time_bucket('1 second', time + '1 second') as interval,
last(val, db_id) as last_value
from table
where time > <date_start> and time < <date_end>
group by interval
order by time;
This will produce a table that has the latest values. last also takes advantage of a column in case another level of sorting is required.
e.g.
time last_value
2020-01-01 08:39:23 | 197.95 |
2020-01-01 08:40:38 | 57.95 |
2020-01-01 08:43:26 | 160.00 |
Note that I shift the time by one second with + '1 second' since I only want data before a particular second - without this it will consider on-the-second data as part of the last price.
creating a table with timestamps per second
select
time_bucket_gapfill('1 second', time) as per_second
from table
where time > <date_start> and time < <date_end>
group by per_second
order by per_second;
Here I produce a table where each row has per second timestamps.
e.g.
per_second
2020-01-01 00:00:00.000
2020-01-01 00:00:01.000
2020-01-01 00:00:02.000
2020-01-01 00:00:03.000
2020-01-01 00:00:04.000
2020-01-01 00:00:05.000
join them together and add a value_partition column
select
per_second,
last_value,
sum(case when last_value is null then 0 else 1 end) over (order by per_second) as value_partition
from
(
select
time_bucket('1 second', time + '1 second') as interval,
last(val, db_id) as last_value
from table
where time > <date_start> and time < <date_end>
group by interval, time
) a
right join
(
select
time_bucket_gapfill('1 second', time) as per_second
from table
where time > <date_start> and time < <date_end>
group by per_second
) b
on a.interval = b.per_second
Inspired by this answer, the goal is to have a counter (value_partition) that increments only if the value is not null.
e.g.
per_second latest_value value_partition
2020-01-01 00:00:00.000 NULL 0
2020-01-01 00:00:01.000 15.82 1
2020-01-01 00:00:02.000 NULL 1
2020-01-01 00:00:03.000 NULL 1
2020-01-01 00:00:04.000 NULL 1
2020-01-01 00:00:05.000 NULL 1
2020-01-01 00:00:06.000 NULL 1
2020-01-01 00:00:07.000 NULL 1
2020-01-01 00:00:08.000 NULL 1
2020-01-01 00:00:09.000 NULL 1
2020-01-01 00:00:10.000 15.72 2
2020-01-01 00:00:10.000 14.67 3
filling in the null values
select
per_second,
first_value(last_value) over (partition by value_partition order by per_second) as latest_value
from
(
select
per_second,
last_value,
sum(case when last_value is null then 0 else 1 end) over (order by per_second) as value_partition
from
(
select
time_bucket('1 second', time + '1 second') as interval,
last(val, db_id) as last_value
from table
where time > <date_start> and time < <date_end>
group by interval
) a
right join
(
select
time_bucket_gapfill('1 second', time) as per_second
from table
where time > <date_start> and time < <date_end>
group by per_second
) b
on a.interval = b.per_second
) as q
This final step brings everything together.
This takes advantage of the value_partition column and overwrites the null values accordingly.
e.g.
per_second latest_value
2020-01-01 00:00:00.000 NULL
2020-01-01 00:00:01.000 15.82
2020-01-01 00:00:02.000 15.82
2020-01-01 00:00:03.000 15.82
2020-01-01 00:00:04.000 15.82
2020-01-01 00:00:05.000 15.82
2020-01-01 00:00:06.000 15.82
2020-01-01 00:00:07.000 15.82
2020-01-01 00:00:08.000 15.82
2020-01-01 00:00:09.000 15.82
2020-01-01 00:00:10.000 15.72
2020-01-01 00:00:10.000 14.67

PostgreSQL: detecting the first/last rows of result set

Is there any way to embed a flag in a select that indicates that it is the first or the last row of a result set? I'm thinking something to the effect of:
> SELECT is_first_row() AS f, is_last_row() AS l FROM blah;
f | l
-----------
t | f
f | f
f | f
f | f
f | t
The answer might be in window functions but I've only just learned about them, and I question their efficiency.
SELECT first_value(unique_column) OVER () = unique_column, last_value(unique_column) OVER () = unique_column, * FROM blah;
seems to do what I want. Unfortunately, I don't even fully understand that syntax, but since unique_column is unique and NOT NULL it should deliver unambiguous results. But if it does sorting, then the cure might be worse than the disease. (Actually, in my tests, unique_column is not sorted, so that's something.)
EXPLAIN ANALYZE doesn't indicate there's an efficiency problem, but when has it ever told me what I needed to know?
And I might need to use this in an aggregate function, but I've just been told window functions aren't allowed there. 😕
Edit:
Actually, I just added ORDER BY unique_column to the above query and the rows identified as first and last were thrown into the middle of the result set. It's as if first_value()/last_value() really means "the first/last value I picked up before I began sorting." I don't think I can safely do this optimally. Not unless a much better understanding of the use of the OVER keyword is to be had.
I'm running PostgreSQL 9.6 in a Debian 9.5 environment.
This isn't a duplicate, because I'm trying to get the first row and last row of the result set to identify themselves, while Postgres: get min, max, aggregate values in one select is just going for the minimum and maximum values for a column in a result set.
You can use the lead() and lag() window functions (over the appropiate window) and compare them to NULL:
-- \i tmp.sql
CREATE TABLE ztable
( id SERIAL PRIMARY KEY
, starttime TIMESTAMP
);
INSERT INTO ztable (starttime) VALUES ( now() - INTERVAL '1 minute');
INSERT INTO ztable (starttime) VALUES ( now() - INTERVAL '2 minute');
INSERT INTO ztable (starttime) VALUES ( now() - INTERVAL '3 minute');
INSERT INTO ztable (starttime) VALUES ( now() - INTERVAL '4 minute');
INSERT INTO ztable (starttime) VALUES ( now() - INTERVAL '5 minute');
INSERT INTO ztable (starttime) VALUES ( now() - INTERVAL '6 minute');
SELECT id, starttime
, ( lead(id) OVER www IS NULL) AS is_first
, ( lag(id) OVER www IS NULL) AS is_last
FROM ztable
WINDOW www AS (ORDER BY id )
ORDER BY id
;
SELECT id, starttime
, ( lead(id) OVER www IS NULL) AS is_first
, ( lag(id) OVER www IS NULL) AS is_last
FROM ztable
WINDOW www AS (ORDER BY starttime )
ORDER BY id
;
SELECT id, starttime
, ( lead(id) OVER www IS NULL) AS is_first
, ( lag(id) OVER www IS NULL) AS is_last
FROM ztable
WINDOW www AS (ORDER BY starttime )
ORDER BY random()
;
Result:
INSERT 0 1
INSERT 0 1
INSERT 0 1
INSERT 0 1
INSERT 0 1
INSERT 0 1
id | starttime | is_first | is_last
----+----------------------------+----------+---------
1 | 2018-08-31 18:38:45.567393 | f | t
2 | 2018-08-31 18:37:45.575586 | f | f
3 | 2018-08-31 18:36:45.587436 | f | f
4 | 2018-08-31 18:35:45.592316 | f | f
5 | 2018-08-31 18:34:45.600619 | f | f
6 | 2018-08-31 18:33:45.60907 | t | f
(6 rows)
id | starttime | is_first | is_last
----+----------------------------+----------+---------
1 | 2018-08-31 18:38:45.567393 | t | f
2 | 2018-08-31 18:37:45.575586 | f | f
3 | 2018-08-31 18:36:45.587436 | f | f
4 | 2018-08-31 18:35:45.592316 | f | f
5 | 2018-08-31 18:34:45.600619 | f | f
6 | 2018-08-31 18:33:45.60907 | f | t
(6 rows)
id | starttime | is_first | is_last
----+----------------------------+----------+---------
2 | 2018-08-31 18:37:45.575586 | f | f
4 | 2018-08-31 18:35:45.592316 | f | f
6 | 2018-08-31 18:33:45.60907 | f | t
5 | 2018-08-31 18:34:45.600619 | f | f
1 | 2018-08-31 18:38:45.567393 | t | f
3 | 2018-08-31 18:36:45.587436 | f | f
(6 rows)
[updated: added a randomly sorted case]
It is simple using window functions with particular frames:
with t(x, y) as (select generate_series(1,5), random())
select *,
count(*) over (rows between unbounded preceding and current row),
count(*) over (rows between current row and unbounded following)
from t;
┌───┬───────────────────┬───────┬───────┐
│ x │ y │ count │ count │
├───┼───────────────────┼───────┼───────┤
│ 1 │ 0.543995119165629 │ 1 │ 5 │
│ 2 │ 0.886343683116138 │ 2 │ 4 │
│ 3 │ 0.124682310037315 │ 3 │ 3 │
│ 4 │ 0.668972567655146 │ 4 │ 2 │
│ 5 │ 0.266671542543918 │ 5 │ 1 │
└───┴───────────────────┴───────┴───────┘
As you can see count(*) over (rows between unbounded preceding and current row) returns rows count from the data set beginning to current row and count(*) over (rows between current row and unbounded following) returns rows count from the current to data set end. 1 indicates the first/last rows.
It works until you ordering your data set by order by. In this case you need to duplicate it in the frames definitions:
with t(x, y) as (select generate_series(1,5), random())
select *,
count(*) over (order by y rows between unbounded preceding and current row),
count(*) over (order by y rows between current row and unbounded following)
from t order by y;
┌───┬───────────────────┬───────┬───────┐
│ x │ y │ count │ count │
├───┼───────────────────┼───────┼───────┤
│ 1 │ 0.125781774986535 │ 1 │ 5 │
│ 4 │ 0.25046408502385 │ 2 │ 4 │
│ 5 │ 0.538880597334355 │ 3 │ 3 │
│ 3 │ 0.802807193249464 │ 4 │ 2 │
│ 2 │ 0.869908029679209 │ 5 │ 1 │
└───┴───────────────────┴───────┴───────┘
PS: As mentioned by a_horse_with_no_name in the comment:
there is no such thing as the "first" or "last" row without sorting.
In fact, Window Functions are a great approach and for that requirement of yours, they are awesome.
Regarding efficiency, window functions work over the data set already at hand. Which means the DBMS will just add extra processing to infer first/last values.
Just one thing I'd like to suggest: I like to put an ORDER BY criteria inside the OVER clause, just to ensure the data set order is the same between multiple executions, thus returning the same values to you.
Try using
SELECT columns
FROM mytable
Join conditions
WHERE conditions ORDER BY date DESC LIMIT 1
UNION ALL
SELECT columns
FROM mytable
Join conditions
WHERE conditions ORDER BY date ASC LIMIT 1
SELECT just cut half of the processing time. You can go for indexing also.

Implementing a sort column in Postgres that acts like a linked list

Say I have a database table teams that has an ordering column position, the position can either be null if it is the last result, or the id of next team that is positioned one higher than that team. This would result in a list that is always strictly sorted (if you use ints you have to manage all the other position values when inserting a new team, ie increment them all by one), and the insertion becomes less complicated...
But to retrieve this table as a sorted query has proved tricky, here is where I'm at so far:
WITH RECURSIVE teams AS (
SELECT *, 1 as depth FROM team
UNION
SELECT t.*, ts.depth + 1 as depth
FROM team t INNER JOIN teams ts ON ts.order = t.id
SELECT
id, order, depth
FROM
teams
;
Which gets me something like:
id | order | depth
----+-------+-------
53 | 55 | 1
55 | 52 | 1
55 | 52 | 2
52 | 54 | 2
52 | 54 | 3
54 | | 3
54 | | 4
Which kind of reflects where I need to get to in terms of ordering (the max of depth represents the ordering I want...) however I cant work out how to alter the query to get something like:
id | order | depth
----+-------+-------
53 | 55 | 1
55 | 52 | 2
52 | 54 | 3
54 | | 4
It seems however I change the query it complains at me about applying a GROUP BY across both id and depth... How do I get from where I am now to where I want to be?
Your recursive query should to start somewhere (for now you selecting whole table in the first subquery). I propose to start from the last record where order column is null and walk to the first record:
with recursive team(id, ord) as (values(53,55),(55,52),(52,54),(54,null)),
teams as (
select *, 1 as depth from team where ord is null -- select the last record here
union all
select t.*, ts.depth + 1 as depth
from team t join teams ts on ts.id = t.ord) -- note that the JOIN condition reversed comparing to the original query
select * from teams order by depth desc; -- finally reverse the order
┌────┬──────┬───────┐
│ id │ ord │ depth │
├────┼──────┼───────┤
│ 53 │ 55 │ 4 │
│ 55 │ 52 │ 3 │
│ 52 │ 54 │ 2 │
│ 54 │ ░░░░ │ 1 │
└────┴──────┴───────┘

How can I calculate the number of months between YYYYMM integer values in PostgreSQL?

How can I calculate the number of months between two YYYYMM integer values in Postgresql?
DATA:
| Date1 | Date2 |
|--------|--------|
| 201608 | 201702 |
| 201609 | 201610 |
DESIRED OUTPUT:
| Date1 | Date2 | MonthsBetweenInclusive | MonthsBetweenExclusive |
|--------|--------|------------------------|------------------------|
| 201608 | 201702 | 7 | 6 |
| 201609 | 201610 | 2 | 1 |
I have looked at the PostgreSQL date function documentation but I'm unable to find a solution that operates on YYYYMM integer values.
with t(d1,d2) as (values(201608,201702),(201609,201610))
select
*,
((d2/100*12)+(d2-d2/100*100))-((d1/100*12)+(d1-d1/100*100))
from t;
There are more ways - what is correct, depends on your case:
select extract( months from (justify_interval(to_timestamp('201610','YYYYMM') -
to_timestamp('201609','YYYYMM'))));
┌───────────┐
│ date_part │
╞═══════════╡
│ 1 │
└───────────┘
(1 row)
or
CREATE OR REPLACE FUNCTION month_number(date)
RETURNS int AS $$
SELECT ((EXTRACT(year FROM $1) - 1900) * 12 +
EXTRACT(month FROM $1))::int
$$ LANGUAGE sql;
SELECT month_number(to_date('201702','YYYYMM')) -
month_number(to_date('201608','YYYYMM'));
┌──────────┐
│ ?column? │
╞══════════╡
│ 6 │
└──────────┘
(1 row)
or
SELECT (to_date('201702','YYYYMM') -
to_date('201608','YYYYMM'))/30;
┌──────────┐
│ ?column? │
╞══════════╡
│ 6 │
└──────────┘
(1 row)
with t (date1, date2) as (values
(201608,201702),(201609,201610)
)
select array_length(
array((
select generate_series(
to_date(date1::text, 'YYYYMM'),
to_date(date2::text, 'YYYYMM'),
'1 month'
)
)), 1
)
from t
;
array_length
--------------
7
2
SELECT date1, date2,
1 + extract(year from age(to_timestamp(date1::text,'YYYYMM'),to_timestamp(date2::text,'YYYYMM'))) * 12
+ extract(month from age(to_timestamp(date1::text,'YYYYMM'),to_timestamp(date2::text,'YYYYMM'))) AS MonthsBetweenInclusive,
extract(year from age(to_timestamp(date1::text,'YYYYMM'),to_timestamp(date2::text,'YYYYMM'))) * 12
+ extract(month from age(to_timestamp(date1::text,'YYYYMM'),to_timestamp(date2::text,'YYYYMM'))) AS MonthsBetweenExclusive
FROM datetable;
select yt.date1,
yt.date2,
trunc(EXTRACT(EPOCH from age(to_timestamp(yt.date2::TEXT, 'YYYYMM'),to_timestamp(yt.date1::TEXT, 'YYYYMM')))/(3600*24*30)),
trunc(EXTRACT(EPOCH from age(to_timestamp(yt.date2::TEXT, 'YYYYMM'),to_timestamp(yt.date1::TEXT, 'YYYYMM')))/(3600*24*30)) +1
from your_table yt