My algorithm was working fine but with a new big database, my integers variable exceed the max limit size. (I use the powerset algorithm : https://www.postgresql.org/message-id/20060924054759.GA71934%40winnie.fuhr.org)
So I've decided to change all of my integer to bigint, but now I have a problem on the comparison operator... I don't know how to manage it :
CREATE OR REPLACE FUNCTION powerset(a anyarray)
RETURNS SETOF anyarray AS
$BODY$
DECLARE
retval a%TYPE;
alower bigint := array_lower(a, 1);
aupper bigint := array_upper(a, 1);
j bigint;
k bigint;
BEGIN
FOR i IN 1 .. COALESCE((CAST(1 AS BIGINT) << (aupper - alower + 1)) - 1, 0) LOOP
retval := '{}';
j := alower;
k := i;
WHILE k > CAST(0 AS BIGINT) LOOP
IF k & CAST(1 AS BIGINT) = CAST(1 AS BIGINT) THEN
retval := array_append(retval, a[j]);
END IF;
j := j + CAST(1 AS BIGINT);
k := k >> CAST(1 AS BIGINT);
END LOOP;
RETURN NEXT retval;
END LOOP;
RETURN;
END;
$BODY$
LANGUAGE plpgsql IMMUTABLE STRICT
COST 100
ROWS 1000;
ALTER FUNCTION powerset(anyarray)
OWNER TO postgres;
I've got the error on line :
FOR i IN 1 .. COALESCE((CAST(1 AS BIGINT) << (aupper - alower + 1)) - 1, 0) LOOP
Error 42883
Postgresql ERROR: operator does not exist: bigint << bigint
The type of the right operand of bitwise shift operators is integer. Unfortunately, this is not mentioned in the documentation.
You should cast the right operand of shift operators to integer:
-- instead of
-- COALESCE((CAST(1 AS BIGINT) << (aupper - alower + 1)) - 1, 0)
-- use
select COALESCE(1 << (aupper - alower + 1)::int- 1, 0)::bigint
-- instead of
-- k := k >> CAST(1 AS BIGINT);
--- use
k := k >> 1;
-- etc
You can check possible types of operands by querying the system catalog pg_operator, e.g.:
select oprname, oprleft::regtype, oprright::regtype
from pg_operator
where oprname = '<<'
and oprcode::text like '%shl%' -- shift left functions
oprname | oprleft | oprright
---------+----------+----------
<< | smallint | integer
<< | integer | integer
<< | bigint | integer
(3 rows)
The above result shows that left operand of the operator << (bitwise shift left) can be smallint, integer or bigint and right operand must be integer.
Related
I have a table like this:
CREATE TABLE DateInsert(
DateInsert timestamp without time zone,
DateInt integer NOT NULL
);
I want insert list day from 2018-01-01 to 2045-05-18 but it give me an erro
"invalid input syntax for type interval:"
CREATE OR REPLACE FUNCTION insertdate() RETURNS integer AS $$
DECLARE i integer := 0;
d timestamp without time zone := '2018-01-01';
di integer := 0;
BEGIN
while i <10000
LOOP
d := d + INTERVAL ''+ i::character varying + ' day';
di := to_char(d , 'yyyymmdd')::int;
insert into DateInsert(DateInsert,DateInt) values(d, di);
i := i+1;
END LOOP ;
return i;
END;
$$ LANGUAGE plpgsql;
How can I insert to db with timestamp increase 1 in n day loop?
Code In sql server has been working.
declare #i int=0
declare #d datetime
declare #di int = 0
while #i <10000
begin
set #d = DATEADD(DAY, #i, '2018-01-01')
set #di = cast(CONVERT(VARCHAR(10), #d, 112) as int)
insert into DateInsert(DateInsert,DateInt) values(#d, #di)
set #i = #i+1
end
The concatenation operator is || not +. And the prefixed form doesn't seem to like anything else than literals. But you can cast the concatenation expression.
So changing
...
d := d + INTERVAL ''+ i::character varying + ' day';
...
to
...
d := d + (i || ' day')::interval;
...
should work for you.
Is it possible to get the maximum value for BIGINT type without hardcoding it?
I know that the limit is well known, however I wouldn't like to hardcode it.
You can use pg_column_size, it'll give you bytes size of bigint. Using it you can get its max and min sizes:
select (2^(8*pg_column_size(1::bigint)-2))::bigint << 1 as min_bigint_value;
select -(((2^(8*pg_column_size(1::bigint)-2))::bigint << 1)+1) as max_bigint_value;
So if somehow in the future bigint max value will change (very very very unlikely) your code will still works if you will rely on these calculated values.
You can write functions for convenience:
CREATE OR REPLACE FUNCTION "get_max_bigint_value"() RETURNS BIGINT as $$
DECLARE
max_bigint_value BIGINT;
BEGIN
SELECT -(((2^(8*pg_column_size(1::BIGINT)-2))::BIGINT << 1)+1) INTO max_bigint_value;
RETURN max_bigint_value;
END
$$ LANGUAGE "plpgsql";
CREATE OR REPLACE FUNCTION "get_min_bigint_value"() RETURNS BIGINT as $$
DECLARE
min_bigint_value BIGINT;
BEGIN
SELECT (2^(8*pg_column_size(1::bigint)-2))::bigint << 1 INTO min_bigint_value;
RETURN min_bigint_value;
END
$$ LANGUAGE "plpgsql";
and then:
SELECT get_min_bigint_value();
SELECT get_max_bigint_value();
The "ideal" solution would be to create some C functions that just expose the relevant defines (below). But assuming that you'd rather have something in SQL, I think the best you'll be able to do is:
CREATE OR REPLACE FUNCTION bigint_min() RETURNS bigint LANGUAGE sql AS 'SELECT 1::bigint<<63';
CREATE OR REPLACE FUNCTION bigint_max() RETURNS bigint LANGUAGE sql AS 'SELECT ~bigint_min()';
CREATE OR REPLACE FUNCTION int_min() RETURNS int LANGUAGE sql AS 'SELECT 1<<31';
CREATE OR REPLACE FUNCTION int_max() RETURNS int LANGUAGE sql AS 'SELECT ~int_min()';
These are what's defined at the C level. bigint uses PG_INT64_*:
#define PG_INT8_MIN (-0x7F-1)
#define PG_INT8_MAX (0x7F)
#define PG_UINT8_MAX (0xFF)
#define PG_INT16_MIN (-0x7FFF-1)
#define PG_INT16_MAX (0x7FFF)
#define PG_UINT16_MAX (0xFFFF)
#define PG_INT32_MIN (-0x7FFFFFFF-1)
#define PG_INT32_MAX (0x7FFFFFFF)
#define PG_UINT32_MAX (0xFFFFFFFFU)
#define PG_INT64_MIN (-INT64CONST(0x7FFFFFFFFFFFFFFF) - 1)
#define PG_INT64_MAX INT64CONST(0x7FFFFFFFFFFFFFFF)
#define PG_UINT64_MAX UINT64CONST(0xFFFFFFFFFFFFFFFF)
Just for fun: Building on alexpods' answer, we could define a "generic" function (assuming two's complement) for all postgres integers:
create or replace function
minint(a anyelement)
returns anyelement
immutable
language sql
as $$
select ((a - a) - 1) << (8 * pg_column_size(a) - 1);
$$
;
create or replace
function maxint(a anyelement)
returns anyelement
immutable
language sql
as $$
select ~minint(a)
$$
;
Usage:
select
minint(0::smallint)
, maxint(0::smallint)
, minint(0::int)
, maxint(0::int)
, minint(0::bigint)
, maxint(0::bigint)
;
Result:
minint | maxint | minint | maxint | minint | maxint
--------+--------+-------------+------------+----------------------+---------------------
-32768 | 32767 | -2147483648 | 2147483647 | -9223372036854775808 | 9223372036854775807
(1 row)
You can reach it in ~100ms and use in code or save to temporary table - here's example of more or less fast way to get the max value to raise it:
do
$$
declare
i bigint =2;
r record;
begin
begin
for r in 2::int..999::int loop
i=power(2,r);
raise info '%', i::text;
end loop;
exception when others then raise info '%', 'fast roll on (power to 2) ended - switching to slow (multiplying by 2)'||i::text;
end;
begin
for r in 2::int..999::int loop
i=i*2;
raise info '%', i::text;
end loop;
exception when others then raise info '%', 'max reached:'||(1+(i-1)*2)::text;
end;
end;
$$
;
On my database i keep a numbers in specific notation as varchar values. Is there any way to typecast this values into decimals with selected notation?
What i basically looking here should look like this:
SELECT to_integer_with_notation('d', '20')
int4
---------
13
One more example:
SELECT to_integer_with_notation('d3', '23')
int4
---------
302
Unfortunately, there is no built-in function for that in PostgreSQL, but can be written fairly easy:
CREATE OR REPLACE FUNCTION number_from_base(num TEXT, base INTEGER)
RETURNS NUMERIC
LANGUAGE sql
IMMUTABLE
STRICT
AS $function$
SELECT sum(exp * cn)
FROM (
SELECT base::NUMERIC ^ (row_number() OVER () - 1) exp,
CASE
WHEN ch BETWEEN '0' AND '9' THEN ascii(ch) - ascii('0')
WHEN ch BETWEEN 'a' AND 'z' THEN 10 + ascii(ch) - ascii('a')
END cn
FROM regexp_split_to_table(reverse(lower(num)), '') ch(ch)
) sub
$function$;
Note: I used numeric as a return type, as int4 is not enough in many cases (with longer string input).
Edit: Here is a sample reverse function, which can convert a bigint to its text representation within a custom base:
CREATE OR REPLACE FUNCTION number_to_base(num BIGINT, base INTEGER)
RETURNS TEXT
LANGUAGE sql
IMMUTABLE
STRICT
AS $function$
WITH RECURSIVE n(i, n, r) AS (
SELECT -1, num, 0
UNION ALL
SELECT i + 1, n / base, (n % base)::INT
FROM n
WHERE n > 0
)
SELECT string_agg(ch, '')
FROM (
SELECT CASE
WHEN r BETWEEN 0 AND 9 THEN r::TEXT
WHEN r BETWEEN 10 AND 35 THEN chr(ascii('a') + r - 10)
ELSE '%'
END ch
FROM n
WHERE i >= 0
ORDER BY i DESC
) ch
$function$;
Example usage:
SELECT number_to_base(1248, 36);
-- +----------------+
-- | number_to_base |
-- +----------------+
-- | yo |
-- +----------------+
I'm trying to extract the timestamp from a Version 1 UUID, naively wished this worked:
SELECT '3efe0a20-f1b3-11e3-bb44-14109fec739e'::uuid::timestamp;
Here is a quick example showing how to extract the time in Go, but I'm hoping PostgreSQL has something built-in rather than creating a one-off PL/pgSql function:
http://play.golang.org/p/XRCooLgfaG
I've tested this with uuid's from my database and it seems to work very well, even without the unsigned bigints
CREATE FUNCTION uuid_timestamp(id uuid) RETURNS timestamptz AS $$
select TIMESTAMP WITH TIME ZONE 'epoch' +
(((('x' || lpad(split_part(id::text, '-', 1), 16, '0'))::bit(64)::bigint) +
(('x' || lpad(split_part(id::text, '-', 2), 16, '0'))::bit(64)::bigint << 32) +
((('x' || lpad(split_part(id::text, '-', 3), 16, '0'))::bit(64)::bigint&4095) << 48) - 122192928000000000) / 10000000 ) * INTERVAL '1 second';
$$ LANGUAGE SQL
IMMUTABLE
RETURNS NULL ON NULL INPUT;
a V1 uuid I created in the 2099 future!
select uuid_timestamp('6d248400-65b7-1243-a57a-14109fec739e');
uuid_timestamp
------------------------
2099-08-01 11:30:00-07
(1 row)
An alternative to #Krut's implementation that is quite a bit faster based in our testing:
CREATE OR REPLACE FUNCTION uuid_timestamp(uuid UUID) RETURNS TIMESTAMPTZ AS $$
DECLARE
bytes bytea;
BEGIN
bytes := uuid_send(uuid);
RETURN to_timestamp(
(
(
(get_byte(bytes, 0)::bigint << 24) |
(get_byte(bytes, 1)::bigint << 16) |
(get_byte(bytes, 2)::bigint << 8) |
(get_byte(bytes, 3)::bigint << 0)
) + (
((get_byte(bytes, 4)::bigint << 8 |
get_byte(bytes, 5)::bigint)) << 32
) + (
(((get_byte(bytes, 6)::bigint & 15) << 8 | get_byte(bytes, 7)::bigint) & 4095) << 48
) - 122192928000000000
) / 10000 / 1000::double precision
);
END
$$ LANGUAGE plpgsql
IMMUTABLE PARALLEL SAFE
RETURNS NULL ON NULL INPUT;
Note, it will only do millisecond precision so you may want to tweak the function's "/ 10000 / 1000::double precision" bit to just "/ 10000000::double precision" instead.
Example:
=> select uuid_timestamp(uuid_generate_v1()), now();
uuid_timestamp | now
----------------------------+-------------------------------
2020-04-29 17:40:54.519+00 | 2020-04-29 17:40:54.518204+00
(1 row)
Also, it is assuming the input is a v1. If you attempt to give it something like a v4, expect to get weird answers or alter the function to RAISE if it isn't v1.
=> select uuid_timestamp(uuid_generate_v4());
uuid_timestamp
----------------------------
4251-12-19 17:38:34.866+00
(1 row)
Here is a rough pl/pgsql implementation which cast (timestamp, clock_seq, macaddr) into a version 1 uuid.
-- Build UUIDv1 via RFC 4122.
-- clock_seq is a random 14bit unsigned int with range [0,16384)
CREATE OR REPLACE FUNCTION form_uuid_v1(ts TIMESTAMPTZ, clock_seq INTEGER, mac MACADDR)
RETURNS UUID AS $$
DECLARE
t BIT(60) := (extract(EPOCH FROM ts) * 10000000 + 122192928000000000) :: BIGINT :: BIT(60);
uuid_hi BIT(64) := substring(t FROM 29 FOR 32) || substring(t FROM 13 FOR 16) || b'0001' ||
substring(t FROM 1 FOR 12);
BEGIN
RETURN lpad(to_hex(uuid_hi :: BIGINT) :: TEXT, 16, '0') ||
(to_hex((b'10' || clock_seq :: BIT(14)) :: BIT(16) :: INTEGER)) :: TEXT ||
replace(mac :: TEXT, ':', '');
END
$$ LANGUAGE plpgsql;
-- Usage
select form_uuid_v1(now(), 666, '44:88:AA:DD:BB:88');
I am trying to understand how to work with binary data in postgresql (v 8.3).
Let's say I have a following table
Table "public.message"
Column | Type | Modifiers
---------+---------+-----------
id | integer |
message | bytea |
I would like to store a packet in the message field in this format:
version (1 byte), identifier (1 byte), epoch (4 bytes)
I would like to pack this data into the message field. Lets say I have version=1, identifier=8 and epoch=123456. How would I pack this data into the message field? How would I convert my integer values to hex.. or octal?
I also need to get the message back and parse it. I was looking at the get_byte function, unless there is another way to parse the data out..
Thanks!
Here is some sample code showing how to do it with server-side Perl. Annoyingly, pack/unpack are considered untrusted operations by PG so this has to be created with plperlu by a superuser and then access granted with GRANT EXECUTE to non superusers.
On the other hand, this choice of language makes it easy to deal with more complex packed structures, which is a significant advantage over code that would be based on the SQL get_bytes()/set_bytes() functions. See Perl's pack() features.
1) first step: define a SQL composite type representing an non-packed record.
create type comp as (a smallint, b smallint, c int);
2) make a function to pack the record value into bytea:
create function pack_comp(comp) returns bytea
as $body$
my $arg=shift;
my $retval = pack("CCL", $arg->{a},$arg->{b},$arg->{c});
# encode bytea according to PG doc. For PG>=9.0, use encode_bytea() instead
$retval =~ s!(\\|[^ -~])!sprintf("\\%03o",ord($1))!ge; # from PG doc
return $retval;
$body$ language plperlu;
3) make a function to unpack bytea into the composite type:
create or replace function unpack_comp(bytea) returns comp
as $body$
my $arg=shift;
# decode bytea according to PG doc. For PG>=9.0, use decode_bytea() instead
$arg =~ s!\\(?:\\|(\d{3}))!$1 ? chr(oct($1)) : "\\"!ge;
my ($v,$i,$e)= unpack("CCL", $arg);
return {"a"=>$v, "b"=>$i, "c"=>$e};
$body$ language plperlu;
4) usage:
# select encode(pack_comp((254,14,1000000)::comp), 'hex');
encode
--------------
fe0e40420f00
# select unpack_comp(decode('fe0e40420f00','hex'));
unpack_comp
------------------
(254,14,1000000)
# select * from unpack_comp(decode('fe0e40420f00','hex'));
a | b | c
-----+----+---------
254 | 14 | 1000000
So I was able to figure out how to do it in plpg
Here's the code to pack
CREATE FUNCTION pack_numeric_bytes(i_values NUMERIC[], i_byte_sizes NUMERIC[], i_big_endian BOOLEAN)
RETURNS BYTEA
DECLARE
v_bytes BYTEA := NULL;
v_start INTEGER := 1;
v_byte BYTEA;
v_byte_size INTEGER;
v_value NUMERIC;
v_binary_value TEXT;
v_num NUMERIC;
i INTEGER;
x INTEGER;
v_sql TEXT;
BEGIN
IF array_upper(i_values, 1) != array_upper(i_byte_sizes, 1) THEN
RETURN v_bytes;
END IF;
FOR x IN array_lower(i_values, 1) .. array_upper(i_values, 1) LOOP
/* Given value and size at x position */
v_byte_size := i_byte_sizes[x]::INTEGER;
v_value := i_values[x];
/* Convert number to binary form */
v_sql := $$SELECT $$|| v_value ||$$::bit($$|| v_byte_size*8 ||$$);$$;
EXECUTE v_sql INTO v_binary_value;
IF i_big_endian IS TRUE THEN
/* Convert each byte at a time */
FOR i IN 1 .. v_byte_size LOOP
/* Extract byte from our binary value.
Big endian starts at 1 and then increments of 8 */
v_byte := substring(v_binary_value, v_start, 8);
/* Convert binary 8 bits to an integer */
v_sql := $$SELECT B$$||quote_literal(v_byte)||$$::int8$$;
EXECUTE v_sql INTO v_num;
/* Build bytea of bytes */
v_bytes := COALESCE(v_bytes, '') || set_byte(E' '::BYTEA, 0, v_num::INTEGER);
v_start := v_start + 8;
END LOOP;
ELSE
/* Small endian is extracted starting from last byte */
v_start := (v_byte_size * 8) + 1;
/* Convert each byte at a time */
FOR i IN 1 .. v_byte_size LOOP
v_start := v_start - 8;
v_byte := substring(v_binary_value, v_start, 8);
/* Convert binary 8 bits to an integer */
v_sql := $$SELECT B$$||quote_literal(v_byte)||$$::int8$$;
EXECUTE v_sql INTO v_num;
/* Build bytea of bytes */
v_bytes := COALESCE(v_bytes, '') || set_byte(E' '::BYTEA, 0, v_num::INTEGER);
END LOOP;
END IF; /* END endian check */
v_start := 1;
END LOOP;
RETURN v_bytes;
END;
And here's the code to unpack:
CREATE OR REPLACE FUNCTION public.unpack_numeric_bytes(i_bytes bytea, i_byte_sizes INTEGER[], i_big_endian BOOLEAN)
RETURNS NUMERIC[]
SECURITY DEFINER AS
DECLARE
v_bytes BYTEA;
v_start INTEGER := 1;
v_byte_index INTEGER := 0;
v_bit_shift INTEGER := 0;
v_length INTEGER;
v_size INTEGER;
v_sum_byte_sizes INTEGER;
v_vals NUMERIC[] := '{}';
v_val BIGINT := 0;
i INTEGER;
x INTEGER;
v_sql TEXT;
BEGIN
v_sql := $$SELECT $$|| array_to_string(i_byte_sizes, '+')||$$;$$;
EXECUTE v_sql INTO v_sum_byte_sizes;
IF length(i_bytes) != v_sum_byte_sizes::INTEGER THEN
RETURN v_vals;
END IF;
/* Loop through values of bytea (split by their sizes) */
FOR x IN array_lower(i_byte_sizes, 1) .. array_upper(i_byte_sizes, 1) LOOP
v_size := i_byte_sizes[x];
v_bytes := substring(i_bytes, v_start, v_size);
v_length := length(v_bytes);
IF i_big_endian IS TRUE THEN
v_byte_index := v_length - 1;
FOR i IN 1..v_length LOOP
v_val := v_val + (get_byte(v_bytes, v_byte_index) << v_bit_shift);
v_bit_shift := v_bit_shift + 8;
v_byte_index := v_byte_index - 1;
END LOOP;
ELSE
FOR i IN 1..v_length LOOP
v_val := v_val + (get_byte(v_bytes, v_byte_index) << v_bit_shift);
v_bit_shift := v_bit_shift + 8;
v_byte_index := v_byte_index + 1;
END LOOP;
END IF;
v_vals := array_append(v_vals, v_val::NUMERIC);
/* Calculate next value start index */
v_start := v_start + v_size;
v_byte_index := 0;
v_bit_shift := 0;
v_val := 0;
END LOOP;
RETURN v_vals;
END;
I hope this will help someone.