Related
How can the PBKDF2 function be done in PostgreSQL? There does not appear to be a native implementation.
Moved Answer out of Question to adhere to Stack Overflow guidelines. See original revision of post.
Original post (Revision link)
Not able to find it natively, and based on PHP code found on the 'net, I came up with this PBKDF2 function for PostgreSQL. Enjoy.
create or replace function PBKDF2
(salt bytea, pw text, count integer, desired_length integer, algorithm text)
returns bytea
immutable
language plpgsql
as $$
declare
hash_length integer;
block_count integer;
output bytea;
the_last bytea;
xorsum bytea;
i_as_int32 bytea;
i integer;
j integer;
k integer;
begin
algorithm := lower(algorithm);
case algorithm
when 'md5' then
hash_length := 16;
when 'sha1' then
hash_length = 20;
when 'sha256' then
hash_length = 32;
when 'sha512' then
hash_length = 64;
else
raise exception 'Unknown algorithm "%"', algorithm;
end case;
block_count := ceil(desired_length::real / hash_length::real);
for i in 1 .. block_count loop
i_as_int32 := E'\\000\\000\\000'::bytea || chr(i)::bytea;
i_as_int32 := substring(i_as_int32, length(i_as_int32) - 3);
the_last := salt::bytea || i_as_int32;
xorsum := HMAC(the_last, pw::bytea, algorithm);
the_last := xorsum;
for j in 2 .. count loop
the_last := HMAC(the_last, pw::bytea, algorithm);
--
-- xor the two
--
for k in 1 .. length(xorsum) loop
xorsum := set_byte(xorsum, k - 1, get_byte(xorsum, k - 1) # get_byte(the_last, k - 1));
end loop;
end loop;
if output is null then
output := xorsum;
else
output := output || xorsum;
end if;
end loop;
return substring(output from 1 for desired_length);
end $$;
I've tested against other implementations without deviation, but be sure to test it yourself.
I need to insert a big batch rows to my PostgreSQL(v9.5) table.
Here is the table:
create table TaskLog(
id serial,
taskid integer,
x double precision,
y double precision,
loc character varying(50),
speed smallint,
gpstime timestamp without time zone,
veh character varying(16),
vin character(17),
regdate date,
enabled boolean,
remake character varying(100),
isdel boolean,
alarm integer,
CONSTRAINT pk_delphi_id PRIMARY KEY (id)
)
I use FireDAC(Delphi XE10.1) to insert the rows:
function RandomStr(aLength : Integer) : string;
var
X: Integer;
begin
if aLength <= 0 then exit;
SetLength(Result, aLength);
for X:=1 to aLength do
Result[X] := Chr(Random(26) + 65);
end;
procedure TForm7.Button6Click(Sender: TObject);
var
i: Integer;
Watch: TStopwatch;
begin
Watch := TStopwatch.StartNew;
try
FDQuery1.SQL.Text :=
'insert into TaskLog values(default, :f1, :f2, :f3, :f4, :f5, :f6, ' +
':f7, :f8, :f9, :f10, :f11, :f12, :f13)';
FDQuery1.Params.ArraySize := StrToInt(Edit1.text); //<--- Change the ArraySize
for i := 0 to FDQuery1.Params.ArraySize - 1 do
begin
FDQuery1.Params[0].AsIntegers[i] := Random(9999999);
FDQuery1.Params[1].AsFloats[i] := Random(114) + Random;
FDQuery1.Params[2].AsFloats[i] := Random(90) + Random;
FDQuery1.Params[3].AsStrings[i] := RandomStr(Random(50));
FDQuery1.Params[4].AsSmallInts[i] := Random(1990);
FDQuery1.Params[5].AsDateTimes[i] := IncSecond(IncDay(Now, -(Random(100) + 1)), Random(99999));
FDQuery1.Params[6].AsStrings[i] := RandomStr(Random(16));
FDQuery1.Params[7].AsStrings[i] := RandomStr(Random(17));
FDQuery1.Params[8].AsDates[i] := IncDay(Now, -(Random(365) + 1));
FDQuery1.Params[9].AsBooleans[i] := Odd(Random(200));
FDQuery1.Params[10].AsStrings[i] := RandomStr(Random(100));
FDQuery1.Params[11].AsBooleans[i] := Odd(Random(100));
FDQuery1.Params[12].AsIntegers[i] := Random(100000);
end;
FDQuery1.Execute(FDQuery1.Params.ArraySize);
Watch.Stop;
Memo1.Lines.Add('Should be inserted ' + IntToStr(FDQuery1.Params.ArraySize) + ' lines');
Memo1.Lines.Add('Actually inserted ' + IntToStr(FDQuery1.RowsAffected) + ' lines');
Memo1.Lines.Add('Take ' + Watch.ElapsedMilliseconds.ToString + ' seconds');
except
Memo1.Lines.Add(Exception(ExceptObject).Message);
end;
end;
It work fine when I set the FDQuery1.Params.ArraySize:=1000,
It work failed when I set the FDQuery1.Params.ArraySize:=10000, no record can been inserted.
Is the ArraySize property have a size limit with PostgreSQL?
const batch_part = 2000;
****
PostgreSQL.Query.Params.ArraySize := Total;
****
// отправка пакетами в цикле / sending packets in a loop
if Total > batch_part then begin
j:= Total div batch_part;
for n := 1 to j do begin
PostgreSQL.Query.Execute(n*batch_part, (n-1)*batch_part); // #Batch INSERT operation
LogFile.Write(Format('FDPostgreSQL.Query.Execute(%d,%d)', [n*batch_part, (n-1)*batch_part]));
end;
if (Total mod batch_part) > 0 then begin // остаток / remnant
PostgreSQL.Query.Execute((Total mod batch_part)+(n-1)*batch_part, (n-1)*batch_part); // #Batch INSERT operation
LogFile.Write(Format('FDPostgreSQL.Query.Execute(%d,%d)', [(Total mod batch_part)+(n-1)*batch_part, (n-1)*batch_part]));
end;
end
else begin
PostgreSQL.Query.Execute(Total, 0);
LogFile.Write(Format('FDPostgreSQL.Query.Execute(%d,0)', [Total]));
end;
LogFile:
[04.08.2022 10:23:21.678] Batch INSERT in FDPostgreSQL...
[04.08.2022 10:23:22.079] FDPostgreSQL Execute(2000,0)
[04.08.2022 10:23:22.154] FDPostgreSQL Execute(4000,2000)
[04.08.2022 10:23:22.227] FDPostgreSQL Execute(6000,4000)
[04.08.2022 10:23:22.299] FDPostgreSQL Execute(8000,6000)
[04.08.2022 10:23:22.373] FDPostgreSQL Execute(10000,8000)
[04.08.2022 10:23:22.443] FDPostgreSQL Execute(12000,10000)
[04.08.2022 10:23:22.516] FDPostgreSQL Execute(13014,12000)
[04.08.2022 10:23:22.540] Batch INSERT finished: 13014 records.
Server side SQL:
INSERT INTO propose_sid VALUES(DEFAULT, $1, $2, $3, $4, $5, $6, $7, $8),(DEFAULT, $9, $10, $11, $12, $13, $14, $15, $16),(DEFAULT, $17, $18, $19, $20, $21, $22, $23, $24)... -- 1999 times
im using this code and I had 1 problem, the o_besteltijden cursor has this as output: 01-06-15 ( a date), but I prefer output like: 16:00 (so only time) Or 01-06-15 16:00:00 (date + time). is that possible?
SET SERVEROUTPUT ON;
CREATE OR REPLACE TYPE t_openingstijd IS TABLE OF DATE;
CREATE OR REPLACE PROCEDURE zoekMogelijkeBesteltijden(p_winkelId IN INTEGER, p_datum IN DATE, p_periode IN INTEGER DEFAULT 21, p_bezorgen IN BOOLEAN,
o_open OUT BOOLEAN, o_besteltijden OUT SYS_REFCURSOR)
AS
v_foundWinkel NUMBER := 0;
v_winkel Winkel%ROWTYPE;
TYPE arrayVarchar IS VARRAY(7) OF VARCHAR2(2);
v_dagen arrayVarchar := arrayVarchar('ma', 'di', 'wo', 'do', 'vr', 'za', 'zo');
CURSOR v_openingstijden(p_id IN NUMBER, p_dag IN VARCHAR2) IS
SELECT * FROM Openingstijd
WHERE winkel_id = p_id
AND dag = p_dag;
v_besteltijden t_openingstijd := t_openingstijd();
v_eindUur NUMBER := 0;
v_eindMinuten NUMBER := 0;
v_beginUur NUMBER := 0;
v_beginMinuten NUMBER := 0;
-- Exceptions
v_winkelNotFound EXCEPTION;
BEGIN
-- Kijken of winkel wel bestaat.
SELECT COUNT(1) INTO v_foundWinkel
FROM Winkel
WHERE id = p_winkelId;
IF(v_foundWinkel = 0) THEN
RAISE v_winkelNotFound;
END IF;
-- Alle data krijgen van de winkel. Exclusief de openingstijden, producten en coupons.
SELECT * INTO v_winkel
FROM Winkel
WHERE id = p_winkelId;
FOR i_tijd IN v_openingstijden(p_winkelId,
v_dagen(TO_NUMBER(TO_CHAR(p_datum, 'D')))) LOOP
-- Instellen van huidige tijd om mee door te lopen.
v_eindUur := TO_NUMBER(TO_CHAR(i_tijd.gesloten, 'HH24'));
IF(p_bezorgen = true) THEN
IF(TO_NUMBER(TO_CHAR(i_tijd.gesloten, 'MI')) < 30) THEN
IF(TO_NUMBER(TO_CHAR(i_tijd.gesloten, 'MI')) < 15) THEN
v_eindMinuten := 30;
ELSE
v_eindMinuten := 45;
END IF;
v_eindUur := v_eindUur - 1;
ELSE
IF(TO_NUMBER(TO_CHAR(i_tijd.gesloten, 'MI')) < 45) THEN
v_eindMinuten := 0;
ELSE
v_eindMinuten := 15;
END IF;
END IF;
ELSE
IF(TO_NUMBER(TO_CHAR(i_tijd.gesloten, 'MI')) < 15) THEN
v_eindMinuten := 45;
v_eindUur := v_eindUur - 1;
ELSE
IF(TO_NUMBER(TO_CHAR(i_tijd.gesloten, 'MI')) < 30) THEN
v_eindMinuten := 0;
ELSIF(TO_NUMBER(TO_CHAR(i_tijd.gesloten, 'MI')) < 45) THEN
v_eindMinuten := 15;
ELSE
v_eindMinuten := 30;
END IF;
END IF;
END IF;
-- Begin tijd berekenen
v_beginUur := TO_NUMBER(TO_CHAR(i_tijd.open, 'HH24'));
IF(TO_NUMBER(TO_CHAR(i_tijd.open, 'MI')) < 15) THEN
v_beginMinuten := 0;
ELSIF(TO_NUMBER(TO_CHAR(i_tijd.open, 'MI')) < 30) THEN
v_beginMinuten := 15;
ELSIF(TO_NUMBER(TO_CHAR(i_tijd.open, 'MI')) < 45) THEN
v_beginMinuten := 30;
ELSE
v_beginMinuten := 45;
END IF;
-- Eerste uur vol maken.
IF(v_beginMinuten != 0) THEN
FOR i IN 1 .. ((60 - v_beginMinuten) / 15) LOOP
v_besteltijden.extend;
v_besteltijden(v_besteltijden.count) := TO_DATE( v_beginUur || ':' || v_beginMinuten * i, 'HH24:MI');
END LOOP;
v_beginUur := v_beginUur + 1;
v_beginMinuten := 0;
END IF;
-- Tot het laatste uur volmaken.
IF(v_beginUur != v_eindUur) THEN
FOR x IN 1 .. (v_eindUur - v_beginUur) LOOP
FOR i IN 1 .. 4 LOOP
v_besteltijden.extend;
IF(i = 1) THEN
v_besteltijden(v_besteltijden.count) := TO_DATE( v_beginUur || ':00', 'HH24:MI');
ELSE
v_besteltijden(v_besteltijden.count) := TO_DATE( v_beginUur || ':' || 15 * (i - 1), 'HH24:MI');
END IF;
END LOOP;
v_beginUur := v_beginUur + 1;
END LOOP;
END IF;
-- Laatste uur ook maar eens volmaken.
IF(v_beginUur = v_eindUur AND v_eindMinuten >= v_beginMinuten) THEN
FOR i IN 1 .. (v_eindMinuten / 15) + 1 LOOP
v_besteltijden.extend;
IF(i = 1) THEN
v_besteltijden(v_besteltijden.count) := TO_DATE( v_beginUur || ':00', 'HH24:MI');
ELSE
v_besteltijden(v_besteltijden.count) := TO_DATE( v_beginUur || ':' || 15 * (i - 1), 'HH24:MI');
END IF;
END LOOP;
END IF;
END LOOP;
-- Data terug in output cursor.
OPEN o_besteltijden FOR
SELECT * FROM TABLE(CAST(v_besteltijden AS t_openingstijd));
-- Is de winkel momenteel open?
FOR i_openingstijd IN v_openingstijden(p_winkelId,
v_dagen(TO_NUMBER(TO_CHAR(SYSDATE, 'D')))) LOOP
IF(o_open = true) THEN
IF(TO_NUMBER(TO_CHAR(SYSDATE, 'HH24MI')) >= TO_NUMBER(TO_CHAR(i_openingstijd.open, 'HH24MI'))
AND TO_NUMBER(TO_CHAR(SYSDATE, 'HH24MI')) <= TO_NUMBER(TO_CHAR(i_openingstijd.gesloten, 'HH24MI'))) THEN
o_open := true;
ELSE
o_open := false;
END IF;
END IF;
END LOOP;
EXCEPTION
WHEN v_winkelNotFound THEN
dbms_output.put_line('Winkel niet gevonden.');
END;
/
The problem here seems to be as follows:
You are setting o_besteltijden as a value selected from CAST(v_besteltijden AS t_openingstijd) where v_besteltijden is calculated from to_date
t_openingstijd is a table of date type (CREATE OR REPLACE TYPE t_openingstijd IS TABLE OF DATE;). So your answer will be in date format.
Instead if you need the timestamp type you have to make changes to these variables and get timestamp stored in them to get the output as desired date+time or time through formatting.
I am trying to understand how to work with binary data in postgresql (v 8.3).
Let's say I have a following table
Table "public.message"
Column | Type | Modifiers
---------+---------+-----------
id | integer |
message | bytea |
I would like to store a packet in the message field in this format:
version (1 byte), identifier (1 byte), epoch (4 bytes)
I would like to pack this data into the message field. Lets say I have version=1, identifier=8 and epoch=123456. How would I pack this data into the message field? How would I convert my integer values to hex.. or octal?
I also need to get the message back and parse it. I was looking at the get_byte function, unless there is another way to parse the data out..
Thanks!
Here is some sample code showing how to do it with server-side Perl. Annoyingly, pack/unpack are considered untrusted operations by PG so this has to be created with plperlu by a superuser and then access granted with GRANT EXECUTE to non superusers.
On the other hand, this choice of language makes it easy to deal with more complex packed structures, which is a significant advantage over code that would be based on the SQL get_bytes()/set_bytes() functions. See Perl's pack() features.
1) first step: define a SQL composite type representing an non-packed record.
create type comp as (a smallint, b smallint, c int);
2) make a function to pack the record value into bytea:
create function pack_comp(comp) returns bytea
as $body$
my $arg=shift;
my $retval = pack("CCL", $arg->{a},$arg->{b},$arg->{c});
# encode bytea according to PG doc. For PG>=9.0, use encode_bytea() instead
$retval =~ s!(\\|[^ -~])!sprintf("\\%03o",ord($1))!ge; # from PG doc
return $retval;
$body$ language plperlu;
3) make a function to unpack bytea into the composite type:
create or replace function unpack_comp(bytea) returns comp
as $body$
my $arg=shift;
# decode bytea according to PG doc. For PG>=9.0, use decode_bytea() instead
$arg =~ s!\\(?:\\|(\d{3}))!$1 ? chr(oct($1)) : "\\"!ge;
my ($v,$i,$e)= unpack("CCL", $arg);
return {"a"=>$v, "b"=>$i, "c"=>$e};
$body$ language plperlu;
4) usage:
# select encode(pack_comp((254,14,1000000)::comp), 'hex');
encode
--------------
fe0e40420f00
# select unpack_comp(decode('fe0e40420f00','hex'));
unpack_comp
------------------
(254,14,1000000)
# select * from unpack_comp(decode('fe0e40420f00','hex'));
a | b | c
-----+----+---------
254 | 14 | 1000000
So I was able to figure out how to do it in plpg
Here's the code to pack
CREATE FUNCTION pack_numeric_bytes(i_values NUMERIC[], i_byte_sizes NUMERIC[], i_big_endian BOOLEAN)
RETURNS BYTEA
DECLARE
v_bytes BYTEA := NULL;
v_start INTEGER := 1;
v_byte BYTEA;
v_byte_size INTEGER;
v_value NUMERIC;
v_binary_value TEXT;
v_num NUMERIC;
i INTEGER;
x INTEGER;
v_sql TEXT;
BEGIN
IF array_upper(i_values, 1) != array_upper(i_byte_sizes, 1) THEN
RETURN v_bytes;
END IF;
FOR x IN array_lower(i_values, 1) .. array_upper(i_values, 1) LOOP
/* Given value and size at x position */
v_byte_size := i_byte_sizes[x]::INTEGER;
v_value := i_values[x];
/* Convert number to binary form */
v_sql := $$SELECT $$|| v_value ||$$::bit($$|| v_byte_size*8 ||$$);$$;
EXECUTE v_sql INTO v_binary_value;
IF i_big_endian IS TRUE THEN
/* Convert each byte at a time */
FOR i IN 1 .. v_byte_size LOOP
/* Extract byte from our binary value.
Big endian starts at 1 and then increments of 8 */
v_byte := substring(v_binary_value, v_start, 8);
/* Convert binary 8 bits to an integer */
v_sql := $$SELECT B$$||quote_literal(v_byte)||$$::int8$$;
EXECUTE v_sql INTO v_num;
/* Build bytea of bytes */
v_bytes := COALESCE(v_bytes, '') || set_byte(E' '::BYTEA, 0, v_num::INTEGER);
v_start := v_start + 8;
END LOOP;
ELSE
/* Small endian is extracted starting from last byte */
v_start := (v_byte_size * 8) + 1;
/* Convert each byte at a time */
FOR i IN 1 .. v_byte_size LOOP
v_start := v_start - 8;
v_byte := substring(v_binary_value, v_start, 8);
/* Convert binary 8 bits to an integer */
v_sql := $$SELECT B$$||quote_literal(v_byte)||$$::int8$$;
EXECUTE v_sql INTO v_num;
/* Build bytea of bytes */
v_bytes := COALESCE(v_bytes, '') || set_byte(E' '::BYTEA, 0, v_num::INTEGER);
END LOOP;
END IF; /* END endian check */
v_start := 1;
END LOOP;
RETURN v_bytes;
END;
And here's the code to unpack:
CREATE OR REPLACE FUNCTION public.unpack_numeric_bytes(i_bytes bytea, i_byte_sizes INTEGER[], i_big_endian BOOLEAN)
RETURNS NUMERIC[]
SECURITY DEFINER AS
DECLARE
v_bytes BYTEA;
v_start INTEGER := 1;
v_byte_index INTEGER := 0;
v_bit_shift INTEGER := 0;
v_length INTEGER;
v_size INTEGER;
v_sum_byte_sizes INTEGER;
v_vals NUMERIC[] := '{}';
v_val BIGINT := 0;
i INTEGER;
x INTEGER;
v_sql TEXT;
BEGIN
v_sql := $$SELECT $$|| array_to_string(i_byte_sizes, '+')||$$;$$;
EXECUTE v_sql INTO v_sum_byte_sizes;
IF length(i_bytes) != v_sum_byte_sizes::INTEGER THEN
RETURN v_vals;
END IF;
/* Loop through values of bytea (split by their sizes) */
FOR x IN array_lower(i_byte_sizes, 1) .. array_upper(i_byte_sizes, 1) LOOP
v_size := i_byte_sizes[x];
v_bytes := substring(i_bytes, v_start, v_size);
v_length := length(v_bytes);
IF i_big_endian IS TRUE THEN
v_byte_index := v_length - 1;
FOR i IN 1..v_length LOOP
v_val := v_val + (get_byte(v_bytes, v_byte_index) << v_bit_shift);
v_bit_shift := v_bit_shift + 8;
v_byte_index := v_byte_index - 1;
END LOOP;
ELSE
FOR i IN 1..v_length LOOP
v_val := v_val + (get_byte(v_bytes, v_byte_index) << v_bit_shift);
v_bit_shift := v_bit_shift + 8;
v_byte_index := v_byte_index + 1;
END LOOP;
END IF;
v_vals := array_append(v_vals, v_val::NUMERIC);
/* Calculate next value start index */
v_start := v_start + v_size;
v_byte_index := 0;
v_bit_shift := 0;
v_val := 0;
END LOOP;
RETURN v_vals;
END;
I hope this will help someone.
Is there a function in PostgreSQL that can convert a base 10 number like 30 into a base 36 representation like u?
There are base-64 functions (such as encode) but nothing for base-36. But you could write one of your own or use this one:
CREATE OR REPLACE FUNCTION base36_encode(IN digits bigint, IN min_width int = 0) RETURNS varchar AS $$
DECLARE
chars char[];
ret varchar;
val bigint;
BEGIN
chars := ARRAY['0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'];
val := digits;
ret := '';
IF val < 0 THEN
val := val * -1;
END IF;
WHILE val != 0 LOOP
ret := chars[(val % 36)+1] || ret;
val := val / 36;
END LOOP;
IF min_width > 0 AND char_length(ret) < min_width THEN
ret := lpad(ret, min_width, '0');
END IF;
RETURN ret;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
I think you should ask yourself if the database is the right place for dealing with this sort of data formatting though, presentational issues like this might be better handled closer to final viewing level of your stack.
here's a version that can take numbers of any size, it uses the data type "numeric" which is the postgresql implementation of bignum.
CREATE OR REPLACE FUNCTION base36_encode(IN digits numeric, IN min_width int = 0) RETURNS text AS $$
DECLARE
chars char[] := ARRAY['0','1','2','3','4','5','6','7','8','9','A','B'
,'C','D','E','F','G','H','I','J','K','L','M','N'
,'O','P','Q','R','S','T','U','V','W','X','Y','Z' ] ;
ret text:='';
val numeric:= digits;
BEGIN
IF digits < 0 THEN
val := -val;
END IF;
WHILE val > 0 OR min_width > 0 LOOP
ret := chars[(mod(val,36))+1] || ret;
val := div(val,36);
min_width := min_width-1;
END LOOP;
IF digits < 0 THEN
ret := '-'||ret;
END IF;
RETURN ret;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
Modified Implementation
Modified from other implementation to increase the readability. Any kind of update or modification or suggestion appreciated to increase the readability.
CREATE OR REPLACE FUNCTION fn_base36_encode(IN base10 bigint)
RETURNS varchar
LANGUAGE plpgsql
AS $BODY$
DECLARE
base36 varchar := '';
intval bigint := abs(base10);
char0z char[] := regexp_split_to_array('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', '');
BEGIN
WHILE intval != 0 LOOP
base36 := char0z[(intval % 36)+1] || base36;
intval := intval / 36;
END LOOP;
IF base10 = 0 THEN base36 := '0'; END IF;
RETURN base36;
END;
$BODY$;