How to hash aggregated results in postgresql incrementally without concatenating everything first - postgresql

Question

I want to calculate a hash (sha256) of as set of results, I know how to do to it:
SELECT digest( string_agg(id_,':'), 'sha256') from mytable order by id_;
It works but it first concatenates all records and finally calculate the hash. I want it to be incremental to avoid possible memory problems with large datasets (millions of rows), something like:
SELECT digest_agg(id_, ':', 'sha256') from mytable order by id_;
Ensuring that the hash is calculated incrementally row by row on the way.

Answer 1

Ok, I have finally coded a c extension for this. I post it here in case it would be useful for someone.
/* ------- OS Includes ------------ */
#include <stdio.h>
#include <openssl/rsa.h>
#include <openssl/pem.h>
#include <openssl/err.h>
/* ----- PostgreSQL Includes -------*/
#include "postgres.h"
#include "fmgr.h"
#include "funcapi.h"
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
PG_MODULE_MAGIC;
typedef struct sha256_state
{
SHA256_CTX* sha256;
bool has_data;
} SHA256_STATE;
/* --------- Prototypes ---------- */
void _PG_init(void);
void _PG_fini(void);
Datum dv_sha256_agg_sfunc(PG_FUNCTION_ARGS);
Datum dv_sha256_agg_final(PG_FUNCTION_ARGS);
SHA256_STATE* sha256_init(void);
void sha256_update(SHA256_STATE* state, char* string, int32 string_size);
void sha256_final(SHA256_STATE* state, char outputBuffer[65]);
/* ------ Version1 convention ---- */
PG_FUNCTION_INFO_V1(dv_sha256_agg_sfunc);
PG_FUNCTION_INFO_V1(dv_sha256_agg_final);
// -----------------------------------------------------------
// Implementations
// -----------------------------------------------------------
void _PG_init(void) {/* Do nothing */}
void _PG_fini(void) {/* Do nothing */}
Datum dv_sha256_agg_sfunc(PG_FUNCTION_ARGS) {
SHA256_STATE* state;
bool is_first = PG_ARGISNULL(0);
text* string;
int32 string_size;
char* data;
char SEP[] = {'\n'}; // Hardcoded separator (can be improved)
if (is_first) {
// First iteration: Create the state
state = sha256_init();
}
else {
// Next iterations: Restore the state
state = (SHA256_STATE*) PG_GETARG_POINTER(0);
}
if (PG_ARGISNULL(1)) {
PG_RETURN_POINTER(state);
}
// Get current chunk
string = PG_GETARG_TEXT_PP(1);
string_size = VARSIZE_ANY_EXHDR(string);
data = VARDATA_ANY(string);
// Add separator
if (state->has_data) {
sha256_update(state, SEP, 1);
}
// Update state
sha256_update(state, data, string_size);
state->has_data = true;
// Updated state
PG_RETURN_POINTER(state);
}
Datum dv_sha256_agg_final(PG_FUNCTION_ARGS) {
SHA256_STATE* state;
char out[65];
text* hash = (text*) palloc(65 + VARHDRSZ);
if (PG_ARGISNULL(0)) {
PG_RETURN_NULL();
}
else {
state = (SHA256_STATE*) PG_GETARG_POINTER(0);
sha256_final(state, out);
SET_VARSIZE(hash, 65 + VARHDRSZ);
memcpy(VARDATA(hash), out, 65);
PG_RETURN_TEXT_P(hash);
}
}
SHA256_STATE* sha256_init() {
SHA256_STATE* state = (SHA256_STATE*) palloc(sizeof(SHA256_STATE));
state->sha256 = (SHA256_CTX*) palloc(sizeof(SHA256_CTX));
SHA256_Init(state->sha256);
state->has_data = false;
return state;
}
void sha256_update(SHA256_STATE* state, char* string, int32 string_size) {
SHA256_Update(state->sha256, string, string_size);
}
void sha256_final(SHA256_STATE* state, char outputBuffer[65]) {
int i;
unsigned char hash[SHA256_DIGEST_LENGTH];
SHA256_Final(hash, state->sha256);
for(i = 0; i < SHA256_DIGEST_LENGTH; i++) {
sprintf(outputBuffer + (i * 2), "%02x", hash[i]);
}
outputBuffer[64] = 0;
}
The aggregate definition:
CREATE FUNCTION dv_sha256_agg_sfunc(state internal, input text)
RETURNS internal AS 'MODULE_PATHNAME',
'dv_sha256_agg_sfunc'
LANGUAGE C VOLATILE;
CREATE FUNCTION dv_sha256_agg_final(state internal)
RETURNS text AS 'MODULE_PATHNAME',
'dv_sha256_agg_final'
LANGUAGE C VOLATILE;
CREATE AGGREGATE dv_sha256_agg(input text) (
SFUNC = dv_sha256_agg_sfunc,
STYPE = internal,
FINALFUNC = dv_sha256_agg_final
);
Test:
select
dv_sha256_agg(id_::text),
encode(digest(string_agg(id_::text,E'\n'),'sha256'),'hex')
from
generate_series(1,100) id_(id_);
Result
dv_sha256_agg | encode
------------------------------------------------------------------+------------------------------------------------------------------
4187fe63fa78d8b4333e6ffc9122e0273ddf90251ced32e1e5b398639c193c87 | 4187fe63fa78d8b4333e6ffc9122e0273ddf90251ced32e1e5b398639c193c87
(1 row)
Notes:
Separator is hardcoded as \n
Null values are ignored

Answer 2

In PostgreSQL you can create your own aggregate function using CREATE AGGREGATE. That way you can write digest_agg yourself.
The challenge is to come up with a good state transition function (SFUNC) that combines the aggregate of the previous values with the next value.

Answer 3

It is possible to implement such a function yourself. Without touching any C code, I wrote one up in plperlu. It is memory efficient, but horrifically slow. If you want it fast, you will probably need to code it up in C. I suspect that the getstate and putstate functions have had no performance consideration whatsoever, as they weren't intended to be used in loops like this.
CREATE LANGUAGE plperlu;
CREATE FUNCTION public.sha256_final(state text) RETURNS bytea
LANGUAGE plperlu
AS $_X$
use Digest::SHA;
my $sha=Digest::SHA->new(256);
if (defined $_[0]) { $sha->putstate($_[0])};
$sha->add($_[1]);
return encode_bytea($sha->digest());
$_X$;
CREATE FUNCTION public.sha256_sfunc(state text, input text) RETURNS text
LANGUAGE plperlu
AS $_X$
use Digest::SHA;
my $sha=Digest::SHA->new(256);
if (defined $_[0]) { $sha->putstate($_[0])};
$sha->add($_[1]);
return $sha->getstate();
$_X$;
CREATE AGGREGATE public.perl_sha256(input text) (
SFUNC = public.sha256_sfunc,
STYPE = text,
FINALFUNC = public.sha256_final
);
--The built in sha256 on pre-aggregate gives the same answer...
select sha256(string_agg::bytea) from (select string_agg(id_::text,'') from generate_series(1,101) id_(id_)) foobar;
--- ...as my own streaming aggregate.
select perl_sha256(id_::text) from generate_series(1,101) id_(id_);
Of course you have to make sure rows are delivered in the correct order. And I didn't implement the ':' delimiter from string_agg, just ''.