Pgpool problem with TLS, certificate verify failed - postgresql

Question

I have a pod running a PgPool image and a Postgresql cluster with writer and read-only instance. I am using PgPool just for loadbalancing read and write queries and have a single endpoint.
This all setup is working great, however when I try to setup TLS it does not work.
This what I have done:
Create a CA:
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: "*",
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour * 24 * 180),
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
IsCA: true,
}
caCertBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &caPrivateKey.PublicKey, caPrivateKey)
if err != nil {
return certResp{}, fmt.Errorf("could not generate ca certificate: %v", err)
}
caPEM := new(bytes.Buffer)
if err := pem.Encode(caPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: caCertBytes,
}); err != nil {
return certResp{}, fmt.Errorf("could not encode ca certificate: %v", err)
}
caPrivateKeyPEM := new(bytes.Buffer)
if err := pem.Encode(caPrivateKeyPEM, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(caPrivateKey),
}); err != nil {
return certResp{}, fmt.Errorf("could not encode ca key: %v", err)
}
Create server and key using the CA template and CA private key generated above:
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: "*",
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour * 24 * 180),
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return certResp{}, err
}
certBytes, err := x509.CreateCertificate(rand.Reader, &template, &caCertTemplate, &privateKey.PublicKey, caPrivateKey)
if err != nil {
return certResp{}, err
}
certPEM := new(bytes.Buffer)
if err := pem.Encode(certPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
}); err != nil {
return certResp{}, err
}
certPrivateKeyPEM := new(bytes.Buffer)
if err := pem.Encode(certPrivateKeyPEM, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
}); err != nil {
return certResp{}, err
}
Update pgpool.conf ( using master slave mode)
ssl=on
ssl_key = '/tls/server.key'
ssl_cert = '/tls/server.crt'
Updated postgresql.conf
ssl_key = '/tls/server.key'
ssl_cert = '/tls/server.crt'
ssl_ca_cert = '/tls/root.crt'
If I set those in postgresql and try to connect to the instance directly it will work:
psql "sslmode=verify-ca sslrootcert=root.crt host=localhost user=postgres port=5432"
psql (14.2 (Ubuntu 14.2-1.pgdg20.04+1), server 13.4 (Ubuntu 13.4-4.pgdg18.04+1))
SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, bits: 256, compression: off)
However if I try connecting via pgpool it does not and I get this error:
psql: error: connection to server at "localhost" (::1), port 5432 failed: ERROR: backend authentication failed
DETAIL: backend response with kind '' when expecting 'R'
And from the pgpool logs:
2022-04-07 06:06:28: pid 51: LOG: pool_ssl: "SSL_connect": "certificate verify failed"
2022-04-07 06:06:28: pid 51: LOG: pool_ssl: "SSL_connect": "certificate verify failed"
2022-04-07 06:06:28: pid 51: ERROR: backend authentication failed
2022-04-07 06:06:28: pid 51: DETAIL: backend response with kind '' when expecting 'R'
This is my pgpool config:
listen_addresses = '*'
port = 5432
socket_dir = '/var/run/pgpool'
pcp_listen_addresses = '*'
pcp_port = 9898
pcp_socket_dir = '/var/run/pgpool'
backend_hostname0 = '%v'
backend_port0 = 5432
backend_weight0 = 1
backend_flag0 = 'ALWAYS_PRIMARY|DISALLOW_TO_FAILOVER'
backend_hostname1 = '%v'
backend_port1 = 5432
backend_weight1 = 1
backend_flag1 = 'DISALLOW_TO_FAILOVER'
sr_check_period = 0
enable_pool_hba = off
backend_clustering_mode = 'streaming_replication'
num_init_children = 32
max_pool = 4
child_life_time = 300
child_max_connections = 0
connection_life_time = 0
client_idle_limit = 0
connection_cache = on
load_balance_mode = on
ssl = on
ssl_key = '/tls/server.key'
ssl_cert = '/tls/server.crt'
ssl_ca_cert = '/tls/root.crt'
failover_on_backend_error = off
allow_clear_text_frontend_auth = true
Seems there is something wrong with pgpool, but I am quite sure I am doing something wrong? Thanks