PostgresDB init in docker-compose - postgresql

I want init my PostgresDb during starting Docker environment.
docker-compose.yml :
version: '3.1'
services:
app:
container_name: springboot-postgresql
image: sringboot-postgresql
build:
context: .
dockerfile: ./java/Dockerfile
ports:
- "8080:8080"
depends_on:
- posgresqldb
posgresqldb:
image: postgres
build:
context: .
dockerfile: ./pg/Dockerfile
ports:
- "5432:5432"
environment:
- POSTGRES_PASSWORD=postgres
- POSTGRES_USER=postgres
- POSTGRES_DB=postgres
volumes:
- ./postgres-data:/var/lib/postgresql/data
- ./pg/init.sql:/docker-entrypoint-initdb.d/init.sql
Dockerfile:
FROM postgres:12-alpine
COPY pg/init.sql /docker-entrypoint-initdb.d/
init.sql:
CREATE USER docker;
CREATE DATABASE postgres;
CREATE TABLE public.usr (
id varchar NOT NULL,
username varchar NOT NULL,
"password" varchar NOT NULL,
active bool NULL,
email varchar NULL,
authorities varchar NULL,
accountnonexpired bool NULL,
accountnonlocked bool NULL,
credentialsnonexpired bool NULL,
enabled bool NULL,
CONSTRAINT id_pkey PRIMARY KEY (id)
);
GRANT ALL PRIVILEGES ON DATABASE postgres TO docker;
INSERT INTO public.usr
(id, username, "password", active, email, authorities, accountnonexpired, accountnonlocked, credentialsnonexpired, enabled)
VALUES('1', 'User_1', '*****', false, '', '', false, false, false, false);
init.db mounts succeeds and Docker containers create successfully, but I don't have changes in DB.
Can tou tell me what should I do else. Do I need any additional commands to initialize init.sql?
Thank you.

You don't need to copy the init.sql in Dockerfile since you already mount it inside docker-compose.yml
The container only checks docker-entrypoint-initdb.d if the database is not already initialized. That means the data folder should be empty for the init script to be run.
In your case, make sure the host directory ./postgres-data is empty before you run docker-compose up. As long as data is present in this folder then postgres will use it and not try to initialize the database again.

Related

Running keycloak along with postgres db via docker compose

I currently have a docker compose file which sets up a psql database and runs around 80 sql files to create the relations for my application. What I want to do now is add a keycloak layer to the application. I'm trying to do this via the compose file as well but I keep getting an error saying that one of my relation already exists. What I think is happening is that when I keycloak service is running its trying to overwrite the database and I don't want that to happen. I'm not sure how to fix this, I've tried a bunch of other methods found online but none of them work.
Here is my compose file (removing some of the sql scripts to make it simpler):
---
version: "3.8"
services:
database:
container_name: database
environment:
# add multiple schemas
# POSTGRES_MULTIPLE_DATABASES: ${POSTGRESQL_DATABASE},${POSTGRESQL_KEYCLOAK_DATABASE}
POSTGRES_USER: ${POSTGRESQL_USER}
POSTGRES_PASSWORD: ${POSTGRESQL_PASSWORD}
POSTGRES_DB: ${POSTGRESQL_DATABASE}
# POSTGRES_KEYCLOAK_USER: ${POSTGRESQL_KEYCLOAK_USER}
# POSTGRES_KEYCLOAK_PASSWORD: ${POSTGRESQL_KEYCLOAK_PASSWORD}
# POSTGRES_DB2: ${POSTGRESQL_KEYCLOAK_DATABASE}
hostname: local
image: postgres:12
restart: always
volumes:
- /pgdata
- ./sql/access_attempt.sql:/docker-entrypoint-initdb.d/A.sql
- ./sql/bceid.sql:/docker-entrypoint-initdb.d/B.sql
- ./sql/lookup_activitytype.sql:/docker-entrypoint-initdb.d/C.sql
- ./sql/lookup_gender_pronoun.sql:/docker-entrypoint-initdb.d/D.sql
- ./sql/client.sql:/docker-entrypoint-initdb.d/E.sql
ports:
- "5432:5432"
networks:
- db-keycloak
backend:
container_name: backend
entrypoint:
- "sh"
- "-c"
- "npm i && npm run start"
environment:
NODE_ENV: development
POSTGRESQL_HOST: ${POSTGRESQL_HOST}
POSTGRESQL_USER: ${POSTGRESQL_USER}
POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3000"]
interval: 1m30s
timeout: 10s
retries: 3
start_period: 40s
hostname: backend
image: node:14
links:
- database
ports:
- "3001:3000"
volumes:
- ./backend:/app:z
- /app/node_modules
working_dir: "/app"
keycloak:
image: jboss/keycloak:13.0.1
container_name: keycloak-service
environment:
DB_VENDOR: POSTGRES
DB_ADDR: database
DB_SCHEMA: public
DB_DATABASE: ${POSTGRESQL_DATABASE}
DB_USER: ${POSTGRESQL_USER}
DB_PASSWORD: ${POSTGRESQL_PASSWORD}
KEYCLOAK_USER: ${POSTGRESQL_KEYCLOAK_USER}
KEYCLOAK_PASSWORD: ${POSTGRESQL_KEYCLOAK_PASSWORD}
ports:
- "28080:8080"
depends_on:
- database
restart: always
links:
- database
networks:
- db-keycloak
frontend:
container_name: frontend
entrypoint:
- "sh"
- "-c"
- "npm i && npm run start"
environment:
NODE_ENV: development
BACKEND_URL: backend
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3000"]
interval: 1m30s
timeout: 10s
retries: 3
start_period: 40s
hostname: frontend
image: node:14
links:
- backend
ports:
- "3000:3000"
volumes:
- ./frontend:/app:z
- /app/node_modules
working_dir: "/app"
networks:
db-keycloak:
And here is the error:
18:05:57,142 ERROR [org.keycloak.connections.jpa.updater.liquibase.conn.DefaultLiquibaseConnectionProvider] (ServerService Thread Pool -- 69) Change Set META-INF/jpa-changelog-1.0.0.Final.xml::1.0.0.Final-KEYCLOAK-5461::sthorger#redhat.com failed. Error: ERROR: relation "client" already exists [Failed SQL: CREATE TABLE public.CLIENT (DTYPE VARCHAR(31) NOT NULL, ID VARCHAR(36) NOT NULL, ALLOWED_CLAIMS_MASK BIGINT, ENABLED BOOLEAN DEFAULT FALSE NOT NULL, FULL_SCOPE_ALLOWED BOOLEAN DEFAULT FALSE NOT NULL, NAME VARCHAR(255), NOT_BEFORE INT, PUBLIC_CLIENT BOOLEAN DEFAULT FALSE NOT NULL, SECRET VARCHAR(255), BASE_URL VARCHAR(255), BEARER_ONLY BOOLEAN DEFAULT FALSE NOT NULL, MANAGEMENT_URL VARCHAR(255), SURROGATE_AUTH_REQUIRED BOOLEAN DEFAULT FALSE NOT NULL, DIRECT_GRANTS_ONLY BOOLEAN DEFAULT FALSE NOT NULL, REALM_ID VARCHAR(36))]
18:05:57,142 ERROR
[org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider] (ServerService Thread Pool -- 69) Error has occurred while updating the database: liquibase.exception.MigrationFailedException: Migration failed for change set META-INF/jpa-changelog-1.0.0.Final.xml::1.0.0.Final-KEYCLOAK-5461::sthorger#redhat.com:
Reason: liquibase.exception.DatabaseException: ERROR: relation "client" already exists [Failed SQL: CREATE TABLE public.CLIENT (DTYPE VARCHAR(31) NOT NULL, ID VARCHAR(36) NOT NULL, ALLOWED_CLAIMS_MASK BIGINT, ENABLED BOOLEAN DEFAULT FALSE NOT NULL, FULL_SCOPE_ALLOWED BOOLEAN DEFAULT FALSE NOT NULL, NAME VARCHAR(255), NOT_BEFORE INT, PUBLIC_CLIENT BOOLEAN DEFAULT FALSE NOT NULL, SECRET VARCHAR(255), BASE_URL VARCHAR(255), BEARER_ONLY BOOLEAN DEFAULT FALSE NOT NULL, MANAGEMENT_URL VARCHAR(255), SURROGATE_AUTH_REQUIRED BOOLEAN DEFAULT FALSE NOT NULL, DIRECT_GRANTS_ONLY BOOLEAN DEFAULT FALSE NOT NULL, REALM_ID VARCHAR(36))]
According to error your existing postgres database already have table/relation named client. But Keycloak is trying to create client table of its own which is causing this error.
To fix this issue, I would suggest to use two different postgres databases/schemas, one for your application and other for keycloak instance.

Flyway migrating successfully but postgres says there are no relations using \dt

I'm currently using Docker Compose to setup Flyway to handle table creation and migration.
When all my docker images run, Flyway tells me it successfully validated my migrations but when I check the wallet database and list tables, no relations exist.
Here is my docker compose file:
version: '3'
services:
web:
build:
dockerfile: ./Dockerfile
container_name: wallet-api-web
depends_on:
- flyway
ports:
- "8080:8080"
db:
build:
dockerfile: ./utils/db.Dockerfile
container_name: postgres-db
environment:
POSTGRES_PASSWORD: testuser1234
POSTGRES_USER: test_user
POSTGRES_DB: wallet
image: postgres:13
ports:
- "5432:5432"
restart: always
flyway:
command: -url=jdbc:postgresql://db:5432/wallet -schemas=wallet -user=test_user -password=testuser1234 -connectRetries=60 migrate
container_name: flyway-migrations
depends_on:
- db
image: flyway/flyway
volumes:
- ./migration:/flyway/sql
Here is the dockerfile used to build web.
FROM node:15
ENV NODE_ENV=production
WORKDIR /app
COPY ["package.json", "package-lock.json*", "./"]
RUN npm install --production
COPY . .
RUN npm run build
ENTRYPOINT npm run start
Here is the Dockerfile used to build the database image.
FROM postgres:latest
ENV POSTGRES_DB wallet
COPY ../init.sql /docker-entrypoint-initdb.d/
Here is the init file used by the postgres image.
CREATE USER test_user;
CREATE DATABASE wallet;
GRANT ALL PRIVILEGES ON DATABASE wallet TO test_user;
Here is my migration file:
CREATE TABLE wallet.transaction_events (
transaction_version int,
transaction_type VARCHAR(200),
coins int,
transaction_id VARCHAR(500),
wallet_id VARCHAR(500) PRIMARY KEY
)
This is my current file structure (directories use - and files use +)
base directory
- migration
- sql
+ V1__transaction_events.sql
- utils
- db.Dockerfile
+ Dockerfile
+ init.sql
+ docker-compose.yml
+ package-lock.json
+ package.json
Any assistance would be greatly appreciated as i'm new to docker and flyway and this has been stumping me for awhile. If any follow up information is required ill be sticking around so feel free to ask. Thank you again.

SQL schema files not running on init in Dockererized PostgreSQL instance

I am trying to create table 'User' in my database with a docker container with PostgreSQL and pgAdmin on docker-compose up.
I am following this StackOveflow article here.
DockerFile
FROM postgres:latest
COPY schema.sql /docker-entrypoint-initdb.d/
docker-compose.yaml
version: "3.8"
services:
db:
build: .
restart: always
environment:
POSTGRES_DB: xxx
POSTGRES_USER: admin
POSTGRES_PASSWORD: xxx
PGDATA: /var/lib/postgresql/data
ports:
- "5432:5432"
volumes:
- db-data:/var/lib/postgresql/data
pgadmin:
image: dpage/pgadmin4:latest
restart: always
environment:
PGADMIN_DEFAULT_EMAIL: admin#xxx.com
PGADMIN_DEFAULT_PASSWORD: xxx
PGADMIN_LISTEN_PORT: 80
ports:
- "8080:80"
volumes:
- pgadmin-data:/var/lib/pgadmin
links:
- "db:pgsql-server"
volumes:
db-data:
pgadmin-data:
schema.sql
create Table Users (
id SERIAL,
uuid VARCHAR(100) NOT NULL
)
The container and admin console starts up fine. However, the table is not generated and cannot be found in pgAdmin.
I exec-ed into the container and the file schema.sql is well copied into /docker-entrypoint-initdb.d/ directory.
In the PostgreSQL Docker repository, the documentation mentions that my data directory has to be empty or the files in the init folder will not be run. Could this be my issue?
How can I create my User table in my dockerized PostgreSQL when I run the container?

How to create table postgresql when start by docker compose

I know this question is already asked and also answer given. But it is not working for me. I follow the same. My postgres container is running fine. I checked inside the container that /docker-entrypoint-initdb.d/init.sql exist.I used the following docker-compose.yml.
version: "3"
services:
postgres:
image: postgres:latest
network_mode: bridge
container_name: postgres
expose:
- 5432
ports:
- 5432:5432
volumes:
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
environment:
- POSTGRES_PASSWORD=admin
- POSTGRES_USER=postgres
- POSTGRES_DB=dev
restart: unless-stopped
# APP
profile_editor2:
build:
context: .
dockerfile: ./Dockerfile
network_mode: bridge
container_name: profile_editor2
volumes:
- ./image:/app/image
expose:
- 8080
ports:
- 8080:8080
restart: unless-stopped
depends_on:
- postgres
links:
- postgres
volumes:
postgres-data:
init.sql:-
create table sometable(id int);
No table created. I can see only the database is created. How do I create a table and also if possible insert some data into the table?
You can write your queries inside init.sql, in this squence:-
DROP DATABASE IF EXISTS test_db;
CREATE DATABASE test_db;
\c test_db;
CREATE TABLE Role(
RoleID SERIAL PRIMARY KEY,
RoleName varchar(50),
);
insert into Role(RoleName)
values ('Admin'),('User');

Copy csv into container to use in init.sql

i need to create a container with PostgreSQL and i have a file init.sql which contains the structure of the DB and at the end a command to copy data from a CSV.
My init.sql is:
CREATE SEQUENCE aditivo_id_seq;
CREATE TABLE public.aditivo
(
id BIGINT NOT NULL DEFAULT nextval ('aditivo_id_seq'::regclass),
clasificacion CHARACTER VARYING (200) NOT NULL,
descripcion CHARACTER VARYING (4000) NOT NULL,
id_aditivo CHARACTER VARYING (10) NOT NULL,
nombre CHARACTER VARYING (200) NOT NULL,
origen CHARACTER VARYING (40) NOT NULL,
peligro CHARACTER VARYING (200) NOT NULL,
CONSTRAINT aditivo_pkey PRIMARY KEY (id)
NOT DEFERRABLE INITIALLY IMMEDIATE,
CONSTRAINT uk_c1t6nik8nbqfei52k7rlo28lv UNIQUE (id_aditivo)
NOT DEFERRABLE INITIALLY IMMEDIATE
);
COPY aditivo (clasificacion, descripcion, id_aditivo, nombre, origen, peligro) FROM '/var/lib/postgresql/csvs/aditivos.csv' DELIMITER ',' CSV HEADER;
And my docker-compose.yml is:
version: "3.3"
services:
postgreSQL:
restart: on-failure
image: postgres:latest
volumes:
- ./postgres/aditivos.csv:/var/lib/postgresql/csvs
- ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
environment:
POSTGRES_USER: UALappDitivos
POSTGRES_PASSWORD: TFGappDitivosUAL!
POSTGRES_DB: db
ports:
- "5432:5432"
The init.sql is copied ok, but the csv no.
I get the error:
psql:/docker-entrypoint-initdb.d/init.sql:147: ERROR: could not open file "/var/lib/postgresql/csvs/user.csv" for reading: Not a directory
How can i do this correctly?
To solve this, i copy the folder instead to copy the file.csv. So my docker-compose.yml looks like:
version: "3.3"
services:
postgreSQL:
restart: on-failure
image: postgres:latest
volumes:
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
- ./postgres/csvs:/var/lib/postgresql/csvs
environment:
POSTGRES_USER: UALappDitivos
POSTGRES_PASSWORD: TFGappDitivosUAL!
POSTGRES_DB: db
ports:
- "5432:5432"
rest:
build: rest
restart: on-failure
depends_on:
- postgreSQL
ports:
- "8080:8080"