Running Postgres in GitHub Actions to test my Go API - postgresql

I've set up a GitHub workflow to start a Postgres instance inside a docker container. I then execute my web API to simply prove it connects. This satisfies the workflow.
My workflow
name: Build
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:10.8
env:
POSTGRES_USER: test
POSTGRES_PASSWORD: test
POSTGRES_DB: test
POSTGRES_PORT: 5432
ports:
- 5432/tcp
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
steps:
- uses: actions/checkout#v2
- name: Set up Go
uses: actions/setup-go#v2
with:
go-version: 1.15
- name: Run
env:
# These are the expected envs in my code
DB_HOST: localhost
DB_USER: test
DB_PASSWORD: test
DB_NAME: test
DB_PORT: 5432
DB_DIALECT: postgres
PORT: 8080
run: make run
For brevity I've trimmed irrelavent bits of code out, this is my database package:
// Config Model
type Config struct {
Host string
Name string
User string
Password string
Port string
Dialect string
}
// Open a new connection to a database
func Open(c Config) (*gorm.DB, error) {
dsn := fmt.Sprintf("host=%s port=%s dbname=%s password=%s user=%s sslmode=disable",
c.Host,
c.Port,
c.Name,
c.Password,
c.User,
)
db, err := gorm.Open(postgres.New(postgres.Config{
DriverName: c.Dialect,
DSN: dsn,
}), &gorm.Config{})
if err != nil {
return nil, fmt.Errorf("sql.Open: %v", err)
}
return db, nil
}
And this is called from my main package
func main() {
// Create Database Configuration
dbConfig := database.Config{
Host: os.Getenv("DB_HOST"),
Name: os.Getenv("DB_NAME"),
User: os.Getenv("DB_USER"),
Port: os.Getenv("DB_PORT"),
Password: os.Getenv("DB_PASSWORD"),
Dialect: os.Getenv("DB_DIALECT"),
}
// Connect to database
db, err := database.Open(dbConfig)
if err != nil {
log.Fatalf("failed to connect to database: %s", err)
}
}
The code fails to connect during the workflow with the given error:
go run -ldflags "-X main.Version=afc0042" cmd/api/main.go
2021/02/10 09:55:12 Running Version: afc0042
2021/02/10 09:55:12 failed to connect to database: sql.Open: dial tcp [::1]:5432: connect: connection refused
2021/02/10 09:55:12 /home/runner/work/database/database.go:32
[error] failed to initialize database, got error dial tcp [::1]:5432: connect: connection refused

you can use localhost when only you are trying to connect your server locally but as docker is not local in your machine so you can't use localhost except you are inside docker container.
try with port forward.
ports:
- 5432:5432 //update this one

Related

Golang container cannot connect to postgres container in docker-compose environment

panic: dial tcp 192.168.0.2:5432: connect: connection refused
package database
import (
"database/sql"
"fmt"
_ "github.com/lib/pq"
"WebApp/config"
)
func Connect() (*sql.DB, error) {
psqlInfo := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
config.DB_HOST, config.DB_PORT, config.DB_USER, config.DB_PASSWORD, config.DB_NAME)
db, err := sql.Open("postgres", psqlInfo)
if err != nil {
panic(err)
}
defer db.Close()
err = db.Ping()
if err != nil {
panic(err)
}
fmt.Printf("\nSuccessfully connected to database!\n")
return db, nil
}
version: '3'
services:
backend:
build:
context: ../RestAPI-Golang
dockerfile: Dockerfile.dev
environment:
- DB_USER=username
- DB_PASSWORD=password
- DB_NAME=default_database
- DB_PORT=5432
- DB_HOST=database
ports:
- "3000:3000"
volumes:
- ../RestAPI-Golang:/app
depends_on:
- database
database:
image: postgres
restart: always
volumes:
- ./db-data/:/var/lib/postgresql/data/
environment:
- POSTGRES_USER=username
- POSTGRES_PASSWORD=password
- POSTGRES_DB=default_database
ports:
- "5432:5432"
db-admin:
image: adminer
ports:
- 8080:8080
I've tried using local host instead of the database container name, using the container ip, and other options online but I have had no luck. I believe the docker container name resolves to the ip of the container? And since it's communication within the same docker network I do not need to expose any additional ports?
use wait-for.sh to docker-compose like this:
entrypoint:
[
"/app/wait-for.sh",
"postgres:5432",
"--",
"/app/start.sh"
]
Remove the port mapping to your host machine in the database service:
ports:
- "5432:5432"
When you bind port 5432 of the database container to your host machine, it’s blocking any other connections on 5432. When Docker tries to establish a database connection the ip aliased as “database” internally, that port in the container is already bound to your host machine.

docker-compose postgres dial error (dial tcp 172.23.0.3:5432: connect: connection refused)

I try to connect from backend container to postgres container.
Here is my docker-compose file:
version: "3.9"
services:
imgress-producer:
build:
context: ./producer
dockerfile: Dockerfile.producer
target: prod
container_name: imgress-producer
ports:
- 8080:8080
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
- DATABASE_PORT=${DATABASE_PORT}
depends_on:
- imgress-db
volumes:
- ./:/app
networks:
- imgress-network
imgress-db:
image: postgres
container_name: imgress-db
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
volumes:
- postgres-data:/var/lib/postgresql/data
ports:
- 5432:5432
networks:
- imgress-network
restart: always
volumes:
postgres-data:
networks:
imgress-network:
driver: bridge
The .env file:
POSTGRES_USER=postgres
POSTGRES_PASSWORD=root
POSTGRES_DB=imgress
DATABASE_HOST=imgress-db
DATABASE_PORT=5432
And here is how I try to connect to db:
package database
import (
"fmt"
"log"
"os"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
var GDB *gorm.DB
func ConnectDB() {
var err error
pgPort := os.Getenv("DATABASE_PORT")
pgHost := os.Getenv("DATABASE_HOST")
pgUser := os.Getenv("POSTGRES_USER")
pgPassword := os.Getenv("POSTGRES_PASSWORD")
pgName := os.Getenv("POSTGRES_DB")
configData := fmt.Sprintf("postgres://%v:%v#%v:%v/%v?sslmode=disable",
pgUser,
pgPassword,
pgHost,
pgPort,
pgName,
)
for i := 0; i < 5; i++ {
GDB, err = gorm.Open(postgres.Open(configData), &gorm.Config{})
if err == nil {
break
}
time.Sleep(10 * time.Second)
}
if err != nil {
log.Println("Producer: Error Connecting to Database")
} else {
log.Println("Producer: Connection Opened to Database")
}
}
So, in the last part, I retry until db container is ready. So it should log an error when db connection is unsuccessful. But instead, it fails to connect and logs a success.
imgress-producer | 2022/12/03 20:28:24 /go/pkg/mod/gorm.io/gorm#v1.24.1/gorm.go:206
imgress-producer | [error] failed to initialize database, got error failed to connect to `host=imgress-db user=postgres database=imgress`: dial error (dial tcp 172.23.0.3:5432: connect: connection refused)
imgress-producer | 2022/12/03 20:28:34 Producer: Connection Opened to Database
There is a lot of connection refused related questions asked on SO, but none of them helped me. So any kind of help is appreciated.
I simplified your code a little bit and I was able to make it work on my machine. Let's see it. The repo structure is:
.env
docker-compose.yaml
Dockerfile
main.go
Let's start with the main.go file.
main.go
package main
import (
"fmt"
"log"
"os"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
var GDB *gorm.DB
func ConnectDB() {
var err error
pgPort := os.Getenv("DATABASE_PORT")
pgHost := os.Getenv("DATABASE_HOST")
pgUser := os.Getenv("POSTGRES_USER")
pgPassword := os.Getenv("POSTGRES_PASSWORD")
pgName := os.Getenv("POSTGRES_DB")
configData := fmt.Sprintf("postgres://%v:%v#%v:%v/%v?sslmode=disable",
pgUser,
pgPassword,
pgHost,
pgPort,
pgName,
)
for i := 0; i < 5; i++ {
GDB, err = gorm.Open(postgres.Open(configData), &gorm.Config{})
if err == nil {
break
}
time.Sleep(1 * time.Second) // change back to 10s
}
if err != nil {
log.Println("Producer: Error Connecting to Database")
} else {
log.Println("Producer: Connection Opened to Database")
}
}
func main() {
ConnectDB()
}
No relevant changes here.
.env
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_DB=postgres
DATABASE_HOST=imgress-db
DATABASE_PORT=5432
Also here nothing to mention.
Dockerfile
FROM golang:1.19.3-alpine3.17 AS build
WORKDIR /go/src/app
COPY ./main.go ./main.go
RUN go mod init pgdockercompose
RUN go mod tidy
RUN go build -o ./bin/webserver ./main.go
FROM alpine:3.17
COPY --from=build /go/src/app/bin /go/bin
EXPOSE 8080
ENTRYPOINT go/bin/webserver
Here we used the multi-staged build to build and copy our Go program. In the build stage we used a bigger image to initialize a go module, restore the dependencies and, to build the source code. While in the leaner image we copy the outcome of our build process.
docker-compose.yaml
version: "3.9"
services:
imgress-producer:
build: "."
ports:
- 8080:8080
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
- DATABASE_PORT=${DATABASE_PORT}
depends_on:
- imgress-db
networks:
- imgress-network
imgress-db:
image: postgres
container_name: imgress-db
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
ports:
- 5432:5432
restart: always
networks:
- imgress-network
networks:
imgress-network:
driver: bridge
For the sake of the demo, I'll leave out the volumes. It shouldn't big a pain to integrate also them too.
Hope that this clarifies a little bit your doubt!
A workaround solution.
if you looked here I bet you have been through a log of "changing hostname from localhost to DB". None of that worked for me. I have gone through every documentation for the docker-compose network but it just does not work. Here is the solution
app:
// other stuff
- network_mode: "host"
db:
- expose:
- "5432" // or the port you want
this will expose your DB to the host machine network and your app will connect to your DB through the host machine. No very ideal, but the best solution found so far. If anyone has a real solution and has tried it yourself. Please let me know.

Docker-compose Postgres connection refused

I'm running Postgres DB with pg-admin and GO on the docker-compose.
Problem: I can connect from pg-admin to Postgres. But cannot establish a connection from Go.
I tried different combinations of authentication string but it does not work. String format same as here https://github.com/karlkeefer/pngr - but different container name - database
(ERROR) Connection URl:
backend_1 | 2021/08/08 14:24:40 DB connection: database://main:fugZwypczB94m0LP7CcH#postgres:5432/temp_db?sslmode=disable
backend_1 | 2021/08/08 14:24:40 Unalble to open DB connection: dial tcp 127.0.0.1:5432: connect: connection refused
(URI generation same as here https://github.com/karlkeefer/pngr)
Docker:
version: '3.8'
services:
backend:
restart: always
build:
context: backend
target: dev
volumes:
- ./backend:/root
ports:
- "5000:5000"
env_file: .env
depends_on:
- database
database:
build: database
restart: always
environment:
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
PGDATA: /var/lib/postgresql/data
volumes:
- ./database/data:/var/lib/postgresql/data
- ./logs/databse:/var/log/postgresql
- ./database/migrations:/docker-entrypoint-initdb.d/migrations
ports:
- "5432:5432"
database-admin:
image: dpage/pgadmin4:5.5
restart: always
environment:
PGADMIN_DEFAULT_EMAIL: ${PG_ADMIN_EMAIL}
PGADMIN_DEFAULT_PASSWORD: ${PG_ADMIN_PASSWORD}
PGADMIN_LISTEN_PORT: 80
ports:
- "8080:80"
volumes:
- ./database/admin:/var/lib/pgadmin
links:
- "database:pgsql-server"
depends_on:
- database
volumes:
database:
database-admin:
Environment:
POSTGRES_HOST=postgres
POSTGRES_PORT=5432
POSTGRES_DB=temp_db
POSTGRES_USER=main
POSTGRES_PASSWORD=fugZwypczB94m0LP7CcH
PG_ADMIN_EMAIL=admin#temp.com
PG_ADMIN_PASSWORD=ayzi2ta8f1TnX3vKQSN1
PG_ADMIN_PORT=80
GO Code:
db, err = sqlx.Open("postgres", str)
str
func buildConnectionString() string {
user := os.Getenv("POSTGRES_USER")
pass := os.Getenv("POSTGRES_PASSWORD")
if user == "" || pass == "" {
log.Fatalln("You must include POSTGRES_USER and POSTGRES_PASSWORD environment variables")
}
host := os.Getenv("POSTGRES_HOST")
port := os.Getenv("POSTGRES_PORT")
dbname := os.Getenv("POSTGRES_DB")
if host == "" || port == "" || dbname == "" {
log.Fatalln("You must include POSTGRES_HOST, POSTGRES_PORT, and POSTGRES_DB environment variables")
}
str := fmt.Sprintf("database://%s:%s#%s:%s/%s?sslmode=disable", user, pass, host, port, dbname)
log.Println("DB connection: " + str)
return str
}
Thanks in advance!
You reference the database hostname as postgres (POSTGRES_HOST=postgres) which is fine, but the container/service name is database.
Either change the name in your compose.yaml from database to postgres or add an explicit hostname field:
database:
build: database
restart: always
hostname: postgres # <- add this
You may also want to add a dedicated network for multiple container services to talk to one another (or prevent others from). To do this, add this to each service your want to use a specific network e.g.
database:
# ...
networks:
- mynet
backend:
# ...
networks:
- mynet
and define the network at the end of your compose.yaml
networks:
mynet:
name: my-shared-db-network

How does one attach to just the primary node of a mongo instance?

This might be me misunderstanding how Mongo works/new Go dev - but I'm not able to connect to my mongo instance from Go. When I connect to my Mongo instance using Studio 3T, I can connect just fine, browse the tables, etc. But If I try to connect using the Go module, it complains about not being able to find all the nodes. Is it necessary for it to be able to access all nodes? I thought the replica set itself was supposed to handle the replication?
For example, I have this Go code:
package main
import (
"context"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
"log"
"time"
)
func main() {
log.Println("Hello World!")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://MongoInsance:27017"))
if err != nil {
log.Fatal("Failed to connect to Mongo DB", err)
}
if err := client.Ping(ctx, readpref.Primary()); err != nil {
log.Fatal("Not actually connected ", err)
}
res, err := client.ListDatabases(ctx, nil)
if err != nil{
log.Fatal("Failed to list databases")
}
for _, val := range res.Databases {
log.Println(val.Name)
}
defer disconnect(client, &ctx)
}
func disconnect(client *mongo.Client, ctx *context.Context) {
if err := client.Disconnect(*ctx); err != nil {
panic(err)
}
}
But the response I get when running said code gives the error:
Not actually connected server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: mqtt-ingester-db-mast:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup mqtt-ingester-db-mast: no such host }, { Addr: mqtt-ingester-db-rep1:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup mqtt-ingester-db-rep1: no such host }, { Addr: mqtt-ingester-db-rep2:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup mqtt-ingester-db-rep2: no such host }, ] }
Do I actually need to expose all the replica sets as well?
Currently I have the primary node and 2 secondary nodes running in docker on host MongoInstance, with the primary node attached to port 27017:
docker-compose.yml
services:
...
mqtt-ingester-db-mast:
container_name: mqtt-ingester-db-mast
restart: always
image: mongo:latest
ports:
- 27017:27017
volumes:
- 'mongo_main:/data/db'
entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ]
mqtt-ingester-db-rep1:
container_name: mqtt-ingester-db-rep1
restart: always
image: mongo:latest
expose:
- 27017
volumes:
- 'mongo_rep1:/data/db'
entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ]
mqtt-ingester-db-rep2:
container_name: mqtt-ingester-db-rep2
restart: always
image: mongo:latest
expose:
- 27017
volumes:
- 'mongo_rep2:/data/db'
entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ]
mqtt-ingester-setup:
image: mongo:latest
container_name: mqtt-ingester-setup
links:
- mqtt-ingester-db-mast:mqtt-ingester-db-mast
- mqtt-ingester-db-rep1:mqtt-ingester-db-rep1
- mqtt-ingester-db-rep2:mqtt-ingester-db-rep2
depends_on:
- mqtt-ingester-db-mast
- mqtt-ingester-db-rep1
- mqtt-ingester-db-rep2
volumes:
- ./mqtt-ingester:/scripts
restart: "no"
entrypoint: [ "bash", "/scripts/mqtt_ingester_setup.sh" ]
mqtt-ingester-explorer:
container_name: mqtt-ingester-explorer
restart: always
image: mongo-express:latest
ports:
- '8081:8081'
depends_on:
- mqtt-ingester-db-mast
links:
- mqtt-ingester-db-mast:mongo
...
Do I actually need to expose all the replica sets as well?
Yes. Clients need to see all nodes in a replica set, so they can fail over when master goes down.

how to connect golang in gitlab to mongodb container in gitlab-ci

I want to connect my golang code (without a container) to the monogdb container, when it is local it works.
when I push it to gitlab.ci using container, connection refused
previously I used to use testing in dockerfile, but I don't use that.
the code is like this
image: docker:latest
services:
- docker:dind
stages:
- test
variables:
REPO_NAME: $REPO_URL
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
test:
stage: test
before_script:
- apk add go make bash docker-compose
# - make service-up-test
script:
- make mongodb-test-up
- go clean -testcache && go test -v ./app/test
and golang test :
package codetify
import (
"context"
"log"
"testing"
"github.com/stretchr/testify/assert"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
var credential = options.Credential{
Username: "usernametest",
Password: "passwordtest",
}
var addr = "mongodb://0.0.0.0:27018"
func InitMongoDB() *mongo.Database {
clientOpts := options.Client().ApplyURI(addr).SetAuth(credential)
clientOpts.SetDirect(true)
client, err := mongo.Connect(context.TODO(), clientOpts)
if err != nil {
log.Println("client", client)
return nil
}
return client.Database("databasetest")
}
func TestPingMongoDBServer(t *testing.T) {
clientOpts := options.Client().ApplyURI(addr).SetAuth(credential)
clientOpts.SetDirect(true)
client, err := mongo.Connect(context.TODO(), clientOpts)
assert.Equal(t, err, nil, "Shoudl be not error")
err = client.Ping(context.Background(), readpref.Primary())
assert.Equal(t, err, nil, "Shoudl be not error")
}
[Next]
this is a docker-compose.yml for mongodb, for testing, i use another port
version: '3'
services:
database:
image: 'mongo:latest'
container_name: '${APP_NAME}-mongodb-test'
environment:
MONGO_INITDB_ROOT_USERNAME: usernametest
MONGO_INITDB_ROOT_PASSWORD: passwordtest
MONGO_INITDB_DATABASE: databasetest
command: mongod
ports:
- '27018:27017'
restart: always
volumes:
- ./resources/mongo-initdb.js:/docker-entrypoint-initdb.d/mongo-initdb.js
networks:
- codetify-net-test
networks:
codetify-net-test