how to connect golang in gitlab to mongodb container in gitlab-ci - mongodb

I want to connect my golang code (without a container) to the monogdb container, when it is local it works.
when I push it to gitlab.ci using container, connection refused
previously I used to use testing in dockerfile, but I don't use that.
the code is like this
image: docker:latest
services:
- docker:dind
stages:
- test
variables:
REPO_NAME: $REPO_URL
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
test:
stage: test
before_script:
- apk add go make bash docker-compose
# - make service-up-test
script:
- make mongodb-test-up
- go clean -testcache && go test -v ./app/test
and golang test :
package codetify
import (
"context"
"log"
"testing"
"github.com/stretchr/testify/assert"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
var credential = options.Credential{
Username: "usernametest",
Password: "passwordtest",
}
var addr = "mongodb://0.0.0.0:27018"
func InitMongoDB() *mongo.Database {
clientOpts := options.Client().ApplyURI(addr).SetAuth(credential)
clientOpts.SetDirect(true)
client, err := mongo.Connect(context.TODO(), clientOpts)
if err != nil {
log.Println("client", client)
return nil
}
return client.Database("databasetest")
}
func TestPingMongoDBServer(t *testing.T) {
clientOpts := options.Client().ApplyURI(addr).SetAuth(credential)
clientOpts.SetDirect(true)
client, err := mongo.Connect(context.TODO(), clientOpts)
assert.Equal(t, err, nil, "Shoudl be not error")
err = client.Ping(context.Background(), readpref.Primary())
assert.Equal(t, err, nil, "Shoudl be not error")
}

[Next]
this is a docker-compose.yml for mongodb, for testing, i use another port
version: '3'
services:
database:
image: 'mongo:latest'
container_name: '${APP_NAME}-mongodb-test'
environment:
MONGO_INITDB_ROOT_USERNAME: usernametest
MONGO_INITDB_ROOT_PASSWORD: passwordtest
MONGO_INITDB_DATABASE: databasetest
command: mongod
ports:
- '27018:27017'
restart: always
volumes:
- ./resources/mongo-initdb.js:/docker-entrypoint-initdb.d/mongo-initdb.js
networks:
- codetify-net-test
networks:
codetify-net-test

Related

Golang container cannot connect to postgres container in docker-compose environment

panic: dial tcp 192.168.0.2:5432: connect: connection refused
package database
import (
"database/sql"
"fmt"
_ "github.com/lib/pq"
"WebApp/config"
)
func Connect() (*sql.DB, error) {
psqlInfo := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
config.DB_HOST, config.DB_PORT, config.DB_USER, config.DB_PASSWORD, config.DB_NAME)
db, err := sql.Open("postgres", psqlInfo)
if err != nil {
panic(err)
}
defer db.Close()
err = db.Ping()
if err != nil {
panic(err)
}
fmt.Printf("\nSuccessfully connected to database!\n")
return db, nil
}
version: '3'
services:
backend:
build:
context: ../RestAPI-Golang
dockerfile: Dockerfile.dev
environment:
- DB_USER=username
- DB_PASSWORD=password
- DB_NAME=default_database
- DB_PORT=5432
- DB_HOST=database
ports:
- "3000:3000"
volumes:
- ../RestAPI-Golang:/app
depends_on:
- database
database:
image: postgres
restart: always
volumes:
- ./db-data/:/var/lib/postgresql/data/
environment:
- POSTGRES_USER=username
- POSTGRES_PASSWORD=password
- POSTGRES_DB=default_database
ports:
- "5432:5432"
db-admin:
image: adminer
ports:
- 8080:8080
I've tried using local host instead of the database container name, using the container ip, and other options online but I have had no luck. I believe the docker container name resolves to the ip of the container? And since it's communication within the same docker network I do not need to expose any additional ports?
use wait-for.sh to docker-compose like this:
entrypoint:
[
"/app/wait-for.sh",
"postgres:5432",
"--",
"/app/start.sh"
]
Remove the port mapping to your host machine in the database service:
ports:
- "5432:5432"
When you bind port 5432 of the database container to your host machine, it’s blocking any other connections on 5432. When Docker tries to establish a database connection the ip aliased as “database” internally, that port in the container is already bound to your host machine.

docker-compose postgres dial error (dial tcp 172.23.0.3:5432: connect: connection refused)

I try to connect from backend container to postgres container.
Here is my docker-compose file:
version: "3.9"
services:
imgress-producer:
build:
context: ./producer
dockerfile: Dockerfile.producer
target: prod
container_name: imgress-producer
ports:
- 8080:8080
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
- DATABASE_PORT=${DATABASE_PORT}
depends_on:
- imgress-db
volumes:
- ./:/app
networks:
- imgress-network
imgress-db:
image: postgres
container_name: imgress-db
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
volumes:
- postgres-data:/var/lib/postgresql/data
ports:
- 5432:5432
networks:
- imgress-network
restart: always
volumes:
postgres-data:
networks:
imgress-network:
driver: bridge
The .env file:
POSTGRES_USER=postgres
POSTGRES_PASSWORD=root
POSTGRES_DB=imgress
DATABASE_HOST=imgress-db
DATABASE_PORT=5432
And here is how I try to connect to db:
package database
import (
"fmt"
"log"
"os"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
var GDB *gorm.DB
func ConnectDB() {
var err error
pgPort := os.Getenv("DATABASE_PORT")
pgHost := os.Getenv("DATABASE_HOST")
pgUser := os.Getenv("POSTGRES_USER")
pgPassword := os.Getenv("POSTGRES_PASSWORD")
pgName := os.Getenv("POSTGRES_DB")
configData := fmt.Sprintf("postgres://%v:%v#%v:%v/%v?sslmode=disable",
pgUser,
pgPassword,
pgHost,
pgPort,
pgName,
)
for i := 0; i < 5; i++ {
GDB, err = gorm.Open(postgres.Open(configData), &gorm.Config{})
if err == nil {
break
}
time.Sleep(10 * time.Second)
}
if err != nil {
log.Println("Producer: Error Connecting to Database")
} else {
log.Println("Producer: Connection Opened to Database")
}
}
So, in the last part, I retry until db container is ready. So it should log an error when db connection is unsuccessful. But instead, it fails to connect and logs a success.
imgress-producer | 2022/12/03 20:28:24 /go/pkg/mod/gorm.io/gorm#v1.24.1/gorm.go:206
imgress-producer | [error] failed to initialize database, got error failed to connect to `host=imgress-db user=postgres database=imgress`: dial error (dial tcp 172.23.0.3:5432: connect: connection refused)
imgress-producer | 2022/12/03 20:28:34 Producer: Connection Opened to Database
There is a lot of connection refused related questions asked on SO, but none of them helped me. So any kind of help is appreciated.
I simplified your code a little bit and I was able to make it work on my machine. Let's see it. The repo structure is:
.env
docker-compose.yaml
Dockerfile
main.go
Let's start with the main.go file.
main.go
package main
import (
"fmt"
"log"
"os"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
var GDB *gorm.DB
func ConnectDB() {
var err error
pgPort := os.Getenv("DATABASE_PORT")
pgHost := os.Getenv("DATABASE_HOST")
pgUser := os.Getenv("POSTGRES_USER")
pgPassword := os.Getenv("POSTGRES_PASSWORD")
pgName := os.Getenv("POSTGRES_DB")
configData := fmt.Sprintf("postgres://%v:%v#%v:%v/%v?sslmode=disable",
pgUser,
pgPassword,
pgHost,
pgPort,
pgName,
)
for i := 0; i < 5; i++ {
GDB, err = gorm.Open(postgres.Open(configData), &gorm.Config{})
if err == nil {
break
}
time.Sleep(1 * time.Second) // change back to 10s
}
if err != nil {
log.Println("Producer: Error Connecting to Database")
} else {
log.Println("Producer: Connection Opened to Database")
}
}
func main() {
ConnectDB()
}
No relevant changes here.
.env
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_DB=postgres
DATABASE_HOST=imgress-db
DATABASE_PORT=5432
Also here nothing to mention.
Dockerfile
FROM golang:1.19.3-alpine3.17 AS build
WORKDIR /go/src/app
COPY ./main.go ./main.go
RUN go mod init pgdockercompose
RUN go mod tidy
RUN go build -o ./bin/webserver ./main.go
FROM alpine:3.17
COPY --from=build /go/src/app/bin /go/bin
EXPOSE 8080
ENTRYPOINT go/bin/webserver
Here we used the multi-staged build to build and copy our Go program. In the build stage we used a bigger image to initialize a go module, restore the dependencies and, to build the source code. While in the leaner image we copy the outcome of our build process.
docker-compose.yaml
version: "3.9"
services:
imgress-producer:
build: "."
ports:
- 8080:8080
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
- DATABASE_PORT=${DATABASE_PORT}
depends_on:
- imgress-db
networks:
- imgress-network
imgress-db:
image: postgres
container_name: imgress-db
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DATABASE_HOST=${DATABASE_HOST}
ports:
- 5432:5432
restart: always
networks:
- imgress-network
networks:
imgress-network:
driver: bridge
For the sake of the demo, I'll leave out the volumes. It shouldn't big a pain to integrate also them too.
Hope that this clarifies a little bit your doubt!
A workaround solution.
if you looked here I bet you have been through a log of "changing hostname from localhost to DB". None of that worked for me. I have gone through every documentation for the docker-compose network but it just does not work. Here is the solution
app:
// other stuff
- network_mode: "host"
db:
- expose:
- "5432" // or the port you want
this will expose your DB to the host machine network and your app will connect to your DB through the host machine. No very ideal, but the best solution found so far. If anyone has a real solution and has tried it yourself. Please let me know.

I can't connect to my MongoDB, which I run in docker compose

Here is my docker compose:
version: "3.9"
services:
app:
container_name: app
build:
context: ./..
dockerfile: deployments/Dockerfile
env_file:
- ../configs/config.env
ports:
- ${APP_PORT:-8080}:8080
networks:
- network
restart: always
db:
image: mongo:latest
container_name: mongodb
restart: always
env_file:
- ../configs/config.env
ports:
- ${MONGODB_PORT:-27017}:27017
environment:
MONGO_INITDB_ROOT_USERNAME: ${MONGODB_ROOT_USER:-admin}
MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_ROOT_PASSWORD:-admin}
volumes:
- ../assets/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh
adminer:
image: adminer
container_name: db-adminer
restart: always
ports:
- ${ADMINER_PORT:-17860}:8080
networks:
- network
depends_on:
- db
networks:
network:
driver: bridge
Here is my mongo-init.sh file:
use newdb
db.createUser(
{
user: admin,
pwd: admin,
roles: [
{
role: "readWrite",
db: "newdb"
}
]
}
);
Here I am trying to connect to the database, but I get the following error: topology is connected or connecting
client, err := mongo.Connect(ctx, options.Client().ApplyURI(fmt.Sprintf("mongodb://%s:%s#%s:%s",
cfg.DbUsername, cfg.DbPassword, cfg.DbHost, cfg.DbPort)))
I don't understand what the error might be, since everything is fine in the env file and all the variables match the authentication parameters in mongo
check this out:
services:
productinfo:
container_name: product_cntnr
build: ./api-productinfo-service
image: arc1999/api-productinfo-service
ports:
- '8090:8090'
depends_on:
db:
condition: service_healthy
links:
- db
scraping:
build: ./api-scraping-service
image: arc1999/api-scraping-service
ports:
- '8080:8080'
depends_on:
db:
condition: service_healthy
links:
- db
db:
image: mongo:latest
container_name: mongo_db
# environment:
# MONGO_INITDB_ROOT_USERNAME: root
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
ports:
- 27017:27017
volumes:
- mongodb_data_container:/data/db
healthcheck:
test: echo 'db.runCommand("ping").ok'
interval: 10s
timeout: 10s
retries: 5
volumes:
mongodb_data_container:
func InitDb() {
host := "mongodb://mongo_db:27017"
rb := bson.NewRegistryBuilder()
rb.RegisterTypeMapEntry(bsontype.EmbeddedDocument, reflect.TypeOf(bson.M{}))
clientOptions := options.Client().ApplyURI(host).SetRegistry(rb.Build())
// Connect to MongoDB
client, err := mongo.Connect(context.TODO(), clientOptions)
if err != nil {
log.Panicln(err)
}
err = client.Ping(context.TODO(), nil)
if err != nil {
log.Panicln(err)
}
db = client.Database(os.Getenv("MONGO_DB_NAME"))
fmt.Println("Connected to MongoDB!")
}

How does one attach to just the primary node of a mongo instance?

This might be me misunderstanding how Mongo works/new Go dev - but I'm not able to connect to my mongo instance from Go. When I connect to my Mongo instance using Studio 3T, I can connect just fine, browse the tables, etc. But If I try to connect using the Go module, it complains about not being able to find all the nodes. Is it necessary for it to be able to access all nodes? I thought the replica set itself was supposed to handle the replication?
For example, I have this Go code:
package main
import (
"context"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
"log"
"time"
)
func main() {
log.Println("Hello World!")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://MongoInsance:27017"))
if err != nil {
log.Fatal("Failed to connect to Mongo DB", err)
}
if err := client.Ping(ctx, readpref.Primary()); err != nil {
log.Fatal("Not actually connected ", err)
}
res, err := client.ListDatabases(ctx, nil)
if err != nil{
log.Fatal("Failed to list databases")
}
for _, val := range res.Databases {
log.Println(val.Name)
}
defer disconnect(client, &ctx)
}
func disconnect(client *mongo.Client, ctx *context.Context) {
if err := client.Disconnect(*ctx); err != nil {
panic(err)
}
}
But the response I get when running said code gives the error:
Not actually connected server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: mqtt-ingester-db-mast:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup mqtt-ingester-db-mast: no such host }, { Addr: mqtt-ingester-db-rep1:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup mqtt-ingester-db-rep1: no such host }, { Addr: mqtt-ingester-db-rep2:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup mqtt-ingester-db-rep2: no such host }, ] }
Do I actually need to expose all the replica sets as well?
Currently I have the primary node and 2 secondary nodes running in docker on host MongoInstance, with the primary node attached to port 27017:
docker-compose.yml
services:
...
mqtt-ingester-db-mast:
container_name: mqtt-ingester-db-mast
restart: always
image: mongo:latest
ports:
- 27017:27017
volumes:
- 'mongo_main:/data/db'
entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ]
mqtt-ingester-db-rep1:
container_name: mqtt-ingester-db-rep1
restart: always
image: mongo:latest
expose:
- 27017
volumes:
- 'mongo_rep1:/data/db'
entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ]
mqtt-ingester-db-rep2:
container_name: mqtt-ingester-db-rep2
restart: always
image: mongo:latest
expose:
- 27017
volumes:
- 'mongo_rep2:/data/db'
entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ]
mqtt-ingester-setup:
image: mongo:latest
container_name: mqtt-ingester-setup
links:
- mqtt-ingester-db-mast:mqtt-ingester-db-mast
- mqtt-ingester-db-rep1:mqtt-ingester-db-rep1
- mqtt-ingester-db-rep2:mqtt-ingester-db-rep2
depends_on:
- mqtt-ingester-db-mast
- mqtt-ingester-db-rep1
- mqtt-ingester-db-rep2
volumes:
- ./mqtt-ingester:/scripts
restart: "no"
entrypoint: [ "bash", "/scripts/mqtt_ingester_setup.sh" ]
mqtt-ingester-explorer:
container_name: mqtt-ingester-explorer
restart: always
image: mongo-express:latest
ports:
- '8081:8081'
depends_on:
- mqtt-ingester-db-mast
links:
- mqtt-ingester-db-mast:mongo
...
Do I actually need to expose all the replica sets as well?
Yes. Clients need to see all nodes in a replica set, so they can fail over when master goes down.

Could not detect default username. Please provide one explicitly

Getting this error when conneccting golang container to postgres.
pq: Could not detect default username. Please provide one explicitly
I have tried many usernames and password combinations with no luck. What username combination should I be using?
docker-compose code is:
version: '3.6'
services:
postgres:
image: postgres:11.1-alpine
ports:
- '5432:5432'
#network_mode: bridge
container_name: postgres
environment:
POSTGRES_USER: 'user'
POSTGRES_PASSWORD: 'password'
POSTGRESS_DB: 'db_amex01'
volumes:
- ./init:/docker-entrypoint-initdb.d/
todo:
build: ./go_amex/
depends_on:
- "postgres"
restart: always
# ports:
# - "8000:8080"
./go_amex/main.go
db, err := gorm.Open("postgres", "host='postgres'&user=user#localhost&port=5432&dbname='todo'&password='password'&sslmode=disable")
if err != nil {
fmt.Println("============ exiting ==========")
fmt.Println(err)
panic("failed to connect database cc")
}
defer db.Close()
Github code for the challenge
Try this:
version: '3'
services:
postgres:
image: kartoza/postgis:9.6-2.4
ports:
- "5432:5432"
environment:
POSTGRES_USER: developer
POSTGRES_DBNAME: somedb
POSTGRES_PASS: somepass
volumes:
- $HOME/data/postgres:/var/lib/postgresql/data
config for database in yaml
sqlBind := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable",
"developer",
"somepass",
"somedb",
)