It's been few weeks since I joined in Gophers team. So far so good.
I started a new project using a fiber web framework to build backend APIs.
I am using MongoDB as my database.
database/db.go
package database
import (
"context"
"log"
"time"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
var DB *mongo.Database
// InitMongo : Initialize mongodb...
func connectToMongo() {
log.Printf("Initializing database")
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://localhost:27017"))
if err != nil {
log.Fatal("Could not able to connect to the database, Reason:", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
err = client.Connect(ctx)
if err != nil {
log.Fatal("Context error, mongoDB:", err)
}
//Cancel context to avoid memory leak
defer cancel()
defer client.Disconnect(ctx)
// Ping our db connection
err = client.Ping(context.Background(), readpref.Primary())
if err != nil {
log.Fatal("Ping, mongoDB:", err)
}
log.Printf("Database connected!")
// Create a database
DB = client.Database("golang-test")
return
}
// In Golang, init() functions always initialize whenever the package is called.
// So, whenever DB variable called, the init() function initialized
func init() {
connectToMongo()
}
controllers/mongo.controller/mongo.controller.go
package mongocontroller
import (
"log"
"github.com/gofiber/fiber/v2"
service "gitlab.com/.../services/mongoservice"
)
// GetPersons godoc
// #Summary Get persons.
// #Description Get persons
// #Tags persons
// #Produce json
// #Success 200 {object} []service.Person
// #Failure 400 {object} httputil.HTTPError
// #Failure 404 {object} httputil.HTTPError
// #Failure 500 {object} httputil.HTTPError
// #Router /v1/persons [get]
func GetPersons(c *fiber.Ctx) error {
res, err := service.GetPersons()
if err != nil {
log.Fatal("ERROR: in controller...", err)
}
return c.JSON(res)
}
services/mongoservice/mongo.service.go
package mongoservice
import (
"context"
"log"
database "gitlab.com/.../database"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// Person : ...
type Person struct {
ID primitive.ObjectID `bson:"_id,omitempty" json:"_id,omitempty"`
Name string `bson:"name,omitempty" json:"name,omitempty"`
Age int `bson:"age,omitempty" json:"age,omitempty"`
}
func GetPersons() ([]Person, error) {
ctx := context.Background()
persons := []Person{}
log.Printf("mongo data...", ctx)
cur, err := database.DB.Collection("persons").Find(ctx, bson.M{})
if err != nil {
log.Fatal(err)
}
// Iterate through the returned cursor.
for cur.Next(ctx) {
var person Person
cur.Decode(&person)
persons = append(persons, person)
}
defer cur.Close(ctx)
return persons, err
}
Here is my data stored in the database:
The problem is, the line cur, err := database.DB.Collection("persons").Find(ctx, bson.M{}) from service always throwing Client is disconnected.
Any help is appreciated!
Thank you.
You are calling defer client.Disconnect(ctx) in the same function which is creating the connection (connectToMongo ) so it will close the connection after calling the function. You should return the connection and close after finishing your task. I mean these parts:
defer cancel()
defer client.Disconnect(ctx)
Related
I have a global instance to Postgres database which is being used throughout the code.
There are 500 goroutines inserting some data into the database using the same instance.
So when there is just one instance (client) why does it fail with panic: FATAL #53300 sorry, too many clients already?
My understanding is that, there is just one instance of the database which is being created by ConnectToDB() method when it is called for the first time & on the subsequent calls, it merely returns this instance instead of creating a new one.
Finally when main ends, that one single instance is closed.
I am not able to understand how could it possibly create too many instances as indicated by the error message.
package main
import (
"github.com/go-pg/pg/v10"
"github.com/go-pg/pg/v10/orm"
"sync"
)
var db *pg.DB
var once sync.Once
type DummyTable struct {
IdOne int
IdTwo int
}
func main() {
var wg sync.WaitGroup
db := ConnectToDB()
defer db.Close()
for i := 0; i < 500; i++ {
wg.Add(1)
go insertIntoDB(&wg)
}
wg.Wait()
}
func insertIntoDB(wg *sync.WaitGroup) {
defer wg.Done()
localDb := ConnectToDB()
_, err := localDb.Model(&DummyTable{
IdOne: 2,
IdTwo: 3,
}).Insert()
if err != nil {
panic(err)
}
}
func createSchema(db *pg.DB) error {
models := []interface{}{
(*DummyTable)(nil),
}
for _, model := range models {
err := db.Model(model).CreateTable(&orm.CreateTableOptions{
Temp: false,
})
if err != nil {
return err
}
}
return nil
}
func ConnectToDB() *pg.DB {
once.Do(func() {
db = pg.Connect(&pg.Options{
User: "username",
Database: "dbName",
})
err := createSchema(db)
if err != nil {
panic(err)
}
})
return db
}
Here I read it's necessary to cancel the context. This is how my db.go looks like:
package db
import (
"context"
"fmt"
"log"
"os"
"time"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
connectionStringTemplate = "mongodb://%s:%s#%s"
)
var DB *mongo.Client
var Ctx context.Context
// Connect with create the connection to MongoDB
func Connect() {
username := os.Getenv("MONGODB_USERNAME")
password := os.Getenv("MONGODB_PASSWORD")
clusterEndpoint := os.Getenv("MONGODB_ENDPOINT")
connectionURI := fmt.Sprintf(connectionStringTemplate, username, password, clusterEndpoint)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client, err := mongo.NewClient(options.Client().ApplyURI(connectionURI))
if err != nil {
log.Printf("Failed to create client: %v", err)
}
err = client.Connect(ctx)
if err != nil {
log.Printf("Failed to connect to cluster: %v", err)
}
// Force a connection to verify our connection string
err = client.Ping(ctx, nil)
if err != nil {
log.Printf("Failed to ping cluster: %v", err)
}
DB = client
Ctx = ctx
log.Printf("Connected to MongoDB!")
}
I execute this in the main.go:
func main() {
// Configure
db.Connect()
defer db.DB.Disconnect(context.Background())
r := gin.Default()
// Routes
//r.GET("/movies", handlers.GetAllMoviesHandler)
r.POST("/movies", handlers.AddMovieHandler)
// listen and serve on 0.0.0.0:8080
r.Run()
}
and it works fine if my local mongodb is running.
Now when I use the client like this (I know the naming is still bad):
func AddMovie(movie *Movie) (primitive.ObjectID, error) {
movie.ID = primitive.NewObjectID()
result, err := db.DB.Database("movies").Collection("movies").InsertOne(db.Ctx, movie)
if err != nil {
log.Printf("Could not create movie: %v", err)
return primitive.NilObjectID, err
}
oid := result.InsertedID.(primitive.ObjectID)
return oid, nil
}
I get an error like this
{"msg":{"Code":0,"Message":"connection(localhost:27017[-4]) failed to write: context canceled","Labels":["NetworkError","RetryableWriteError"],"Name":"","Wrapped":{"ConnectionID":"localhost:27017[-4]","Wrapped":{}}}}
It works fine when I put the defer cancel() in comment but I suppose this is not correct?. What am I doing wrong here.
Update:
It works when I use:
result, err :=db.DB.Database("movies").Collection("movies").InsertOne(context.Background(), movie) instead of ctx. I don't fully understand the usage of the context in these use case. I'm not sure if I should do the Ctx = ctx in the Connect function either.
As far as I understand, you have to disconnect from MongoDB after you finish using it, but I'm not entirely sure how to do it right
var collection *mongo.Collection
var ctx = context.TODO()
func init() {
clientOptions := options.Client().ApplyURI("mongodb://localhost:27017/")
client, err := mongo.Connect(ctx, clientOptions)
if err != nil {
log.Fatal(err)
}
//defer client.Disconnect(ctx)
err = client.Ping(ctx, nil)
if err != nil {
log.Fatal(err)
}
fmt.Println("Connected successfully")
collection = client.Database("testDB").Collection("testCollection") //create DB
}
There is commented out function call
defer client.Disconnect(ctx)
Which would work fine if all code happens in main() function, but since defer gets called right after init() executes, DB in main() function is already disconnected.
So the question is - what would be the right way to approach this case?
Your application needs the connected MongoDB client in your all services or repositories and therefore it is easier, if you have separated MongoDB client connect and disconnect functions in application package. You don't need to connect MongoDB client, if your server is starting, you can connect first if your services or repositories need the MongoDB client connection.
// db.go
package application
import (
"context"
"fmt"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"log"
"os"
)
var client *mongo.Client
func ResolveClientDB() *mongo.Client {
if client != nil {
return client
}
var err error
// TODO add to your .env.yml or .config.yml MONGODB_URI: mongodb://localhost:27017
clientOptions := options.Client().ApplyURI(os.Getenv("MONGODB_URI"))
client, err = mongo.Connect(context.Background(), clientOptions)
if err != nil {
log.Fatal(err)
}
// check the connection
err = client.Ping(context.Background(), nil)
if err != nil {
log.Fatal(err)
}
// TODO optional you can log your connected MongoDB client
return client
}
func CloseClientDB() {
if client == nil {
return
}
err := client.Disconnect(context.TODO())
if err != nil {
log.Fatal(err)
}
// TODO optional you can log your closed MongoDB client
fmt.Println("Connection to MongoDB closed.")
}
In main:
func main() {
// TODO add your main code here
defer application.CloseClientDB()
}
In your repositories or services you can get now your MongoDB client easily:
// account_repository.go
// TODO add here your account repository interface
func (repository *accountRepository) getClient() *mongo.Client {
if repository.client != nil {
return repository.client
}
repository.client = application.ResolveClientDB()
return repository.client
}
func (repository *accountRepository) FindOneByFilter(filter bson.D) (*model.Account, error) {
var account *model.Account
collection := repository.getClient().Database("yourDB").Collection("account")
err := collection.FindOne(context.Background(), filter).Decode(&account)
return account, err
}
edit: solved, see first answer. Should have really noticed that while I was creating this example..
I'm trying to create a function that takes an interface instead of a specific type and calls the FindOne function. Does anyone know why the printFirstTrainerByInterface function does not work properly?
I'm using the official Go Mongo-Driver and sample snippets from mongodb-go-driver-tutorial.
package main
import (
"context"
"fmt"
"log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type Trainer struct {
Name string
Age int
City string
}
var db *mongo.Database
func main() {
opts := options.Client().ApplyURI("mongodb://localhost:27017")
client, err := mongo.Connect(context.TODO(), opts)
if err != nil {
log.Fatal(err)
}
err = client.Ping(context.TODO(), nil)
if err != nil {
log.Fatal(err)
}
db = client.Database("test")
insertTestDocument()
var result Trainer
printFirstTrainer(result)
var result2 Trainer
printFirstTrainerByInterface(&result2)
}
func insertTestDocument() {
ash := Trainer{"Ash", 10, "Pallet Town"}
res, err := db.Collection("trainers").InsertOne(context.TODO(), ash)
if err != nil {
log.Fatal(err)
}
fmt.Println("Inserted a test document: ", res.InsertedID)
}
func printFirstTrainer(result Trainer) {
collection := db.Collection("trainers")
err := collection.FindOne(context.TODO(), bson.M{}).Decode(&result)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found a single document: %+v\n", result)
}
func printFirstTrainerByInterface(result interface{}) {
collection := db.Collection("trainers")
err := collection.FindOne(context.TODO(), bson.M{}).Decode(&result)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found a single document: %+v\n", result)
}
Output:
Inserted a test document: ObjectID("5e8216f74f41a13f01061d61")
Found a single document: {Name:Ash Age:10 City:Pallet Town}
Found a single document: [{Key:_id Value:ObjectID("5e8216f74f41a13f01061d61")} {Key:name Value:Ash} {Key:age Value:10} {Key:city Value:Pallet Town}]
You are passing in the address of the struct you want to decode into as an interface. You have to pass that as the argument to decode, not the address of the interface. Try:
err := collection.FindOne(context.TODO(), bson.M{}).Decode(result)
I'm new to Golang, i have been created an api in Golang and MongoDB.
After a hard struggle successfully separate the controller and model packages ,Now i want to define routes in a separate package of routers and access them in main package same like controllers and models.I'm using gorilla/mux package for routing.Anyone can help me please, thanks in Advance!
and here is all of my code:
RESTMONGOMVC/main.go
package main
import (
"RESTMONGOMVC/controllers"
"log"
"net/http"
"github.com/gorilla/mux"
"gopkg.in/mgo.v2"
)
var (
session *mgo.Session
collection *mgo.Collection
err error
)
func getSession() *mgo.Session {
// Connect to our local mongo
s, err := mgo.Dial("mongodb://localhost")
// Check if connection error, is mongo running?
if err != nil {
panic(err)
}
// Deliver session
return s
}
func main() {
var err error
r := mux.NewRouter()
uc := controllers.NewNoteController(getSession())
r.HandleFunc("/api/notes", uc.GetNotes).Methods("GET")
r.HandleFunc("/api/notes", uc.CreateNote).Methods("POST")
r.HandleFunc("/api/notes/{id}", uc.UpdateNote).Methods("PUT")
r.HandleFunc("/api/notes/{id}", uc.DeleteNote).Methods("DELETE")
http.Handle("/api/", r)
http.Handle("/", http.FileServer(http.Dir(".")))
log.Println("Starting Mongodb Session")
session, err = mgo.Dial("localhost")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
collection = session.DB("notesdb").C("notes")
log.Println("Listening on 8080")
http.ListenAndServe(":8080", nil)
}
controllers/note.go
package controllers
import (
"RESTMONGOMVC/models"
"encoding/json"
"log"
"net/http"
"time"
"github.com/gorilla/mux"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
var (
session *mgo.Session
collection *mgo.Collection
err error
)
type (
// UserController represents the controller for operating on the User resource
NoteController struct {
session *mgo.Session
}
)
// NewUserController provides a reference to a UserController with provided mongo session
func NewNoteController(s *mgo.Session) *NoteController {
return &NoteController{s}
}
func (uc NoteController) GetNotes(w http.ResponseWriter, r *http.Request) {
var notes []models.Note
iter := collection.Find(nil).Iter()
result := models.Note{}
for iter.Next(&result) {
notes = append(notes, result)
}
w.Header().Set("Content-Type", "application/json")
j, err := json.Marshal(models.NotesResource{Notes: notes})
if err != nil {
panic(err)
}
w.Write(j)
}
func (uc NoteController) CreateNote(w http.ResponseWriter, r *http.Request) {
var noteResource models.NoteResource
err := json.NewDecoder(r.Body).Decode(¬eResource)
if err != nil {
panic(err)
}
note := noteResource.Note
//get a new Id
obj_id := bson.NewObjectId()
note.Id = obj_id
note.CreatedOn = time.Now()
//Insert into document collection
err = collection.Insert(¬e)
if err != nil {
panic(err)
} else {
log.Printf("Inserted New Record with Title :%s", note.Title)
}
j, err := json.Marshal(models.NoteResource{Note: note})
if err != nil {
panic(err)
}
w.Header().Set("Content-Type", "application/json")
w.Write(j)
}
func (uc NoteController) UpdateNote(w http.ResponseWriter, r *http.Request) {
var err error
//get id from incoming url
vars := mux.Vars(r)
id := bson.ObjectIdHex(vars["id"])
//decode the incoming Note into json
var noteResource models.NoteResource
err = json.NewDecoder(r.Body).Decode(¬eResource)
if err != nil {
panic(err)
}
//partial update on mongodb
err = collection.Update(bson.M{"_id": id},
bson.M{"$set": bson.M{
"title": noteResource.Note.Title,
"decription": noteResource.Note.Description,
}})
if err == nil {
log.Printf("Updated Note : %s", id, noteResource.Note.Title)
} else {
panic(err)
}
w.WriteHeader(http.StatusNoContent)
}
func (uc NoteController) DeleteNote(w http.ResponseWriter, r *http.Request) {
var err error
vars := mux.Vars(r)
id := vars["id"]
//Remove from database
err = collection.Remove(bson.M{"_id": bson.ObjectIdHex(id)})
if err != nil {
log.Printf("Could not find the Note %s to delete", id)
}
w.WriteHeader(http.StatusNoContent)
}
models/note.go
package models
import (
"time"
"gopkg.in/mgo.v2/bson"
)
type Note struct {
Id bson.ObjectId `bson:"_id" json:"id"`
Title string `json:"title"`
Description string `json:"description"`
CreatedOn time.Time `json:"craetedOn"`
}
type NoteResource struct {
Note Note `json:"note"`
}
type NotesResource struct {
Notes []Note `json:"notes"`
}
Not an programming expert but this is how I manage my routes/handlers.
routes/routes.go
package routes
import (
"github.com/gorilla/mux"
)
//NewRouter is main routing entry point
func NewRouter() *mux.Router {
r := mux.NewRouter()
indexHandler(r) // Index handler
fileServer(r) // Fileserver to serve static files
otherLogicalHandler(r) // Other domain/business logic scoped handler
return r
}
routes/indexHandler.go
package routes
import (
"fmt"
"net/http"
"github.com/gorilla/mux"
"github.com/myusername/project/models"
)
func indexHandler(r *mux.Router) {
r.HandleFunc("/", indexMainHandler).Methods("GET")
// Other endpoints goes there if you want to list it in this current indexHandler.go file
// Example: r.HandleFunc("/signup", signupMainHandler).Methods("GET")
}
// Handlers
func indexMainHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=UTF-8")
// Call your model/s there
mydata, err := models.GetMyDataFunction()
if err != nil {
// Handle your error there
return
}
utils.ExecuteTemplate(w, "index.html", struct {
Title string
// Use your model data for templates there
MyData []models.MyData
// Other models/data can go there if multiple data objects used per page.
}{
Title: "Main Page",
MyData: mydata,
})
}
// func signupMainHandler(w http.ResponseWriter, r *http.Request) ...
// Basically repeat the same logic as in indexMainHandler function
routes/fileServer.go
package routes
import (
"net/http"
"github.com/gorilla/mux"
)
func fileServer(r *mux.Router) {
fs := http.FileServer(http.Dir("static"))
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", fs))
}
routes/otherLogicalHandler.go
... and so on.
As you can see, all they belong to package routes but are divided into multiple files. File names doesn't actually matter. You can name them as you want.
Models lives in models directory and also belongs to single package models package.
Every time you create new routes file, remember to call it in routes.go file.
Hope this will help for somebody.