Using mgo with context - mongodb

I have been using mgo for my API but I'm seeing many current connections in my MongoDB (while using less than 5 devices for testing). By executing db.serverStatus().connections in my Mongo server I get: { "current" : 641, "available" : 838219, "totalCreated" : 1136 }. Below I transcript my issue in mgo's Github (Issue #429):
Is my way of using mgo in a web server the correct way? If not, can you give me a full example?
This code is not functional, take it as almost pseudo code (because of the missing parts like the imports or where the configs come from and models), but it is exactly how I am using mgo.
I must clarify that I'm building an API which is used by several mobile devices and webapps.
main.go
func main() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{config.DatabaseURL},
Timeout: 60 * time.Second,
Database: config.DatabaseName,
Username: config.DatabaseUsername,
Password: config.DatabasePassword,
}
db, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
log.Fatal("Cannot Dial Mongo: ", err)
}
defer db.Close()
db.SetMode(mgo.Monotonic, true)
phoneIndex := mgo.Index{
Key: []string{"pp"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err = db.DB(config.DatabaseName).C("users").EnsureIndex(phoneIndex)
if err != nil {
panic(err)
}
router := mux.NewRouter()
router.HandleFunc("/login", publicWithDB(login, db)).Methods("POST")
if err := http.ListenAndServe(":5000", router); err != nil {
log.Fatal(err)
}
}
func publicWithDB(fn http.HandlerFunc, db *mgo.Session) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
dbsession := db.Copy()
defer dbsession.Close()
fn(w, r.WithContext(context.WithValue(r.Context(), contextKeyDatabase, dbsession)))
}
}
func login(w http.ResponseWriter, r *http.Request) {
r.ParseForm() // Parses the request body
device := r.Form.Get("device")
var deviceid bson.ObjectId
if bson.IsObjectIdHex(device) {
deviceid = bson.ObjectIdHex(device)
}
db := r.Context().Value(contextKeyDatabase).(*mgo.Session)
var device models.Device
err := db.DB(config.DatabaseName).C("devices").FindId(deviceid).One(&device)
w.WriteHeader(200)
w.Write([]byte(utils.ResponseToString(models.Response{Status: 200, Message: "asdasd", Data: device})))
}
I'm posting this because I couldn't find a complete implementation.

Here's an example of how I have seen myself and others structure web apps in Go. This code is untested and is purely for example. It's missing imports and potentially has errors.
EDIT Added a middleware example.
main.go
package main
func main() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{config.DatabaseURL},
Timeout: 60 * time.Second,
Database: config.DatabaseName,
Username: config.DatabaseUsername,
Password: config.DatabasePassword,
}
db, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
log.Fatal("Cannot Dial Mongo: ", err)
}
defer db.Close()
db.SetMode(mgo.Monotonic, true)
phoneIndex := mgo.Index{
Key: []string{"pp"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err = db.DB(config.DatabaseName).C("users").EnsureIndex(phoneIndex)
if err != nil {
panic(err)
}
mgoAdapter := mongo.NewAdapter(db, config.DatabaseName)
deviceStore := mongo.NewDeviceStore(mgoAdapter)
userStore := mongo.NewUserStore(mgoAdapter)
loginController := controllers.NewLoginController(deviceStore)
router := mux.NewRouter()
router.HandleFunc("/login", middleware.AuthorizeUser(userStore)(http.HandlerFunc(loginController.Login)).Methods("POST")
if err := http.ListenAndServe(":5000", router); err != nil {
log.Fatal(err)
}
}
controllers/login.go
package controllers
type LoginController struct {
store DeviceStore
}
func NewLoginController(store stores.DeviceStore) *LoginController {
return &LoginController{
store: store,
}
}
func (c *LoginController) Login(w http.ResponseWriter, r *http.Request) {
r.ParseForm() // Parses the request body
device := r.Form.Get("device")
data, err := c.store.FindByDevice(device)
var respose models.Response
if err != nil {
w.WriteHeader(500)
response = models.Response{Status: 500, Message: fmt.Sprintf("error: %s", err)}
} else if data == nil {
w.WriteHeader(404)
response = models.Response{Status: 404, Message: "device not found"}
} else {
response = models.Response{Status: 200, Message: "device found", Data: data}
}
// Write sets header to 200 if it hasn't been set already
w.Write([]byte(utils.ResponseToString(response)))
}
stores/stores.go
package stores
type DeviceStore interface {
FindByDevice(device string) (*models.Device, error)
}
type UserStore interface {
FindByToken(token string) (*models.User, error)
}
middleware/auth.go
package middleware
func AuthorizeUser(store stores.UserStore) func(h *http.Handler) http.Handler {
return func(h *http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Logic for authorizing user
// Could also store user in the request context
})
}
}
mongo/mongo.go
package mongo
type Adapter struct {
session *mgo.Session
databaseName string
}
func NewAdapter(session *mgo.Session, dbName string) *Adapter {
return &Adapter{session: session, databaseName: dbName}
}
type deviceStore struct {
*Adapter
}
func NewDeviceStore(adapter *Adapter) stores.DeviceStore {
return &deviceStore{adapter}
}
const devices = "devices"
func (s *deviceStore) FindByDevice(d string) (*models.Device, err) {
sess := s.session.copy()
defer sess.close()
var deviceID bson.ObjectId
if bson.IsObjectIdHex(d) {
deviceID = bson.ObjectIdHex(d)
}
var device models.Device
err := db.DB(s.databaseName).C(devices).FindId(deviceID).One(&device)
if err == mgo.ErrNotFound {
return nil, nil
}
return &device, err
}
type userStore struct {
*Adapter
}
const users = "users"
func NewUserStore(adapter *Adapter) stores.UserStore {
return &userStore{adapter}
}
func (s *userStore) GetUserByToken(token string) (*models.User, error) {
sess := s.session.copy()
defer sess.close()
var user models.User
err := db.DB(s.databaseName).C(users).Find(bson.M{"token": token}).One(&user)
if err == mgo.ErrNotFound {
return nil, nil
}
return &user, err
}

Related

How to Handle Dynamic Database Connections in Go?

I am currently building a Go application that needs to connect to multiple databases dynamically.
For context I have 22 Databases (db1, db2, db3...) and the dbUser, dbPass and dbPort remains the same. To determine which database to connect to, I need access to the query param in echo before database connection.
I need a solution to connect to the right database efficiently. What are some best practices and methods for achieving this in Go?
Main.go
package main
import (
"database/sql"
"fmt"
"log"
"time"
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
"github.com/labstack/echo"
"github.com/spf13/viper"
_variantHttpDelivery "backend/server/variant/delivery/http"
_variantHttpDeliveryMiddleware "backend/server/variant/delivery/http/middleware"
_variantRepo "backend/server/variant/repository/postgres"
_variantUcase "backend/server/variant/usecase"
)
func init() {
viper.SetConfigFile(`config.json`)
err := viper.ReadInConfig()
if err != nil {
panic(err)
}
if viper.GetBool(`debug`) {
log.Println("Service RUN on DEBUG mode")
}
}
func main() {
dbHost := viper.GetString(`database.host`)
dbPort := viper.GetString(`database.port`)
dbUser := viper.GetString(`database.user`)
dbPass := viper.GetString(`database.pass`)
dbName := viper.GetString(`database.name`)
connection := fmt.Sprintf("postgresql://%s:%s#%s:%s/%s", dbUser, dbPass, dbHost, dbPort, dbName)
dsn := fmt.Sprintf("%s?%s", connection)
dbConn, err := sql.Open(`postgres`, dsn)
log.Println("Connection Successful 👍")
if err != nil {
log.Fatal(err)
}
err = dbConn.Ping()
if err != nil {
log.Fatal(err)
}
defer func() {
err := dbConn.Close()
if err != nil {
log.Fatal(err)
}
}()
e := echo.New()
middL := _variantHttpDeliveryMiddleware.InitMiddleware()
e.Use(middL.CORS)
variantRepo := _variantRepo.NewPsqlVariantRepository(dbConn)
timeoutContext := time.Duration(viper.GetInt("context.timeout")) * time.Second
au := _variantUcase.NewVariantUsecase(variantRepo, timeoutContext)
_variantHttpDelivery.NewVariantHandler(e, au)
log.Fatal(e.Start(viper.GetString("server.address"))) //nolint
}
Repository which handles all the database logic
package postgres
import (
"backend/server/domain"
"context"
"database/sql"
"github.com/sirupsen/logrus"
"reflect"
)
type psqlVariantRepository struct {
Conn *sql.DB
}
// NewPsqlVariantRepository will create an object that represent the variant.Repository interface
func NewPsqlVariantRepository(conn *sql.DB) domain.VariantRepository {
return &psqlVariantRepository{conn}
}
func (m *psqlVariantRepository) GetByVCF(ctx context.Context, vcf string) (res domain.Variant, err error) {
query := `SELECT * FROM main1 WHERE variant_vcf = $1`
list, err := m.fetch(ctx, query, vcf)
if err != nil {
return domain.Variant{}, err
}
if len(list) > 0 {
res = list[0]
} else {
return res, domain.ErrNotFound
}
return
}
func (m *psqlVariantRepository) fetch(ctx context.Context, query string, args ...interface{}) (result []domain.Variant, err error) {
rows, err := m.Conn.QueryContext(ctx, query, args...)
if err != nil {
logrus.Error(err)
return nil, err
}
defer func() {
errRow := rows.Close()
if errRow != nil {
logrus.Error(errRow)
}
}()
result = make([]domain.Variant, 0)
for rows.Next() {
t := domain.Variant{}
values := make([]interface{}, 0, reflect.TypeOf(t).NumField())
v := reflect.ValueOf(&t).Elem()
for i := 0; i < v.NumField(); i++ {
if v.Type().Field(i).Type.Kind() == reflect.String {
values = append(values, new(sql.NullString))
} else {
values = append(values, v.Field(i).Addr().Interface())
}
}
err = rows.Scan(values...)
if err != nil {
logrus.Error(err)
return nil, err
}
for i, value := range values {
if ns, ok := value.(*sql.NullString); ok {
v.Field(i).SetString(ns.String)
}
}
result = append(result, t)
}
logrus.Info("Successfully fetched results from database 👍")
return result, nil
}
So far I couldn't find any solution

session was not created by this client - MongoDB, Golang

I am trying to creat transaction in MongoDB with Golang and Iris. Problem is that transaction did not accept iris context and Con, I don't know why this thing happened. Can you tell me what I am doing wrong here?
Main.go Using Iris
func main() {
app := iris.New()
app.Logger().SetLevel("debug")
app.Use(recover.New())
app.Use(logger.New())
// Resource: http://localhost:8080
app.Get("/", func(ctx iris.Context) {
ctx.JSON(iris.Map{"message": "Welcome to Woft Bank"})
})
// API endpoints
router.SetRoute(app)
app.Listen(PORT)}
Router
func SetRoute(app *iris.Application) {
userRoute := app.Party("/user")
{
userRoute.Post("/register", middleware.UserValidator, controller.CreateUser)
userRoute.Get("/detail", middleware.UserValidator, controller.GetUserBalanceWithUserID)
userRoute.Patch("/transfer", middleware.TransferValidator, controller.Transfer)
}}
Transacion function (session was not created by this client)
func Transfer(ctx iris.Context) {
senderID := ctx.URLParam("from")
receiverID := ctx.URLParam("to")
amount, _ := strconv.ParseInt(ctx.URLParam("amount"), 10, 64)
session, err := Config.DB().StartSession()
if err != nil {
handleErr(ctx, err)
return
}
defer session.EndSession(ctx)
callback := func(sessCtx mongo.SessionContext) (interface{}, error) {
upsert := false
after := options.After
opt := options.FindOneAndUpdateOptions{
ReturnDocument: &after,
Upsert: &upsert,
}
sender := Models.User{}
filter := bson.M{"username": senderID}
update := bson.M{"$inc": bson.M{"balance": -amount}}
//FindOneAndUpdate did not accept sessCtx
err := UserCollection.FindOneAndUpdate(sessCtx, filter, update, &opt).Decode(&sender)
if err != nil {
return nil, err
}
if sender.Balance < 0 {
return nil, errors.New("sender's balance is not enough")
}
filter = bson.M{"username": receiverID}
update = bson.M{"$inc": bson.M{"balance": +amount}}
_, err = UserCollection.UpdateOne(sessCtx, filter, update)
if err != nil {
return nil, err
}
return sender, nil
}
result, err := session.WithTransaction(ctx, callback)
if err != nil {
handleErr(ctx, err)
return
}
response(result, "success", ctx)}

Golang beginner not able solve the problem

I would like to make a user login from this method. This has to be in three parts. The same as user registration, but I am not understanding how to do this? Could you please write how I can write a user login logic this same way? It is created using gorilla mux.
One method has to be in db_service.go then one method has to be in login_service.go
one method has to be in login.go.
This is db_service.go code:
/* Used to create a singleton object of MongoDB client.
Initialized and exposed through GetMongoClient().*/
var clientInstance *mongo.Client
//Used during creation of singleton client object in GetMongoClient().
var clientInstanceError error
//Used to execute client creation procedure only once.
var mongoOnce sync.Once
//I have used below constants just to hold required database config's.
const (
CONNECTIONSTRING = "http://127.0.0.1:27017"
AUTH_DB = "Cluster0"
USER_COLLECTION = "user"
)
//GetMongoClient - Return mongodb connection to work with
func GetMongoClient() (*mongo.Client, error) {
//Perform connection creation operation only once.
mongoOnce.Do(func() {
// Set client options
clientOptions := options.Client().ApplyURI(CONNECTIONSTRING)
// Connect to MongoDB
client, err := mongo.Connect(context.TODO(), clientOptions)
if err != nil {
clientInstanceError = err
}
// Check the connection
err = client.Ping(context.TODO(), nil)
if err != nil {
clientInstanceError = err
}
log.Println("Connected Mongodb!")
clientInstance = client
})
return clientInstance, clientInstanceError
}
//CreateIssue - Insert a new document in the collection.
func User_Collection(user *model.User) (*mongo.InsertOneResult, error) {
//Create a handle to the respective collection in the database.
collection := clientInstance.Database(AUTH_DB).Collection(USER_COLLECTION)
//Perform InsertOne operation & validate against the error.
return collection.InsertOne(context.TODO(), user)
}
login_service.go contained code for user registration:
func Register_User(user *model.User) (interface{}, error) {
user.CreatedAt = time.Now().UTC()
user.UpdatedAt = time.Now().UTC()
if result, err := util.User_Collection(user); err == nil {
return result.InsertedID, err
} else {
return nil, err
}
}
User registration code:
func Register(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
var user model.User
if err := json.NewDecoder(r.Body).Decode(&user); err == nil {
if _, err := service.Register_User(&user); err == nil {
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(user)
} else {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(err)
}
} else {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
Something like this for the handler
func Login(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
var user model.User
if err := json.NewDecoder(r.Body).Decode(&user); err == nil {
if _, err := service.Login_User(&user); err == nil {
json.NewEncoder(w).Encode(user)
} else {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(err)
}
} else {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
And something like this for the DB (Need to implement the query instead of insert)
func Login_User(user *model.User) (interface{}, error) {
// Implement this query
if result, err := util.Find_User(user); err == nil {
return result, err
} else {
return nil, err
}
}

Go Gin Setting and Accessing context value from middleware

I am trying to set my user context, in the middleware then trying to check if user have permission in other handler functions. But for some reason when I try to access the user from context it is coming back as nils. The middleware code seems to be working, when I pass a valid jwt token, it is showing the user is being set in context in the middleware function. But as soon as I hit getCurrentUser function it says it's nil.
Here is the code:
Middleware
// Middleware wraps the request with auth middleware
func Middleware(path string, sc *cfg.Server, orm *orm.ORM) gin.HandlerFunc {
logger.Info("[Auth.Middleware] Applied to path: ", path)
return gin.HandlerFunc(func(c *gin.Context) {
t, err := ParseToken(c, sc)
if err != nil {
authError(c, err)
} else {
if claims, ok := t.Claims.(jwt.MapClaims); ok {
if claims["exp"] != nil {
issuer := claims["iss"].(string)
userid := claims["jti"].(string)
email := claims["email"].(string)
if claims["aud"] != nil {
audiences := claims["aud"].(interface{})
logger.Warnf("\n\naudiences: %s\n\n", audiences)
}
if claims["alg"] != nil {
algo := claims["alg"].(string)
logger.Warnf("\n\nalgo: %s\n\n", algo)
}
if user, err := orm.FindUserByJWT(email, issuer, userid); err != nil {
authError(c, ErrForbidden)
} else {
if user != nil {
c.Request = addToContext(c, consts.ProjectContextKeys.UserCtxKey, user)
logger.Debug("User: ", user.ID)
}
c.Next()
}
} else {
authError(c, ErrMissingExpField)
}
} else {
authError(c, err)
}
}
})
}
routes
// User routes
func User(sc *cfg.Server, r *gin.Engine, orm *orm.ORM) error {
// OAuth handlers
mw := auth.Middleware(sc.VersionedEndpoint("/user/:id"), sc, orm)
g := r.Group(sc.VersionedEndpoint("/user"))
g.Use(mw)
g.GET("/:id", mw, user.Get(orm))
g.PUT("/:id", mw, user.Update(orm))
g.POST("/", user.Create(orm))
return nil
}
handler
func Get(orm *orm.ORM) gin.HandlerFunc {
return func(ctx *gin.Context) {
cu := getCurrentUser(ctx)
if ok, err := cu.HasPermission(consts.Permissions.Create, consts.EntityNames.Users); !ok || err != nil {
ctx.String(http.StatusUnauthorized, "BAD")
}
}
}
addToContext:
func addToContext(c *gin.Context, key consts.ContextKey, value interface{}) *http.Request {
return c.Request.WithContext(context.WithValue(c.Request.Context(), key, value))
}
getCurrentUser:
func getCurrentUser(ctx context.Context) *dbm.User {
cu := ctx.Value(utils.ProjectContextKeys.UserCtxKey).(*dbm.User)
logger.Debugf("currentUser: %s - %s", cu.Email, cu.ID)
return cu
}
The problem is that you're storing the user in one context but then you're attempting to retrieve the user from another context. The value *gin.Context and the value *gin.Context.Request.Context are two separate context values.
You're using the Request's context to store the user:
c.Request.WithContext(context.WithValue(c.Request.Context(), key, value))
And then you're using the gin context to retrieve the user:
func getCurrentUser(ctx context.Context) *dbm.User {
cu := ctx.Value(utils.ProjectContextKeys.UserCtxKey).(*dbm.User)
// ...
func Get(orm *orm.ORM) gin.HandlerFunc {
return func(ctx *gin.Context) {
cu := getCurrentUser(ctx) // here you're passing *gin.Context to the function.
// ...
So to fix that change the value that's passed in to the getCurrentUser call to:
func Get(orm *orm.ORM) gin.HandlerFunc {
return func(ctx *gin.Context) {
cu := getCurrentUser(ctx.Request.Context())
if ok, err := cu.HasPermission(consts.Permissions.Create, consts.EntityNames.Users); !ok || err != nil {
ctx.String(http.StatusUnauthorized, "BAD")
}
}
}

Golang Mongodb %!(EXTRA

I'm trying to marshal a struct into JSON and then insert it into my Mongo database, but keep on getting this error: %!(EXTRA main.Test={575590180 Me}). What am I doing wrong? I took this code exactly from another project I worked on which could insert documents without any problems.
package main
import (
"utils"
"hash/fnv"
"log"
"gopkg.in/mgo.v2"
"encoding/json"
)
type Test struct {
Id uint32
Name string
}
func ConnectDB() *mgo.Session {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
return session
}
func SaveMgoDoc(dbName string, collectionName string, file Test) bool {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
defer session.Close()
fileJson, err := json.Marshal(file)
if err != nil {
log.Printf("failed to marshal struct to json...\n", file)
return false
}
collection := session.DB(dbName).C(collectionName)
err = collection.Insert(&fileJson)
if err != nil {
log.Printf("failed to insert doc into database...\n", file)
return false
}
return true
}
func hash(s string) uint32 {
h := fnv.New32a()
h.Write([]byte(s))
return h.Sum32()
}
func main() {
utils.SaveMgoDoc("mydb", "mydoc", Test{hash("Me"), "Me"})
}
Insert expects a pointer to a struct, not a json string. So, in this case, just use:
err = collection.Insert(&file)