Testing gin based REST API not getting params while using net/http/httptest - rest

I am developing a REST API based on Gin Go, the endpoint looks something like below:
func carsByType(c *gin.Context) {
fmt.Println("Go Request in Handler...")
carType := c.Params.ByName("type")
fmt.Println(carType)
if carType != "" {
}
c.JSON(http.StatusBadRequest, gin.H{"result": "Bad request"})
return
}
func main() {
router := gin.Default()
router.GET("/cars/:type", carsByType)
router.Run(":80")
}
When I am making request to the endpoint via browser and cURL its just working fine, getting the carType value but when I am running the tests its returning bad request and getting carType is "".
For testing the endpoint my test code looks like this:
func TestGetcarsByType(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := carsByType
router := gin.Default()
router.GET("/cars/1", handler)
req, err := http.NewRequest("GET", "/cars/1", nil)
if err != nil {
fmt.Println(err)
}
resp := httptest.NewRecorder()
router.ServeHTTP(resp, req)
assert.Equal(t, resp.Code, 200)
}
What am I doing wrong?

router.GET("/cars/1", handler) should be router.GET("/cars/:type", handler) in your test.
Note that a better (more testable; less duplication of routes) would be to create a SetupRouter() *gin.Engine function that returns all of your routes. e.g.
// package main
// server.go
// This is where you create a gin.Default() and add routes to it
func SetupRouter() *gin.Engine {
router := gin.Default()
router.GET("/cars/:type", carsByType)
return router
}
func main() {
router := SetupRouter()
router.Run()
}
// router_test.go
testRouter := SetupRouter()
req, err := http.NewRequest("GET", "/cars/1", nil)
if err != nil {
fmt.Println(err)
}
resp := httptest.NewRecorder()
testRouter.ServeHTTP(resp, req)
assert.Equal(t, resp.Code, 200)

Related

Consuming Paginated REST API

I am trying to consume an API that on each successful request has a json key like the following:
{
"hasNextPage": true,
"endCursor": "some_guid_present_here"
}
The way the API works (I have used it many times in Python but am trying with Go for separate use case) is to basically check if there is a next page, then use the appropriate cursor to continue to paginate.
However, every time I use this structure, it will sometimes keep looping even though the response.Paginations.HasNextPage will be false.
I am trying to understand if it is my structure of consuming a paginated API like this that is causing this or else.
Also, I have say 4-5 requests to start off with, which I sent separately via goroutines. I am not sure if this is causing an issue but I've attached that as well in etl.go.
The main request making structs are in api.go.
I've confirmed that I do receive responses and they are unmarshaling properly, but am trying to wrangle this uncertain behavior.
api.go
package models
import (
"encoding/json"
"io/ioutil"
"net/http"
"fmt"
)
type Request struct {
Url string
ApiKey string
}
type Response struct {
...some fields...
Paginations Pagination `json:"pagination"`
}
type Pagination struct {
EndCursor string `json:"endCursor"`
HasNextPage bool `json:"hasNextPage"`
}
func (request *Request) Get() ([]Response, error) {
var responses []Response
var response Response
// Set up new request
req, err := http.NewRequest("GET", request.Url, nil)
if err != nil {
fmt.Println("Error creating request...")
return responses, err
}
// Add request headers
req.Header = http.Header{
"accept": {"application/json"},
"authorization": {"Bearer " + request.ApiKey},
}
// Get our initial response from the API and capture status code
resp, _ := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
// Read the response body and Unmarshal into struct
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// This field will be in the response payload
// It is verified to be of type bool (not string)
fmt.Printf("Has Next Page? %t\n", resp.Paginations.HasNextPage)
// Append response to our slice of responses
responses = append(responses, response)
// If paginations are available, continue to loop through until all paginations are exhausted
for response.Paginations.HasNextPage == true {
req, err := http.NewRequest("GET", request.Url, nil)
if err != nil {
return responses, err
}
// Append "after" cursor to query in order to redirect to paginated response
qry := req.URL.Query()
qry.Set("after", response.Paginations.EndCursor)
req.URL.RawQuery = qry.Encode()
fmt.Println("Paginated request query: ", req.URL.String())
// Make request
resp, err := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
fmt.Printf("Status Code: %d\n", response.Status)
// Read response and deserialize it
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
fmt.Println("Pagination Cursor: ", response.Paginations.EndCursor)
fmt.Printf("Has Next Page? %t\n", response.Paginations.HasNextPage)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// Append response to our slice of responses
responses = append(responses, response)
}
return responses, nil
}
etl.go
package models
import (
"fmt"
"sync"
)
type Etl struct {
Requests []Request
}
func (etl *Etl) Extract() [][]Response {
var wg sync.WaitGroup
ch := make(chan []Response)
defer close(ch)
for _, req := range etl.Requests {
wg.Add(1) // Increment
fmt.Println("Incremented wait group")
go func(i Request) {
defer wg.Done() // Decrement
resp, err := req.Get()
if err != nil {
fmt.Println(err)
}
ch <- resp
fmt.Println("Decremented wait group")
}(req)
}
results := make([][]Response, len(etl.Requests))
for i, _ := range results {
results[i] = <-ch
//fmt.Println(results[i])
}
wg.Wait()
return nil
}
I believe I found the issue. In my pagination loop for response.Paginations.HasNextPage == true I was creating a new request object (http.NewRequest) on each iteration which did not have headers added from the prior (initial request).
This caused a 401 unauthorized error to return and continuous querying of the API since it was not receiving a new response.Paginations.HasNextPage value.
My solution was to simply alter the for loop a bit like so:
package models
import (
"encoding/json"
"io/ioutil"
"net/http"
"fmt"
)
type Request struct {
Url string
ApiKey string
}
type Response struct {
...some fields...
Paginations Pagination `json:"pagination"`
}
type Pagination struct {
EndCursor string `json:"endCursor"`
HasNextPage bool `json:"hasNextPage"`
}
func (request *Request) Get() ([]Response, error) {
var responses []Response
var response Response
// Set up new request
req, err := http.NewRequest("GET", request.Url, nil)
if err != nil {
fmt.Println("Error creating request...")
return responses, err
}
// Add request headers
req.Header = http.Header{
"accept": {"application/json"},
"authorization": {"Bearer " + request.ApiKey},
}
// Get our initial response from the API and capture status code
resp, _ := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
// Read the response body and Unmarshal into struct
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// This field will be in the response payload
// It is verified to be of type bool (not string)
fmt.Printf("Has Next Page? %t\n", resp.Paginations.HasNextPage)
// Append response to our slice of responses
responses = append(responses, response)
// If paginations are available, continue to loop through until all paginations are exhausted
for response.Paginations.HasNextPage == true {
// Append "after" cursor to query in order to redirect to paginated response
qry := req.URL.Query()
qry.Set("after", response.Paginations.EndCursor)
req.URL.RawQuery = qry.Encode()
fmt.Println("Paginated request query: ", req.URL.String())
// Make request
resp, err := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
fmt.Printf("Status Code: %d\n", response.Status)
// Read response and deserialize it
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
fmt.Println("Pagination Cursor: ", response.Paginations.EndCursor)
fmt.Printf("Has Next Page? %t\n", response.Paginations.HasNextPage)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// Append response to our slice of responses
responses = append(responses, response)
}
return responses, nil
}

best practice for passing client / collection connections when using goji

I have some golang that uses the now de-supported mgo driver and goji, here is an excerpt of the code:
func main() {
session, err := mgo.Dial("mongo")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
ensureIndex(session)
mux := goji.NewMux()
mux.HandleFunc(pat.Get("/cars"), allCars(session))
mux.HandleFunc(pat.Post("/cars"), addCar(session))
mux.HandleFunc(pat.Get("/cars/:vin"), carByVIN(session))
mux.HandleFunc(pat.Delete("/cars/:vin"), deleteCar(session))
http.ListenAndServe(":8080", mux)
}
func allCars(s *mgo.Session) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
session := s.Copy()
defer session.Close()
c := session.DB("carsupermarket").C("cars")
var cars []vehicle
err := c.Find(bson.M{}).All(&cars)
if err != nil {
errorWithJSON(w, "Database error", http.StatusInternalServerError)
log.Println("Failed get all cars: ", err)
return
}
respBody, err := json.MarshalIndent(cars, "", " ")
if err != nil {
log.Fatal(err)
}
responseWithJSON(w, respBody, http.StatusOK)
}
}
I want to replace mgo with the official mongo driver for golang, my question is this, what is the best practice, if there is such a thing, for instantiating client connections and collections in the main function and passing them into other functions, should I be doing this at all, at present I am passing a pointer to a collection into the functions in my code, i.e.:
func ensureIndex(c *mongo.Collection) {
.
.
.

Golang beginner not able solve the problem

I would like to make a user login from this method. This has to be in three parts. The same as user registration, but I am not understanding how to do this? Could you please write how I can write a user login logic this same way? It is created using gorilla mux.
One method has to be in db_service.go then one method has to be in login_service.go
one method has to be in login.go.
This is db_service.go code:
/* Used to create a singleton object of MongoDB client.
Initialized and exposed through GetMongoClient().*/
var clientInstance *mongo.Client
//Used during creation of singleton client object in GetMongoClient().
var clientInstanceError error
//Used to execute client creation procedure only once.
var mongoOnce sync.Once
//I have used below constants just to hold required database config's.
const (
CONNECTIONSTRING = "http://127.0.0.1:27017"
AUTH_DB = "Cluster0"
USER_COLLECTION = "user"
)
//GetMongoClient - Return mongodb connection to work with
func GetMongoClient() (*mongo.Client, error) {
//Perform connection creation operation only once.
mongoOnce.Do(func() {
// Set client options
clientOptions := options.Client().ApplyURI(CONNECTIONSTRING)
// Connect to MongoDB
client, err := mongo.Connect(context.TODO(), clientOptions)
if err != nil {
clientInstanceError = err
}
// Check the connection
err = client.Ping(context.TODO(), nil)
if err != nil {
clientInstanceError = err
}
log.Println("Connected Mongodb!")
clientInstance = client
})
return clientInstance, clientInstanceError
}
//CreateIssue - Insert a new document in the collection.
func User_Collection(user *model.User) (*mongo.InsertOneResult, error) {
//Create a handle to the respective collection in the database.
collection := clientInstance.Database(AUTH_DB).Collection(USER_COLLECTION)
//Perform InsertOne operation & validate against the error.
return collection.InsertOne(context.TODO(), user)
}
login_service.go contained code for user registration:
func Register_User(user *model.User) (interface{}, error) {
user.CreatedAt = time.Now().UTC()
user.UpdatedAt = time.Now().UTC()
if result, err := util.User_Collection(user); err == nil {
return result.InsertedID, err
} else {
return nil, err
}
}
User registration code:
func Register(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
var user model.User
if err := json.NewDecoder(r.Body).Decode(&user); err == nil {
if _, err := service.Register_User(&user); err == nil {
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(user)
} else {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(err)
}
} else {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
Something like this for the handler
func Login(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
var user model.User
if err := json.NewDecoder(r.Body).Decode(&user); err == nil {
if _, err := service.Login_User(&user); err == nil {
json.NewEncoder(w).Encode(user)
} else {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(err)
}
} else {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
And something like this for the DB (Need to implement the query instead of insert)
func Login_User(user *model.User) (interface{}, error) {
// Implement this query
if result, err := util.Find_User(user); err == nil {
return result, err
} else {
return nil, err
}
}

Can I have insecure GET HTTP requests whilst having MTLS securing all other HTTP requests?

I have a HTTP REST service written in golang demonstrating what I'm attempting.
I want GET requests insecure and all other REST requests secured with MTLS.
My implementation already uses the gin web server library so I'd like to stick with that if possible.
My issue is that I have only been able to apply the tlsConfig to both groups or neither. I've been unable to find a way to apply this at the group level.
package main
import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io/ioutil"
"net/http"
"log"
"github.com/gin-gonic/gin"
)
func main() {
router := gin.Default()
// Unprotected public router for GET requests
public := router.Group("/")
// Private router with MTLS
private := router.Group("/")
public.GET("/insecure-ping", func(c *gin.Context) {
c.JSON(200, gin.H{
"message": "insecure pong",
})
})
private.POST("/secure-ping", func(c *gin.Context) {
c.JSON(200, gin.H{
"message": "secure pong",
})
})
// Get the SystemCertPool, continue with an empty pool on error
rootCAs, err := x509.SystemCertPool()
if err != nil {
log.Fatal(err)
}
if rootCAs == nil {
rootCAs = x509.NewCertPool()
}
// Create a CA certificate pool and add cacert.pem to it
caCert, err := ioutil.ReadFile("cacert.pem")
if err != nil {
log.Fatal(err)
}
if ok := rootCAs.AppendCertsFromPEM(caCert); !ok {
err := errors.New("failed to append CA cert to local system certificate pool")
log.Fatal(err)
}
server := http.Server{
Addr: fmt.Sprintf(":%v", 8080),
Handler: router,
}
server.TLSConfig = &tls.Config{
RootCAs: rootCAs,
}
err = server.ListenAndServeTLS("certificate.crt", "privateKey.key")
if err != nil {
log.Fatal(err)
}
}
Just create two Server instances and run them both, one with ListenAndServe and one with ListenAndServeTLS, configured with the same routes. Because HTTP and HTTPS operate on different ports, they have to have different listeners, but both listeners can use the same (or different) handlers. For example:
publicRouter := gin.Default()
// Unprotected public router for GET requests
public := publicRouter.Group("/")
public.GET("/insecure-ping", func(c *gin.Context) {
c.JSON(200, gin.H{
"message": "insecure pong",
})
})
server := http.Server{
Addr: fmt.Sprintf(":%v", 8081), // Or whatever
Handler: publicRouter,
}
go func() {
err = tlsServer.ListenAndServe()
if err != nil {
log.Fatal(err)
}
}()
// Private router with MTLS
router := gin.Default()
private := router.Group("/")
private.POST("/secure-ping", func(c *gin.Context) {
c.JSON(200, gin.H{
"message": "secure pong",
})
})
// Get the SystemCertPool, continue with an empty pool on error
rootCAs, err := x509.SystemCertPool()
if err != nil {
log.Fatal(err)
}
if rootCAs == nil {
rootCAs = x509.NewCertPool()
}
// Create a CA certificate pool and add cacert.pem to it
caCert, err := ioutil.ReadFile("cacert.pem")
if err != nil {
log.Fatal(err)
}
if ok := rootCAs.AppendCertsFromPEM(caCert); !ok {
err := errors.New("failed to append CA cert to local system certificate pool")
log.Fatal(err)
}
tlsServer := http.Server{
Addr: fmt.Sprintf(":%v", 8080),
Handler: router,
}
tlsServer.TLSConfig = &tls.Config{
RootCAs: rootCAs,
}
err = tlsServer.ListenAndServeTLS("certificate.crt", "privateKey.key")
if err != nil {
log.Fatal(err)
}

Using mgo with context

I have been using mgo for my API but I'm seeing many current connections in my MongoDB (while using less than 5 devices for testing). By executing db.serverStatus().connections in my Mongo server I get: { "current" : 641, "available" : 838219, "totalCreated" : 1136 }. Below I transcript my issue in mgo's Github (Issue #429):
Is my way of using mgo in a web server the correct way? If not, can you give me a full example?
This code is not functional, take it as almost pseudo code (because of the missing parts like the imports or where the configs come from and models), but it is exactly how I am using mgo.
I must clarify that I'm building an API which is used by several mobile devices and webapps.
main.go
func main() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{config.DatabaseURL},
Timeout: 60 * time.Second,
Database: config.DatabaseName,
Username: config.DatabaseUsername,
Password: config.DatabasePassword,
}
db, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
log.Fatal("Cannot Dial Mongo: ", err)
}
defer db.Close()
db.SetMode(mgo.Monotonic, true)
phoneIndex := mgo.Index{
Key: []string{"pp"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err = db.DB(config.DatabaseName).C("users").EnsureIndex(phoneIndex)
if err != nil {
panic(err)
}
router := mux.NewRouter()
router.HandleFunc("/login", publicWithDB(login, db)).Methods("POST")
if err := http.ListenAndServe(":5000", router); err != nil {
log.Fatal(err)
}
}
func publicWithDB(fn http.HandlerFunc, db *mgo.Session) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
dbsession := db.Copy()
defer dbsession.Close()
fn(w, r.WithContext(context.WithValue(r.Context(), contextKeyDatabase, dbsession)))
}
}
func login(w http.ResponseWriter, r *http.Request) {
r.ParseForm() // Parses the request body
device := r.Form.Get("device")
var deviceid bson.ObjectId
if bson.IsObjectIdHex(device) {
deviceid = bson.ObjectIdHex(device)
}
db := r.Context().Value(contextKeyDatabase).(*mgo.Session)
var device models.Device
err := db.DB(config.DatabaseName).C("devices").FindId(deviceid).One(&device)
w.WriteHeader(200)
w.Write([]byte(utils.ResponseToString(models.Response{Status: 200, Message: "asdasd", Data: device})))
}
I'm posting this because I couldn't find a complete implementation.
Here's an example of how I have seen myself and others structure web apps in Go. This code is untested and is purely for example. It's missing imports and potentially has errors.
EDIT Added a middleware example.
main.go
package main
func main() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{config.DatabaseURL},
Timeout: 60 * time.Second,
Database: config.DatabaseName,
Username: config.DatabaseUsername,
Password: config.DatabasePassword,
}
db, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
log.Fatal("Cannot Dial Mongo: ", err)
}
defer db.Close()
db.SetMode(mgo.Monotonic, true)
phoneIndex := mgo.Index{
Key: []string{"pp"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err = db.DB(config.DatabaseName).C("users").EnsureIndex(phoneIndex)
if err != nil {
panic(err)
}
mgoAdapter := mongo.NewAdapter(db, config.DatabaseName)
deviceStore := mongo.NewDeviceStore(mgoAdapter)
userStore := mongo.NewUserStore(mgoAdapter)
loginController := controllers.NewLoginController(deviceStore)
router := mux.NewRouter()
router.HandleFunc("/login", middleware.AuthorizeUser(userStore)(http.HandlerFunc(loginController.Login)).Methods("POST")
if err := http.ListenAndServe(":5000", router); err != nil {
log.Fatal(err)
}
}
controllers/login.go
package controllers
type LoginController struct {
store DeviceStore
}
func NewLoginController(store stores.DeviceStore) *LoginController {
return &LoginController{
store: store,
}
}
func (c *LoginController) Login(w http.ResponseWriter, r *http.Request) {
r.ParseForm() // Parses the request body
device := r.Form.Get("device")
data, err := c.store.FindByDevice(device)
var respose models.Response
if err != nil {
w.WriteHeader(500)
response = models.Response{Status: 500, Message: fmt.Sprintf("error: %s", err)}
} else if data == nil {
w.WriteHeader(404)
response = models.Response{Status: 404, Message: "device not found"}
} else {
response = models.Response{Status: 200, Message: "device found", Data: data}
}
// Write sets header to 200 if it hasn't been set already
w.Write([]byte(utils.ResponseToString(response)))
}
stores/stores.go
package stores
type DeviceStore interface {
FindByDevice(device string) (*models.Device, error)
}
type UserStore interface {
FindByToken(token string) (*models.User, error)
}
middleware/auth.go
package middleware
func AuthorizeUser(store stores.UserStore) func(h *http.Handler) http.Handler {
return func(h *http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Logic for authorizing user
// Could also store user in the request context
})
}
}
mongo/mongo.go
package mongo
type Adapter struct {
session *mgo.Session
databaseName string
}
func NewAdapter(session *mgo.Session, dbName string) *Adapter {
return &Adapter{session: session, databaseName: dbName}
}
type deviceStore struct {
*Adapter
}
func NewDeviceStore(adapter *Adapter) stores.DeviceStore {
return &deviceStore{adapter}
}
const devices = "devices"
func (s *deviceStore) FindByDevice(d string) (*models.Device, err) {
sess := s.session.copy()
defer sess.close()
var deviceID bson.ObjectId
if bson.IsObjectIdHex(d) {
deviceID = bson.ObjectIdHex(d)
}
var device models.Device
err := db.DB(s.databaseName).C(devices).FindId(deviceID).One(&device)
if err == mgo.ErrNotFound {
return nil, nil
}
return &device, err
}
type userStore struct {
*Adapter
}
const users = "users"
func NewUserStore(adapter *Adapter) stores.UserStore {
return &userStore{adapter}
}
func (s *userStore) GetUserByToken(token string) (*models.User, error) {
sess := s.session.copy()
defer sess.close()
var user models.User
err := db.DB(s.databaseName).C(users).Find(bson.M{"token": token}).One(&user)
if err == mgo.ErrNotFound {
return nil, nil
}
return &user, err
}