Golang test of REST API dumps entire database - rest

I wrote a small web service to learn unit testing. There is one endpoint to get data with a three-letter string. My code runs fine. The right query is http://localhost:8000/iata/thu, with the last bit thu being the three-letter string. I can get the correct data with it. I can also successfully get 404 with wrong ones. Then I wrote the test. It fails and dumps the entire database.
The SQLite3 database, main.go, and main_test.go are in the same directory.
Here's main_test.go:
package main
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestIata(t *testing.T) {
// "thu" is the three-letter code.
// I also tried "http://localhost:8000/iata/thu"
req, err := http.NewRequest("GET", "/iata/thu", nil)
if err != nil {
t.Fatal(err)
}
rr := httptest.NewRecorder()
handler := http.HandlerFunc(iata)
handler.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
t.Errorf("handler returned wrong status code: got %v want %v",
status, http.StatusOK)
}
expected := `[{"airport_id":"10","name":"Thule Air Base","city":"Thule","country":"Greenland","iata":"THU","icao":"BGTL","latitude":"76.5311965942","longitude":"-68.7032012939","altitude":"251","timezone":"-4","dst":"E","tz_db":"America/Thule","type":"airport","source":"OurAirports"}]`
if rr.Body.String() != expected {
t.Errorf("handler returned unexpected body: got %v want %v",
rr.Body.String(), expected)
}
}
Here's main.go:
package main
import (
"database/sql"
"encoding/json"
"log"
"net/http"
"github.com/gorilla/mux"
_ "github.com/mattn/go-sqlite3"
)
type datum struct {
AirportID string `json:"airport_id,omitempty"`
...
}
func check(err error) {
...
}
// Accesses the database and gets relevant rows.
func getRows(column string, searchTerm string) *sql.Rows {
db, err := sql.Open("sqlite3", "airports.db")
check(err)
stmt := `SELECT * FROM airports WHERE ` + column + ` LIKE ? COLLATE NOCASE;`
rows, err := db.Query(stmt, `%`+searchTerm+`%`)
check(err)
return rows
}
// Processes the data into a slice so it can be sent out as JSON.
func processData(rows *sql.Rows) []datum {
data := []datum{}
// For each row, insert data into a datum instance and then append to data slice.
for rows.Next() {
datum := datum{}
rows.Scan(&datum.AirportID,
...)
data = append(data, datum)
}
rows.Close()
return data
}
// Uses the above code to get data from the database, process it, and send it.
func getAndSendData(w http.ResponseWriter, r *http.Request, searchType string) {
params := mux.Vars(r)
searchTerm := params[searchType]
datum := getRows(searchType, searchTerm)
processed := processData(datum)
if len(processed) == 0 {
http.Error(w, "Data not found.", 404)
return
}
json.NewEncoder(w).Encode(processed)
}
func main() {
router := mux.NewRouter()
router.HandleFunc("/iata/{iata}", iata).Methods("GET")
log.Fatal(http.ListenAndServe(":8000", router))
}
func iata(w http.ResponseWriter, r *http.Request) {
searchType := "iata"
getAndSendData(w, r, searchType)
}
Running the test gets the entire database dumped in the result:
=== RUN TestIata
--- FAIL: TestIata (0.21s)
main_test.go:46: handler returned unexpected body: got [ENTIRE DATABASE DUMPED HERE] want [CORRECT DATA HERE]
FAIL
exit status 1
I've looked through a lot of tutorials, such as this one, which I feel is pretty clear. As far as I can tell, my test code is correct. I've also tried running main.go before doing the test. But that shouldn't matter, right?
What am I missing?

Related

How do I place the AVG of a row into a new column?

Hey everyone I am using golang to build a super simple API. I have this json data being passed from a POST request and being stored in my DB. I would like to take the tts data which is an integer array and average that array and place it in the ttc column and return that number on the json response. I am having a hard time doing that any help would be greatly appreciated. My source code is below as well as my DB Model. I know I would have to use the AVG() function somehow in postgres but I am brand new to postgres so I am super confused.
main.go
package main
import (
"encoding/json"
"github.com/gorilla/mux"
"github.com/jinzhu/gorm"
"github.com/lib/pq"
"github.com/rs/cors"
"log"
"net/http"
_ "github.com/jinzhu/gorm/dialects/postgres"
)
type Resource struct {
gorm.Model
Name string
TTS pq.Int64Array `gorm:"type:integer[]"`
TTC int
}
var db *gorm.DB
var err error
func main() {
router := mux.NewRouter()
db, err = gorm.Open(
"postgres",
"host=localhost"+" user=postgres"+
" dbname=Shoes"+" sslmode=disable password=root")
if err != nil {
panic("failed to connect database")
}
defer db.Close()
db.AutoMigrate(&Resource{})
router.HandleFunc("/resources", GetResources).Methods("GET")
router.HandleFunc("/resources/{id}", GetResource).Methods("GET")
router.HandleFunc("/resources", CreateResource).Methods("POST")
router.HandleFunc("/resources/{id}", DeleteResource).Methods("DELETE")
handler := cors.Default().Handler(router)
log.Fatal(http.ListenAndServe(":8080", handler))
}
func GetResources(w http.ResponseWriter, r *http.Request) {
var resources []Resource
db.Find(&resources)
json.NewEncoder(w).Encode(&resources)
}
func GetResource(w http.ResponseWriter, r *http.Request) {
params := mux.Vars(r)
var resource Resource
db.First(&resource, params["id"])
json.NewEncoder(w).Encode(&resource)
}
func CreateResource(w http.ResponseWriter, r *http.Request) {
var resource Resource
json.NewDecoder(r.Body).Decode(&resource)
db.Create(&resource)
json.NewEncoder(w).Encode(&resource)
}
func DeleteResource(w http.ResponseWriter, r *http.Request) {
params := mux.Vars(r)
var resource Resource
db.First(&resource, params["id"])
db.Delete(&resource)
var resources []Resource
db.Find(&resources)
json.NewEncoder(w).Encode(&resources)
}
I am thinking I could do something like
db.Select("AVG(tts)")
I am just not sure how to put that result in the column ttc
Since the json of the post request already contains the tts_data, you can get the average before setting it in the database
sum := 0
for _, i := range tts_data {
sum += i
}
avg := sum / len(tts_data)
// save the data in your db
rs := Ressource{Name: "name", TTS: tts_data, ttc: avg}
b := db.Create(&rs)
if b {
// send all the resource
json.NewEncoder(w).Encode(&rs)
// or send only the avg
json.NewEncoder(w).Encode(struct{avg: int}{avg: avg})
} else {
// handle error
}

Generic REST API Golang

Searching SO for Generic REST API Golang gives 0 results. Searching Google gives 2 results. So this question is maybe not correctly formulated or it is impossible to achieve in Golang.
My goal is to avoid repeating similar code over and over again. So I am trying to make the code in Golang as generic as possible. Write once, use many.
This is my first attempt to create a generic REST API for select in Golang. The code below gives almost what I want:
But the result is presented in the Terminal. I have no idea how to redirect the result to the browser.
package main
import (
"fmt"
"log"
"net/http"
"database/sql"
"time"
_ "github.com/lib/pq"
)
var db *sql.DB
func main() {
Connect()
http.HandleFunc("/", Query)
log.Fatal(http.ListenAndServe(":8080", nil))
}
func Connect() {
const (
host = "127.0.0.1"
port = 5432
user = "test"
password = "test"
dbname = "Test")
login := fmt.Sprintf("host=%s port=%d user=%s "+"password=%s dbname=%s sslmode=require", host, port, user, password, dbname)
var err error
db, err = sql.Open("postgres", login)
if err != nil {
log.Fatalln(err)
}
err = db.Ping()
if err != nil {
panic(err)
}
func Query(w http.ResponseWriter, r *http.Request) {
var query string
switch r.URL.String() {
case "/getuser":
query = "select * from getuser()"
case "/getco":
query = "select * from getco()"
case "/etc"
query = "select * from etc"
default:
query = ""
}
var err error
var rows *sql.Rows
rows, err = db.Query(query)
if err != nil {
http.Error(w, http.StatusText(500), 500)
return
}
defer rows.Close()
cols, err := rows.Columns()
vals := make([]interface{}, len(cols))
for i := 0; i < len(cols); i++ {
vals[i] = new(interface{})
if i != 0 {
fmt.Print("\t")
}
fmt.Print(cols[i])
}
fmt.Println()
for rows.Next() {
err = rows.Scan(vals...)
if err != nil {
fmt.Println(err)
continue
}
for i := 0; i < len(vals); i++ {
if i != 0 {
fmt.Print("\t")
}
printValue(vals[i].(*interface{}))
}
fmt.Println()
}
func printValue(pval *interface{}) {
switch v := (*pval).(type) {
case nil:
fmt.Print("NULL")
case bool:
if v {
fmt.Print("1")
} else {
fmt.Print("0")
}
case []byte:
fmt.Print(string(v))
case time.Time:
fmt.Print(v.Format("2006-01-02"))
default:
fmt.Print(v)
}
}
Every attempt to write to the browser gives various type of errors:
fmt.Printf("%s\n", vals...)
My questions are
How do I redirect the result to the browser?
Is there any better way to achieve this? (reuse generic code)
My recommendation would be to look at using existing packages like "mux" for calling REST APIs in browser. As a quick demo how you would do it as as follows:
your restapi.go cound have APIs as follows:
func SampleAPI(w http.ResponseWriter, r *http.Request) { //Assuming this is a POST request
var example SomeSruct
_ = json.NewDecoder(r.Body).Decode(&example) //Decode the POST body
result := someLogicFunction(example) //call your generic function
json.NewEncoder(w).Encode(result) //encode the result to pass it back to browser
}
Now say you write a main.go and you are using mux package here is an example of how you would call this
main.go
func main() {
router := mux.NewRouter()
router.HandleFunc("/testFunc",restapi.SampleAPI).Methods("POST") //This creates the route for your http request
handler := cros.Default().Handler(router) //You will need this if you plan to deploy it in a server and call it externally for testing locally you don't need this
log.Fatal(http.ListenAndServe(":8080", handler)) //Port that the router is listening to
}
Now note that you will have to import the "github.com/gorilla/mux" and the "github.com/rs/cors" packages to use these but this way you can create REST APIs whic can be accessed by te browser. Similarly you could create a GET method and use parameters which you can grab in your function and perform any logical step.
If you build and install the above code you can POST to localhost:8080/testFunc over http using any web app and get results i your browser. If you had a GET request you could directly type the Url in the browser and see the result.
write response with appropriahe HTTP hearers && status code
import "net/http"
func writeResponse(w http.ResponseWriter, contents []byte) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, contents)
}
sounds a little unclear, sorry

too many open files in mgo go server

I'm getting these errors in the logs:
Accept error: accept tcp [::]:80: accept4: too many open files;
for a mongodb server on ubuntu, written in go using mgo. They start appearing after it's been running for about a day.
code:
package main
import (
"encoding/json"
"io"
"net/http"
"gopkg.in/mgo.v2/bson"
)
var (
Database *mgo.Database
)
func hello(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "hello")
}
func setTile(w http.ResponseWriter, r *http.Request) {
var requestJSON map[string]interface{}
err := json.NewDecoder(r.Body).Decode(&requestJSON)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
collection := Database.C("tiles")
if requestJSON["tileId"] != nil {
query := bson.M{"tileId": requestJSON["tileId"]}
collection.RemoveAll(query)
collection.Insert(requestJSON)
w.WriteHeader(200)
w.Header().Set("Content-Type", "application/json")
js, _ := json.Marshal(map[string]string{"result": "ok"})
w.Write(js)
} else {
w.WriteHeader(200)
w.Header().Set("Content-Type", "application/json")
w.Write(js)
}
}
func getTile(w http.ResponseWriter, r *http.Request) {
var requestJSON map[string]interface{}
err := json.NewDecoder(r.Body).Decode(&requestJSON)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
collection := Database.C("tiles")
var result []map[string]interface{}
if requestJSON["tileId"] != nil {
query := bson.M{"tileId": requestJSON["tileId"]}
collection.Find(query).All(&result)
}
if len(result) > 0 {
w.WriteHeader(200)
w.Header().Set("Content-Type", "application/json")
js, _ := json.Marshal(result[0])
w.Write(js)
} else {
w.WriteHeader(200)
w.Header().Set("Content-Type", "application/json")
js, _ := json.Marshal(map[string]string{"result": "tile id not found"})
w.Write(js)
}
}
func main() {
session, _ := mgo.Dial("localhost")
Database = session.DB("mapdb")
mux := http.NewServeMux()
mux.HandleFunc("/", hello)
mux.HandleFunc("/setTile", setTile)
mux.HandleFunc("/getTile", getTile)
http.ListenAndServe(":80", mux)
}
Is there something in there that needs closing? Or is it structured wrong in some way?
There seems to be lots of places to set the open file limits, so i'm not sure how to find out what the limits actually are. But it seems like increasing the limit isn't the problem anyway, surely something is being opened on every request and not closed.
This is not how you store and use a MongoDB connection in Go.
You have to store an mgo.Session, not an mgo.Database instance. And whenever you need to interact with the MongoDB, you acquire a copy or a clone of the session (e.g. with Session.Copy() or Session.Clone()), and you close it when you don't need it (preferable using a defer statement). This will ensure you don't leak connections.
You also religiously omit checking for errors, please don't do that. Whatever returns an error, do check it and act on it properly (the least you can do is print / log it).
So basically what you need to do is something like this:
var session *mgo.Session
func init() {
var err error
if session, err = mgo.Dial("localhost"); err != nil {
log.Fatal(err)
}
}
func someHandler(w http.ResponseWriter, r *http.Request) {
sess := session.Copy()
defer sess.Close() // Must close!
c := sess.DB("mapdb").C("tiles")
// Do something with the collection, e.g.
var tile bson.M
if err := c.FindId("someTileID").One(&result); err != nil {
// Tile does not exist, send back error, e.g.:
log.Printf("Tile with ID not found: %v, err: %v", "someTileID", err)
http.NotFound(w, r)
return
}
// Do something with tile
}
See related questions:
mgo - query performance seems consistently slow (500-650ms)
Concurrency in gopkg.in/mgo.v2 (Mongo, Go)
You are missing:
defer r.Body.Close()
Make sure it is used before return statement.

Golang how can I make sql row a string

I am using Golang and Postgres, Postgres has an advance feature where it can return your queries in Json format. What I want to do is get that Json query results and return it but I am having trouble since it has to be a String in order to return it. This is my code
package main
import(
"fmt"
"database/sql"
_ "github.com/lib/pq"
"log"
)
func HelloServer(w http.ResponseWriter, req *http.Request) {
db, err := sql.Open("postgres", "user=postgres password=password dbname=name sslmode=disable")
if err != nil {
log.Fatal(err)
}
defer db.Close()
rows, err := db.Query("select To_Json(t) (SELECT * from cars)t")
io.WriteString(w, "hello, world!\n")
}
func main() {
http.HandleFunc("/hello", HelloServer)
log.Fatal(http.ListenAndServe(":12345", nil))
}
The Rows element returns a Json array how can I turn that Rows element into a String ? For C# and Java I would just append the .ToString() method to it and it would make it a string . As you can see from the code above the io.WriteString takes a String as a second parameter so I want to make the Rows variable a String after it has the Json returned so that I can display it in the browser by passing it to the method. I want to replace the Hello World with the String Rows.
Rows is a sql.Rows type. In order to use the data returned by your database query you will have to iterated over the "rows".
An example from the docs
age := 27
rows, err := db.Query("SELECT name FROM users WHERE age=?", age)
if err != nil {
log.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
log.Fatal(err)
}
fmt.Printf("%s is %d\n", name, age)
}
if err := rows.Err(); err != nil {
log.Fatal(err)
}
You should instead use QueryRow because you are expecting the database to return one result. In either case once you have used "Scan" to put the data into your own variable then you can either parse the JSON or print it out.

Can't get Go http to return an error; 'No data recieved'

I am doing attempting to build a basic API using Go which returns the results of a SQL query using the PostgreSQL library.
At the moment I can make the program return the values, but I can't get it to return a failed message to the user i.e. some JSON with an error message.
I have an error function as follows :
func handleError(w http.ResponseWriter, err error) {
if err != nil {
log.Print(err.Error() + "\r\n") // Logging
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
However the http.Error method doesn't appear to ever return anything. The error thrown is a table that doesn't exist in the database (which gets logged to a text file: i.e. 2016/01/11 23:28:19 pq: relation "building_roof" does not exist
My programmes query code looks like this:
table := pq.QuoteIdentifier(table)
identifier := pq.QuoteIdentifier("ID")
rows, err := db.Query( fmt.Sprintf("SELECT %s, ST_AsText(geom) FROM %s WHERE %s = $1", identifier, table, identifier), feature)
handleError(w, err)
Causing an error just gives a Chrome error:
No data received
ERR_EMPTY_RESPONSE
EDIT Full Code:
package main
import (
"fmt"
"encoding/json"
"os"
"log"
"net/http"
"database/sql"
"strings"
"time"
"github.com/lib/pq"
)
func handler(w http.ResponseWriter, r *http.Request) {
f, err := os.OpenFile("pgdump_errorlog.txt", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
log.Print("Couldn't open file")
defer f.Close()
log.SetOutput(f)
// Timing
start := time.Now()
// Postgres Credentials
const (
DB_USER = "postgres"
DB_PASSWORD = "OMITTED" // Removed details !
DB_PORT = "OMITTED"
DB_NAME = "OMITTED"
)
// Postgres Connect
dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s port=%s sslmode=disable",
DB_USER, DB_PASSWORD, DB_NAME, DB_PORT)
db, err := sql.Open("postgres", dbinfo)
handleError(w, err)
defer db.Close()
table := r.FormValue("table")
feature := r.FormValue("id")
if table != "" {
//Postgres Query
var (
id int
geom string
)
table := pq.QuoteIdentifier(table)
identifier := pq.QuoteIdentifier("ID")
rows, qerr := db.Query( fmt.Sprintf("SELECT %s, ST_AsText(geom) FROM %s WHERE %s = $1", identifier, table, identifier), feature)
handleError(w, err)
defer rows.Close()
for rows.Next() {
err := rows.Scan(&id, &geom)
handleError(w, err)
}
err = rows.Err()
handleError(w, err)
// Maniplate Strings
returngeom := strings.Replace(geom, "1.#QNAN", "", -1)
i := strings.Index(returngeom, "(")
wkt := strings.TrimSpace(returngeom[:i])
returngeom = returngeom[i:]
type WTKJSON struct {
WTKType string
Geometry string
Elapsed time.Duration
}
returnjson := WTKJSON{Geometry: returngeom, WTKType: wkt , Elapsed: time.Since(start)/1000000.0}
json.NewEncoder(w).Encode(returnjson)
}
}
func handleError(w http.ResponseWriter, err error) {
if err != nil {
log.Print(err.Error() + "\r\n") // Logging
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func main() {
http.HandleFunc("/", handler)
http.ListenAndServe(":8080", nil)
}
The following appeared to allow me to return JSON Errors:
func handleError(w http.ResponseWriter, err string) {
type APIError struct {
Error string
}
re, _ := json.Marshal(APIError{Error: err})
io.WriteString(w, string(re))
}
Used like so:
rows, err := db.Query( fmt.Sprintf("SELECT %s, ST_AsText(geom) FROM %s WHERE %s = $1", identifier, table, identifier), feature)
if err != nil {
handleError(w, err.Error())
return
}
Not suggesting this is the best method, but worked in my case.
When you are using the http.Error function the error message should a string. For simple testing I would suggest take this line
http.Error(w, err.Error(), http.StatusInternalServerError)
and change it to something like
http.Error(w,"there was an error", http.StatusInternalServerError)
and see if that response comes through. If it does its likely that you are trying to pass something that isn't a string in http.Error()