I'm using chi with our Go webservices.
How to configure the logger (middleware) so it only logs requests that ended up with errors (like 404) but it doesn't log successful requests (with status code 200)?
Here's our current implementation (with no logging at all)
r := chi.NewRouter()
if DEBUG_LOGS {
r.Use(middleware.Logger)
} else {
}
The easiest way is to implement the logging function by yourself using the example from the chi package (for simplicity, I removed the colors).
package main
import (
"bytes"
"fmt"
"log"
"net/http"
"os"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
)
const DEBUG_LOGS = true
func main() {
api := &http.Server{Addr: ":8000"}
r := chi.NewRouter()
if DEBUG_LOGS {
// create default logger/zerolog/logrus
logger := log.New(os.Stdout, "", log.LstdFlags)
r.Use(middleware.RequestLogger(&StructuredLogger{logger}))
}
r.Get("/tea", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTeapot) })
r.Get("/ok", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
api.Handler = r
err := api.ListenAndServe()
if err != nil {
log.Fatal(err)
}
}
// below is the implementation of the custom logger.
type StructuredLogger struct {
Logger middleware.LoggerInterface
}
type LogEntry struct {
*StructuredLogger
request *http.Request
buf *bytes.Buffer
useColor bool
}
func (l *StructuredLogger) NewLogEntry(r *http.Request) middleware.LogEntry {
entry := &LogEntry{
StructuredLogger: l,
request: r,
buf: &bytes.Buffer{},
useColor: false,
}
reqID := middleware.GetReqID(r.Context())
if reqID != "" {
fmt.Fprintf(entry.buf, "[%s] ", reqID)
}
fmt.Fprintf(entry.buf, "\"")
fmt.Fprintf(entry.buf, "%s ", r.Method)
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
fmt.Fprintf(entry.buf, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto)
entry.buf.WriteString("from ")
entry.buf.WriteString(r.RemoteAddr)
entry.buf.WriteString(" - ")
return entry
}
func (l *LogEntry) Write(status, bytes int, header http.Header, elapsed time.Duration, extra interface{}) {
// Do nothing if status code is 200/201/eg
if status < 300 {
return
}
fmt.Fprintf(l.buf, "%03d", status)
fmt.Fprintf(l.buf, " %dB", bytes)
l.buf.WriteString(" in ")
if elapsed < 500*time.Millisecond {
fmt.Fprintf(l.buf, "%s", elapsed)
} else if elapsed < 5*time.Second {
fmt.Fprintf(l.buf, "%s", elapsed)
} else {
fmt.Fprintf(l.buf, "%s", elapsed)
}
l.Logger.Print(l.buf.String())
}
func (l *LogEntry) Panic(v interface{}, stack []byte) {
middleware.PrintPrettyStack(v)
}
Related
Go version: go1.14.4 darwin/amd64
TCP server
package main
import (
"fmt"
"io"
"net"
"time"
)
const (
maxBufLen int = 4 * 1024
)
type Server struct {
}
func (s *Server) Start() {
listener, err := net.Listen("tcp", "localhost:9001")
if err != nil {
panic(err)
}
var connections []net.Conn
defer func() {
for _, conn := range connections {
// ignore
_ = conn.Close()
}
}()
// loop...
for {
connected, err := listener.Accept()
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Temporary() {
// temporary err will bi ignored
continue
} else {
fmt.Println(err)
return
}
}
go handleConn(connected, true)
connections = append(connections, connected)
}
}
func handleConn(conn net.Conn, server bool) {
buf := make([]byte, maxBufLen)
for {
// read
setTimeout(conn)
_, err := conn.Read(buf)
if err != nil {
if ne, ok := err.(net.Error); ok && (ne.Timeout() || ne.Temporary()) {
fmt.Println("need continue...")
continue
}
if err == io.EOF {
fmt.Println("EOF")
break
}
// other...
panic(err)
}
// handle recv msg.
s := string(buf)
if server {
//fmt.Println("server recv req ", s)
} else {
fmt.Println("client recv resp ", s)
}
if server {
output := "hi " + s
ob := []byte(output)
_, err := conn.Write(ob)
if err != nil {
fmt.Println(err)
break
}
}
}
}
func setTimeout(conn net.Conn) {
setErr := conn.SetReadDeadline(time.Now().Add(20 * time.Second))
if setErr != nil {
panic(setErr)
}
}
TCP client
package main
import (
"net"
"time"
)
type Client struct {
Exit chan struct{}
}
func (c *Client) Start() {
conn, err := net.Dial("tcp", "localhost:9001")
if err != nil {
panic(err)
}
defer conn.Close()
go handleWrite(conn)
go handleConn(conn, false)
<-c.Exit
}
func handleWrite(conn net.Conn) {
for {
input := "carryxyh"
_, err := conn.Write([]byte(input))
if err != nil {
panic(err)
}
<-time.After(100 * time.Second)
}
}
Main function
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
)
type Starter interface {
Start()
}
func main() {
var s Server
var c Client
go s.Start()
go c.Start()
sigs := make(chan os.Signal)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
<-sigs
c.Exit <- struct{}{}
fmt.Println("exit")
}
Running package main will print :
client recv resp hi carryxyh
client recv resp carryxyh
Printed twice. But from the program point of view, the server only writes information to the client once, and the content of the information should be hi carryxyh. But the client printed carryxyh in addition to hi carryxyh, which made me very confused.
During the investigation, I accidentally modified a line of code, and the program immediately returned to normal:
modeifycode
As shown above, I modified the server response information: output := "hi "+ s => output := "hi ", at this time the program only prints client recv resp hi.
This makes me completely confused, can anyone help me solve this problem? It would be better if there is a troubleshooting idea.
enter image description here
这里你忽略了读取的字节长度,可能返回0字节,你又把buf的内容 又回写给client了.
_, err := conn.Read(buf) 改成 bytesRead, err := conn.Read(buf) ;
if bytesRead <= 0 { // 没有buf 可以读取 }
--
eg:
if bytesRead == 0 && err == nil {
err = io.EOF
log.Errorf("[network] ReadOnce maybe always return (0, nil) and causes dead loop, Connection = %d, Local Address = %+v, Remote Address = %+v",
c.id, c.rawConnection.LocalAddr(), c.RemoteAddr())
}
I have been using mgo for my API but I'm seeing many current connections in my MongoDB (while using less than 5 devices for testing). By executing db.serverStatus().connections in my Mongo server I get: { "current" : 641, "available" : 838219, "totalCreated" : 1136 }. Below I transcript my issue in mgo's Github (Issue #429):
Is my way of using mgo in a web server the correct way? If not, can you give me a full example?
This code is not functional, take it as almost pseudo code (because of the missing parts like the imports or where the configs come from and models), but it is exactly how I am using mgo.
I must clarify that I'm building an API which is used by several mobile devices and webapps.
main.go
func main() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{config.DatabaseURL},
Timeout: 60 * time.Second,
Database: config.DatabaseName,
Username: config.DatabaseUsername,
Password: config.DatabasePassword,
}
db, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
log.Fatal("Cannot Dial Mongo: ", err)
}
defer db.Close()
db.SetMode(mgo.Monotonic, true)
phoneIndex := mgo.Index{
Key: []string{"pp"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err = db.DB(config.DatabaseName).C("users").EnsureIndex(phoneIndex)
if err != nil {
panic(err)
}
router := mux.NewRouter()
router.HandleFunc("/login", publicWithDB(login, db)).Methods("POST")
if err := http.ListenAndServe(":5000", router); err != nil {
log.Fatal(err)
}
}
func publicWithDB(fn http.HandlerFunc, db *mgo.Session) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
dbsession := db.Copy()
defer dbsession.Close()
fn(w, r.WithContext(context.WithValue(r.Context(), contextKeyDatabase, dbsession)))
}
}
func login(w http.ResponseWriter, r *http.Request) {
r.ParseForm() // Parses the request body
device := r.Form.Get("device")
var deviceid bson.ObjectId
if bson.IsObjectIdHex(device) {
deviceid = bson.ObjectIdHex(device)
}
db := r.Context().Value(contextKeyDatabase).(*mgo.Session)
var device models.Device
err := db.DB(config.DatabaseName).C("devices").FindId(deviceid).One(&device)
w.WriteHeader(200)
w.Write([]byte(utils.ResponseToString(models.Response{Status: 200, Message: "asdasd", Data: device})))
}
I'm posting this because I couldn't find a complete implementation.
Here's an example of how I have seen myself and others structure web apps in Go. This code is untested and is purely for example. It's missing imports and potentially has errors.
EDIT Added a middleware example.
main.go
package main
func main() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{config.DatabaseURL},
Timeout: 60 * time.Second,
Database: config.DatabaseName,
Username: config.DatabaseUsername,
Password: config.DatabasePassword,
}
db, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
log.Fatal("Cannot Dial Mongo: ", err)
}
defer db.Close()
db.SetMode(mgo.Monotonic, true)
phoneIndex := mgo.Index{
Key: []string{"pp"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err = db.DB(config.DatabaseName).C("users").EnsureIndex(phoneIndex)
if err != nil {
panic(err)
}
mgoAdapter := mongo.NewAdapter(db, config.DatabaseName)
deviceStore := mongo.NewDeviceStore(mgoAdapter)
userStore := mongo.NewUserStore(mgoAdapter)
loginController := controllers.NewLoginController(deviceStore)
router := mux.NewRouter()
router.HandleFunc("/login", middleware.AuthorizeUser(userStore)(http.HandlerFunc(loginController.Login)).Methods("POST")
if err := http.ListenAndServe(":5000", router); err != nil {
log.Fatal(err)
}
}
controllers/login.go
package controllers
type LoginController struct {
store DeviceStore
}
func NewLoginController(store stores.DeviceStore) *LoginController {
return &LoginController{
store: store,
}
}
func (c *LoginController) Login(w http.ResponseWriter, r *http.Request) {
r.ParseForm() // Parses the request body
device := r.Form.Get("device")
data, err := c.store.FindByDevice(device)
var respose models.Response
if err != nil {
w.WriteHeader(500)
response = models.Response{Status: 500, Message: fmt.Sprintf("error: %s", err)}
} else if data == nil {
w.WriteHeader(404)
response = models.Response{Status: 404, Message: "device not found"}
} else {
response = models.Response{Status: 200, Message: "device found", Data: data}
}
// Write sets header to 200 if it hasn't been set already
w.Write([]byte(utils.ResponseToString(response)))
}
stores/stores.go
package stores
type DeviceStore interface {
FindByDevice(device string) (*models.Device, error)
}
type UserStore interface {
FindByToken(token string) (*models.User, error)
}
middleware/auth.go
package middleware
func AuthorizeUser(store stores.UserStore) func(h *http.Handler) http.Handler {
return func(h *http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Logic for authorizing user
// Could also store user in the request context
})
}
}
mongo/mongo.go
package mongo
type Adapter struct {
session *mgo.Session
databaseName string
}
func NewAdapter(session *mgo.Session, dbName string) *Adapter {
return &Adapter{session: session, databaseName: dbName}
}
type deviceStore struct {
*Adapter
}
func NewDeviceStore(adapter *Adapter) stores.DeviceStore {
return &deviceStore{adapter}
}
const devices = "devices"
func (s *deviceStore) FindByDevice(d string) (*models.Device, err) {
sess := s.session.copy()
defer sess.close()
var deviceID bson.ObjectId
if bson.IsObjectIdHex(d) {
deviceID = bson.ObjectIdHex(d)
}
var device models.Device
err := db.DB(s.databaseName).C(devices).FindId(deviceID).One(&device)
if err == mgo.ErrNotFound {
return nil, nil
}
return &device, err
}
type userStore struct {
*Adapter
}
const users = "users"
func NewUserStore(adapter *Adapter) stores.UserStore {
return &userStore{adapter}
}
func (s *userStore) GetUserByToken(token string) (*models.User, error) {
sess := s.session.copy()
defer sess.close()
var user models.User
err := db.DB(s.databaseName).C(users).Find(bson.M{"token": token}).One(&user)
if err == mgo.ErrNotFound {
return nil, nil
}
return &user, err
}
I'm new to Golang, i have been created an api in Golang and MongoDB.
After a hard struggle successfully separate the controller and model packages ,Now i want to define routes in a separate package of routers and access them in main package same like controllers and models.I'm using gorilla/mux package for routing.Anyone can help me please, thanks in Advance!
and here is all of my code:
RESTMONGOMVC/main.go
package main
import (
"RESTMONGOMVC/controllers"
"log"
"net/http"
"github.com/gorilla/mux"
"gopkg.in/mgo.v2"
)
var (
session *mgo.Session
collection *mgo.Collection
err error
)
func getSession() *mgo.Session {
// Connect to our local mongo
s, err := mgo.Dial("mongodb://localhost")
// Check if connection error, is mongo running?
if err != nil {
panic(err)
}
// Deliver session
return s
}
func main() {
var err error
r := mux.NewRouter()
uc := controllers.NewNoteController(getSession())
r.HandleFunc("/api/notes", uc.GetNotes).Methods("GET")
r.HandleFunc("/api/notes", uc.CreateNote).Methods("POST")
r.HandleFunc("/api/notes/{id}", uc.UpdateNote).Methods("PUT")
r.HandleFunc("/api/notes/{id}", uc.DeleteNote).Methods("DELETE")
http.Handle("/api/", r)
http.Handle("/", http.FileServer(http.Dir(".")))
log.Println("Starting Mongodb Session")
session, err = mgo.Dial("localhost")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
collection = session.DB("notesdb").C("notes")
log.Println("Listening on 8080")
http.ListenAndServe(":8080", nil)
}
controllers/note.go
package controllers
import (
"RESTMONGOMVC/models"
"encoding/json"
"log"
"net/http"
"time"
"github.com/gorilla/mux"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
var (
session *mgo.Session
collection *mgo.Collection
err error
)
type (
// UserController represents the controller for operating on the User resource
NoteController struct {
session *mgo.Session
}
)
// NewUserController provides a reference to a UserController with provided mongo session
func NewNoteController(s *mgo.Session) *NoteController {
return &NoteController{s}
}
func (uc NoteController) GetNotes(w http.ResponseWriter, r *http.Request) {
var notes []models.Note
iter := collection.Find(nil).Iter()
result := models.Note{}
for iter.Next(&result) {
notes = append(notes, result)
}
w.Header().Set("Content-Type", "application/json")
j, err := json.Marshal(models.NotesResource{Notes: notes})
if err != nil {
panic(err)
}
w.Write(j)
}
func (uc NoteController) CreateNote(w http.ResponseWriter, r *http.Request) {
var noteResource models.NoteResource
err := json.NewDecoder(r.Body).Decode(¬eResource)
if err != nil {
panic(err)
}
note := noteResource.Note
//get a new Id
obj_id := bson.NewObjectId()
note.Id = obj_id
note.CreatedOn = time.Now()
//Insert into document collection
err = collection.Insert(¬e)
if err != nil {
panic(err)
} else {
log.Printf("Inserted New Record with Title :%s", note.Title)
}
j, err := json.Marshal(models.NoteResource{Note: note})
if err != nil {
panic(err)
}
w.Header().Set("Content-Type", "application/json")
w.Write(j)
}
func (uc NoteController) UpdateNote(w http.ResponseWriter, r *http.Request) {
var err error
//get id from incoming url
vars := mux.Vars(r)
id := bson.ObjectIdHex(vars["id"])
//decode the incoming Note into json
var noteResource models.NoteResource
err = json.NewDecoder(r.Body).Decode(¬eResource)
if err != nil {
panic(err)
}
//partial update on mongodb
err = collection.Update(bson.M{"_id": id},
bson.M{"$set": bson.M{
"title": noteResource.Note.Title,
"decription": noteResource.Note.Description,
}})
if err == nil {
log.Printf("Updated Note : %s", id, noteResource.Note.Title)
} else {
panic(err)
}
w.WriteHeader(http.StatusNoContent)
}
func (uc NoteController) DeleteNote(w http.ResponseWriter, r *http.Request) {
var err error
vars := mux.Vars(r)
id := vars["id"]
//Remove from database
err = collection.Remove(bson.M{"_id": bson.ObjectIdHex(id)})
if err != nil {
log.Printf("Could not find the Note %s to delete", id)
}
w.WriteHeader(http.StatusNoContent)
}
models/note.go
package models
import (
"time"
"gopkg.in/mgo.v2/bson"
)
type Note struct {
Id bson.ObjectId `bson:"_id" json:"id"`
Title string `json:"title"`
Description string `json:"description"`
CreatedOn time.Time `json:"craetedOn"`
}
type NoteResource struct {
Note Note `json:"note"`
}
type NotesResource struct {
Notes []Note `json:"notes"`
}
Not an programming expert but this is how I manage my routes/handlers.
routes/routes.go
package routes
import (
"github.com/gorilla/mux"
)
//NewRouter is main routing entry point
func NewRouter() *mux.Router {
r := mux.NewRouter()
indexHandler(r) // Index handler
fileServer(r) // Fileserver to serve static files
otherLogicalHandler(r) // Other domain/business logic scoped handler
return r
}
routes/indexHandler.go
package routes
import (
"fmt"
"net/http"
"github.com/gorilla/mux"
"github.com/myusername/project/models"
)
func indexHandler(r *mux.Router) {
r.HandleFunc("/", indexMainHandler).Methods("GET")
// Other endpoints goes there if you want to list it in this current indexHandler.go file
// Example: r.HandleFunc("/signup", signupMainHandler).Methods("GET")
}
// Handlers
func indexMainHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=UTF-8")
// Call your model/s there
mydata, err := models.GetMyDataFunction()
if err != nil {
// Handle your error there
return
}
utils.ExecuteTemplate(w, "index.html", struct {
Title string
// Use your model data for templates there
MyData []models.MyData
// Other models/data can go there if multiple data objects used per page.
}{
Title: "Main Page",
MyData: mydata,
})
}
// func signupMainHandler(w http.ResponseWriter, r *http.Request) ...
// Basically repeat the same logic as in indexMainHandler function
routes/fileServer.go
package routes
import (
"net/http"
"github.com/gorilla/mux"
)
func fileServer(r *mux.Router) {
fs := http.FileServer(http.Dir("static"))
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", fs))
}
routes/otherLogicalHandler.go
... and so on.
As you can see, all they belong to package routes but are divided into multiple files. File names doesn't actually matter. You can name them as you want.
Models lives in models directory and also belongs to single package models package.
Every time you create new routes file, remember to call it in routes.go file.
Hope this will help for somebody.
I'm attempting to do a resumable upload as described here.
When I perform the upload, I receive a response status of 400 Bad Request, and a response body of:
{
"error": {
"errors": [
{
"domain": "global",
"reason": "badContent",
"message": "Unsupported content with type: application/octet-stream"
}
],
"code": 400,
"message": "Unsupported content with type: application/octet-stream"
}
}
The script I'm using to perform the upload is in Go, here:
package main
import(
"bytes"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
func main(){
// config
accessToken := "a valid token"
acctId := "a valid account id"
webPropertyId := "a valid web property id"
customDataSourceId := "a valid custom data source id"
path := "/a/path/to/a/validly/formatted/file.csv"
params := map[string]string{
"title": "My Document",
"author": "Becca Petrin",
"description": "Riveting stuff",
}
paramName := "file"
url := fmt.Sprintf("https://www.googleapis.com/upload/analytics/v3/management/accounts/%s/webproperties/%s/customDataSources/%s/uploads?uploadType=resumable", acctId, webPropertyId, customDataSourceId)
// create the body
file, err := os.Open(path)
if err != nil {
fmt.Println("Err opening file:", err.Error())
return
}
defer file.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(paramName, filepath.Base(path))
if err != nil {
fmt.Println("Err creating form file:", err.Error())
return
}
_, err = io.Copy(part, file)
if err != nil {
fmt.Println("Err copying:", err.Error())
return
}
for k, v := range params {
_ = writer.WriteField(k, v)
}
if err := writer.Close(); err != nil {
fmt.Println("Err closing writer:", err.Error())
return
}
req, err := http.NewRequest("POST", url, body)
if err != nil {
fmt.Printf("Err creating request:", err.Error())
return
}
// add authorization
req.Header.Set("Authorization", "Bearer "+accessToken)
// add headers
// no multipart headers work, and "application/octet-stream"" doesn't work
// uncommenting and using "text/plain" results in a 200 without the expected response body
//req.Header.Add("Content-Type", "text/plain")
// execute request
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println("Err doing request:", err.Error())
return
}
fmt.Println("Response status:", resp.Status)
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println("Err reading resp body:", err.Error())
return
}
fmt.Printf("Response body: %s", b)
}
As noted in the comments, I receive the unsupported content type response if I don't include a Content-Type header. I also receive it if I use any multipart content types, or application/octet-stream. If I use text/plain, I receive a 200, but I don't receive the expected body.
What am I doing wrong? Thanks in advance!
I decided to take an alternate approach to this by trying through the Google Go client library, and I got it working with this snippet:
package main
import (
"errors"
"fmt"
"os"
"time"
google "google.golang.org/api/analytics/v3"
"golang.org/x/oauth2"
)
var (
accessToken = "a valid token"
acctId = "a valid account ID"
webPropertyId = "a valid web property ID"
customDataSourceId = "a valid custom data source ID"
filePath = "/path/to/file.csv"
)
func main(){
ctx1 := NewCallContext()
tokenSource := TokenSource{}
httpClient := oauth2.NewClient(ctx1, &tokenSource)
service, err := google.New(httpClient)
if err != nil {
fmt.Println("Err making client:", err.Error())
return
}
r, err := NewFileReader(filePath)
if err != nil {
fmt.Println("Err making reader:", err.Error())
return
}
ctx2 := NewCallContext()
call := service.Management.Uploads.UploadData(acctId, webPropertyId, customDataSourceId)
call = call.ResumableMedia(ctx2, r, 10, "application/octet-stream")
upload, err := call.Do()
if err != nil {
fmt.Println("Err doing call: %v", err)
return
}
fmt.Printf("%s", upload)
}
// http://golang.org/pkg/io/#ReaderAt
func NewFileReader(pathToFile string) (*FileReader, error) {
f, err := os.Open(pathToFile)
if err != nil {
return nil, err
}
return &FileReader{f}, nil
}
type FileReader struct {
f *os.File
}
func (s *FileReader) ReadAt(p []byte, off int64) (int, error) {
return s.f.ReadAt(p, off)
}
// https://godoc.org/golang.org/x/net/context#Context
func NewCallContext() *CallContext {
c := make(<-chan struct{})
return &CallContext{c}
}
type CallContext struct {
doneChan <-chan struct{}
}
func (s *CallContext) Deadline() (time.Time, bool) {
return time.Now().Add(time.Duration(10) * time.Second), true
}
func (s *CallContext) Done() <-chan struct{} {
return s.doneChan
}
func (s *CallContext) Err() error {
select {
case <- s.doneChan:
return errors.New("Done")
default:
}
return nil
}
func (s *CallContext) Value(key interface{}) interface{} {
return nil
}
// satisfies the oauth2 tokensource interface
type TokenSource struct {}
func (t *TokenSource) Token() (*oauth2.Token, error) {
return &oauth2.Token{AccessToken:accessToken}, nil
}
I'm new to Go and evaluating it for a project.
I'm trying to write a custom handler to serve files with net/http.
I can't use the default http.FileServer() handler because I need to have access to the underlying socket (the internal net.Conn) so I can perform some informational platform specific "syscall" calls on it (mainly TCP_INFO).
More precisly: I need to access the underlying socket of the http.ResponseWriter in the handler function:
func myHandler(w http.ResponseWriter, r *http.Request) {
...
// I need the net.Conn of w
...
}
used in
http.HandleFunc("/", myHandler)
Is there a way to this. I looked at how websocket.Upgrade does this but it uses Hijack() which is 'too much' because then I have to code 'speaking http' over the raw tcp socket I get. I just want a reference to the socket and not taking over completely.
After Issue #30694 is completed, it looks like Go 1.13 will probably support storing the net.Conn in the Request Context, which makes this fairly clean and simple:
package main
import (
"net/http"
"context"
"net"
"log"
)
type contextKey struct {
key string
}
var ConnContextKey = &contextKey{"http-conn"}
func SaveConnInContext(ctx context.Context, c net.Conn) (context.Context) {
return context.WithValue(ctx, ConnContextKey, c)
}
func GetConn(r *http.Request) (net.Conn) {
return r.Context().Value(ConnContextKey).(net.Conn)
}
func main() {
http.HandleFunc("/", myHandler)
server := http.Server{
Addr: ":8080",
ConnContext: SaveConnInContext,
}
server.ListenAndServe()
}
func myHandler(w http.ResponseWriter, r *http.Request) {
conn := GetConn(r)
...
}
Until then ... For a server listening on a TCP port, net.Conn.RemoteAddr().String() is unique for each connection and is available to the http.Handler as r.RemoteAddr, so it can be used as a key to a global map of Conns:
package main
import (
"net/http"
"net"
"fmt"
"log"
)
var conns = make(map[string]net.Conn)
func ConnStateEvent(conn net.Conn, event http.ConnState) {
if event == http.StateActive {
conns[conn.RemoteAddr().String()] = conn
} else if event == http.StateHijacked || event == http.StateClosed {
delete(conns, conn.RemoteAddr().String())
}
}
func GetConn(r *http.Request) (net.Conn) {
return conns[r.RemoteAddr]
}
func main() {
http.HandleFunc("/", myHandler)
server := http.Server{
Addr: ":8080",
ConnState: ConnStateEvent,
}
server.ListenAndServe()
}
func myHandler(w http.ResponseWriter, r *http.Request) {
conn := GetConn(r)
...
}
For a server listening on a UNIX socket, net.Conn.RemoteAddr().String() is always "#", so the above doesn't work. To make this work, we can override net.Listener.Accept(), and use that to override net.Conn.RemoteAddr().String() so that it returns a unique string for each connection:
package main
import (
"net/http"
"net"
"os"
"golang.org/x/sys/unix"
"fmt"
"log"
)
func main() {
http.HandleFunc("/", myHandler)
listenPath := "/var/run/go_server.sock"
l, err := NewUnixListener(listenPath)
if err != nil {
log.Fatal(err)
}
defer os.Remove(listenPath)
server := http.Server{
ConnState: ConnStateEvent,
}
server.Serve(NewConnSaveListener(l))
}
func myHandler(w http.ResponseWriter, r *http.Request) {
conn := GetConn(r)
if unixConn, isUnix := conn.(*net.UnixConn); isUnix {
f, _ := unixConn.File()
pcred, _ := unix.GetsockoptUcred(int(f.Fd()), unix.SOL_SOCKET, unix.SO_PEERCRED)
f.Close()
log.Printf("Remote UID: %d", pcred.Uid)
}
}
var conns = make(map[string]net.Conn)
type connSaveListener struct {
net.Listener
}
func NewConnSaveListener(wrap net.Listener) (net.Listener) {
return connSaveListener{wrap}
}
func (self connSaveListener) Accept() (net.Conn, error) {
conn, err := self.Listener.Accept()
ptrStr := fmt.Sprintf("%d", &conn)
conns[ptrStr] = conn
return remoteAddrPtrConn{conn, ptrStr}, err
}
func GetConn(r *http.Request) (net.Conn) {
return conns[r.RemoteAddr]
}
func ConnStateEvent(conn net.Conn, event http.ConnState) {
if event == http.StateHijacked || event == http.StateClosed {
delete(conns, conn.RemoteAddr().String())
}
}
type remoteAddrPtrConn struct {
net.Conn
ptrStr string
}
func (self remoteAddrPtrConn) RemoteAddr() (net.Addr) {
return remoteAddrPtr{self.ptrStr}
}
type remoteAddrPtr struct {
ptrStr string
}
func (remoteAddrPtr) Network() (string) {
return ""
}
func (self remoteAddrPtr) String() (string) {
return self.ptrStr
}
func NewUnixListener(path string) (net.Listener, error) {
if err := unix.Unlink(path); err != nil && !os.IsNotExist(err) {
return nil, err
}
mask := unix.Umask(0777)
defer unix.Umask(mask)
l, err := net.Listen("unix", path)
if err != nil {
return nil, err
}
if err := os.Chmod(path, 0660); err != nil {
l.Close()
return nil, err
}
return l, nil
}
Note that although in current implementation http.ResponseWriter is a *http.response (note the lowercase!) which holds the connection, the field is unexported and you can't access it.
Instead take a look at the Server.ConnState hook: you can "register" a function which will be called when the connection state changes, see http.ConnState for details. For example you will get the net.Conn even before the request enters the handler (http.StateNew and http.StateActive states).
You can install a connection state listener by creating a custom Server like this:
func main() {
http.HandleFunc("/", myHandler)
s := &http.Server{
Addr: ":8081",
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
ConnState: ConnStateListener,
}
panic(s.ListenAndServe())
}
func ConnStateListener(c net.Conn, cs http.ConnState) {
fmt.Printf("CONN STATE: %v, %v\n", cs, c)
}
This way you will have exactly the desired net.Conn even before (and also during and after) invoking the handler. The downside is that it is not "paired" with the ResponseWriter, you have to do that manually if you need that.
You can use an HttpHijacker to take over the TCP connection from the ResponseWriter. Once you've done that you're free to use the socket to do whatever you want.
See http://golang.org/pkg/net/http/#Hijacker, which also contains a good example.
This can be done with reflection. it's a bit "dirty" but it works:
package main
import "net/http"
import "fmt"
import "runtime"
import "reflect"
func myHandler(w http.ResponseWriter, r *http.Request) {
ptrVal := reflect.ValueOf(w)
val := reflect.Indirect(ptrVal)
// w is a "http.response" struct from which we get the 'conn' field
valconn := val.FieldByName("conn")
val1 := reflect.Indirect(valconn)
// which is a http.conn from which we get the 'rwc' field
ptrRwc := val1.FieldByName("rwc").Elem()
rwc := reflect.Indirect(ptrRwc)
// which is net.TCPConn from which we get the embedded conn
val1conn := rwc.FieldByName("conn")
val2 := reflect.Indirect(val1conn)
// which is a net.conn from which we get the 'fd' field
fdmember := val2.FieldByName("fd")
val3 := reflect.Indirect(fdmember)
// which is a netFD from which we get the 'sysfd' field
netFdPtr := val3.FieldByName("sysfd")
fmt.Printf("netFDPtr= %v\n", netFdPtr)
// which is the system socket (type is plateform specific - Int for linux)
if runtime.GOOS == "linux" {
fd := int(netFdPtr.Int())
fmt.Printf("fd = %v\n", fd)
// fd is the socket - we can call unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(fd),....) on it for instance
}
fmt.Fprintf(w, "Hello World")
}
func main() {
http.HandleFunc("/", myHandler)
err := http.ListenAndServe(":8081", nil)
fmt.Println(err.Error())
}
Ideally the library should be augmented with a method to get the underlying net.Conn
Expanding on KGJV's answer, a working solution using reflection to maintain a map of connections indexed by net.Conn instance memory addresses.
Instances of net.Conn can be looked up by pointer, and pointers derived using reflection against http.Response.
It's a bit nasty, but given you can't access unpublished fields with reflection it's the only way I could see of doing it.
// Connection array indexed by connection address
var conns = make(map[uintptr]net.Conn)
var connMutex = sync.Mutex{}
// writerToConnPrt converts an http.ResponseWriter to a pointer for indexing
func writerToConnPtr(w http.ResponseWriter) uintptr {
ptrVal := reflect.ValueOf(w)
val := reflect.Indirect(ptrVal)
// http.conn
valconn := val.FieldByName("conn")
val1 := reflect.Indirect(valconn)
// net.TCPConn
ptrRwc := val1.FieldByName("rwc").Elem()
rwc := reflect.Indirect(ptrRwc)
// net.Conn
val1conn := rwc.FieldByName("conn")
val2 := reflect.Indirect(val1conn)
return val2.Addr().Pointer()
}
// connToPtr converts a net.Conn into a pointer for indexing
func connToPtr(c net.Conn) uintptr {
ptrVal := reflect.ValueOf(c)
return ptrVal.Pointer()
}
// ConnStateListener bound to server and maintains a list of connections by pointer
func ConnStateListener(c net.Conn, cs http.ConnState) {
connPtr := connToPtr(c)
connMutex.Lock()
defer connMutex.Unlock()
switch cs {
case http.StateNew:
log.Printf("CONN Opened: 0x%x\n", connPtr)
conns[connPtr] = c
case http.StateClosed:
log.Printf("CONN Closed: 0x%x\n", connPtr)
delete(conns, connPtr)
}
}
func HandleRequest(w http.ResponseWriter, r *http.Request) {
connPtr := writerToConnPtr(w)
connMutex.Lock()
defer connMutex.Unlock()
// Requests can access connections by pointer from the responseWriter object
conn, ok := conns[connPtr]
if !ok {
log.Printf("error: no matching connection found")
return
}
// Do something with connection here...
}
// Bind with http.Server.ConnState = ConnStateListener
It looks like you cannot "pair" a socket (or net.Conn) to either http.Request or http.ResponseWriter.
But you can implement your own Listener:
package main
import (
"fmt"
"net"
"net/http"
"time"
"log"
)
func main() {
// init http server
m := &MyHandler{}
s := &http.Server{
Handler: m,
}
// create custom listener
nl, err := net.Listen("tcp", ":8080")
if err != nil {
log.Fatal(err)
}
l := &MyListener{nl}
// serve through custom listener
err = s.Serve(l)
if err != nil {
log.Fatal(err)
}
}
// net.Conn
type MyConn struct {
nc net.Conn
}
func (c MyConn) Read(b []byte) (n int, err error) {
return c.nc.Read(b)
}
func (c MyConn) Write(b []byte) (n int, err error) {
return c.nc.Write(b)
}
func (c MyConn) Close() error {
return c.nc.Close()
}
func (c MyConn) LocalAddr() net.Addr {
return c.nc.LocalAddr()
}
func (c MyConn) RemoteAddr() net.Addr {
return c.nc.RemoteAddr()
}
func (c MyConn) SetDeadline(t time.Time) error {
return c.nc.SetDeadline(t)
}
func (c MyConn) SetReadDeadline(t time.Time) error {
return c.nc.SetReadDeadline(t)
}
func (c MyConn) SetWriteDeadline(t time.Time) error {
return c.nc.SetWriteDeadline(t)
}
// net.Listener
type MyListener struct {
nl net.Listener
}
func (l MyListener) Accept() (c net.Conn, err error) {
nc, err := l.nl.Accept()
if err != nil {
return nil, err
}
return MyConn{nc}, nil
}
func (l MyListener) Close() error {
return l.nl.Close()
}
func (l MyListener) Addr() net.Addr {
return l.nl.Addr()
}
// http.Handler
type MyHandler struct {
// ...
}
func (h *MyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Hello World")
}