How does golang's net.Conn.Read when to stop reading? - sockets

I'm trying to write a simple sockets based go server. I'm just wondering how does the connection.Read below knows when to stop reading.
(Note: this is not my code, I copied it from Unix Sockets in Go as example)
package main
import (
"log"
"net"
)
func echoServer(c net.Conn) {
for {
buf := make([]byte, 512)
nr, err := c.Read(buf)
if err != nil {
return
}
data := buf[0:nr]
println("Server got:", string(data))
_, err = c.Write(data)
if err != nil {
log.Fatal("Write: ", err)
}
}
}
func main() {
l, err := net.Listen("unix", "/tmp/echo.sock")
if err != nil {
log.Fatal("listen error:", err)
}
for {
fd, err := l.Accept()
if err != nil {
log.Fatal("accept error:", err)
}
go echoServer(fd)
}
}
Is it the EOF character or there's something else?
It would be really helpful if someone can point me to a link official go docs. Thanks.

This is the implementation of the default Read method on net.Conn.Read:
// Read implements the Conn Read method.
func (c *conn) Read(b []byte) (int, error) {
if !c.ok() {
return 0, syscall.EINVAL
}
n, err := c.fd.Read(b)
if err != nil && err != io.EOF {
err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
}
return n, err
}
This is the implementation of the c.fd.Read(b) that is called within the function above:
func (fd *netFD) Read(p []byte) (n int, err error) {
if err := fd.readLock(); err != nil {
return 0, err
}
defer fd.readUnlock()
if len(p) == 0 {
// If the caller wanted a zero byte read, return immediately
// without trying. (But after acquiring the readLock.) Otherwise
// syscall.Read returns 0, nil and eofError turns that into
// io.EOF.
// TODO(bradfitz): make it wait for readability? (Issue 15735)
return 0, nil
}
if err := fd.pd.prepareRead(); err != nil {
return 0, err
}
if fd.isStream && len(p) > 1<<30 {
p = p[:1<<30]
}
for {
n, err = syscall.Read(fd.sysfd, p)
if err != nil {
n = 0
if err == syscall.EAGAIN {
if err = fd.pd.waitRead(); err == nil {
continue
}
}
}
err = fd.eofError(n, err)
break
}
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("read", err)
}
return
}
So, yes, an EOF will make it stop reading. But so will plenty of other non-nil errors.

It will stop reading when its underlying implementation hits any error.
Error may be an actual I/O error, or it could be the operating system signaling connection closed with io.EOF, or it could be a timeout, so on.

Related

Is there a way with GRPC to notify the stream of CRUD operations to give realtime updates to the client?

I am new to GRPC and I am trying to implement a basic CRUD + listing. I use unary rpc's for the CRUD and a server stream for the listing. What I would like to do however is update the client whenever someone changes a record in the database that you are listing.
So for example user A is listing 10 companies. And user B is updating one of those companies. I want user A's client to be updated once the update rpc is called.
This is what I have for now
func RegisterCompanyServer(l hclog.Logger, gs *grpc.Server) {
r := postgres.NewPostgresCompanyRepository()
cs := NewCompanyServer(l, r)
pb.RegisterCompanyServiceServer(gs, cs)
}
type CompanyServer struct {
logger hclog.Logger
repo repo.CompanyRepository
pb.UnimplementedCompanyServiceServer
}
func NewCompanyServer(l hclog.Logger, r repo.CompanyRepository) *CompanyServer {
return &CompanyServer{
logger: l,
repo: r,
}
}
func (c *CompanyServer) ListCompany(req *pb.CompanyListRequest, stream pb.CompanyService_ListCompanyServer) error {
//Somehow listen to CreateCompany() and update the client
companies, err := c.repo.List(req.Query)
if err != nil {
return err
}
for _, c := range companies {
bytes, err := json.Marshal(c)
if err != nil {
return err
}
out := &pb.Company{}
if err = jsonEnc.Unmarshal(bytes, out); err != nil {
return err
}
res := &pb.CompanyListResponse{
Company: out,
}
err = stream.Send(res)
if err != nil {
return err
}
}
return nil
}
func (c *CompanyServer) CreateCompany(context context.Context, req *pb.CompanyCreateRequest) (*pb.CompanyCreateResponse, error) {
input := req.GetCompany()
if input == nil {
return nil, errors.New("Parsing Error")
}
bytes, err := jsonEnc.Marshal(input)
if err != nil {
return nil, err
}
company := &myCompany.Company{}
if err = json.Unmarshal(bytes, company); err != nil {
return nil, err
}
result, err := c.repo.Create(company)
if err != nil {
return nil, err
}
res := &pb.CompanyCreateResponse{
Id: result,
}
//Somehow notify the stream that a company was created
return res, nil
}
Is this even feasable with GRPC? What techniques are out there to do this? I am currently working with a postgresql database.

golang net/smtp getting smtp server response DSN

I am using golang net/smtp to send mails
Whenever I send to my smtp server I need to capture the response from the server
Especially the DSN
For example my local smtp server gives a "ok queued as " at the end of the mail
I need to capture this and print in the logs
How can I do this
package main
import (
"log"
"net/smtp"
)
func sendEmail(msg []byte) {
c, err := smtp.Dial("localhost:25")
if err != nil {
log.Fatal(err)
}
if err := c.Mail("sender#example.org"); err != nil {
log.Fatal(err)
}
if err := c.Rcpt("recipient#example.net"); err != nil {
log.Fatal(err)
}
wc, err := c.Data()
if err != nil {
log.Fatal(err)
}
_, err = wc.Write(msg)
if err != nil {
log.Fatal(err)
}
//How do I get the response here ??
err = wc.Close()
if err != nil {
log.Fatal(err)
}
err = c.Quit()
if err != nil {
log.Fatal(err)
}
}
As mentioned in the comments you can use c.Text.ReadResponse():
package main
import (
"net/smtp"
)
func sendEmail(msg []byte) (code int, message string, err error) {
c, err := smtp.Dial("localhost:25")
if err != nil {
return
}
defer c.Quit() // make sure to quit the Client
if err = c.Mail("sender#example.org"); err != nil {
return
}
if err = c.Rcpt("recipient#example.net"); err != nil {
return
}
wc, err := c.Data()
if err != nil {
return
}
defer wc.Close() // make sure WriterCloser gets closed
_, err = wc.Write(msg)
if err != nil {
return
}
code, message, err = c.Text.ReadResponse(0)
return
}
The code, message and any err are now passed to the caller, don't use log.Fatal throughout your code, handle the error on the calling side.
package main
import (
"net/smtp"
)
func sendEmail(msg []byte) (code int, message string, err error) {
c, err := smtp.Dial("localhost:25")
if err != nil {
return
}
defer c.Quit() // make sure to quit the Client
if err = c.Mail("sender#example.org"); err != nil {
return
}
if err = c.Rcpt("recipient#example.net"); err != nil {
return
}
wc, err := c.Data()
if err != nil {
return
}
_, err = wc.Write(msg)
if err != nil {
return
}
code, message, err = closeData(c)
if err != nil {
return 0, "", err
}
return code, message, err
}
func closeData(client *smtp.Client) error {
d := &dataCloser{
c: client,
WriteCloser: client.Text.DotWriter(),
}
return d.Close()
}
type dataCloser struct {
c *smtp.Client
io.WriteCloser
}
func (d *dataCloser) Close() (int, string, error) {
d.WriteCloser.Close() // make sure WriterCloser gets closed
code, message, err := d.c.Text.ReadResponse(250)
fmt.Printf("Message %v, Error %v\n", message, err)
return code, message, err
}

Golang Channel Streams

I am relatively new to Golang and don't fully understand streams. I have a function (circuit breaker function) that is making Rest calls. I have it working but it is only streaming the "responseBody" back. I would actually like to stream back the entire request of stream back both the Body and the Header together.
When I try to use a similar approach on the "header" then I get an error that the header is not streamable.
Is there a good/best way to accomplish this? Thank you. Below is my function.
func CallWithRetries(req *http.Request, output chan []byte) error {
r := retrier.New(retrier.ConstantBackoff(RETRIES, 100 * time.Millisecond), nil)
attempt := 0
err := r.Run(func() error {
attempt++
resp, err := Client.Do(req)
if err == nil && resp.StatusCode < 299 {
responseBody, err := ioutil.ReadAll(resp.Body)
if err == nil {
output <- responseBody
return err
}
return err
} else if err == nil {
customLogger.LogDebug("Status code was: " , transactionId)
err = fmt.Errorf("Status was %v", resp.StatusCode)
}
return err
})
return err
}
You are looking for the httputil.DumpResponse function.
The code might be changed to something similar to
func CallWithRetries(req *http.Request, output chan []byte) error {
r := retrier.New(retrier.ConstantBackoff(RETRIES, 100*time.Millisecond), nil)
attempt := 0
err := r.Run(func() error {
attempt++
resp, err := Client.Do(req)
if err == nil && resp.StatusCode < 299 {
dump, err := httputil.DumpResponse(resp, true)
if err == nil {
output <- dump
return err
}
return err
} else if err == nil {
customLogger.LogDebug("Status code was: ", transactionId)
err = fmt.Errorf("Status was %v", resp.StatusCode)
}
return err
})
return err
}
Side notes,
you might want to consider to close the response body as mentioned in the documentation https://golang.org/pkg/net/http/#Client.Get
Looks likes the err variable is shadowed, this should be modified to avoid any surprises.
This version of the code attempts to return errors early, and to close the response body. It was not tested, only written on the fly, to use with care.
func CallWithRetries(req *http.Request, output chan []byte) error {
r := retrier.New(retrier.ConstantBackoff(RETRIES, 100*time.Millisecond), nil)
attempt := 0
return r.Run(func() error {
attempt++
var resp *http.Response
{
r, err := Client.Do(req)
if err != nil {
return err
}
defer r.Body.Close()
if resp.StatusCode > 299 {
customLogger.LogDebug("Status code was: ", transactionId)
return fmt.Errorf("Status was %v", resp.StatusCode)
}
resp = r
}
var out []byte
{
x, err := httputil.DumpResponse(resp, true)
if err != nil {
return err
}
out = x
}
output <- out
return nil
})
}

Implementing ICMP ping in Go

Is it possible to implement an ICMP ping in Go? The alternative is to fork a 'ping' process, but I'd rather write it in Go.
The following code shows how to perform a ping over IPv4 using a raw socket (requires root privs):
package main
import (
"log"
"net"
"os"
"golang.org/x/net/icmp"
"golang.org/x/net/ipv4"
)
const targetIP = "8.8.8.8"
func main() {
c, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0")
if err != nil {
log.Fatalf("listen err, %s", err)
}
defer c.Close()
wm := icmp.Message{
Type: ipv4.ICMPTypeEcho, Code: 0,
Body: &icmp.Echo{
ID: os.Getpid() & 0xffff, Seq: 1,
Data: []byte("HELLO-R-U-THERE"),
},
}
wb, err := wm.Marshal(nil)
if err != nil {
log.Fatal(err)
}
if _, err := c.WriteTo(wb, &net.IPAddr{IP: net.ParseIP(targetIP)}); err != nil {
log.Fatalf("WriteTo err, %s", err)
}
rb := make([]byte, 1500)
n, peer, err := c.ReadFrom(rb)
if err != nil {
log.Fatal(err)
}
rm, err := icmp.ParseMessage(ipv4.ICMPTypeEchoReply.Protocol(), rb[:n])
if err != nil {
log.Fatal(err)
}
switch rm.Type {
case ipv4.ICMPTypeEchoReply:
log.Printf("got reflection from %v", peer)
default:
log.Printf("got %+v; want echo reply", rm)
}
}
Code based on the example found here: https://godoc.org/golang.org/x/net/icmp#PacketConn
In order to ping from Linux as a non-privileged user, see this post
Currently, the ICMP Echo (Ping) function isn't supported in the Go net package.
There's no support for sending ICMP
echo requests. You'd have to add
support to package net. ping
To perform this without root requirement you can use
package main
import (
"fmt"
"net"
"os"
"time"
"golang.org/x/net/icmp"
"golang.org/x/net/ipv4"
)
const target = "google.com"
func main() {
for {
time.Sleep(time.Second * 1)
Ping(target)
}
}
func Ping(target string) {
ip, err := net.ResolveIPAddr("ip4", target)
if err != nil {
panic(err)
}
conn, err := icmp.ListenPacket("udp4", "0.0.0.0")
if err != nil {
fmt.Printf("Error on ListenPacket")
panic(err)
}
defer conn.Close()
msg := icmp.Message{
Type: ipv4.ICMPTypeEcho, Code: 0,
Body: &icmp.Echo{
ID: os.Getpid() & 0xffff, Seq: 1,
Data: []byte(""),
},
}
msg_bytes, err := msg.Marshal(nil)
if err != nil {
fmt.Printf("Error on Marshal %v", msg_bytes)
panic(err)
}
// Write the message to the listening connection
if _, err := conn.WriteTo(msg_bytes, &net.UDPAddr{IP: net.ParseIP(ip.String())}); err != nil {
fmt.Printf("Error on WriteTo %v", err)
panic(err)
}
err = conn.SetReadDeadline(time.Now().Add(time.Second * 1))
if err != nil {
fmt.Printf("Error on SetReadDeadline %v", err)
panic(err)
}
reply := make([]byte, 1500)
n, _, err := conn.ReadFrom(reply)
if err != nil {
fmt.Printf("Error on ReadFrom %v", err)
panic(err)
}
parsed_reply, err := icmp.ParseMessage(1, reply[:n])
if err != nil {
fmt.Printf("Error on ParseMessage %v", err)
panic(err)
}
switch parsed_reply.Code {
case 0:
// Got a reply so we can save this
fmt.Printf("Got Reply from %s\n", target)
case 3:
fmt.Printf("Host %s is unreachable\n", target)
// Given that we don't expect google to be unreachable, we can assume that our network is down
case 11:
// Time Exceeded so we can assume our network is slow
fmt.Printf("Host %s is slow\n", target)
default:
// We don't know what this is so we can assume it's unreachable
fmt.Printf("Host %s is unreachable\n", target)
}
}

Unix Sockets in Go

I'm trying to make a simple echo client and server that uses Unix sockets. In this example, the connection seems to be unidirectional. The server can receive data from the client, but it can't send the data back. If I use tcp connections instead, it works great. Does anyone know what's wrong?
Server
package main
import "net"
import "fmt"
func echoServer(c net.Conn) {
for {
buf := make([]byte, 512)
nr, err := c.Read(buf)
if err != nil {
return
}
data := buf[0:nr]
fmt.Printf("Received: %v", string(data))
_, err = c.Write(data)
if err != nil {
panic("Write: " + err.String())
}
}
}
func main() {
l, err := net.Listen("unix", "/tmp/echo.sock")
if err != nil {
println("listen error", err.String())
return
}
for {
fd, err := l.Accept()
if err != nil {
println("accept error", err.String())
return
}
go echoServer(fd)
}
}
Client
package main
import "net"
import "time"
func main() {
c,err := net.Dial("unix","", "/tmp/echo.sock")
if err != nil {
panic(err.String())
}
for {
_,err := c.Write([]byte("hi\n"))
if err != nil {
println(err.String())
}
time.Sleep(1e9)
}
}
In your example client, you don't seem to be reading the result from the server. When I add code to do that, I see the results from the server.
Also, take a look at how I used defer and break to make the reader goroutine exit.
Server
package main
import (
"log"
"net"
)
func echoServer(c net.Conn) {
for {
buf := make([]byte, 512)
nr, err := c.Read(buf)
if err != nil {
return
}
data := buf[0:nr]
println("Server got:", string(data))
_, err = c.Write(data)
if err != nil {
log.Fatal("Write: ", err)
}
}
}
func main() {
l, err := net.Listen("unix", "/tmp/echo.sock")
if err != nil {
log.Fatal("listen error:", err)
}
for {
fd, err := l.Accept()
if err != nil {
log.Fatal("accept error:", err)
}
go echoServer(fd)
}
}
Client
package main
import (
"io"
"log"
"net"
"time"
)
func reader(r io.Reader) {
buf := make([]byte, 1024)
for {
n, err := r.Read(buf[:])
if err != nil {
return
}
println("Client got:", string(buf[0:n]))
}
}
func main() {
c, err := net.Dial("unix", "/tmp/echo.sock")
if err != nil {
panic(err)
}
defer c.Close()
go reader(c)
for {
_, err := c.Write([]byte("hi"))
if err != nil {
log.Fatal("write error:", err)
break
}
time.Sleep(1e9)
}
}