Match regexp email in website in go - email

I try to find email match in a website in goland with a file include url, for example, if i put "http://facebook.com" in the file, he will try to find all email find in the website, but he always result 0. I think I choose the wrong function but i try to find other function but i've got the same result. Here the code :
package main
import (
"bufio"
"bytes"
"fmt"
"log"
"net/http"
"os"
"regexp"
"sync"
)
func main() {
var wg sync.WaitGroup
wg.Add(1)
go emailWeb(os.Args[1], &wg)
wg.Wait()
}
func emailWeb(name string, wg *sync.WaitGroup) {
file, err := os.Open(name)
if err != nil {
log.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
str := scanner.Text()
nb_arobase := numberEmail(str)
fmt.Println("URL : ", str, " nb email: ", nb_arobase)
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
(*wg).Done()
}
func numberEmail(url string) int {
count := 0
reg := regexp.MustCompile(`[a-z0-9._%+\-]+#[a-z0-9.\-]+\.[a-z]{2,4}`)
response, err := http.Get(url)
if err != nil {
log.Fatal(err)
} else {
str := response.Body
buf := new(bytes.Buffer)
buf.ReadFrom(str)
bodyStr := buf.String()
for i := 0; i < len(bodyStr); i++ {
if reg.MatchString(string(bodyStr[i])) {
count += 1
}
}
}
return count
}

You're trying to match the regexp against each individual character in the http response body. You can count the matches in the entire body if you want by counting the matched indexes.
resp, err := http.Get(url)
if err != nil {
log.Println(err)
return 0
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return 0
}
return len(reg.FindAllIndex(body))

Related

How to Handle Dynamic Database Connections in Go?

I am currently building a Go application that needs to connect to multiple databases dynamically.
For context I have 22 Databases (db1, db2, db3...) and the dbUser, dbPass and dbPort remains the same. To determine which database to connect to, I need access to the query param in echo before database connection.
I need a solution to connect to the right database efficiently. What are some best practices and methods for achieving this in Go?
Main.go
package main
import (
"database/sql"
"fmt"
"log"
"time"
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
"github.com/labstack/echo"
"github.com/spf13/viper"
_variantHttpDelivery "backend/server/variant/delivery/http"
_variantHttpDeliveryMiddleware "backend/server/variant/delivery/http/middleware"
_variantRepo "backend/server/variant/repository/postgres"
_variantUcase "backend/server/variant/usecase"
)
func init() {
viper.SetConfigFile(`config.json`)
err := viper.ReadInConfig()
if err != nil {
panic(err)
}
if viper.GetBool(`debug`) {
log.Println("Service RUN on DEBUG mode")
}
}
func main() {
dbHost := viper.GetString(`database.host`)
dbPort := viper.GetString(`database.port`)
dbUser := viper.GetString(`database.user`)
dbPass := viper.GetString(`database.pass`)
dbName := viper.GetString(`database.name`)
connection := fmt.Sprintf("postgresql://%s:%s#%s:%s/%s", dbUser, dbPass, dbHost, dbPort, dbName)
dsn := fmt.Sprintf("%s?%s", connection)
dbConn, err := sql.Open(`postgres`, dsn)
log.Println("Connection Successful 👍")
if err != nil {
log.Fatal(err)
}
err = dbConn.Ping()
if err != nil {
log.Fatal(err)
}
defer func() {
err := dbConn.Close()
if err != nil {
log.Fatal(err)
}
}()
e := echo.New()
middL := _variantHttpDeliveryMiddleware.InitMiddleware()
e.Use(middL.CORS)
variantRepo := _variantRepo.NewPsqlVariantRepository(dbConn)
timeoutContext := time.Duration(viper.GetInt("context.timeout")) * time.Second
au := _variantUcase.NewVariantUsecase(variantRepo, timeoutContext)
_variantHttpDelivery.NewVariantHandler(e, au)
log.Fatal(e.Start(viper.GetString("server.address"))) //nolint
}
Repository which handles all the database logic
package postgres
import (
"backend/server/domain"
"context"
"database/sql"
"github.com/sirupsen/logrus"
"reflect"
)
type psqlVariantRepository struct {
Conn *sql.DB
}
// NewPsqlVariantRepository will create an object that represent the variant.Repository interface
func NewPsqlVariantRepository(conn *sql.DB) domain.VariantRepository {
return &psqlVariantRepository{conn}
}
func (m *psqlVariantRepository) GetByVCF(ctx context.Context, vcf string) (res domain.Variant, err error) {
query := `SELECT * FROM main1 WHERE variant_vcf = $1`
list, err := m.fetch(ctx, query, vcf)
if err != nil {
return domain.Variant{}, err
}
if len(list) > 0 {
res = list[0]
} else {
return res, domain.ErrNotFound
}
return
}
func (m *psqlVariantRepository) fetch(ctx context.Context, query string, args ...interface{}) (result []domain.Variant, err error) {
rows, err := m.Conn.QueryContext(ctx, query, args...)
if err != nil {
logrus.Error(err)
return nil, err
}
defer func() {
errRow := rows.Close()
if errRow != nil {
logrus.Error(errRow)
}
}()
result = make([]domain.Variant, 0)
for rows.Next() {
t := domain.Variant{}
values := make([]interface{}, 0, reflect.TypeOf(t).NumField())
v := reflect.ValueOf(&t).Elem()
for i := 0; i < v.NumField(); i++ {
if v.Type().Field(i).Type.Kind() == reflect.String {
values = append(values, new(sql.NullString))
} else {
values = append(values, v.Field(i).Addr().Interface())
}
}
err = rows.Scan(values...)
if err != nil {
logrus.Error(err)
return nil, err
}
for i, value := range values {
if ns, ok := value.(*sql.NullString); ok {
v.Field(i).SetString(ns.String)
}
}
result = append(result, t)
}
logrus.Info("Successfully fetched results from database 👍")
return result, nil
}
So far I couldn't find any solution

How do I convert a string type url param into an acceptable type for my mongodb query?

I'm trying to find and retrieve a related number of fields from my mongodb via a url param sent from a get request.
func main() {
r := mux.NewRouter()
r.HandleFunc("/user", createUser).Methods("POST")
r.HandleFunc("/suggest", searchCity).Methods("GET") // The route for the function
fmt.Println("Server running at port 8080")
log.Fatal(http.ListenAndServe(":8080", r))
}
func searchCity(w http.ResponseWriter, r *http.Request) {
//ctx := context.Background()
DB := setup() // setup() returns a mongo.Database type
values := r.URL.Query()
city := values.Get("city_name") // so the route would ultimately be `/suggest?city_name=<cityname>` (I think?)
cityCollection := DB.Collection("city")
cursor, err := cityCollection.Find(r.Context(), city) // options.Find().SetProjection(projection))
if err != nil {
log.Fatal(err)
}
var cityList []bson.M
if err = cursor.All(r.Context(), &cityList); err != nil {
log.Fatal(err)
}
for _, cityList := range cityList {
fmt.Println(cityList["all_names"])
fmt.Println(cityList["country_name"])
}
}
I am specifically trying to return the all_names and country_name fields from the collection and tried to use SetProjection() but I may have been using it incorrectly so not sure about that one.
However, I keep recieveing this error:
cannot transform type string to a BSON Document: WriteString can only write while positioned on a Element or Value but is positioned on a TopLevel
Is there a way to convert this into a format that the mongodb drivers would accept?
As documented and indicated by the error, you need to use the bson package to use a bson object to query mongodb.
collection.Find(ctx, bson.D{{"name", city}})
Below is a working example, which I have tested based on your comment.
package main
import (
"context"
"encoding/json"
"log"
"net/http"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func cityHandler(collection *mongo.Collection) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
city := r.URL.Query().Get("city_name")
var q bson.D
if city != "" {
q = bson.D{{"name", city}}
} else {
q = bson.D{{}}
}
cur, currErr := collection.Find(ctx, q)
if currErr != nil {
http.Error(w, currErr.Error(), http.StatusInternalServerError)
}
defer cur.Close(ctx)
var result []primitive.M
if err := cur.All(ctx, &result); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(result)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
}
func main() {
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://root:example#localhost:27017"))
if err != nil {
panic(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
err = client.Connect(ctx)
if err != nil {
panic(err)
}
defer client.Disconnect(ctx)
docs := []interface{}{
bson.D{{"name", "Wuhu"}},
bson.D{{"name", "Shexian"}},
}
collection := client.Database("china").Collection("cities")
_, err = collection.InsertMany(ctx, docs)
if err != nil {
panic(err)
}
http.HandleFunc("/", cityHandler(collection))
log.Fatal(http.ListenAndServe(":8080", nil))
}

Double read from socket

Go version: go1.14.4 darwin/amd64
TCP server
package main
import (
"fmt"
"io"
"net"
"time"
)
const (
maxBufLen int = 4 * 1024
)
type Server struct {
}
func (s *Server) Start() {
listener, err := net.Listen("tcp", "localhost:9001")
if err != nil {
panic(err)
}
var connections []net.Conn
defer func() {
for _, conn := range connections {
// ignore
_ = conn.Close()
}
}()
// loop...
for {
connected, err := listener.Accept()
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Temporary() {
// temporary err will bi ignored
continue
} else {
fmt.Println(err)
return
}
}
go handleConn(connected, true)
connections = append(connections, connected)
}
}
func handleConn(conn net.Conn, server bool) {
buf := make([]byte, maxBufLen)
for {
// read
setTimeout(conn)
_, err := conn.Read(buf)
if err != nil {
if ne, ok := err.(net.Error); ok && (ne.Timeout() || ne.Temporary()) {
fmt.Println("need continue...")
continue
}
if err == io.EOF {
fmt.Println("EOF")
break
}
// other...
panic(err)
}
// handle recv msg.
s := string(buf)
if server {
//fmt.Println("server recv req ", s)
} else {
fmt.Println("client recv resp ", s)
}
if server {
output := "hi " + s
ob := []byte(output)
_, err := conn.Write(ob)
if err != nil {
fmt.Println(err)
break
}
}
}
}
func setTimeout(conn net.Conn) {
setErr := conn.SetReadDeadline(time.Now().Add(20 * time.Second))
if setErr != nil {
panic(setErr)
}
}
TCP client
package main
import (
"net"
"time"
)
type Client struct {
Exit chan struct{}
}
func (c *Client) Start() {
conn, err := net.Dial("tcp", "localhost:9001")
if err != nil {
panic(err)
}
defer conn.Close()
go handleWrite(conn)
go handleConn(conn, false)
<-c.Exit
}
func handleWrite(conn net.Conn) {
for {
input := "carryxyh"
_, err := conn.Write([]byte(input))
if err != nil {
panic(err)
}
<-time.After(100 * time.Second)
}
}
Main function
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
)
type Starter interface {
Start()
}
func main() {
var s Server
var c Client
go s.Start()
go c.Start()
sigs := make(chan os.Signal)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
<-sigs
c.Exit <- struct{}{}
fmt.Println("exit")
}
Running package main will print :
client recv resp hi carryxyh
client recv resp carryxyh
Printed twice. But from the program point of view, the server only writes information to the client once, and the content of the information should be hi carryxyh. But the client printed carryxyh in addition to hi carryxyh, which made me very confused.
During the investigation, I accidentally modified a line of code, and the program immediately returned to normal:
modeifycode
As shown above, I modified the server response information: output := "hi "+ s => output := "hi ", at this time the program only prints client recv resp hi.
This makes me completely confused, can anyone help me solve this problem? It would be better if there is a troubleshooting idea.
enter image description here
这里你忽略了读取的字节长度,可能返回0字节,你又把buf的内容 又回写给client了.
_, err := conn.Read(buf) 改成 bytesRead, err := conn.Read(buf) ;
if bytesRead <= 0 { // 没有buf 可以读取 }
--
eg:
if bytesRead == 0 && err == nil {
err = io.EOF
log.Errorf("[network] ReadOnce maybe always return (0, nil) and causes dead loop, Connection = %d, Local Address = %+v, Remote Address = %+v",
c.id, c.rawConnection.LocalAddr(), c.RemoteAddr())
}

How to find by id in golang and mongodb

I need get values using ObjectIdHex and do update and also view the result. I'm using mongodb and golang.But following code doesn't work as expected
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
Id bson.ObjectId `json:"id" bson:"_id,omitempty"`
Name string
Phone string
}
func checkError(err error) {
if err != nil {
panic(err)
}
}
const (
DB_NAME = "gotest"
DB_COLLECTION = "pepole_new1"
)
func main() {
session, err := mgo.Dial("localhost")
checkError(err)
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(DB_NAME).C(DB_COLLECTION)
err = c.DropCollection()
checkError(err)
ale := Person{Name:"Ale", Phone:"555-5555"}
cla := Person{Name:"Cla", Phone:"555-1234-2222"}
kasaun := Person{Name:"kasaun", Phone:"533-12554-2222"}
chamila := Person{Name:"chamila", Phone:"533-545-6784"}
fmt.Println("Inserting")
err = c.Insert(&ale, &cla, &kasaun, &chamila)
checkError(err)
fmt.Println("findbyID")
var resultsID []Person
//err = c.FindId(bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")).One(&resultsID)
err = c.FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID)
checkError(err)
if err != nil {
panic(err)
}
fmt.Println("Phone:", resultsID)
fmt.Println("Queryingall")
var results []Person
err = c.Find(nil).All(&results)
if err != nil {
panic(err)
}
fmt.Println("Results All: ", results)
}
FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID) didn't work for me and giving me following output
Inserting
Queryingall
Results All: [{ObjectIdHex("56bddee2cfa93bfe3d3504a1") Ale 555-5555} {ObjectIdHex("56bddee2cfa93bfe3d3504a2") Cla 555-1234-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a3") kasaun 533-12554-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a4") chamila 533-545-6784}]
findbyID
panic: not found
goroutine 1 [running]:
main.checkError(0x7f33d524b000, 0xc8200689b0)
How can i fix this problem? i need get value using oid and do update also how can i do that
Use can do the same with Golang official driver as follows:
// convert id string to ObjectId
objectId, err := primitive.ObjectIDFromHex("5b9223c86486b341ea76910c")
if err != nil{
log.Println("Invalid id")
}
// find
result:= client.Database(database).Collection("user").FindOne(context.Background(), bson.M{"_id": objectId})
user := model.User{}
result.Decode(user)
It should be _id not Id:
c.FindId(bson.M{"_id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")})
Some sample code that i use.
func (model *SomeModel) FindId(id string) error {
db, ctx, client := Drivers.MongoCollection("collection")
defer client.Disconnect(ctx)
objID, err := primitive.ObjectIDFromHex(id)
if err != nil {
return err
}
filter := bson.M{"_id": bson.M{"$eq": objID}}
if err := db.FindOne(ctx, filter).Decode(&model); err != nil {
//fmt.Println(err)
return err
}
fmt.Println(model)
return nil
}

Unix Sockets in Go

I'm trying to make a simple echo client and server that uses Unix sockets. In this example, the connection seems to be unidirectional. The server can receive data from the client, but it can't send the data back. If I use tcp connections instead, it works great. Does anyone know what's wrong?
Server
package main
import "net"
import "fmt"
func echoServer(c net.Conn) {
for {
buf := make([]byte, 512)
nr, err := c.Read(buf)
if err != nil {
return
}
data := buf[0:nr]
fmt.Printf("Received: %v", string(data))
_, err = c.Write(data)
if err != nil {
panic("Write: " + err.String())
}
}
}
func main() {
l, err := net.Listen("unix", "/tmp/echo.sock")
if err != nil {
println("listen error", err.String())
return
}
for {
fd, err := l.Accept()
if err != nil {
println("accept error", err.String())
return
}
go echoServer(fd)
}
}
Client
package main
import "net"
import "time"
func main() {
c,err := net.Dial("unix","", "/tmp/echo.sock")
if err != nil {
panic(err.String())
}
for {
_,err := c.Write([]byte("hi\n"))
if err != nil {
println(err.String())
}
time.Sleep(1e9)
}
}
In your example client, you don't seem to be reading the result from the server. When I add code to do that, I see the results from the server.
Also, take a look at how I used defer and break to make the reader goroutine exit.
Server
package main
import (
"log"
"net"
)
func echoServer(c net.Conn) {
for {
buf := make([]byte, 512)
nr, err := c.Read(buf)
if err != nil {
return
}
data := buf[0:nr]
println("Server got:", string(data))
_, err = c.Write(data)
if err != nil {
log.Fatal("Write: ", err)
}
}
}
func main() {
l, err := net.Listen("unix", "/tmp/echo.sock")
if err != nil {
log.Fatal("listen error:", err)
}
for {
fd, err := l.Accept()
if err != nil {
log.Fatal("accept error:", err)
}
go echoServer(fd)
}
}
Client
package main
import (
"io"
"log"
"net"
"time"
)
func reader(r io.Reader) {
buf := make([]byte, 1024)
for {
n, err := r.Read(buf[:])
if err != nil {
return
}
println("Client got:", string(buf[0:n]))
}
}
func main() {
c, err := net.Dial("unix", "/tmp/echo.sock")
if err != nil {
panic(err)
}
defer c.Close()
go reader(c)
for {
_, err := c.Write([]byte("hi"))
if err != nil {
log.Fatal("write error:", err)
break
}
time.Sleep(1e9)
}
}