How to find maximum value of a field in mongodb golang? - mongodb

This is my code. I always get maximum value of 999, even when there are more blocks (e.g 3000 blocks).
This is what the document(s) look like.
func GetLatestBlockFromMongoDB() int{
if contains(ReturnBlocksExists(), "blocks") == true {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
var blockheights []models.Block
defer cancel()
options := options.Find().SetProjection(bson.M{"block.header.blockNumber": 1})
options.SetSort(bson.D{{"block.header.blockNumber", -1}})
options.SetLimit(1)
results, err := blocksCollections.Find(context.TODO(), bson.D{}, options)
if err != nil {
fmt.Println(err)
}
//reading from the db in an optimal way
defer results.Close(ctx)
for results.Next(ctx) {
var singleBlock models.Block
if err = results.Decode(&singleBlock); err != nil {
fmt.Println(err)
}
blockheights = append(blockheights, singleBlock)
}
fmt.Println("%v", blockheights[0].Block.Header.BlockNumber)
if len(strings.TrimSpace(blockheights[0].Block.Header.BlockNumber)) > 0{
i, err := strconv.Atoi(blockheights[0].Block.Header.BlockNumber)
if err != nil {
glog.Fatalf("%v", err)
}
return i
} else {
return 0
}
} else {
return 0
}
}
How can i get the maximum value of blockNumber? I think the problem might be because blockNumber is a string but not an integer. I'm not sure. I did set sort and also limit 1 so it should normally work.

Yes, you are right, from the image, I can see that the blockNumber field is a string, so you are not comparing integers, you are comparing string, where "999" is greater than "3000":
For example:
package main
import (
"fmt"
)
func findMax(a []string) string {
m := a[0]
for _, value := range a {
if value > m {
m = value
}
}
return m
}
func main() {
blocks := []string{"999", "2000", "3000"}
ma := findMax(blocks)
fmt.Println(ma)
}
$ go run .
999
Check out this question too.

Related

How to Handle Dynamic Database Connections in Go?

I am currently building a Go application that needs to connect to multiple databases dynamically.
For context I have 22 Databases (db1, db2, db3...) and the dbUser, dbPass and dbPort remains the same. To determine which database to connect to, I need access to the query param in echo before database connection.
I need a solution to connect to the right database efficiently. What are some best practices and methods for achieving this in Go?
Main.go
package main
import (
"database/sql"
"fmt"
"log"
"time"
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
"github.com/labstack/echo"
"github.com/spf13/viper"
_variantHttpDelivery "backend/server/variant/delivery/http"
_variantHttpDeliveryMiddleware "backend/server/variant/delivery/http/middleware"
_variantRepo "backend/server/variant/repository/postgres"
_variantUcase "backend/server/variant/usecase"
)
func init() {
viper.SetConfigFile(`config.json`)
err := viper.ReadInConfig()
if err != nil {
panic(err)
}
if viper.GetBool(`debug`) {
log.Println("Service RUN on DEBUG mode")
}
}
func main() {
dbHost := viper.GetString(`database.host`)
dbPort := viper.GetString(`database.port`)
dbUser := viper.GetString(`database.user`)
dbPass := viper.GetString(`database.pass`)
dbName := viper.GetString(`database.name`)
connection := fmt.Sprintf("postgresql://%s:%s#%s:%s/%s", dbUser, dbPass, dbHost, dbPort, dbName)
dsn := fmt.Sprintf("%s?%s", connection)
dbConn, err := sql.Open(`postgres`, dsn)
log.Println("Connection Successful 👍")
if err != nil {
log.Fatal(err)
}
err = dbConn.Ping()
if err != nil {
log.Fatal(err)
}
defer func() {
err := dbConn.Close()
if err != nil {
log.Fatal(err)
}
}()
e := echo.New()
middL := _variantHttpDeliveryMiddleware.InitMiddleware()
e.Use(middL.CORS)
variantRepo := _variantRepo.NewPsqlVariantRepository(dbConn)
timeoutContext := time.Duration(viper.GetInt("context.timeout")) * time.Second
au := _variantUcase.NewVariantUsecase(variantRepo, timeoutContext)
_variantHttpDelivery.NewVariantHandler(e, au)
log.Fatal(e.Start(viper.GetString("server.address"))) //nolint
}
Repository which handles all the database logic
package postgres
import (
"backend/server/domain"
"context"
"database/sql"
"github.com/sirupsen/logrus"
"reflect"
)
type psqlVariantRepository struct {
Conn *sql.DB
}
// NewPsqlVariantRepository will create an object that represent the variant.Repository interface
func NewPsqlVariantRepository(conn *sql.DB) domain.VariantRepository {
return &psqlVariantRepository{conn}
}
func (m *psqlVariantRepository) GetByVCF(ctx context.Context, vcf string) (res domain.Variant, err error) {
query := `SELECT * FROM main1 WHERE variant_vcf = $1`
list, err := m.fetch(ctx, query, vcf)
if err != nil {
return domain.Variant{}, err
}
if len(list) > 0 {
res = list[0]
} else {
return res, domain.ErrNotFound
}
return
}
func (m *psqlVariantRepository) fetch(ctx context.Context, query string, args ...interface{}) (result []domain.Variant, err error) {
rows, err := m.Conn.QueryContext(ctx, query, args...)
if err != nil {
logrus.Error(err)
return nil, err
}
defer func() {
errRow := rows.Close()
if errRow != nil {
logrus.Error(errRow)
}
}()
result = make([]domain.Variant, 0)
for rows.Next() {
t := domain.Variant{}
values := make([]interface{}, 0, reflect.TypeOf(t).NumField())
v := reflect.ValueOf(&t).Elem()
for i := 0; i < v.NumField(); i++ {
if v.Type().Field(i).Type.Kind() == reflect.String {
values = append(values, new(sql.NullString))
} else {
values = append(values, v.Field(i).Addr().Interface())
}
}
err = rows.Scan(values...)
if err != nil {
logrus.Error(err)
return nil, err
}
for i, value := range values {
if ns, ok := value.(*sql.NullString); ok {
v.Field(i).SetString(ns.String)
}
}
result = append(result, t)
}
logrus.Info("Successfully fetched results from database 👍")
return result, nil
}
So far I couldn't find any solution

How to update without creating a whole new field using upsert in mongodb golang?

Okay, so. There is this field called bitfield and inside is a map. Everytime i set bits, for example set bit 5000, i'll get key 4096, but if i set bit 1000, then i'll get the key, just 0. However, i set both, 5000 and 0, so it should be both keys 0 and 4096. How can i update the existing field, without deleting the other data, so both keys 0 and 4096 exist?
func UpsertBitsArray(tx models.Transactions, sendingQrlAddress string) (error, string) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
optionss := options.Find().
SetProjection(bson.M{"bitfield": 1}).
SetLimit(1)
results, err := addressesCollections.Find(ctx, bson.M{"id": sendingQrlAddress}, optionss)
if err != nil {
return nil, "Success"
}
var bitfieldStruct []wallet.Wallet
err = results.All(ctx, &bitfieldStruct)
if err != nil {
return nil, "Success"
}
// signature, err := base64.StdEncoding.DecodeString(tx.Signature)
// if err != nil {
// glog.Info("%v", err)
// }
otsKeyIndex := big.NewInt(10000)
for _, bitfieldExist := range bitfieldStruct {
if bitfieldExist.Paged == nil {
fmt.Println("No array/bitfield, creating new one..")
bitfieldExist.Paged = bitfield.NewBig()
bitfieldExist.Paged.Set(otsKeyIndex)
filter := bson.D{{"id", sendingQrlAddress}}
update := bson.D{
{
Key: "$set",
Value: bson.D{
{Key: "bitfield", Value: bitfieldExist.Paged},
},
},
}
opts := options.Update().SetUpsert(true)
result, err := addressesCollections.UpdateOne(context.TODO(), filter, update, opts)
if err != nil {
glog.Info("%v", err)
}
fmt.Printf("Number of documents updated: %v\n", result.ModifiedCount)
fmt.Printf("Number of documents upserted: %v\n", result.UpsertedCount)
} else {
fmt.Println("Bitfield/array exists, setting it..")
bitfieldExist.Paged.Set(otsKeyIndex)
filter := bson.D{{"id", sendingQrlAddress}}
update := bson.D{
{
Key: "$set",
Value: bson.D{
{Key: "bitfield", Value: bitfieldExist.Paged},
},
},
}
opts := options.Update().SetUpsert(true)
result, err := addressesCollections.UpdateOne(context.TODO(), filter, update, opts)
if err != nil {
glog.Info("%v", err)
}
fmt.Printf("Number of documents updated: %v\n", result.ModifiedCount)
fmt.Printf("Number of documents upserted: %v\n", result.UpsertedCount)
}
}
return nil, "Success"
}
Here is what i mean:
Edit:
The wallet struct:
type Wallet struct {
_id primitive.ObjectID `json:"_id,omitempty"`
Id string `json:"id,omitempty"`
Amount int `json:"amount,omitempty"`
Paged bitfield.Big `json:"bitfield"`
}
What i'm storing:
package bitfield
import (
"math/big"
)
const (
fieldSize = 1024
)
type Big map[string]Bitfield
func NewBig() Big {
return make(map[string]Bitfield)
}
func (fields Big) Set(i *big.Int) {
number, pageNumber := indexOfBig(i, fieldSize)
page, exists := fields[pageNumber.String()]
if !exists {
page = New(fieldSize)
fields[pageNumber.String()] = page
}
page.Set(uint(number.Uint64()))
}

Double read from socket

Go version: go1.14.4 darwin/amd64
TCP server
package main
import (
"fmt"
"io"
"net"
"time"
)
const (
maxBufLen int = 4 * 1024
)
type Server struct {
}
func (s *Server) Start() {
listener, err := net.Listen("tcp", "localhost:9001")
if err != nil {
panic(err)
}
var connections []net.Conn
defer func() {
for _, conn := range connections {
// ignore
_ = conn.Close()
}
}()
// loop...
for {
connected, err := listener.Accept()
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Temporary() {
// temporary err will bi ignored
continue
} else {
fmt.Println(err)
return
}
}
go handleConn(connected, true)
connections = append(connections, connected)
}
}
func handleConn(conn net.Conn, server bool) {
buf := make([]byte, maxBufLen)
for {
// read
setTimeout(conn)
_, err := conn.Read(buf)
if err != nil {
if ne, ok := err.(net.Error); ok && (ne.Timeout() || ne.Temporary()) {
fmt.Println("need continue...")
continue
}
if err == io.EOF {
fmt.Println("EOF")
break
}
// other...
panic(err)
}
// handle recv msg.
s := string(buf)
if server {
//fmt.Println("server recv req ", s)
} else {
fmt.Println("client recv resp ", s)
}
if server {
output := "hi " + s
ob := []byte(output)
_, err := conn.Write(ob)
if err != nil {
fmt.Println(err)
break
}
}
}
}
func setTimeout(conn net.Conn) {
setErr := conn.SetReadDeadline(time.Now().Add(20 * time.Second))
if setErr != nil {
panic(setErr)
}
}
TCP client
package main
import (
"net"
"time"
)
type Client struct {
Exit chan struct{}
}
func (c *Client) Start() {
conn, err := net.Dial("tcp", "localhost:9001")
if err != nil {
panic(err)
}
defer conn.Close()
go handleWrite(conn)
go handleConn(conn, false)
<-c.Exit
}
func handleWrite(conn net.Conn) {
for {
input := "carryxyh"
_, err := conn.Write([]byte(input))
if err != nil {
panic(err)
}
<-time.After(100 * time.Second)
}
}
Main function
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
)
type Starter interface {
Start()
}
func main() {
var s Server
var c Client
go s.Start()
go c.Start()
sigs := make(chan os.Signal)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
<-sigs
c.Exit <- struct{}{}
fmt.Println("exit")
}
Running package main will print :
client recv resp hi carryxyh
client recv resp carryxyh
Printed twice. But from the program point of view, the server only writes information to the client once, and the content of the information should be hi carryxyh. But the client printed carryxyh in addition to hi carryxyh, which made me very confused.
During the investigation, I accidentally modified a line of code, and the program immediately returned to normal:
modeifycode
As shown above, I modified the server response information: output := "hi "+ s => output := "hi ", at this time the program only prints client recv resp hi.
This makes me completely confused, can anyone help me solve this problem? It would be better if there is a troubleshooting idea.
enter image description here
这里你忽略了读取的字节长度,可能返回0字节,你又把buf的内容 又回写给client了.
_, err := conn.Read(buf) 改成 bytesRead, err := conn.Read(buf) ;
if bytesRead <= 0 { // 没有buf 可以读取 }
--
eg:
if bytesRead == 0 && err == nil {
err = io.EOF
log.Errorf("[network] ReadOnce maybe always return (0, nil) and causes dead loop, Connection = %d, Local Address = %+v, Remote Address = %+v",
c.id, c.rawConnection.LocalAddr(), c.RemoteAddr())
}

Why does each transaction count as a client?

I am processing a bunch of files and then dumping the results in PostgreSQL. I would like to process many workers at the same time but keep getting the error "pq: sorry, too many clients already". This seems to happen whenever workers is > 100 or so. (For simplicity, the code below demonstrates the process but instead of processing a file I am simply inserting 1M rows in each table).
Since I am reusing the same *db why am I getting this error? Does each transaction count as a client or am I doing something wrong?
package main
import (
"database/sql"
"flag"
"fmt"
"log"
"sync"
"github.com/lib/pq"
)
func process(db *sql.DB, table string) error {
if _, err := db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %v;`, table)); err != nil {
return err
}
col := "age"
s := fmt.Sprintf(`
CREATE TABLE %v (
pk serial PRIMARY KEY,
%v int NOT NULL
)`, table, col)
_, err := db.Exec(s)
if err != nil {
return err
}
tx, err := db.Begin()
if err != nil {
return err
}
defer func() {
if err != nil {
tx.Rollback()
return
}
err = tx.Commit()
}()
stmt, err := tx.Prepare(pq.CopyIn(table, col))
if err != nil {
return err
}
defer func() {
err = stmt.Close()
}()
for i := 0; i < 1e6; i++ {
if _, err = stmt.Exec(i); err != nil {
return err
}
}
return err
}
func main() {
var u string
flag.StringVar(&u, "user", "", "user")
var pass string
flag.StringVar(&pass, "pass", "", "pass")
var host string
flag.StringVar(&host, "host", "", "host")
var database string
flag.StringVar(&database, "database", "", "database")
var workers int
flag.IntVar(&workers, "workers", 10, "workers")
flag.Parse()
db, err := sql.Open("postgres",
fmt.Sprintf(
"user=%s password=%s host=%s database=%s sslmode=require",
u, pass, host, database,
),
)
if err != nil {
log.Fatalln(err)
}
defer db.Close()
db.SetMaxIdleConns(0)
var wg sync.WaitGroup
ch := make(chan int)
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := range ch {
table := fmt.Sprintf("_table%d", i)
log.Println(table)
if err := process(db, table); err != nil {
log.Fatalln(err)
}
}
}()
}
for i := 0; i < 300; i++ {
ch <- i
}
close(ch)
wg.Wait()
}
I realize I can simply increase the posgresql settings but would like to understand the question: How to increase the max connections in postgres?
Since I am reusing the same *db why am I getting this error?
I suspect the Postgress driver is using a separate connections for each of your workers which is a smart decision for most cases.
Does each transaction count as a client or am I doing something wrong?
In your case yes each transaction count as a client, because you are calling process() as a goroutine. You are creating as many concurrent transactions as workers. Since each of your transactions is long all of them are probably using an individual connection to the database at the same time and hence you hit a limit.
go func() {
defer wg.Done()
for i := range ch {
table := fmt.Sprintf("_table%d", i)
log.Println(table)
if err := process(db, table); err != nil {
log.Fatalln(err)
}
}
}()

Match regexp email in website in go

I try to find email match in a website in goland with a file include url, for example, if i put "http://facebook.com" in the file, he will try to find all email find in the website, but he always result 0. I think I choose the wrong function but i try to find other function but i've got the same result. Here the code :
package main
import (
"bufio"
"bytes"
"fmt"
"log"
"net/http"
"os"
"regexp"
"sync"
)
func main() {
var wg sync.WaitGroup
wg.Add(1)
go emailWeb(os.Args[1], &wg)
wg.Wait()
}
func emailWeb(name string, wg *sync.WaitGroup) {
file, err := os.Open(name)
if err != nil {
log.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
str := scanner.Text()
nb_arobase := numberEmail(str)
fmt.Println("URL : ", str, " nb email: ", nb_arobase)
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
(*wg).Done()
}
func numberEmail(url string) int {
count := 0
reg := regexp.MustCompile(`[a-z0-9._%+\-]+#[a-z0-9.\-]+\.[a-z]{2,4}`)
response, err := http.Get(url)
if err != nil {
log.Fatal(err)
} else {
str := response.Body
buf := new(bytes.Buffer)
buf.ReadFrom(str)
bodyStr := buf.String()
for i := 0; i < len(bodyStr); i++ {
if reg.MatchString(string(bodyStr[i])) {
count += 1
}
}
}
return count
}
You're trying to match the regexp against each individual character in the http response body. You can count the matches in the entire body if you want by counting the matched indexes.
resp, err := http.Get(url)
if err != nil {
log.Println(err)
return 0
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return 0
}
return len(reg.FindAllIndex(body))