I have a function that updates records in table and requests it using pgx Postgres driver. This function hangs on commit. Is there any ideas why does it happen? Why I can't use transactions in this case?
Of course, as the query is atomic I can remove transactions. But it's still unclear—why it happens and what to do if I need a transaction.
func (r *Repository) GetUpdatedItems(ctx context.Context, filters []string) ([]Item, error) {
conn, err := r.pool.Acquire(ctx)
// error handling
defer conn.Release()
tx, err := conn.Begin(ctx)
// error handling
defer func() {
closeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_ = tx.Rollback(closeCtx)
}()
query := fmt.Sprintf(`UPDATE %s
SET fieldOne = $1, fieldTwo = $2
WHERE otheField = '' OR otherField IS NULL
RETURNING fieldOne, fieldTwo, otheField, someMoreField;`,
r.tableName, sqlArray(aggregatesTypes))
rows, err := conn.Query(ctx, query, filters[0], filters[1])
// error handling
defer rows.Close()
var retItems []reaper.Item
for rows.Next() {
var fieldOne string
var fieldTwo string
var otheField string
var someMoreField string
if err := rows.Scan(&id, &fieldOne, &fieldTwo, &otheField, &someMoreField); err != nil {
return nil, fmt.Errorf("failed to scan item: %w", err)
}
item := Item{
FieldOne: fieldOne,
FieldTwo: fieldTwo,
OtheField: otheField,
SomeMoreField: someMoreField
}
retItems = append(retItems, item)
}
if err := tx.Commit(ctx); err != nil {
return nil, fmt.Errorf("failed to commit transaction: %w", err)
}
return retItems, nil
}
tx.Query must be used instead of conn.Query.
Thanks for the help!
Related
I use query to Find All DB data, and one of tables need to be array, so i integrated the loop.
EveryTime errors is conn is busy.
`
for rows.Next() {
var ord Order
err = rows.Scan(
and after one more Query() which is uses to find all Item with ord.UID. So the question is, what is wrong in my code? And how to use this. Here is my FindAll Func:
rows, err := r.client.Query(ctx, q)
if err != nil {
return nil, err
}
defer rows.Close()
orders := make([]Order, 0)
for rows.Next() {
var ord Order
err = rows.Scan(
&ord.OrderUID,
&ord.TrackNumber,
&ord.Entry,
&ord.Delivery.Name,
&ord.Delivery.Phone,
&ord.Delivery.Zip,
&ord.Delivery.City,
&ord.Delivery.Address,
&ord.Delivery.Region,
&ord.Delivery.Email,
&ord.Payment.Transaction,
&ord.Payment.RequestID,
&ord.Payment.Currency,
&ord.Payment.Provider,
&ord.Payment.Amount,
&ord.Payment.PaymentDT,
&ord.Payment.Bank,
&ord.Payment.DeliveryCost,
&ord.Payment.GoodsTotal,
&ord.Payment.CustomFee,
&ord.Locale,
&ord.InternalSignature,
&ord.CustomerID,
&ord.DeliveryService,
&ord.ShardKey,
&ord.SmID,
&ord.DateCreated,
&ord.OofShard,
)
if err != nil {
return nil, err
}
iq := ` ... `
itemRows, err := r.client.Query(ctx, iq, ord.OrderUID)
if err != nil {
return nil, err
}
items := make([]item.Item, 0)
for itemRows.Next() {
var item item.Item
err = itemRows.Scan(
&item.ID,
&item.ChrtID,
&item.TrackNumber,
&item.Price,
&item.Rid,
&item.Name,
&item.Sale,
&item.Size,
&item.TotalPrice,
&item.NmID,
&item.Brand,
&item.Status,
)
if err != nil {
return nil, err
}
items = append(items, item)
}
ord.Items = items
orders = append(orders, ord)
}
`
I tried to rows.Close, but then i Can't use row anymore.
I am getting pq: sorry, too many clients already error when I am calling the GetMessages() multiple times.
Please find the updated code:
main() code
func main() {
dbConn, err := InitDB()
if err != nil {
Log.Error("Connection Error: ", err.Error())
return
}
defer dbConn.Close()
go run()
var input string
fmt.Scanln(&input)
}
Database connection code is:
func InitDB()(*sql.DB, error) {
connectionString := fmt.Sprintf("user=%v password='%v' dbname=%v sslmode=disable", USER, PASSWORD, DATABASE)
db, err = sql.Open(DRIVER, connectionString)
return db, err
}
run goroutine:
func run() {
for {
messages, err := GetMessages()
if err != nil {
Log.Error("Connection Error: ", err.Error())
return
}
log.Info(messages)
}
}
GetMessages() function code:
func GetMessages() (messages []string, err error) {
rows, err := db.Query(`SELECT message1, message2, message3, message4, message5,
message6, message7, message8, message9, message10, message11, message12, message13,
message14, message15, message16, message17, message18, message19, message20, message21,
message22, message23, message24, message25, message26, message27, message28, message29,
message30, message31, message32, message33, message34, message35, message36, message37,
message38, message39, message40, message41, message42, message43, message44, message45,
message46, message47, message48 FROM table1 WHERE id=1`)
if err != nil {
Log.Error("Query error", err)
return messages, err
}
var pointers []interface{}
defer rows.Close()
for rows.Next() {
pointers = make([]interface{}, 48)
messages = make([]string, 48)
for i, _ := range pointers {
pointers[i] = &messages[i]
}
err = rows.Scan(pointers...)
if err != nil {
Log.Error("Failed to scan row", err)
return messages, err
}
}
return messages, nil
}
I checked this answer and I have used scan but still it isn't working
UPDATE
Issue was in another function. I was using db.Query without closing the returned rows object and was repeatedly calling that function. I've updated my code; used db.Exec instead of db.Query and it's working now. Thank you so much #mkopriva for this answer. :)
Try setting SetMaxOpenConns. The default is 0 (unlimited). This may be causing the issue. It would help if you also had SetConnMaxLifetime; otherwise, Postgres will start holding connections longer, and you will notice an increase in memory usage.
I've had the same problem with my postgres / golang project.
Eventually, this example worked flawlessly, without "eating" any DB connections:
// example params
firstName := "Jeremy"
lastName := "Baker"
// setup statement
stmt, err := db.Prepare(
`INSERT INTO user (
firstname,
lastname) VALUES($1, $2)
RETURNING id`) // id is the primary key of table: user
if err != nil {
return err
}
defer stmt.Close()
// execute statement
var userID string
err = stmt.QueryRow(
firstName,
lastName).Scan(&userID)
if err != nil {
return err
}
I am processing a bunch of files and then dumping the results in PostgreSQL. I would like to process many workers at the same time but keep getting the error "pq: sorry, too many clients already". This seems to happen whenever workers is > 100 or so. (For simplicity, the code below demonstrates the process but instead of processing a file I am simply inserting 1M rows in each table).
Since I am reusing the same *db why am I getting this error? Does each transaction count as a client or am I doing something wrong?
package main
import (
"database/sql"
"flag"
"fmt"
"log"
"sync"
"github.com/lib/pq"
)
func process(db *sql.DB, table string) error {
if _, err := db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %v;`, table)); err != nil {
return err
}
col := "age"
s := fmt.Sprintf(`
CREATE TABLE %v (
pk serial PRIMARY KEY,
%v int NOT NULL
)`, table, col)
_, err := db.Exec(s)
if err != nil {
return err
}
tx, err := db.Begin()
if err != nil {
return err
}
defer func() {
if err != nil {
tx.Rollback()
return
}
err = tx.Commit()
}()
stmt, err := tx.Prepare(pq.CopyIn(table, col))
if err != nil {
return err
}
defer func() {
err = stmt.Close()
}()
for i := 0; i < 1e6; i++ {
if _, err = stmt.Exec(i); err != nil {
return err
}
}
return err
}
func main() {
var u string
flag.StringVar(&u, "user", "", "user")
var pass string
flag.StringVar(&pass, "pass", "", "pass")
var host string
flag.StringVar(&host, "host", "", "host")
var database string
flag.StringVar(&database, "database", "", "database")
var workers int
flag.IntVar(&workers, "workers", 10, "workers")
flag.Parse()
db, err := sql.Open("postgres",
fmt.Sprintf(
"user=%s password=%s host=%s database=%s sslmode=require",
u, pass, host, database,
),
)
if err != nil {
log.Fatalln(err)
}
defer db.Close()
db.SetMaxIdleConns(0)
var wg sync.WaitGroup
ch := make(chan int)
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := range ch {
table := fmt.Sprintf("_table%d", i)
log.Println(table)
if err := process(db, table); err != nil {
log.Fatalln(err)
}
}
}()
}
for i := 0; i < 300; i++ {
ch <- i
}
close(ch)
wg.Wait()
}
I realize I can simply increase the posgresql settings but would like to understand the question: How to increase the max connections in postgres?
Since I am reusing the same *db why am I getting this error?
I suspect the Postgress driver is using a separate connections for each of your workers which is a smart decision for most cases.
Does each transaction count as a client or am I doing something wrong?
In your case yes each transaction count as a client, because you are calling process() as a goroutine. You are creating as many concurrent transactions as workers. Since each of your transactions is long all of them are probably using an individual connection to the database at the same time and hence you hit a limit.
go func() {
defer wg.Done()
for i := range ch {
table := fmt.Sprintf("_table%d", i)
log.Println(table)
if err := process(db, table); err != nil {
log.Fatalln(err)
}
}
}()
Within a transaction I'm inserting a row.
How can I access and return the ID of the inserted row.
As you can see in the code below(See under comment // Return last Inserted ID.) I tried to use the LastInsertedId() function, but it gives me an error back.
Btw, I'm using Postgres.
What am I missing here?
Thx!
/**
* Creates order and returns its ID.
*/
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
ret, err := stmt.Exec(order.CustomerID)
// Rollback.
if err != nil {
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Return last Inserted ID.
lastID, err := ret.LastInsertId()
if err != nil {
orderID = 0
} else {
orderID = int(lastID)
}
// Commit.
tx.Commit()
return orderID
} // createOrder
Here is a working solution for now, further improvement is welcomed.
/**
* Creates order and returns its ID.
*/
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
_, err = stmt.Exec(order.CustomerID)
// Rollback.
if err != nil {
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Return last Inserted ID.
//lastID, err := ret.LastInsertId()
err = stmt.QueryRow(order.CustomerID).Scan(&orderID)
if err != nil {
orderID = 0
}
// Commit.
tx.Commit()
return orderID
} // createOrder
This happens because the postgresql driver you are using for go doesn't supports the LastInsertedId() function. You didn't say which driver you are using but I have had this issue working with github.com/lib/pq.
The answer to this is to use QueryRow insted of Exec in your original example. Just make sure you are using RETURNING ID on your query and treat it as if it was a select.
Here is an example (I didn't test this and I might be missing something but you get the idea):
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
var orderID int // Or whatever type you are using
err := stmt.QueryRow(order.CustomerID).Scan(&orderID)
// Rollback.
if err != nil {
//if something goes wrong set the orderID to 0 as in your original code
orderID = 0
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Commit.
tx.Commit()
return orderID
} // createOrder
I need get values using ObjectIdHex and do update and also view the result. I'm using mongodb and golang.But following code doesn't work as expected
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
Id bson.ObjectId `json:"id" bson:"_id,omitempty"`
Name string
Phone string
}
func checkError(err error) {
if err != nil {
panic(err)
}
}
const (
DB_NAME = "gotest"
DB_COLLECTION = "pepole_new1"
)
func main() {
session, err := mgo.Dial("localhost")
checkError(err)
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(DB_NAME).C(DB_COLLECTION)
err = c.DropCollection()
checkError(err)
ale := Person{Name:"Ale", Phone:"555-5555"}
cla := Person{Name:"Cla", Phone:"555-1234-2222"}
kasaun := Person{Name:"kasaun", Phone:"533-12554-2222"}
chamila := Person{Name:"chamila", Phone:"533-545-6784"}
fmt.Println("Inserting")
err = c.Insert(&ale, &cla, &kasaun, &chamila)
checkError(err)
fmt.Println("findbyID")
var resultsID []Person
//err = c.FindId(bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")).One(&resultsID)
err = c.FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID)
checkError(err)
if err != nil {
panic(err)
}
fmt.Println("Phone:", resultsID)
fmt.Println("Queryingall")
var results []Person
err = c.Find(nil).All(&results)
if err != nil {
panic(err)
}
fmt.Println("Results All: ", results)
}
FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID) didn't work for me and giving me following output
Inserting
Queryingall
Results All: [{ObjectIdHex("56bddee2cfa93bfe3d3504a1") Ale 555-5555} {ObjectIdHex("56bddee2cfa93bfe3d3504a2") Cla 555-1234-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a3") kasaun 533-12554-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a4") chamila 533-545-6784}]
findbyID
panic: not found
goroutine 1 [running]:
main.checkError(0x7f33d524b000, 0xc8200689b0)
How can i fix this problem? i need get value using oid and do update also how can i do that
Use can do the same with Golang official driver as follows:
// convert id string to ObjectId
objectId, err := primitive.ObjectIDFromHex("5b9223c86486b341ea76910c")
if err != nil{
log.Println("Invalid id")
}
// find
result:= client.Database(database).Collection("user").FindOne(context.Background(), bson.M{"_id": objectId})
user := model.User{}
result.Decode(user)
It should be _id not Id:
c.FindId(bson.M{"_id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")})
Some sample code that i use.
func (model *SomeModel) FindId(id string) error {
db, ctx, client := Drivers.MongoCollection("collection")
defer client.Disconnect(ctx)
objID, err := primitive.ObjectIDFromHex(id)
if err != nil {
return err
}
filter := bson.M{"_id": bson.M{"$eq": objID}}
if err := db.FindOne(ctx, filter).Decode(&model); err != nil {
//fmt.Println(err)
return err
}
fmt.Println(model)
return nil
}