Why is this copyin statement for sqlx hanging? - postgresql

I've written a toy app to experiment with using Postgresql through sqlx. I got a mass insert working using
pq.CopyIn
as content of a prepared statement
stmt, _ := tx.Preparex(pq.CopyIn(tablename, column, column, ...)
I would then proceed to add rows to the mass insert I'm creating.
tx.Exec(..., ..., ...)
then finally execute the prepared statement
stmt.Exec()
This worked perfectly before, but now I've come back to it and try and execute this code, it hangs on the
stmt.Exec
Am I missing something in my code or is this all to do with the Database Engine, being unresponsive.
Here's my full code for this.
package main
import (
_ "database/sql"
"fmt"
"log"
"encoding/json"
"io/ioutil"
"os"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
)
var schema = `
CREATE TABLE IF NOT EXISTS contact (
id Serial,
first_name text,
last_name text,
email text
);`
type Contact struct {
Id int `json:"-"`
First_name string `json:"first_name"`
Last_name string `json:"last_name"`
Email string `json:"email"`
}
type Contacts struct {
Contacts []Contact `json:"contacts"`
}
func (c *Contacts) createFromJSON(json_str []byte) error {
b := []byte(json_str)
err := json.Unmarshal(b, &c)
if err != nil {
log.Fatal(err)
}
return err
}
func (c *Contacts) save(db *sqlx.DB) error {
tx := db.MustBegin()
stmt, _ := tx.Preparex(pq.CopyIn("contact", "first_name", "last_name", "email"))
for _, contact := range c.Contacts {
tx.Exec(contact.First_name, contact.Last_name, contact.Email)
}
_, err := stmt.Exec()
if err != nil {
log.Fatal(err)
return err
}
err = stmt.Close()
if err != nil {
log.Fatal(err)
return err
}
tx.Commit()
return nil
}
func connect() (*sqlx.DB, error) {
db, err := sqlx.Connect("postgres", "user=pqgotest dbname=pqgotest sslmode=disable")
if err != nil {
log.Fatal(err)
}
return db, err
}
func createTables(db *sqlx.DB) {
db.MustExec(schema)
}
func main() {
db, err := connect()
if err != nil {
os.Exit(1)
}
createTables(db)
contactsJson, e := ioutil.ReadFile("./contacts.json")
if e != nil {
fmt.Printf("File error: %v\n", e)
os.Exit(1)
}
tx := db.MustBegin()
tx.MustExec("DELETE FROM contact")
tx.Commit()
contacts := new(Contacts)
contacts.createFromJSON(contactsJson)
contacts.save(db)
people := new(Contacts)
db.Select(people.Contacts, "SELECT * FROM contact ORDER BY email,id ASC")
for _, contact := range people.Contacts {
contact_json, err := json.Marshal(contact)
if err != nil {
log.Fatal(err)
os.Exit(1)
}
fmt.Printf("%s\n", contact_json)
}
}
I could include the contents of the contacts.json file as well, if that will help.
UPDATE
Yes it was obvious in the end. I was creating a statement from tx,
stmt, _ := tx.Preparex(pq.CopyIn(tablename, column, column, ...)
and further additions to this should be to stmt
stmt.Exec(..., ..., ...)
Also another error not directly related to the question is where I insert an array of contacts into the Contacts field of the struct Contacts
people := new(Contacts)
db.Select(people.Contacts, "SELECT * FROM contact ORDER BY email,id ASC")
should be passing a pointer to the Select method of db of the Contacts array field of Contacts, like so
db.Select(&people.Contacts, "SELECT * FROM contact ORDER BY email,id ASC")
In case people try and run this code later and wonder why it's not printing the results to the console.

From Bulk imports part in https://godoc.org/github.com/lib/pq, it should be
stmt.Exec(contact.First_name, contact.Last_name, contact.Email)

Related

Rollback does not work well with Go language transactional wrapper

I have recently started learning Go.
I found the following Github implementation of a wrapper for database transaction processing and decided to try it out.
(source) https://github.com/oreilly-japan/practical-go-programming/blob/master/ch09/transaction/wrapper/main.go
I am using PostgreSQL as the database.
Initially, it contains the following data.
testdb=> select * from products;
product_id | price
------------+-------
0001 | 200
0002 | 100
0003 | 150
0004 | 300
(4 rows)
After Process A succeeds, Process B is intentionally made to fail, and a rollback of transaction A is expected. However, when we run it, the rollback does not occur and we end up with the following
In truth, since B failed, the process A should be rolled back and there should be no change in the database value.
I have inserted Logs in places to confirm this, but I am not sure. Why is the rollback not executed?
package main
import (
"context"
"database/sql"
"fmt"
"log"
_ "github.com/jackc/pgx/v4/stdlib"
)
// transaction-wrapper-start
type txAdmin struct {
*sql.DB
}
type Service struct {
tx txAdmin
}
func (t *txAdmin) Transaction(ctx context.Context, f func(ctx context.Context) (err error)) error {
log.Printf("transaction")
tx, err := t.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := f(ctx); err != nil {
log.Printf("transaction err")
return fmt.Errorf("transaction query failed: %w", err)
}
log.Printf("commit")
return tx.Commit()
}
func (s *Service) UpdateProduct(ctx context.Context, productID string) error {
updateFunc := func(ctx context.Context) error {
log.Printf("first process")
// Process A
if _, err := s.tx.ExecContext(ctx, "UPDATE products SET price = 200 WHERE product_id = $1", productID); err != nil {
log.Printf("first err")
return err
}
log.Printf("second process")
// Process B(They are intentionally failing.)
if _, err := s.tx.ExecContext(ctx, "...", productID); err != nil {
log.Printf("second err")
return err
}
return nil
}
log.Printf("update")
return s.tx.Transaction(ctx, updateFunc)
}
// transaction-wrapper-end
func main() {
data, err := sql.Open("pgx", "host=localhost port=5432 user=testuser dbname=testdb password=password sslmode=disable")
if nil != err {
log.Fatal(err)
}
database := Service {tx: txAdmin{data}}
ctx := context.Background()
database.UpdateProduct(ctx, "0004")
}
output
2022/05/26 13:28:55 update
2022/05/26 13:28:55 transaction
2022/05/26 13:28:55 first process
2022/05/26 13:28:55 second process
2022/05/26 13:28:55 second err
2022/05/26 13:28:55 transaction err
database changes(If the rollback works, the PRICE for id 0004 should remain 300.)
testdb=> select * from products;
product_id | price
------------+-------
0001 | 200
0002 | 100
0003 | 150
0004 | 200
(4 rows)
Please tell me how I can use the wrapper to correctly process transactions.
=========
PS.
The following code without the wrapper worked properly.
package main
import (
"context"
"database/sql"
"log"
_ "github.com/jackc/pgx/v4/stdlib"
)
// transaction-defer-start
type Service struct {
db *sql.DB
}
func (s *Service) UpdateProduct(ctx context.Context, productID string) (err error) {
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if _, err = tx.ExecContext(ctx, "UPDATE products SET price = 200 WHERE product_id = $1", productID); err != nil {
log.Println("update err")
return err
}
if _, err = tx.ExecContext(ctx, "...", productID); err != nil {
log.Println("update err")
return err
}
return tx.Commit()
}
// transaction-defer-end
func main() {
var database Service
dbConn, err := sql.Open("pgx", "host=localhost port=5432 user=testuser dbname=testdb password=passs sslmode=disable")
if nil != err {
log.Fatal(err)
}
database.db = dbConn
ctx := context.Background()
database.UpdateProduct(ctx, "0004")
}
As #Richard Huxton said, pass tx into a function f
here are the steps:
add one field on struct txAdmin to accommodate *sql.Tx, so txAdmin have DB and Tx fields
inside Transaction set tx to *txAdmin.Tx
inside UpdateProduct use *Service.tx.Tx for every query
so the final code looks like this:
package main
import (
"context"
"database/sql"
"fmt"
"log"
_ "github.com/jackc/pgx/v4/stdlib"
)
// transaction-wrapper-start
type txAdmin struct {
*sql.DB
*sql.Tx
}
type Service struct {
tx txAdmin
}
func (t *txAdmin) Transaction(ctx context.Context, f func(ctx context.Context) (err error)) error {
log.Printf("transaction")
tx, err := t.DB.BeginTx(ctx, nil)
if err != nil {
return err
}
// set tx to Tx
t.Tx = tx
defer tx.Rollback()
if err := f(ctx); err != nil {
log.Printf("transaction err")
return fmt.Errorf("transaction query failed: %w", err)
}
log.Printf("commit")
return tx.Commit()
}
func (s *Service) UpdateProduct(ctx context.Context, productID string) error {
updateFunc := func(ctx context.Context) error {
log.Printf("first process")
// Process A
if _, err := s.tx.Tx.ExecContext(ctx, "UPDATE products SET price = 200 WHERE product_id = $1", productID); err != nil {
log.Printf("first err")
return err
}
log.Printf("second process")
// Process B(They are intentionally failing.)
if _, err := s.tx.Tx.ExecContext(ctx, "...", productID); err != nil {
log.Printf("second err")
return err
}
return nil
}
log.Printf("update")
return s.tx.Transaction(ctx, updateFunc)
}
// transaction-wrapper-end
func main() {
data, err := sql.Open("pgx", "host=localhost port=5432 user=testuser dbname=testdb password=password sslmode=disable")
if nil != err {
log.Fatal(err)
}
database := Service{tx: txAdmin{DB: data}}
ctx := context.Background()
database.UpdateProduct(ctx, "0004")
}

Postgres - Golang Error: pq: Users does not exists

I'm trying to connect Golang to a Postgres DB, it is in Gcloud and I already allowed the network IP. I'm trying to access the User DB and then the User table inside the public schema, this is the database structure:
I am getting this error: pq: database "Users" does not exist
I'll add the code:
package ports
import (
"database/sql"
"log"
"github.com/gofiber/fiber/v2"
_ "github.com/lib/pq"
)
func OpenConnection(dbName string) (*sql.DB, error) {
connStr := "<addr>" + dbName
db, err := sql.Open("postgres", connStr)
if err != nil {
log.Fatal(err)
}
return db, err
}
func GetUsers(ctx *fiber.Ctx) error {
db, dbErr := OpenConnection("Users")
if dbErr != nil {
log.Fatalln(dbErr)
}
defer db.Close()
rows, err := db.Query("SELECT * FROM Users")
if err != nil {
log.Fatalln(err)
ctx.JSON("An error ocurred")
}
defer rows.Close()
return ctx.Render("/users", fiber.Map{
"Users": rows,
})
}
I just fixed it and it was a combination of two things:
I wasn't properly combining the strings, when I hardcoded it I got rid of the db error.
After that I encountered an error where it would say the table doesn't exists, the issue is I was sending an unquote table name and I needed it to be case sensitive so I had to wrap the name between quotes to make it work. Like so
package ports
import (
"database/sql"
"log"
"github.com/gofiber/fiber/v2"
_ "github.com/lib/pq"
)
func OpenConnection() (*sql.DB, error) {
connStr := "<connStr>/users"
db, err := sql.Open("postgres", connStr)
if err != nil {
log.Fatal(err)
}
return db, err
}
func GetUsers(ctx *fiber.Ctx, db *sql.DB) error {
defer db.Close()
rows, err := db.Query(`SELECT * FROM "Users"`)
if err != nil {
log.Fatalln(err)
ctx.JSON("An error ocurred")
}
defer rows.Close()
return ctx.Render("/users", fiber.Map{
"Users": rows,
})
}

pq: sorry, too many clients already

I am getting pq: sorry, too many clients already error when I am calling the GetMessages() multiple times.
Please find the updated code:
main() code
func main() {
dbConn, err := InitDB()
if err != nil {
Log.Error("Connection Error: ", err.Error())
return
}
defer dbConn.Close()
go run()
var input string
fmt.Scanln(&input)
}
Database connection code is:
func InitDB()(*sql.DB, error) {
connectionString := fmt.Sprintf("user=%v password='%v' dbname=%v sslmode=disable", USER, PASSWORD, DATABASE)
db, err = sql.Open(DRIVER, connectionString)
return db, err
}
run goroutine:
func run() {
for {
messages, err := GetMessages()
if err != nil {
Log.Error("Connection Error: ", err.Error())
return
}
log.Info(messages)
}
}
GetMessages() function code:
func GetMessages() (messages []string, err error) {
rows, err := db.Query(`SELECT message1, message2, message3, message4, message5,
message6, message7, message8, message9, message10, message11, message12, message13,
message14, message15, message16, message17, message18, message19, message20, message21,
message22, message23, message24, message25, message26, message27, message28, message29,
message30, message31, message32, message33, message34, message35, message36, message37,
message38, message39, message40, message41, message42, message43, message44, message45,
message46, message47, message48 FROM table1 WHERE id=1`)
if err != nil {
Log.Error("Query error", err)
return messages, err
}
var pointers []interface{}
defer rows.Close()
for rows.Next() {
pointers = make([]interface{}, 48)
messages = make([]string, 48)
for i, _ := range pointers {
pointers[i] = &messages[i]
}
err = rows.Scan(pointers...)
if err != nil {
Log.Error("Failed to scan row", err)
return messages, err
}
}
return messages, nil
}
I checked this answer and I have used scan but still it isn't working
UPDATE
Issue was in another function. I was using db.Query without closing the returned rows object and was repeatedly calling that function. I've updated my code; used db.Exec instead of db.Query and it's working now. Thank you so much #mkopriva for this answer. :)
Try setting SetMaxOpenConns. The default is 0 (unlimited). This may be causing the issue. It would help if you also had SetConnMaxLifetime; otherwise, Postgres will start holding connections longer, and you will notice an increase in memory usage.
I've had the same problem with my postgres / golang project.
Eventually, this example worked flawlessly, without "eating" any DB connections:
// example params
firstName := "Jeremy"
lastName := "Baker"
// setup statement
stmt, err := db.Prepare(
`INSERT INTO user (
firstname,
lastname) VALUES($1, $2)
RETURNING id`) // id is the primary key of table: user
if err != nil {
return err
}
defer stmt.Close()
// execute statement
var userID string
err = stmt.QueryRow(
firstName,
lastName).Scan(&userID)
if err != nil {
return err
}

Need the ID of the current row inserted within a transaction

Within a transaction I'm inserting a row.
How can I access and return the ID of the inserted row.
As you can see in the code below(See under comment // Return last Inserted ID.) I tried to use the LastInsertedId() function, but it gives me an error back.
Btw, I'm using Postgres.
What am I missing here?
Thx!
/**
* Creates order and returns its ID.
*/
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
ret, err := stmt.Exec(order.CustomerID)
// Rollback.
if err != nil {
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Return last Inserted ID.
lastID, err := ret.LastInsertId()
if err != nil {
orderID = 0
} else {
orderID = int(lastID)
}
// Commit.
tx.Commit()
return orderID
} // createOrder
Here is a working solution for now, further improvement is welcomed.
/**
* Creates order and returns its ID.
*/
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
_, err = stmt.Exec(order.CustomerID)
// Rollback.
if err != nil {
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Return last Inserted ID.
//lastID, err := ret.LastInsertId()
err = stmt.QueryRow(order.CustomerID).Scan(&orderID)
if err != nil {
orderID = 0
}
// Commit.
tx.Commit()
return orderID
} // createOrder
This happens because the postgresql driver you are using for go doesn't supports the LastInsertedId() function. You didn't say which driver you are using but I have had this issue working with github.com/lib/pq.
The answer to this is to use QueryRow insted of Exec in your original example. Just make sure you are using RETURNING ID on your query and treat it as if it was a select.
Here is an example (I didn't test this and I might be missing something but you get the idea):
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
var orderID int // Or whatever type you are using
err := stmt.QueryRow(order.CustomerID).Scan(&orderID)
// Rollback.
if err != nil {
//if something goes wrong set the orderID to 0 as in your original code
orderID = 0
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Commit.
tx.Commit()
return orderID
} // createOrder

How to find by id in golang and mongodb

I need get values using ObjectIdHex and do update and also view the result. I'm using mongodb and golang.But following code doesn't work as expected
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
Id bson.ObjectId `json:"id" bson:"_id,omitempty"`
Name string
Phone string
}
func checkError(err error) {
if err != nil {
panic(err)
}
}
const (
DB_NAME = "gotest"
DB_COLLECTION = "pepole_new1"
)
func main() {
session, err := mgo.Dial("localhost")
checkError(err)
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(DB_NAME).C(DB_COLLECTION)
err = c.DropCollection()
checkError(err)
ale := Person{Name:"Ale", Phone:"555-5555"}
cla := Person{Name:"Cla", Phone:"555-1234-2222"}
kasaun := Person{Name:"kasaun", Phone:"533-12554-2222"}
chamila := Person{Name:"chamila", Phone:"533-545-6784"}
fmt.Println("Inserting")
err = c.Insert(&ale, &cla, &kasaun, &chamila)
checkError(err)
fmt.Println("findbyID")
var resultsID []Person
//err = c.FindId(bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")).One(&resultsID)
err = c.FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID)
checkError(err)
if err != nil {
panic(err)
}
fmt.Println("Phone:", resultsID)
fmt.Println("Queryingall")
var results []Person
err = c.Find(nil).All(&results)
if err != nil {
panic(err)
}
fmt.Println("Results All: ", results)
}
FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID) didn't work for me and giving me following output
Inserting
Queryingall
Results All: [{ObjectIdHex("56bddee2cfa93bfe3d3504a1") Ale 555-5555} {ObjectIdHex("56bddee2cfa93bfe3d3504a2") Cla 555-1234-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a3") kasaun 533-12554-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a4") chamila 533-545-6784}]
findbyID
panic: not found
goroutine 1 [running]:
main.checkError(0x7f33d524b000, 0xc8200689b0)
How can i fix this problem? i need get value using oid and do update also how can i do that
Use can do the same with Golang official driver as follows:
// convert id string to ObjectId
objectId, err := primitive.ObjectIDFromHex("5b9223c86486b341ea76910c")
if err != nil{
log.Println("Invalid id")
}
// find
result:= client.Database(database).Collection("user").FindOne(context.Background(), bson.M{"_id": objectId})
user := model.User{}
result.Decode(user)
It should be _id not Id:
c.FindId(bson.M{"_id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")})
Some sample code that i use.
func (model *SomeModel) FindId(id string) error {
db, ctx, client := Drivers.MongoCollection("collection")
defer client.Disconnect(ctx)
objID, err := primitive.ObjectIDFromHex(id)
if err != nil {
return err
}
filter := bson.M{"_id": bson.M{"$eq": objID}}
if err := db.FindOne(ctx, filter).Decode(&model); err != nil {
//fmt.Println(err)
return err
}
fmt.Println(model)
return nil
}