type User struct {
Email string `json:"email"`
Password string `json:"password"`
}
db, err := sql.Open("postgres", os.Getenv("DATABASE_URL"))
if err != nil {
log.Fatal(err)
}
fmt.Println("email: ", reflect.TypeOf(usr.Email)) //string
fmt.Println("salt: ", reflect.TypeOf(salt)) //[]uint8
fmt.Println("hash: ", reflect.TypeOf(hash)) //string
sql := `INSERT INTO public."Users" (email, password, salt) VALUES ($1, $2, $3)`
_, err = db.Exec(sql, usr.Email, hash, salt)
throws error: "pq: invalid byte sequence for encoding "UTF8": 0x97"
my table: "email" type: TEXT, "password" type: TEXT, "salt" type: smallint[] (thinking this might be the cause of the error but I am not sure what to use instead)
PostgreSQL bytea = []unit8 Golang
changed type and issue was resolved!
Related
I'm trying to query from multiple databases. Each database is connected using the following function:
func connectDB(dbEnv str) *sql.DB{
// Loading environment variables from local.env file
err1 := godotenv.Load(dbEnv)
if err1 != nil {
log.Fatalf("Some error occured. Err: %s", err1)
}
dialect := os.Getenv("DIALECT")
host := os.Getenv("HOST")
dbPort := os.Getenv("DBPORT")
user := os.Getenv("USER")
dbName := os.Getenv("NAME")
password := os.Getenv("PASSWORD")
// Database connection string
dbURI := fmt.Sprintf("port=%s host=%s user=%s "+"password=%s dbname=%s sslmode=disable", dbPort, host, user, password, dbName)
// Create database object
db, err := sql.Open(dialect,dbURI)
if err != nil {
log.Fatal(err)
}
return db
}
type order struct{
OrderID string `json:"orderID"`
Name string `json:"name"`
}
type book struct{
OrderID string `json:"orderID"`
Name string `json:"name"`
}
func getOrders(db *sql.DB) []order {
var (
orderID string
name string
)
var allRows = []order{}
query := `
SELECT orderID, name
FROM orders.orders;
`
//Get rows using the query
rows, err := db.Query(query)
if err != nil { //Log if error
log.Fatal(err)
}
defer rows.Close()
// Add each row into the "allRows" slice
for rows.Next() {
err := rows.Scan(&orderID, &name, &date)
if err != nil {
log.Fatal(err)
}
//Create new order struct with the received data
row := order{
OrderID: orderID,
Name: name,
}
allRows = append(allRows, row)
}
//Log if error
err = rows.Err()
if err != nil {
log.Fatal(err)
}
return allRows
}
func getBooks(db *sql.DB) []book{
var (
bookID string
name string
)
var allRows = []book{}
query := `
SELECT bookID, name
FROM books.books;
`
//Get rows using the query
rows, err := db.Query(query)
if err != nil { //Log if error
log.Fatal(err)
}
defer rows.Close()
// Add each row into the "allRows" slice
for rows.Next() {
err := rows.Scan(&bookID, &name)
if err != nil {
log.Fatal(err)
}
//Create new book struct with the received data
row := book{
BookID: bookID,
Name: name,
}
allRows = append(allRows, row)
}
//Log if error
err = rows.Err()
if err != nil {
log.Fatal(err)
}
return allRows
}
func main() {
ordersDB:= connectDB("ordersDB.env")
booksDB:= connectDB("booksDB.env")
orders := getOrders(ordersDB)
books := getBooks(booksDB)
}
The issue is that when I use ordersDB first, the program only recognizes the table in ordersDB. And when I use booksDB first, the program only recognizes the table in booksDB.
When I try to query a table in booksDB after using ordersDB, it is giving me "relation "books.books" does not exist" error. When I try to query a table in ordersDB after using booksDB, it gives "relation "orders.orders" does not exist"
Is there a better way to connect to multiple databases?
You are using github.com/joho/godotenv to load the database configuration from the environment. Summarising (and cutting out a lot of detail) what you are doing is:
godotenv.Load("ordersDB.env")
host := os.Getenv("HOST")
// Connect to DB
godotenv.Load("booksDB.env")
host := os.Getenv("HOST")
// Connect to DB 2
However as stated in the docs "Existing envs take precedence of envs that are loaded later". This is also stated more clearly here "It's important to note that it WILL NOT OVERRIDE an env variable that already exists".
So your code will load in the first .env file, populate the environment variables, and connect to the database. You will then load the second .env file but, because the environmental variables are already set, they will not be changed and you will connect to the same database a second time.
As a work around you could use Overload. However it's probably better to reconsider your use of environmental variables (and perhaps use different variables for the second connection).
I am trying to update a record in a postgres table with an array (slice) of values. The table has the following DDL:
CREATE TABLE slm_files (
id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
filename character varying NOT NULL,
status character varying NOT NULL,
original_headers text[]
);
and the Go code I have is as follows:
package main
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"os"
"strings"
"time"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/lib/pq"
)
type message struct {
ID string `json:"id"`
Filename string `json:"filename"`
Status string `json:"status"`
OriginalHeaders []string `json:"OriginalHeaders"`
}
func main() {
host := os.Getenv("PGhost")
port := 5432
user := os.Getenv("PGuser")
password := os.Getenv("PGpassword")
dbname := os.Getenv("PGdbname")
pgConString := fmt.Sprintf("port=%d host=%s user=%s "+
"password=%s dbname=%s sslmode=disable",
port, host, user, password, dbname)
msgBody := `update_headers___
{
"id": "76b67119-d8c1-4a20-b53e-49e4972e2f19",
"filename": "SLM1171_inputData_preNCOA-5babc88b-1d14-468d-bf6e-c3b36ce90d95.csv",
"status": "Submitted",
"OriginalHeaders": [
"city",
"state",
"zipcode",
"full_name",
"individual_id"
]
}`
fmt.Println("Processing file", msgBody)
queryMethod := strings.Split(msgBody, "___")[0]
fieldDict := strings.Split(msgBody, "___")[1]
db, err := sql.Open("postgres", pgConString)
if err != nil {
panic(err)
}
fmt.Println("Connected Successfully")
defer db.Close()
body := message{}
json.Unmarshal([]byte(fieldDict), &body)
fmt.Println(queryMethod)
fmt.Println(body)
var sqlStatement string
switch queryMethod {
case "update_ncoa":
sqlStatement = fmt.Sprintf(`UPDATE slm_files SET status = '%s', updated_at = '%s' where id = '%s';`,
body.Status,
body.UpdatedAt,
body.ID,
)
case "update_headers":
sqlStatement = fmt.Sprintf(`UPDATE slm_files SET original_headers = '%s', updated_at = '%s' where id = '%s';`,
pq.Array(body.OriginalHeaders),
body.UpdatedAt,
body.ID,
)
}
fmt.Println(sqlStatement)
_, err = db.Query(sqlStatement)
if err != nil {
fmt.Println("Failed to run query", err)
return
}
}
fmt.Println("Query executed!")
return
}
but I keep getting the error
pq: malformed array literal: "&[first_name last_name city state zipcode full_name individual_id]": Error
null
I have read a few things on the internet that lead me to using pq.Array() but that doesnt seem to work.
I have read about the difference in format between Go arrays and Postgres arrays, so I had hoped that letting the pq.Array function would sort it out but apparently not.
As Peter advised, there's a lot to fix up with that database handling. And it's definitely worth redoing those SQL statements to not use Sprintf to make the query.
But in terms of just getting something working with postgres arrays and the pq library, you need to use the Value() method of pq.Array to get the postgres format. Change your update statement for the headers to something like this:
arrayVal, _ := pq.Array(body.OriginalHeaders).Value()
sqlStatement = fmt.Sprintf(`UPDATE slm_files SET original_headers = '%s', updated_at = '%s' where id = '%s';`,
arrayVal,
body.UpdatedAt,
body.ID,
)
And it's worth checking the return from the Value() method to make sure there are no errors, I just ignored it for the sake of a simple example.
A peculiar error occurred today when was manipulating data in Postgres using database/sql and driver github.com/lib/pq. I have the following SQL schema created in Postgres:
CREATE TABLE IF NOT EXISTS bench_bytea (
id INT PRIMARY KEY,
name VARCHAR,
data BYTEA
);
A very basic table containing a data blob of type BYTEA.
I then tried to execute simple INSERT statement using the Exec() function provided by database/sql. Here it is:
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
"password=%s dbname=%s sslmode=disable",
host, port, user, password, dbname)
db, err := sql.Open("postgres", psqlInfo)
if err != nil {
panic(err)
}
defer db.Close()
stmt := `INSERT INTO bench_bytea (id, name, data) VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING`
data := `{"title": "Sleeping Beauties", "genres": ["Fiction", "Thriller", "Horror"], "published": false}`
i := 0
_, err = db.Exec(stmt, i, "testing "+string(i), []byte(data))
if err != nil {
panic(err)
}
The key highlight happens on the db.Exec() line where I execute a SQL INSERT statement (in practicality i is an index of an array where I stored different testing data. I didn't want to include the other pieces of data here since it's really long and irrelevant). The error I received is:
pq: invalid byte sequence for encoding "UTF8": 0x00
Now if I change "testing "+string(i) into "testing", the error is gone. That is, if I didn't insert a concatenating strings into the name column, there's no error. What is going on here?
You can't convert an integer to a string like that. The result of string(0) is "\x00", aka a null byte, instead of "0" which is probably you want. You should use strconv.Itoa for the conversion instead.
I'm writing simple models for interacting with postgresql db in my api, and all of them work fine except one table. When i try to run tests i get an error:
=== RUN TestAddUser
time="2018-08-08T10:24:56+03:00" level=info msg="db connection" db: ="&{0xc4200c6dc0 postgres false 0xc4200912c0}"
--- FAIL: TestAddUser (0.00s)
require.go:794:
Error Trace: users_test.go:19
Error: Received unexpected error:
pq: column "user_name" of relation "users" does not exist
Error while trying to add new user
gitlab.com/inn4sci-go/Nesterenko/course/api/models.AddUser
/home/nestor/go/src/gitlab.com/inn4sci-go/Nesterenko/course/api/models/users.go:20
gitlab.com/inn4sci-go/Nesterenko/course/api/models/tests.TestAddUser
/home/nestor/go/src/gitlab.com/inn4sci-go/Nesterenko/course/api/models/tests/users_test.go:18
testing.tRunner
/usr/local/go/src/testing/testing.go:777
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:2361
Test: TestAddUser
FAIL
Process finished with exit code 1
But if i run same query from psql console all works fine:
nestor=> INSERT INTO users (user_name, user_pass) VALUES ('wrerw','wrwer');
INSERT 0 1
nestor=>
Here the code of AddUser func and Users struct:
package models
import (
"github.com/pkg/errors"
"gitlab.com/inn4sci-go/Nesterenko/course/api/db"
)
type Users struct {
Id int `db:"id" json:"id"`
ProfileId int `db:"profile_id" json:"profileId"`
UserName string `db:"user_name" json:"nickname"`
UserPass string `db:"user_pass" json:"password"`
}
func AddUser(db db.DbInt, user Users) (int64, error) {
query := `INSERT INTO users (user_name, user_pass) VALUES (:user_name, :user_pass)`
res, err := db.GetDB().NamedExec(query, &user)
if err != nil {
return 0, errors.Wrap(err, "Error while trying to add new user")
}
id, err := res.LastInsertId()
if err != nil {
return 0, errors.Wrap(err, "Error while trying to obtain userId")
}
return id, nil
}
Here code of test func:
func TestAddUser(t *testing.T) {
mockDB := testutils.CreateFakeDBObj(t) //this func mocks db connection, and works fine in all test
user := models.Users{
UserName: "test",
UserPass: "test",
}
userId, err := models.AddUser(mockDB, user)
require.NoError(t, err)
logrus.WithField("User was successfully added with id = ", userId).Info("Users test")
}
And this is query for creating users table:
nestor=> CREATE TABLE IF NOT EXISTS users (id SERIAL PRIMARY KEY, profile_id INTEGER REFERENCES profile(id), user_name TEXT UNIQUE NOT NULL, user_pass TEXT NOT NULL);
CREATE TABLE
nestor=> \dt users
List of relations
Schema | Name | Type | Owner
--------+-------+-------+-------
public | users | table | ihor
(1 row)
I suppose that i have got a mistake somewhere in AddUser func when trying to execute this query: INSERT INTO users (user_name, user_pass) VALUES (:user_name, :user_pass)
EDITED:
Establishing connection to db:
type MockingDBObj struct {
mock.Mock
}
func (m MockingDBObj) ConnectToDB(c configure.ConfigInt) error {
return nil
}
func (m MockingDBObj) GetDB() *sqlx.DB {
args := m.Called()
return args.Get(0).(*sqlx.DB)
}
func CreateFakeDBObj(t *testing.T) MockingDBObj {
mockObj := new(MockingDBObj)
host := "localhost"
port := 5432
user := "ihor"
password := "ihor"
dbname := "final_project"
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
"password=%s dbname=%s sslmode=disable",
host, port, user, password, dbname)
db, err := sqlx.Connect("postgres", psqlInfo)
require.NoError(t, err)
logrus.WithField("db: ", db).Info("db connection")
mockObj.On("GetDB").Return(db)
return *mockObj
}
I'm using go and the package uuid to generate a uuid of type [16]byte. However when I try to insert that uuid into my postgres column of type uuid I get the error converting argument $1 type: unsupported type [16]uint8, a array. So apparently I should convert the uuid on the client before I insert it into the db. How should I do that? What type should I convert it to?
In short: What go data type will work with uuid in postgres?
Thanks to the link from #sberry, I found success. Here are snippets of the code for your benefit (with a PostgreSQL 9.5 database):
import (
"database/sql"
"net/http"
"github.com/google/uuid"
)
type Thing struct {
ID uuid.UUID `json:"-" sql:",type:uuid"`
Name string `json:"name"`
}
// For a database table created as such:
// CREATE TABLE things (
// id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
// name TEXT DEFAULT ''::text
// )
func selectThingssSQL() ([]Thing, error) {
things := make([]Thing, 0)
rows, err := db.Query("SELECT id, name FROM things")
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
t := &Thing{}
if err := rows.Scan(&t.ID, &t.Name); err != nil {
return nil, err
}
things = append(things, *t)
}
return things, nil
}