Can PostgreSQL's "COPY table FROM file" statement be used in Go? - postgresql

After reading github.com/lib/pq documentation, it is still not clear for me if it is possible to copy data from a CSV file using a simple COPY <table> FROM <file> CSV HEADER command.
This is what I'm trying to do:
func CopyFromCSV(con Con, tableName, fileName string) error {
_, err := con.Exec(fmt.Sprintf("TRUNCATE %s", tableName))
if err != nil {
return err
}
stm, err := con.Prepare(fmt.Sprintf("COPY %s FROM '%s' CSV HEADER", tableName, fileName))
if err != nil {
return err
}
defer stm.Close()
_, err = stm.Exec()
return err
}
Where tableName is an existing table, and fileName the absolute path to an existing csv file.
I'm getting always the following error after the con.Prepare call: pq: unknown response for copy query: 'C'
Is it possible to do this in Go with a postgres database using the github.com/lib/pq driver?

The code below uses https://github.com/lib/pq
import (
"database/sql"
"fmt"
"github.com/lib/pq"
)
func bulkCopyTest() {
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=verify-full",
host, port, user, password, dbname)
db, err := sql.Open("postgres", psqlInfo)
if err != nil {
panic(err)
}
defer db.Close()
tx, err := db.Begin()
if err != nil {
panic(err)
}
stmt, err := tx.Prepare(pq.CopyInSchema("schemaName", "DBName", "columnName", "columnName"))
if err != nil {
panic(err)
}
//loop through an array of struct filled with data, or read from a file
for _, row := range loadCsv {
stmt.Exec(row.value1, row.value2)
if err != nil {
panic(err)
}
}
_, err = stmt.Exec()
if err != nil {
panic(err)
}
err = stmt.Close()
if err != nil {
panic(err)
}
err = tx.Commit()
if err != nil {
panic(err)
}
}

A rough template to execute COPY command can be found here - https://play.golang.org/p/6y5v3IW8kD
The code below should work for you.
func copyTest() {
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=verify-full",host, port, user, password, dbname)
db, err := sql.Open("postgres", psqlInfo)
if err != nil {
panic(err)
}
defer db.Close()
err = db.Ping()
if err != nil {
panic(err)
}
tx, err := db.Begin()
if err != nil {
panic(err)
}
_, err = tx.Exec("copy metadata.charvalues_temp from 'C:\\yourlocation\\yourfile.csv' with csv")
if err != nil {
panic(err)
}
err = tx.Commit()
if err != nil {
panic(err)
}
}

Related

IndexStats Query in DocumentDB returning 0 ops

I'm trying to build a Documentdb Prometheus exporter. I'm trying to get the IndexStats to identify the unused Indices. But I'm getting ops value as 0 even though the ops value is not 0. Can you please help me identify the error?
This the Go code for the same.
databases, err := client.ListDatabaseNames(ctx, bson.D{})
if err != nil{
return make(map[string]map[string]interface{}), err
}
var indexMap map[string]map[string]interface{} = make(map[string]map[string]interface{})
for _, database := range databases {
collections, err := client.Database(database).ListCollectionNames(ctx, bson.D{})
if err != nil {
return make(map[string]map[string]interface{}), err
}
for _, collectionName := range collections {
collection := client.Database(database).Collection(collectionName)
indexStats := bson.D{{Key: "$indexStats", Value: bson.M{}}}
cursor, err := collection.Aggregate(ctx, mongo.Pipeline{indexStats})
if err != nil {
return make(map[string]map[string]interface{}), err
}
var indices []bson.M
err = cursor.All(ctx, &indices)
if err != nil {
return make(map[string]map[string]interface{}), err
}
err = cursor.Close(ctx)
if err != nil {
return make(map[string]map[string]interface{}), err
}
fmt.Print(database + " " + collectionName + " ")
fmt.Println(indices)
indicesBytes, err := json.Marshal(indices)
if err != nil {
return make(map[string]map[string]interface{}), err
}
var indicesJson []interface{}
err = json.Unmarshal(indicesBytes, &indicesJson)
if err != nil {
return make(map[string]map[string]interface{}), err
}
if _, ok := indexMap[database]; !ok {
indexMap[database] = make(map[string]interface{})
}
indexMap[database][collectionName] = indicesJson
}
}
May I know the reason and the fix for this issue?
Thanks in Advance :)

Print collection in a list of a collection in mongodb in golang

To print a collection from mongodb the following is my code in python:
print(list(MongoClient(***).get_database("ChatDB").get_collection("room_members".find({'_id.username': username})))
I am learning Go and I am trying to translate the aforementioned code into golang.
My code is as follows:
client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI("*****"))
if err != nil {
panic(err)
}
likes_collection := client.Database("ChatDB").Collection("likes")
cur, err := likes_collection.Find(context.Background(), bson.D{{}})
if err != nil {
panic(err)
}
defer cur.Close(context.Background())
fmt.Println(cur)
However, I get some hex value
Mongo in go lang different api than mongo.
Find returns cursor not collection.
You should changed your code to :
var items []Items
cur, err := likes_collection.Find(context.Background(), bson.D{{}})
if err != nil {
panic(err)
}
cur.All(context.Background(),&items)
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
client, err := mongo.Connect(ctx, options.Client().ApplyURI("***"))
if err != nil {
panic(err)
}
var likes []bson.M
likes_collection := client.Database("ChatDB").Collection("likes")
defer client.Disconnect(ctx)
cursor, err := likes_collection.Find(context.Background(), bson.D{{}})
if err != nil {
panic(err)
}
if err = cursor.All(ctx, &likes); err != nil {
panic(err)
}
fmt.Println(likes)

golang net/smtp getting smtp server response DSN

I am using golang net/smtp to send mails
Whenever I send to my smtp server I need to capture the response from the server
Especially the DSN
For example my local smtp server gives a "ok queued as " at the end of the mail
I need to capture this and print in the logs
How can I do this
package main
import (
"log"
"net/smtp"
)
func sendEmail(msg []byte) {
c, err := smtp.Dial("localhost:25")
if err != nil {
log.Fatal(err)
}
if err := c.Mail("sender#example.org"); err != nil {
log.Fatal(err)
}
if err := c.Rcpt("recipient#example.net"); err != nil {
log.Fatal(err)
}
wc, err := c.Data()
if err != nil {
log.Fatal(err)
}
_, err = wc.Write(msg)
if err != nil {
log.Fatal(err)
}
//How do I get the response here ??
err = wc.Close()
if err != nil {
log.Fatal(err)
}
err = c.Quit()
if err != nil {
log.Fatal(err)
}
}
As mentioned in the comments you can use c.Text.ReadResponse():
package main
import (
"net/smtp"
)
func sendEmail(msg []byte) (code int, message string, err error) {
c, err := smtp.Dial("localhost:25")
if err != nil {
return
}
defer c.Quit() // make sure to quit the Client
if err = c.Mail("sender#example.org"); err != nil {
return
}
if err = c.Rcpt("recipient#example.net"); err != nil {
return
}
wc, err := c.Data()
if err != nil {
return
}
defer wc.Close() // make sure WriterCloser gets closed
_, err = wc.Write(msg)
if err != nil {
return
}
code, message, err = c.Text.ReadResponse(0)
return
}
The code, message and any err are now passed to the caller, don't use log.Fatal throughout your code, handle the error on the calling side.
package main
import (
"net/smtp"
)
func sendEmail(msg []byte) (code int, message string, err error) {
c, err := smtp.Dial("localhost:25")
if err != nil {
return
}
defer c.Quit() // make sure to quit the Client
if err = c.Mail("sender#example.org"); err != nil {
return
}
if err = c.Rcpt("recipient#example.net"); err != nil {
return
}
wc, err := c.Data()
if err != nil {
return
}
_, err = wc.Write(msg)
if err != nil {
return
}
code, message, err = closeData(c)
if err != nil {
return 0, "", err
}
return code, message, err
}
func closeData(client *smtp.Client) error {
d := &dataCloser{
c: client,
WriteCloser: client.Text.DotWriter(),
}
return d.Close()
}
type dataCloser struct {
c *smtp.Client
io.WriteCloser
}
func (d *dataCloser) Close() (int, string, error) {
d.WriteCloser.Close() // make sure WriterCloser gets closed
code, message, err := d.c.Text.ReadResponse(250)
fmt.Printf("Message %v, Error %v\n", message, err)
return code, message, err
}

How do I import rows to Postgresql from STDIN? [duplicate]

This question already has an answer here:
Using PostgreSQL COPY FROM STDIN
(1 answer)
Closed 3 months ago.
In Python I have the following that will bulk-load rows to Postgresql without using a file:
import csv
import subprocess
mylist, keys = [{'name': 'fred'}, {'name': 'mary'}], ['name']
p = subprocess.Popen(['psql', 'mydb', '-U', 'openupitsme', '-h', 'my.ip.address', '--no-password', '-c',
'\COPY tester(%s) FROM STDIN (FORMAT CSV)' % ', '.join(keys),
'--set=ON_ERROR_STOP=false'
], stdin=subprocess.PIPE
)
for d in mylist:
dict_writer = csv.DictWriter(p.stdin, keys, quoting=csv.QUOTE_MINIMAL)
dict_writer.writerow(d)
p.stdin.close()
I am trying to accomplish the same in Go. I am currently writing the rows to a file then importing them and then deleting that file. I'd like to import the rows from STDIN like I do in Python. I have:
package main
import (
"database/sql"
"log"
"os"
"os/exec"
_ "github.com/lib/pq"
)
var (
err error
db *sql.DB
)
func main() {
var err error
fh := "/path/to/my/file.txt"
f, err := os.Create(fh)
if err != nil {
panic(err)
}
defer f.Close()
defer os.Remove(fh)
rows := []string{"fred", "mary"}
for _, n := range rows {
_, err = f.WriteString(n + "\n")
if err != nil {
panic(err)
}
}
// dump to postgresql
c := exec.Command("psql", "mydb", "-U", "openupitsme", "-h", "my.ip.address", "--no-password",
"-c", `\COPY tester(customer) FROM `+fh)
if out, err := c.CombinedOutput(); err != nil {
log.Println(string(out), err)
}
}
EDIT:
A bit further along but this is not inserting records:
keys := []string{"link", "domain"}
records := [][]string{
{"first_name", "last_name"},
{"Rob", "Pike"},
{"Ken", "Thompson"},
{"Robert", "Griesemer"},
}
cmd := exec.Command("psql")
stdin, err := cmd.StdinPipe()
if err != nil {
log.Println(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Println(err)
}
if err := cmd.Start(); err != nil {
log.Println(err)
}
go func() {
_, err = io.WriteString(stdin, "search -U meyo -h 1.2.3.4 -p 1111 --no-password -c ")
if err != nil {
log.Println(err)
}
_, err := io.WriteString(stdin, fmt.Sprintf("COPY links(%s) FROM STDIN (FORMAT CSV)", strings.Join(keys, ",")))
if err != nil {
log.Println(err)
}
w := csv.NewWriter(stdin)
if err := w.WriteAll(records); err != nil {
log.Fatalln("error writing record to csv:", err)
}
w.Flush()
if err := w.Error(); err != nil {
log.Fatal(err)
}
if err != nil {
log.Println(err)
}
stdin.Close()
}()
done := make(chan bool)
go func() {
_, err := io.Copy(os.Stdout, stdout)
if err != nil {
log.Fatal(err)
}
stdout.Close()
done <- true
}()
<-done
if err := cmd.Wait(); err != nil {
log.Println(err, cmd.Args, stdout)
}
No records are inserted and I get a non-helpful error:
exit status 2
The github.com/lib/pq package docs actually have an example of how to do what you want. Here is the adapted text of the whole program:
package main
import (
"database/sql"
"log"
"github.com/lib/pq"
)
func main() {
records := [][]string{
{"Rob", "Pike"},
{"Ken", "Thompson"},
{"Robert", "Griesemer"},
}
db, err := sql.Open("postgres", "dbname=postgres user=postgres password=postgres")
if err != nil {
log.Fatalf("open: %v", err)
}
if err = db.Ping(); err != nil {
log.Fatalf("open ping: %v", err)
}
defer db.Close()
txn, err := db.Begin()
if err != nil {
log.Fatalf("begin: %v", err)
}
stmt, err := txn.Prepare(pq.CopyIn("test", "first_name", "last_name"))
if err != nil {
log.Fatalf("prepare: %v", err)
}
for _, r := range records {
_, err = stmt.Exec(r[0], r[1])
if err != nil {
log.Fatalf("exec: %v", err)
}
}
_, err = stmt.Exec()
if err != nil {
log.Fatalf("exec: %v", err)
}
err = stmt.Close()
if err != nil {
log.Fatalf("stmt close: %v", err)
}
err = txn.Commit()
if err != nil {
log.Fatalf("commit: %v", err)
}
}
On my machine this imports 1 000 000 records in about 2 seconds.
The following code should point you in the direction you want to go:
package main
import (
"fmt"
"log"
"os"
"os/exec"
"strings"
)
func main() {
keys := []string{"customer"}
sqlCmd := fmt.Sprintf("COPY tester(%s) FROM STDIN (FORMAT CSV)", strings.Join(keys, ","))
cmd := exec.Command("psql", "<dbname>", "-U", "<username>", "-h", "<host_ip>", "--no-password", "-c", sqlCmd)
cmd.Stdin = os.Stdin
output, _ := cmd.CombinedOutput()
log.Println(string(output))
}
If the keys need to be dynamic you can harvest them from os.Args.
Please note that if you plan to use the psql command then you don't need to import database/sql or lib/pq. If you are interested in using lib/pq then have a look at Bulk Imports in the lib/pq documentation.

Passing Audio/ Video File to API

I'm trying to use the Soundcloud API (https://developers.soundcloud.com/docs/api/reference#tracks) to upload an audio file to Soundcloud. The parameter I must pass the file in requires "binary data of the audio file" and I'm unsure how to load such a thing in Go.
My current code is as follows, but the audio file of course does not send properly.
buf := new(bytes.Buffer)
w := multipart.NewWriter(buf)
label, err := w.CreateFormField("oauth_token")
if err != nil {
return err
}
label.Write([]byte(c.Token.AccessToken))
fw, err := w.CreateFormFile("upload", "platform/young.mp3")
if err != nil {
return err
}
fd, err := os.Open("platform/young.mp3")
if err != nil {
return err
}
defer fd.Close()
_, err = io.Copy(fw, fd)
if err != nil {
return err
}
w.Close()
req, err := http.NewRequest("POST", "https://api.soundcloud.com/tracks.json", buf)
if err != nil {
return err
}
req.Header.Set("Content-Type", w.FormDataContentType())
req.SetBasicAuth("email#email.com", "password")
fmt.Println(req.Form)
res, err := c.Client.Do(req)
if err != nil {
return err
}
I haven't tested the code below, as I don't have a valid Oauth token, but it may put you on the right track.
package main
import (
"bytes"
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
func main() {
uri := "https://api.soundcloud.com/tracks.json"
params := map[string]string{
"oauth_token": "************",
"track[title]": "Test Track",
"track[sharing]": "public",
}
trackData := "track[asset_data]"
path := "test_track.mp3"
file, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
defer file.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(trackData, filepath.Base(path))
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(part, file)
for key, val := range params {
err := writer.WriteField(key, val)
if err != nil {
log.Fatal(err)
}
}
err = writer.Close()
if err != nil {
log.Fatal(err)
}
request, err := http.NewRequest("POST", uri, body)
if err != nil {
log.Fatal(err)
}
request.Header.Set("Content-Type", writer.FormDataContentType())
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
log.Fatal(err)
}
resp.Body.Close()
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
fmt.Println(body)
}
}