The problem is:
I have a database in PostgreSQL and I read all the data in Golang and make arrays from it. The question is: How can I take this arrays and put it in influxdb?
package main
import (
"database/sql"
"log"
_ "./pq"
"fmt"
)
type DbInfo struct {
id string
person_id int
timestamp int
age string
gender string
attention string
interest int
happines int
surprise int
anger int
disgust int
fear string
sadness int
neutrall string
}
var DBObject DbInfo
var DbArray []DbInfo
func main() {
db, err := sql.Open("****", "postgres://postgres:********/cam?
sslmode=disable")
if err != nil {
log.Fatal(err)
}
defer db.Close()
rows, err := db.Query(`SELECT avg(age), avg(id), avg(neutrall),avg(fear)
FROM public.stat`) //stringTimeDayAgoQuery )
if err != nil {
log.Fatal(err)
}
defer rows.Close()
for rows.Next() {
//err := rows.Scan(&DBObject.id, &DBObject.person_id,
&DBObject.timestamp,&DBObject.age,
&DBObject.gender,&DBObject.attention,&DBObject.interest,&DBObject.happines,
&DBObject.surprise,&DBObject.anger,&DBObject.disgust,&DBObject.fear,
&DBObject.sadness,&DBObject.neutrall)
err := rows.Scan(&DBObject.age,&DBObject.id,&DBObject.neutrall,
&DBObject.fear)
if err != nil {
log.Fatal(err)
}
DbArray = append(DbArray, DBObject)
}
fmt.Println(DbArray)
}
In this code. I read my data from postgreSQL. I read it as an array, now I want to import my output to Influxdb. Because I want to use Grafana and it doesn't work with postgresql.
Related
edit: solved, see first answer. Should have really noticed that while I was creating this example..
I'm trying to create a function that takes an interface instead of a specific type and calls the FindOne function. Does anyone know why the printFirstTrainerByInterface function does not work properly?
I'm using the official Go Mongo-Driver and sample snippets from mongodb-go-driver-tutorial.
package main
import (
"context"
"fmt"
"log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type Trainer struct {
Name string
Age int
City string
}
var db *mongo.Database
func main() {
opts := options.Client().ApplyURI("mongodb://localhost:27017")
client, err := mongo.Connect(context.TODO(), opts)
if err != nil {
log.Fatal(err)
}
err = client.Ping(context.TODO(), nil)
if err != nil {
log.Fatal(err)
}
db = client.Database("test")
insertTestDocument()
var result Trainer
printFirstTrainer(result)
var result2 Trainer
printFirstTrainerByInterface(&result2)
}
func insertTestDocument() {
ash := Trainer{"Ash", 10, "Pallet Town"}
res, err := db.Collection("trainers").InsertOne(context.TODO(), ash)
if err != nil {
log.Fatal(err)
}
fmt.Println("Inserted a test document: ", res.InsertedID)
}
func printFirstTrainer(result Trainer) {
collection := db.Collection("trainers")
err := collection.FindOne(context.TODO(), bson.M{}).Decode(&result)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found a single document: %+v\n", result)
}
func printFirstTrainerByInterface(result interface{}) {
collection := db.Collection("trainers")
err := collection.FindOne(context.TODO(), bson.M{}).Decode(&result)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found a single document: %+v\n", result)
}
Output:
Inserted a test document: ObjectID("5e8216f74f41a13f01061d61")
Found a single document: {Name:Ash Age:10 City:Pallet Town}
Found a single document: [{Key:_id Value:ObjectID("5e8216f74f41a13f01061d61")} {Key:name Value:Ash} {Key:age Value:10} {Key:city Value:Pallet Town}]
You are passing in the address of the struct you want to decode into as an interface. You have to pass that as the argument to decode, not the address of the interface. Try:
err := collection.FindOne(context.TODO(), bson.M{}).Decode(result)
I'm trying to marshal a struct into JSON and then insert it into my Mongo database, but keep on getting this error: %!(EXTRA main.Test={575590180 Me}). What am I doing wrong? I took this code exactly from another project I worked on which could insert documents without any problems.
package main
import (
"utils"
"hash/fnv"
"log"
"gopkg.in/mgo.v2"
"encoding/json"
)
type Test struct {
Id uint32
Name string
}
func ConnectDB() *mgo.Session {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
return session
}
func SaveMgoDoc(dbName string, collectionName string, file Test) bool {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
defer session.Close()
fileJson, err := json.Marshal(file)
if err != nil {
log.Printf("failed to marshal struct to json...\n", file)
return false
}
collection := session.DB(dbName).C(collectionName)
err = collection.Insert(&fileJson)
if err != nil {
log.Printf("failed to insert doc into database...\n", file)
return false
}
return true
}
func hash(s string) uint32 {
h := fnv.New32a()
h.Write([]byte(s))
return h.Sum32()
}
func main() {
utils.SaveMgoDoc("mydb", "mydoc", Test{hash("Me"), "Me"})
}
Insert expects a pointer to a struct, not a json string. So, in this case, just use:
err = collection.Insert(&file)
I've written a toy app to experiment with using Postgresql through sqlx. I got a mass insert working using
pq.CopyIn
as content of a prepared statement
stmt, _ := tx.Preparex(pq.CopyIn(tablename, column, column, ...)
I would then proceed to add rows to the mass insert I'm creating.
tx.Exec(..., ..., ...)
then finally execute the prepared statement
stmt.Exec()
This worked perfectly before, but now I've come back to it and try and execute this code, it hangs on the
stmt.Exec
Am I missing something in my code or is this all to do with the Database Engine, being unresponsive.
Here's my full code for this.
package main
import (
_ "database/sql"
"fmt"
"log"
"encoding/json"
"io/ioutil"
"os"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
)
var schema = `
CREATE TABLE IF NOT EXISTS contact (
id Serial,
first_name text,
last_name text,
email text
);`
type Contact struct {
Id int `json:"-"`
First_name string `json:"first_name"`
Last_name string `json:"last_name"`
Email string `json:"email"`
}
type Contacts struct {
Contacts []Contact `json:"contacts"`
}
func (c *Contacts) createFromJSON(json_str []byte) error {
b := []byte(json_str)
err := json.Unmarshal(b, &c)
if err != nil {
log.Fatal(err)
}
return err
}
func (c *Contacts) save(db *sqlx.DB) error {
tx := db.MustBegin()
stmt, _ := tx.Preparex(pq.CopyIn("contact", "first_name", "last_name", "email"))
for _, contact := range c.Contacts {
tx.Exec(contact.First_name, contact.Last_name, contact.Email)
}
_, err := stmt.Exec()
if err != nil {
log.Fatal(err)
return err
}
err = stmt.Close()
if err != nil {
log.Fatal(err)
return err
}
tx.Commit()
return nil
}
func connect() (*sqlx.DB, error) {
db, err := sqlx.Connect("postgres", "user=pqgotest dbname=pqgotest sslmode=disable")
if err != nil {
log.Fatal(err)
}
return db, err
}
func createTables(db *sqlx.DB) {
db.MustExec(schema)
}
func main() {
db, err := connect()
if err != nil {
os.Exit(1)
}
createTables(db)
contactsJson, e := ioutil.ReadFile("./contacts.json")
if e != nil {
fmt.Printf("File error: %v\n", e)
os.Exit(1)
}
tx := db.MustBegin()
tx.MustExec("DELETE FROM contact")
tx.Commit()
contacts := new(Contacts)
contacts.createFromJSON(contactsJson)
contacts.save(db)
people := new(Contacts)
db.Select(people.Contacts, "SELECT * FROM contact ORDER BY email,id ASC")
for _, contact := range people.Contacts {
contact_json, err := json.Marshal(contact)
if err != nil {
log.Fatal(err)
os.Exit(1)
}
fmt.Printf("%s\n", contact_json)
}
}
I could include the contents of the contacts.json file as well, if that will help.
UPDATE
Yes it was obvious in the end. I was creating a statement from tx,
stmt, _ := tx.Preparex(pq.CopyIn(tablename, column, column, ...)
and further additions to this should be to stmt
stmt.Exec(..., ..., ...)
Also another error not directly related to the question is where I insert an array of contacts into the Contacts field of the struct Contacts
people := new(Contacts)
db.Select(people.Contacts, "SELECT * FROM contact ORDER BY email,id ASC")
should be passing a pointer to the Select method of db of the Contacts array field of Contacts, like so
db.Select(&people.Contacts, "SELECT * FROM contact ORDER BY email,id ASC")
In case people try and run this code later and wonder why it's not printing the results to the console.
From Bulk imports part in https://godoc.org/github.com/lib/pq, it should be
stmt.Exec(contact.First_name, contact.Last_name, contact.Email)
I need get values using ObjectIdHex and do update and also view the result. I'm using mongodb and golang.But following code doesn't work as expected
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
Id bson.ObjectId `json:"id" bson:"_id,omitempty"`
Name string
Phone string
}
func checkError(err error) {
if err != nil {
panic(err)
}
}
const (
DB_NAME = "gotest"
DB_COLLECTION = "pepole_new1"
)
func main() {
session, err := mgo.Dial("localhost")
checkError(err)
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(DB_NAME).C(DB_COLLECTION)
err = c.DropCollection()
checkError(err)
ale := Person{Name:"Ale", Phone:"555-5555"}
cla := Person{Name:"Cla", Phone:"555-1234-2222"}
kasaun := Person{Name:"kasaun", Phone:"533-12554-2222"}
chamila := Person{Name:"chamila", Phone:"533-545-6784"}
fmt.Println("Inserting")
err = c.Insert(&ale, &cla, &kasaun, &chamila)
checkError(err)
fmt.Println("findbyID")
var resultsID []Person
//err = c.FindId(bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")).One(&resultsID)
err = c.FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID)
checkError(err)
if err != nil {
panic(err)
}
fmt.Println("Phone:", resultsID)
fmt.Println("Queryingall")
var results []Person
err = c.Find(nil).All(&results)
if err != nil {
panic(err)
}
fmt.Println("Results All: ", results)
}
FindId(bson.M{"Id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")}).One(&resultsID) didn't work for me and giving me following output
Inserting
Queryingall
Results All: [{ObjectIdHex("56bddee2cfa93bfe3d3504a1") Ale 555-5555} {ObjectIdHex("56bddee2cfa93bfe3d3504a2") Cla 555-1234-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a3") kasaun 533-12554-2222} {ObjectIdHex("56bddee2cfa93bfe3d3504a4") chamila 533-545-6784}]
findbyID
panic: not found
goroutine 1 [running]:
main.checkError(0x7f33d524b000, 0xc8200689b0)
How can i fix this problem? i need get value using oid and do update also how can i do that
Use can do the same with Golang official driver as follows:
// convert id string to ObjectId
objectId, err := primitive.ObjectIDFromHex("5b9223c86486b341ea76910c")
if err != nil{
log.Println("Invalid id")
}
// find
result:= client.Database(database).Collection("user").FindOne(context.Background(), bson.M{"_id": objectId})
user := model.User{}
result.Decode(user)
It should be _id not Id:
c.FindId(bson.M{"_id": bson.ObjectIdHex("56bdd27ecfa93bfe3d35047d")})
Some sample code that i use.
func (model *SomeModel) FindId(id string) error {
db, ctx, client := Drivers.MongoCollection("collection")
defer client.Disconnect(ctx)
objID, err := primitive.ObjectIDFromHex(id)
if err != nil {
return err
}
filter := bson.M{"_id": bson.M{"$eq": objID}}
if err := db.FindOne(ctx, filter).Decode(&model); err != nil {
//fmt.Println(err)
return err
}
fmt.Println(model)
return nil
}
I am a newbie in Golang.
I am trying to retrieve a PDF file object I have inserted. I am not using GridFS, as the files that I would be storing are under 16 MB.
The object has been inserted (using load_file function) and the object ID I am seeing with the MongoDB visual client is ObjectId("554f98a400afc2dd3cbfb21b").
Unfortunately the file created on disk is of 0 kb.
Please advise how to correctly retrieve the inserted PDF object.
Thank you
package main
import (
"fmt"
"io/ioutil"
"gopkg.in/mgo.v2"
)
type Raw struct {
Kind byte
Data []byte
}
type RawDocElem struct {
Name string
Value Raw
}
func check(err error) {
if err != nil {
panic(err.Error())
}
}
func read_file_content(AFileName string) []byte {
file_contents, err := ioutil.ReadFile(AFileName)
check(err)
return file_contents
}
func save_fetched_file(AFileName RawDocElem) {
ioutil.WriteFile("fisier.pdf", AFileName.Value.Data, 0644)
fmt.Println("FileName:", AFileName.Name)
}
func load_file(AFileName string, ADatabaseName string, ACollection string) {
session, err := mgo.Dial("127.0.0.1,127.0.0.1")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(ADatabaseName).C(ACollection)
the_obj_to_insert := Raw{Kind: 0x00, Data: read_file_content(AFileName)}
err = c.Database.C(ACollection).Insert(&the_obj_to_insert)
check(err)
}
func get_file(ADatabaseName string, ACollection string, The_ID string) RawDocElem {
session, err := mgo.Dial("127.0.0.1,127.0.0.1")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(ADatabaseName).C(ACollection)
result := RawDocElem{}
err = c.FindId(The_ID).One(&result)
return result
}
func main() {
//f_name := "Shortcuts.pdf"
db_name := "teste"
the_collection := "ColectiaDeFisiere"
//load_file(f_name, db_name, the_collection)
fmt.Sprintf(`ObjectIdHex("%x")`, string("554f98a400afc2dd3cbfb21b"))
save_fetched_file(get_file(db_name, the_collection, fmt.Sprintf(`ObjectIdHex("%x")`, string("554f98a400afc2dd3cbfb21b"))))
}
I think your way to build the object ID is wrong. Furthermore, you forgot to test the error following the FindId call.
I would try to import the bson package, and build the object ID using something like:
hexString := "554f98a400afc2dd3cbfb21b" //24 digits hexadecimal string
objid := bson.ObjectIdHex(hexString)
...
err = c.FindId(objid).One(&result)
if err == nil {
// Do something with result
...
} else {
// Error
...
}