Golang - MongoDB (mgo) retrieve inserted file (BSON not GridFS) - mongodb

I am a newbie in Golang.
I am trying to retrieve a PDF file object I have inserted. I am not using GridFS, as the files that I would be storing are under 16 MB.
The object has been inserted (using load_file function) and the object ID I am seeing with the MongoDB visual client is ObjectId("554f98a400afc2dd3cbfb21b").
Unfortunately the file created on disk is of 0 kb.
Please advise how to correctly retrieve the inserted PDF object.
Thank you
package main
import (
"fmt"
"io/ioutil"
"gopkg.in/mgo.v2"
)
type Raw struct {
Kind byte
Data []byte
}
type RawDocElem struct {
Name string
Value Raw
}
func check(err error) {
if err != nil {
panic(err.Error())
}
}
func read_file_content(AFileName string) []byte {
file_contents, err := ioutil.ReadFile(AFileName)
check(err)
return file_contents
}
func save_fetched_file(AFileName RawDocElem) {
ioutil.WriteFile("fisier.pdf", AFileName.Value.Data, 0644)
fmt.Println("FileName:", AFileName.Name)
}
func load_file(AFileName string, ADatabaseName string, ACollection string) {
session, err := mgo.Dial("127.0.0.1,127.0.0.1")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(ADatabaseName).C(ACollection)
the_obj_to_insert := Raw{Kind: 0x00, Data: read_file_content(AFileName)}
err = c.Database.C(ACollection).Insert(&the_obj_to_insert)
check(err)
}
func get_file(ADatabaseName string, ACollection string, The_ID string) RawDocElem {
session, err := mgo.Dial("127.0.0.1,127.0.0.1")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(ADatabaseName).C(ACollection)
result := RawDocElem{}
err = c.FindId(The_ID).One(&result)
return result
}
func main() {
//f_name := "Shortcuts.pdf"
db_name := "teste"
the_collection := "ColectiaDeFisiere"
//load_file(f_name, db_name, the_collection)
fmt.Sprintf(`ObjectIdHex("%x")`, string("554f98a400afc2dd3cbfb21b"))
save_fetched_file(get_file(db_name, the_collection, fmt.Sprintf(`ObjectIdHex("%x")`, string("554f98a400afc2dd3cbfb21b"))))
}

I think your way to build the object ID is wrong. Furthermore, you forgot to test the error following the FindId call.
I would try to import the bson package, and build the object ID using something like:
hexString := "554f98a400afc2dd3cbfb21b" //24 digits hexadecimal string
objid := bson.ObjectIdHex(hexString)
...
err = c.FindId(objid).One(&result)
if err == nil {
// Do something with result
...
} else {
// Error
...
}

Related

(Golang) InvalidBSON when using custom MarshalBSON()

I am currently migrating some of my Python Services to GoLang, where I encountered some problems with the integration with the existing database.
When it comes to saving, the formatting of the date string is different, and I am not planning to change the format of my database entries.
So I decided to implement a time type, with custom (Un)MarshalBSON() methods, to keep pythons time formatting.
However when saving it, I get the following error:
2022/09/10 20:57:29 (InvalidBSON) Unrecognized BSON type 45 in element with field name 'created_at.09-10 20:57:27.798545' in object with _id: ObjectId('631cde192c2aad6a49bc52af')
I don't know how the field created_at.09-10 20:57:27.798545 came up.
Thank you for advance four your advice, here is my source code:
main.go:
package main
import (
"context"
"example/customdate/user"
"fmt"
"log"
)
func main() {
ctx := context.Background()
repo := user.NewRepository(ctx)
u1 := user.NewUser("Max")
u1, err := repo.Add(ctx, u1)
if err != nil {
log.Fatalln(err)
}
fmt.Println(u1)
}
user/repository.go:
package user
import (
"context"
"log"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type Repository interface {
Get(ctx context.Context) ([]User, error)
Add(ctx context.Context, user *User) (*User, error)
}
type repository struct {
client *mongo.Client
}
func NewRepository(ctx context.Context) repository {
c, err := mongo.NewClient(options.Client().ApplyURI("mongodb://127.0.0.1:27017/"))
if err != nil {
log.Fatalln(err)
}
err = c.Connect(ctx)
if err != nil {
log.Fatal(err)
}
return repository{
client: c,
}
}
func (r *repository) Get(ctx context.Context) ([]User, error) {
result, err := r.client.Database("db").Collection("user").Find(ctx, map[string]interface{}{})
if err != nil {
return nil, err
}
var users []User
if err = result.All(ctx, &users); err != nil {
return nil, err
}
return users, nil
}
func (r *repository) Add(ctx context.Context, user *User) (*User, error) {
result, err := r.client.Database("db").Collection("user").InsertOne(ctx, user)
if err != nil {
return nil, err
}
user.ID = result.InsertedID
return user, nil
}
user/dto.go:
package user
import "time"
type CustomTime struct {
time.Time
}
func (t *CustomTime) MarshalBSON() ([]byte, error) {
str := t.Format("2006-01-02 15:04:05.999999")
return []byte(str), nil
}
func (t *CustomTime) UnmarshalBSON(raw []byte) error {
parsed, err := time.Parse("2006-01-02 15:04:05.999999", string(raw))
if err != nil {
return err
}
t = &CustomTime{parsed}
return nil
}
func Now() CustomTime {
return CustomTime{time.Now()}
}
type User struct {
ID interface{}
Name string `bson:"name"`
CreatedAt CustomTime `bson:"created_at"`
}
func NewUser(name string) *User {
return &User{
Name: name,
CreatedAt: Now(),
}
}

Custom BSON marshal and unmarshal using mongo-driver

I have a struct field like this below. I also store raw protobuf of the same struct in db. Now every time fetch or save data to mongo. I have to update ReallyBigRaw, from the proto when I want to save to DB and when fetch I have to unmarshal ReallyBigRaw to ReallyBigObj to give out responses. Is there a way I can implement some interface or provide some callback functions so that the mongo driver does this automatically before saving or fetching data from DB.
Also, I am using the offical golang mongo driver not mgo, I have read some answers where can be done in mgo golang library.
import (
"github.com/golang/protobuf/jsonpb"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
proto "github.com/dinesh/api/go"
)
type ReallyBig struct {
ID string `bson:"_id,omitempty"`
DraftID string `bson:"draft_id,omitempty"`
// Marshaled ReallyBigObj proto to map[string]interface{} stored in DB
ReallyBigRaw map[string]interface{} `bson:"raw,omitempty"`
ReallyBigObj *proto.ReallyBig `bson:"-"`
CreatedAt primitive.DateTime `bson:"created_at,omitempty"`
UpdatedAt primitive.DateTime `bson:"updated_at,omitempty"`
}
func (r *ReallyBig) GetProto() (*proto.ReallyBig, error) {
if r.ReallyBigObj != nil {
return r.ReallyBigObj, nil
}
Obj, err := getProto(r.ReallyBigRaw)
if err != nil {
return nil, err
}
r.ReallyBigObj = Obj
return r.ReallyBigObj, nil
}
func getRaw(r *proto.ReallyBig) (map[string]interface{}, error) {
m := jsonpb.Marshaler{}
b := bytes.NewBuffer([]byte{})
// marshals proto to json format
err := m.Marshal(b, r)
if err != nil {
return nil, err
}
var raw map[string]interface{}
// unmarshal the raw data to an interface
err = json.Unmarshal(b.Bytes(), &raw)
if err != nil {
return nil, err
}
return raw, nil
}
func getProto(raw map[string]interface{}) (*proto.ReallyBig, error) {
b, err := json.Marshal(raw)
if err != nil {
return nil, err
}
u := jsonpb.Unmarshaler{}
var reallyBigProto proto.ReallyBig
err = u.Unmarshal(bytes.NewReader(b), &recipeProto)
if err != nil {
return nil, err
}
return &reallyBigProto, nil
}
I implemented the Marshaler and Unmarshaler interface. Since mongo driver calls MarshalBSON and UnmarshalBSON if the type implements Marshaler and Unmarshaler we also end up in infinite loop. To avoid that we create a Alias of the type. Alias in Golang inherit only the fields not the methods so we end up calling normal bson.Marshal and bson.Unmarshal
func (r *ReallyBig) MarshalBSON() ([]byte, error) {
type ReallyBigAlias ReallyBig
reallyBigRaw, err := getRaw(r.ReallyBigObj)
if err != nil {
return nil, err
}
r.ReallyBigRaw = reallyBigRaw
return bson.Marshal((*ReallyBigAlias)(r))
}
func (r *ReallyBig) UnmarshalBSON(data []byte) error {
type ReallyBigAlias ReallyBig
err := bson.Unmarshal(data, (*ReallyBigAlias)(r))
if err != nil {
return err
}
reallyBigProto, err := getProto(r.ReallyBigRaw)
if err != nil {
return err
}
r.ReallyBigObj = reallyBigProto
return nil
}

Mongo FindOne and interface{}

edit: solved, see first answer. Should have really noticed that while I was creating this example..
I'm trying to create a function that takes an interface instead of a specific type and calls the FindOne function. Does anyone know why the printFirstTrainerByInterface function does not work properly?
I'm using the official Go Mongo-Driver and sample snippets from mongodb-go-driver-tutorial.
package main
import (
"context"
"fmt"
"log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type Trainer struct {
Name string
Age int
City string
}
var db *mongo.Database
func main() {
opts := options.Client().ApplyURI("mongodb://localhost:27017")
client, err := mongo.Connect(context.TODO(), opts)
if err != nil {
log.Fatal(err)
}
err = client.Ping(context.TODO(), nil)
if err != nil {
log.Fatal(err)
}
db = client.Database("test")
insertTestDocument()
var result Trainer
printFirstTrainer(result)
var result2 Trainer
printFirstTrainerByInterface(&result2)
}
func insertTestDocument() {
ash := Trainer{"Ash", 10, "Pallet Town"}
res, err := db.Collection("trainers").InsertOne(context.TODO(), ash)
if err != nil {
log.Fatal(err)
}
fmt.Println("Inserted a test document: ", res.InsertedID)
}
func printFirstTrainer(result Trainer) {
collection := db.Collection("trainers")
err := collection.FindOne(context.TODO(), bson.M{}).Decode(&result)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found a single document: %+v\n", result)
}
func printFirstTrainerByInterface(result interface{}) {
collection := db.Collection("trainers")
err := collection.FindOne(context.TODO(), bson.M{}).Decode(&result)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found a single document: %+v\n", result)
}
Output:
Inserted a test document: ObjectID("5e8216f74f41a13f01061d61")
Found a single document: {Name:Ash Age:10 City:Pallet Town}
Found a single document: [{Key:_id Value:ObjectID("5e8216f74f41a13f01061d61")} {Key:name Value:Ash} {Key:age Value:10} {Key:city Value:Pallet Town}]
You are passing in the address of the struct you want to decode into as an interface. You have to pass that as the argument to decode, not the address of the interface. Try:
err := collection.FindOne(context.TODO(), bson.M{}).Decode(result)

Cannot read time_stamp with type `strfmt.DateTime` from MongoDB

I'm trying to write timestamp with strfmt.DateTime type (https://godoc.org/github.com/go-openapi/strfmt#DateTime) into the mongodb
I can successfully write this format of date into the DB, which looks like this:
{ "_id" : ObjectId("5bcb58f7540ac6d0bc946e22"), "status" : "test", "time_stamp" : {
"data" : "2018-10-21T00:33:59.699+08:00" } }
But I just can't retrieve it from the mongodb, the value of the time_stamp always shows 0001-01-01T00:00:00.000Z, I just don't see why.
Here is my code, any suggestions or opinions are welcome! Thanks
package main
import (
"fmt"
"time"
"github.com/go-openapi/strfmt"
"github.com/op/go-logging"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type TxStatus struct {
Status string `json:"status" bson:"status"`
TimeStamp *strfmt.DateTime `json:"time_stamp" bson:"time_stamp"`
}
type MongoDBOperations struct {
mongoSession *mgo.Session
database string
collection string
}
var log = logging.MustGetLogger("example")
func main() {
mo := MongoDBOperations{}
mo.database = Database
mo.collection = Collection
// We need this object to establish a session to our MongoDB.
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{MongoDBHosts},
Timeout: 60 * time.Second,
Database: AuthDatabase,
Username: AuthUserName,
Password: AuthPassword,
}
// Create a session which maintains a pool of socket connections
// to our MongoDB.
var err error
mo.mongoSession, err = mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
log.Fatalf("CreateSession: %s\n", err)
}
mo.mongoSession.SetMode(mgo.Eventual, true)
write(mo)
read(mo)
}
func write(mo MongoDBOperations) {
log.Info("write operation")
session := mo.mongoSession.Copy()
defer session.Close()
c := session.DB(Database).C(Collection)
timestamp := strfmt.DateTime(time.Now())
txStatus := TxStatus{
Status: "test",
TimeStamp: &timestamp,
}
if err2 := c.Insert(txStatus); err2 != nil {
panic(err2)
}
}
func read(mo MongoDBOperations) {
log.Info("write operation")
session := mo.mongoSession.Copy()
defer session.Close()
c := session.DB(Database).C(Collection)
// Find and Count
var status []TxStatus
err2 := c.Find(bson.M{"status": "test"}).All(&status)
if err2 != nil {
panic(err2)
}
for _, elem := range status {
fmt.Printf("%+v\n", elem)
}
}
I dug through the code of github.com/go-openapi/strfmt and what I've found out is that they are using github.com/globalsign/mgo instead of gopkg.in/mgo.v2. Changing imports to use github.com/globalsign/mgo and github.com/globalsign/mgo/bson has fixed the issue. Globalsign package is fork of mgo.v2 so the methods remain the same and there is no need to change any code besides import.

Golang Mongodb %!(EXTRA

I'm trying to marshal a struct into JSON and then insert it into my Mongo database, but keep on getting this error: %!(EXTRA main.Test={575590180 Me}). What am I doing wrong? I took this code exactly from another project I worked on which could insert documents without any problems.
package main
import (
"utils"
"hash/fnv"
"log"
"gopkg.in/mgo.v2"
"encoding/json"
)
type Test struct {
Id uint32
Name string
}
func ConnectDB() *mgo.Session {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
return session
}
func SaveMgoDoc(dbName string, collectionName string, file Test) bool {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
defer session.Close()
fileJson, err := json.Marshal(file)
if err != nil {
log.Printf("failed to marshal struct to json...\n", file)
return false
}
collection := session.DB(dbName).C(collectionName)
err = collection.Insert(&fileJson)
if err != nil {
log.Printf("failed to insert doc into database...\n", file)
return false
}
return true
}
func hash(s string) uint32 {
h := fnv.New32a()
h.Write([]byte(s))
return h.Sum32()
}
func main() {
utils.SaveMgoDoc("mydb", "mydoc", Test{hash("Me"), "Me"})
}
Insert expects a pointer to a struct, not a json string. So, in this case, just use:
err = collection.Insert(&file)