I am currently migrating some of my Python Services to GoLang, where I encountered some problems with the integration with the existing database.
When it comes to saving, the formatting of the date string is different, and I am not planning to change the format of my database entries.
So I decided to implement a time type, with custom (Un)MarshalBSON() methods, to keep pythons time formatting.
However when saving it, I get the following error:
2022/09/10 20:57:29 (InvalidBSON) Unrecognized BSON type 45 in element with field name 'created_at.09-10 20:57:27.798545' in object with _id: ObjectId('631cde192c2aad6a49bc52af')
I don't know how the field created_at.09-10 20:57:27.798545 came up.
Thank you for advance four your advice, here is my source code:
main.go:
package main
import (
"context"
"example/customdate/user"
"fmt"
"log"
)
func main() {
ctx := context.Background()
repo := user.NewRepository(ctx)
u1 := user.NewUser("Max")
u1, err := repo.Add(ctx, u1)
if err != nil {
log.Fatalln(err)
}
fmt.Println(u1)
}
user/repository.go:
package user
import (
"context"
"log"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type Repository interface {
Get(ctx context.Context) ([]User, error)
Add(ctx context.Context, user *User) (*User, error)
}
type repository struct {
client *mongo.Client
}
func NewRepository(ctx context.Context) repository {
c, err := mongo.NewClient(options.Client().ApplyURI("mongodb://127.0.0.1:27017/"))
if err != nil {
log.Fatalln(err)
}
err = c.Connect(ctx)
if err != nil {
log.Fatal(err)
}
return repository{
client: c,
}
}
func (r *repository) Get(ctx context.Context) ([]User, error) {
result, err := r.client.Database("db").Collection("user").Find(ctx, map[string]interface{}{})
if err != nil {
return nil, err
}
var users []User
if err = result.All(ctx, &users); err != nil {
return nil, err
}
return users, nil
}
func (r *repository) Add(ctx context.Context, user *User) (*User, error) {
result, err := r.client.Database("db").Collection("user").InsertOne(ctx, user)
if err != nil {
return nil, err
}
user.ID = result.InsertedID
return user, nil
}
user/dto.go:
package user
import "time"
type CustomTime struct {
time.Time
}
func (t *CustomTime) MarshalBSON() ([]byte, error) {
str := t.Format("2006-01-02 15:04:05.999999")
return []byte(str), nil
}
func (t *CustomTime) UnmarshalBSON(raw []byte) error {
parsed, err := time.Parse("2006-01-02 15:04:05.999999", string(raw))
if err != nil {
return err
}
t = &CustomTime{parsed}
return nil
}
func Now() CustomTime {
return CustomTime{time.Now()}
}
type User struct {
ID interface{}
Name string `bson:"name"`
CreatedAt CustomTime `bson:"created_at"`
}
func NewUser(name string) *User {
return &User{
Name: name,
CreatedAt: Now(),
}
}
Related
I'm trying to write interfaces for mongo go driver to write unit test for my post_repo.go, but have this error:
cannot use redditclone (type *mongo.Database) as type newPosts.DatabaseHelper in argument to newPosts.NewPostsDatabase:
*mongo.Database does not implement newPosts.DatabaseHelper (wrong type for Client method)
have Client() *mongo.Client
want Client() newPosts.ClientHelper
my interface file, took it from this post https://medium.com/better-programming/unit-testing-code-using-the-mongo-go-driver-in-golang-7166d1aa72c0:
package newPosts
import (
"context"
"rc/pkg/config"
// "localhost/medium-mongo-go-driver/config"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type DatabaseHelper interface {
Collection(name string) CollectionHelper
Client() ClientHelper
}
type CollectionHelper interface {
FindOne(context.Context, interface{}) SingleResultHelper
InsertOne(context.Context, interface{}) (interface{}, error)
DeleteOne(ctx context.Context, filter interface{}) (int64, error)
}
type SingleResultHelper interface {
Decode(v interface{}) error
}
type ClientHelper interface {
Database(string) DatabaseHelper
Connect() error
StartSession() (mongo.Session, error)
}
type mongoClient struct {
cl *mongo.Client
}
type mongoDatabase struct {
db *mongo.Database
}
type mongoCollection struct {
coll *mongo.Collection
}
type mongoSingleResult struct {
sr *mongo.SingleResult
}
type mongoSession struct {
mongo.Session
}
func NewClient(cnf *config.Config) (ClientHelper, error) {
c, err := mongo.NewClient(options.Client().SetAuth(
options.Credential{
Username: cnf.Username,
Password: cnf.Password,
AuthSource: cnf.DatabaseName,
}).ApplyURI(cnf.URL))
return &mongoClient{cl: c}, err
}
func NewDatabase(cnf *config.Config, client ClientHelper) DatabaseHelper {
return client.Database(cnf.DatabaseName)
}
func (mc *mongoClient) Database(dbName string) DatabaseHelper {
db := mc.cl.Database(dbName)
return &mongoDatabase{db: db}
}
func (mc *mongoClient) StartSession() (mongo.Session, error) {
session, err := mc.cl.StartSession()
return &mongoSession{session}, err
}
func (mc *mongoClient) Connect() error {
// mongo client does not use context on connect method. There is a ticket
// with a request to deprecate this functionality and another one with
// explanation why it could be useful in synchronous requests.
// https://jira.mongodb.org/browse/GODRIVER-1031
// https://jira.mongodb.org/browse/GODRIVER-979
return mc.cl.Connect(nil)
}
func (md *mongoDatabase) Collection(colName string) CollectionHelper {
collection := md.db.Collection(colName)
return &mongoCollection{coll: collection}
}
func (md *mongoDatabase) Client() ClientHelper {
client := md.db.Client()
return &mongoClient{cl: client}
}
func (mc *mongoCollection) FindOne(ctx context.Context, filter interface{}) SingleResultHelper {
singleResult := mc.coll.FindOne(ctx, filter)
return &mongoSingleResult{sr: singleResult}
}
func (mc *mongoCollection) InsertOne(ctx context.Context, document interface{}) (interface{}, error) {
id, err := mc.coll.InsertOne(ctx, document)
return id.InsertedID, err
}
func (mc *mongoCollection) DeleteOne(ctx context.Context, filter interface{}) (int64, error) {
count, err := mc.coll.DeleteOne(ctx, filter)
return count.DeletedCount, err
}
func (sr *mongoSingleResult) Decode(v interface{}) error {
return sr.sr.Decode(v)
}
my post_repo.go file:
package newPosts
import (
"context"
// "localhost/medium-mongo-go-driver/models"
)
const collectionName = "posts"
// PostsDatabase user database representation to find, update, delete users
type PostsDatabase interface {
FindOne(context.Context, interface{}) (*Post, error)
Create(context.Context, *Post) error
DeleteByUsername(context.Context, string) error
}
type postsDatabase struct {
// db *mongo.Database
db DatabaseHelper
}
// NewPostsDatabase creates new user database instance
func NewPostsDatabase(db DatabaseHelper) PostsDatabase {
return &postsDatabase{
db: db,
}
}
func (u *postsDatabase) FindOne(ctx context.Context, filter interface{}) (*Post, error) {
user := &Post{}
err := u.db.Collection(collectionName).FindOne(ctx, filter).Decode(user)
if err != nil {
return nil, err
}
return user, nil
}
func (u *postsDatabase) Create(ctx context.Context, usr *Post) error {
_, err := u.db.Collection(collectionName).InsertOne(ctx, usr)
return err
}
func (u *postsDatabase) DeleteByUsername(ctx context.Context, username string) error {
// In this case it is possible to use bson.M{"username":username} but I tend
// to avoid another dependency in this layer and for demonstration purposes
// used omitempty in the model
author := &Author{Username: username}
post := &Post{
Author: author,
}
_, err := u.db.Collection(collectionName).DeleteOne(ctx, post)
return err
}
My posts_repo_test.go tests works fine, but only if I pass newPosts.DatabaseHelper in function newPosts.NewPostsDatabase(). I guess I need to pass *mongo.Database for my program works.
Maybe I misunderstanding how works unit tests.
Any help
I have a struct field like this below. I also store raw protobuf of the same struct in db. Now every time fetch or save data to mongo. I have to update ReallyBigRaw, from the proto when I want to save to DB and when fetch I have to unmarshal ReallyBigRaw to ReallyBigObj to give out responses. Is there a way I can implement some interface or provide some callback functions so that the mongo driver does this automatically before saving or fetching data from DB.
Also, I am using the offical golang mongo driver not mgo, I have read some answers where can be done in mgo golang library.
import (
"github.com/golang/protobuf/jsonpb"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
proto "github.com/dinesh/api/go"
)
type ReallyBig struct {
ID string `bson:"_id,omitempty"`
DraftID string `bson:"draft_id,omitempty"`
// Marshaled ReallyBigObj proto to map[string]interface{} stored in DB
ReallyBigRaw map[string]interface{} `bson:"raw,omitempty"`
ReallyBigObj *proto.ReallyBig `bson:"-"`
CreatedAt primitive.DateTime `bson:"created_at,omitempty"`
UpdatedAt primitive.DateTime `bson:"updated_at,omitempty"`
}
func (r *ReallyBig) GetProto() (*proto.ReallyBig, error) {
if r.ReallyBigObj != nil {
return r.ReallyBigObj, nil
}
Obj, err := getProto(r.ReallyBigRaw)
if err != nil {
return nil, err
}
r.ReallyBigObj = Obj
return r.ReallyBigObj, nil
}
func getRaw(r *proto.ReallyBig) (map[string]interface{}, error) {
m := jsonpb.Marshaler{}
b := bytes.NewBuffer([]byte{})
// marshals proto to json format
err := m.Marshal(b, r)
if err != nil {
return nil, err
}
var raw map[string]interface{}
// unmarshal the raw data to an interface
err = json.Unmarshal(b.Bytes(), &raw)
if err != nil {
return nil, err
}
return raw, nil
}
func getProto(raw map[string]interface{}) (*proto.ReallyBig, error) {
b, err := json.Marshal(raw)
if err != nil {
return nil, err
}
u := jsonpb.Unmarshaler{}
var reallyBigProto proto.ReallyBig
err = u.Unmarshal(bytes.NewReader(b), &recipeProto)
if err != nil {
return nil, err
}
return &reallyBigProto, nil
}
I implemented the Marshaler and Unmarshaler interface. Since mongo driver calls MarshalBSON and UnmarshalBSON if the type implements Marshaler and Unmarshaler we also end up in infinite loop. To avoid that we create a Alias of the type. Alias in Golang inherit only the fields not the methods so we end up calling normal bson.Marshal and bson.Unmarshal
func (r *ReallyBig) MarshalBSON() ([]byte, error) {
type ReallyBigAlias ReallyBig
reallyBigRaw, err := getRaw(r.ReallyBigObj)
if err != nil {
return nil, err
}
r.ReallyBigRaw = reallyBigRaw
return bson.Marshal((*ReallyBigAlias)(r))
}
func (r *ReallyBig) UnmarshalBSON(data []byte) error {
type ReallyBigAlias ReallyBig
err := bson.Unmarshal(data, (*ReallyBigAlias)(r))
if err != nil {
return err
}
reallyBigProto, err := getProto(r.ReallyBigRaw)
if err != nil {
return err
}
r.ReallyBigObj = reallyBigProto
return nil
}
I'd like to build an infrastructure, which is a package for the project. So that other developers can import this package to perform CRUD operations on the DB.
But I've got an error during the test:
type Students struct {
Name string
Age int
}
type InsertOneResult struct {
InsertedID interface{}
}
func dbGetOne(coll, document interface{}) (*InsertOneResult, error) {
...
}
func dbUpdateOne(coll, document interface{}) (*InsertOneResult, error) {
...
}
func dbDeleteOne(coll, document interface{}) (*InsertOneResult, error) {
...
}
func dbInsertOne(coll, document interface{}) (*InsertOneResult, error) {
res, err := coll.InsertOne(context.TODO(), document)
if err != nil {
log.Fatal(err)
}
return &InsertOneResult{InsertedID: res[0]}, err
}
func main() {
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://<user>:<password>#<host>:<port>/<dbname>"))
if err != nil {
log.Fatal(err)
}
ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
err = client.Connect(ctx)
if err != nil {
log.Fatal(err)
}
coll := client.Database("db").Collection("students")
data := Students{"Amy", 10}
res, err := dbInsertOne(coll, data)
if err != nil {
log.Fatal(err)
}
fmt.Printf("inserted document with ID %v\n", res.InsertedID)
}
Here's the error:
./main.go:24:18: coll.InsertOne undefined (type interface {} is interface with no methods)
Is there any way to solve this? Thanks in advance.
Hey it looks like the error could be coming from a type conversion issue. The solution would be to clearly define the type for coll as *mongo.Collection in the dbInsertOne() function. This allows the compiler at compile time to figure out the structure of the input instead of having to rely on an abstract interface.
func dbInsertOne(coll *mongo.Collection, document interface{}) (*InsertOneResult, error) {
res, err := coll.InsertOne(context.TODO(), document)
if err != nil {
log.Fatal(err)
}
return &InsertOneResult{InsertedID: res.InsertedID}, err
}
I would further suggest that the 2nd argument document should also be a typed known term if possible. e.g.
func dbInsertOne(coll *mongo.Collection, document Students)
Static typing will help quite a bit and clear up any confusion.
I'm trying to marshal a struct into JSON and then insert it into my Mongo database, but keep on getting this error: %!(EXTRA main.Test={575590180 Me}). What am I doing wrong? I took this code exactly from another project I worked on which could insert documents without any problems.
package main
import (
"utils"
"hash/fnv"
"log"
"gopkg.in/mgo.v2"
"encoding/json"
)
type Test struct {
Id uint32
Name string
}
func ConnectDB() *mgo.Session {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
return session
}
func SaveMgoDoc(dbName string, collectionName string, file Test) bool {
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
defer session.Close()
fileJson, err := json.Marshal(file)
if err != nil {
log.Printf("failed to marshal struct to json...\n", file)
return false
}
collection := session.DB(dbName).C(collectionName)
err = collection.Insert(&fileJson)
if err != nil {
log.Printf("failed to insert doc into database...\n", file)
return false
}
return true
}
func hash(s string) uint32 {
h := fnv.New32a()
h.Write([]byte(s))
return h.Sum32()
}
func main() {
utils.SaveMgoDoc("mydb", "mydoc", Test{hash("Me"), "Me"})
}
Insert expects a pointer to a struct, not a json string. So, in this case, just use:
err = collection.Insert(&file)
I am a newbie in Golang.
I am trying to retrieve a PDF file object I have inserted. I am not using GridFS, as the files that I would be storing are under 16 MB.
The object has been inserted (using load_file function) and the object ID I am seeing with the MongoDB visual client is ObjectId("554f98a400afc2dd3cbfb21b").
Unfortunately the file created on disk is of 0 kb.
Please advise how to correctly retrieve the inserted PDF object.
Thank you
package main
import (
"fmt"
"io/ioutil"
"gopkg.in/mgo.v2"
)
type Raw struct {
Kind byte
Data []byte
}
type RawDocElem struct {
Name string
Value Raw
}
func check(err error) {
if err != nil {
panic(err.Error())
}
}
func read_file_content(AFileName string) []byte {
file_contents, err := ioutil.ReadFile(AFileName)
check(err)
return file_contents
}
func save_fetched_file(AFileName RawDocElem) {
ioutil.WriteFile("fisier.pdf", AFileName.Value.Data, 0644)
fmt.Println("FileName:", AFileName.Name)
}
func load_file(AFileName string, ADatabaseName string, ACollection string) {
session, err := mgo.Dial("127.0.0.1,127.0.0.1")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(ADatabaseName).C(ACollection)
the_obj_to_insert := Raw{Kind: 0x00, Data: read_file_content(AFileName)}
err = c.Database.C(ACollection).Insert(&the_obj_to_insert)
check(err)
}
func get_file(ADatabaseName string, ACollection string, The_ID string) RawDocElem {
session, err := mgo.Dial("127.0.0.1,127.0.0.1")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB(ADatabaseName).C(ACollection)
result := RawDocElem{}
err = c.FindId(The_ID).One(&result)
return result
}
func main() {
//f_name := "Shortcuts.pdf"
db_name := "teste"
the_collection := "ColectiaDeFisiere"
//load_file(f_name, db_name, the_collection)
fmt.Sprintf(`ObjectIdHex("%x")`, string("554f98a400afc2dd3cbfb21b"))
save_fetched_file(get_file(db_name, the_collection, fmt.Sprintf(`ObjectIdHex("%x")`, string("554f98a400afc2dd3cbfb21b"))))
}
I think your way to build the object ID is wrong. Furthermore, you forgot to test the error following the FindId call.
I would try to import the bson package, and build the object ID using something like:
hexString := "554f98a400afc2dd3cbfb21b" //24 digits hexadecimal string
objid := bson.ObjectIdHex(hexString)
...
err = c.FindId(objid).One(&result)
if err == nil {
// Do something with result
...
} else {
// Error
...
}