I might have found a weird bug..
I do update once in a while my mongodb-driver package as I use for year now I layer I've made a long time ago.
However, today, for some personal reasons, I have refactored my tests and now have e2e full tests suite over my layer. The thing is, while doing my tests, I could reproduce a bug I had in the past, which is when I don't set an ID for my document (_id bson field)
Situation:
For obvious reason, some piece of code are missing but I am quite sure you will be able to guess really easily ;)
The commented out code part is the tests that are creating the weird behavior
If I create a document (really simple struct) and set the _id field before inserting, it works perfectly.
If I create a document (really simple struct) and don't set the _id field before inserting, it inserts something that results as "null" while observing the database using Robo3T.
Code:
To reproduce that bug, there is my tests and the source code of my layer
Tests:
type e2eMongoDBSuite struct {
suite.Suite
_mongodb mongodb.IMongoDB
}
type e2eTestDocument struct {
ID *primitive.ObjectID `json:"_id" bson:"_id"`
ValueStr string `json:"value_str" bson:"value_str"`
ValueInt32 int32 `json:"value_int32" bson:"value_int32"`
}
const (
//// ci
connectionString = "mongodb://localhost:27017"
databaseName = "mongodb-database-e2e-tests"
collection = "mongodb-collection-e2e-tests"
defaultTimeout = 5 * time.Second
)
func (e2eSuite *e2eMongoDBSuite) SetupSuite() {
var err error
if e2eSuite._mongodb, err = mongodb.New(&mongodb.Config{
ConnectionString: connectionString,
DatabaseName: databaseName,
DefaultTimeout: defaultTimeout,
}); err != nil {
panic(fmt.Errorf("couldn't setup suite: %w", err).Error())
}
}
func (e2eSuite *e2eMongoDBSuite) TestInsertOneWithID_OK() {
a := assert.New(e2eSuite.T())
// given
docID := e2eSuite._mongodb.GenerateNewObjectID()
doc := &e2eTestDocument{ID: docID}
// when
insertedID, err := e2eSuite._mongodb.InsertOne(collection, doc)
// expected
a.NotNil(insertedID, "the inserted document ID should be returned")
a.Equal(docID, insertedID, "the inserted document ID should be the same as the given one at creation")
a.Nil(err, "there should be no error for this insert")
// then
_ = e2eSuite._mongodb.DeleteMany(collection, bson.D{})
}
//func (e2eSuite *e2eMongoDBSuite) TestInsertOneWithNoID_OK() {
// a := assert.New(e2eSuite.T())
//
// // given
// doc := &e2eTestDocument{}
//
// // when
// insertedID, err := e2eSuite._mongodb.InsertOne(collection, doc)
//
// // expected
// a.NotNil(insertedID, "the inserted document ID should be returned")
// //a.Equal(docID, insertedID, "the inserted document ID should be the same as the given one at creation")
// a.Nil(err, "there should be no error for this insert")
//
// // then
// _ = e2eSuite._mongodb.DeleteMany(collection, bson.D{})
//}
func (e2eSuite *e2eMongoDBSuite) TestInsertManyWithIDs_OK() {
a := assert.New(e2eSuite.T())
// given
docID1 := e2eSuite._mongodb.GenerateNewObjectID()
docID2 := e2eSuite._mongodb.GenerateNewObjectID()
doc1 := &e2eTestDocument{ID: docID1}
doc2 := &e2eTestDocument{ID: docID2}
// when
insertedIDs, err := e2eSuite._mongodb.InsertMany(collection, []*e2eTestDocument{doc1, doc2})
// expected
a.NotNil(insertedIDs, "the inserted document IDs should be returned")
a.Equal(2, len(insertedIDs), "the inserted document IDs amount should be of two")
a.EqualValues([]*primitive.ObjectID{docID1, docID2}, insertedIDs, "the inserted document IDs should be the same as the given ones at creation")
a.Nil(err, "there should be no error for this insert")
// then
_ = e2eSuite._mongodb.DeleteMany(collection, bson.D{})
}
//func (e2eSuite *e2eMongoDBSuite) TestInsertManyWithNoIDs_OK() {
// a := assert.New(e2eSuite.T())
//
// // given
// doc1 := &e2eTestDocument{}
// doc2 := &e2eTestDocument{}
//
// // when
// insertedIDs, err := e2eSuite._mongodb.InsertMany(collection, []*e2eTestDocument{doc1, doc2})
//
// // expected
// a.NotNil(insertedIDs, "the inserted document IDs should be returned")
// a.Equal(2, len(insertedIDs), "the inserted document IDs amount should be of two")
// a.Nil(err, "there should be no error for this insert")
//
//// then
//_ = e2eSuite._mongodb.DeleteMany(collection, bson.D{})
//}
func (e2eSuite *e2eMongoDBSuite) TearDownSuite() {
_ = e2eSuite._mongodb.DropDatabase()
}
func TestE2eMongoDBSuite(t *testing.T) {
suite.Run(t, new(e2eMongoDBSuite))
}
The layer i've developed:
// InsertOne inserts a document in a given collection and returns the inserted ObjectID.
func (m *mongoDB) InsertOne(collectionName string, document interface{}) (*primitive.ObjectID, error) {
res, err := m.GetDatabase().Collection(collectionName).InsertOne(context.Background(), document)
if err != nil {
return nil, err
}
if oid, ok := res.InsertedID.(primitive.ObjectID); ok {
return &oid, nil
}
return nil, nil
}
// InsertMany inserts documents in a given collection and returns the inserted ObjectIDs.
func (m *mongoDB) InsertMany(collectionName string, documents interface{}) ([]*primitive.ObjectID, error) {
s := reflect.ValueOf(documents)
if s.Kind() != reflect.Slice {
panic("Documents given a non-slice type")
}
slice := make([]interface{}, s.Len())
for i := 0; i < s.Len(); i++ {
slice[i] = s.Index(i).Interface()
}
res, err := m.GetDatabase().Collection(collectionName).InsertMany(context.Background(), slice)
if err != nil {
return nil, err
}
var insertedIDs []*primitive.ObjectID
for _, insertedID := range res.InsertedIDs {
if oid, ok := insertedID.(primitive.ObjectID); ok {
insertedIDs = append(insertedIDs, &oid)
}
}
return insertedIDs, nil
}
Conclusion
I don't know if the behavior is logic or not but an id should be generated if my document as no ID :confused:
Note
A topic as been opened here: https://developer.mongodb.com/community/forums/t/inserting-a-document-in-go-with-no-id-set-results-into-a-weird-behavior/100359 (don't know if it's a bug or not so I thought I should post there too)
Thanks!
Max
If you insert a document containing the _id field, mongodb will use that as the document id, even if it is null. To have the _id auto-generated, you have to insert the document without the _id field. This should work:
type Doc struct {
ID *primitive.ObjectID `json:"_id" bson:"_id,omitempty"`
...
This will not marshal the _id field if it is nil, focing it to be auto-generated.
Related
I try to read and write and delete data from a Go application with the official mongodb driver for go (go.mongodb.org/mongo-driver).
Here is my struct I want to use:
Contact struct {
ID xid.ID `json:"contact_id" bson:"contact_id"`
SurName string `json:"surname" bson:"surname"`
PreName string `json:"prename" bson:"prename"`
}
// xid is https://github.com/rs/xid
I omit code to add to the collection as this is working find.
I can get a list of contacts with a specific contact_id using the following code (abbreviated):
filter := bson.D{}
cursor, err := contactCollection.Find(nil, filter)
for cur.Next(context.TODO()) {
...
}
This works and returns the documents. I thought about doing the same for delete or a matched get:
// delete - abbreviated
filter := bson.M{"contact_id": id}
result, _ := contactCollection.DeleteMany(nil, filter)
// result.DeletedCount is always 0, err is nil
if err != nil {
sendError(c, err) // helper function
return
}
c.JSON(200, gin.H{
"ok": true,
"message": fmt.Sprintf("deleted %d patients", result.DeletedCount),
}) // will be called, it is part of a webservice done with gin
// get complete
func Get(c *gin.Context) {
defer c.Done()
id := c.Param("id")
filter := bson.M{"contact_id": id}
cur, err := contactCollection.Find(nil, filter)
if err != nil {
sendError(c, err) // helper function
return
} // no error
contacts := make([]types.Contact, 0)
for cur.Next(context.TODO()) { // nothing returned
// create a value into which the single document can be decoded
var elem types.Contact
err := cur.Decode(&elem)
if err != nil {
sendError(c, err) // helper function
return
}
contacts = append(contacts, elem)
}
c.JSON(200, contacts)
}
Why does the same filter does not work on delete?
Edit: Insert code looks like this:
_, _ = contactCollection.InsertOne(context.TODO(), Contact{
ID: "abcdefg",
SurName: "Demo",
PreName: "on stackoverflow",
})
Contact.ID is of type xid.ID, which is a byte array:
type ID [rawLen]byte
So the insert code you provided where you use a string literal to specify the value for the ID field would be a compile-time error:
_, _ = contactCollection.InsertOne(context.TODO(), Contact{
ID: "abcdefg",
SurName: "Demo",
PreName: "on stackoverflow",
})
Later in your comments you clarified that the above insert code was just an example, and not how you actually do it. In your real code you unmarshal the contact (or its ID field) from a request.
xid.ID has its own unmarshaling logic, which might interpret the input data differently, and might result in an ID representing a different string value than your input. ID.UnmarshalJSON() defines how the string ID will be converted to xid.ID:
func (id *ID) UnmarshalJSON(b []byte) error {
s := string(b)
if s == "null" {
*id = nilID
return nil
}
return id.UnmarshalText(b[1 : len(b)-1])
}
As you can see, the first byte is cut off, and ID.UnmarshalText() does even more "magic" on it (check the source if you're interested).
All-in-all, to avoid such "transformations" happen in the background without your knowledge, use a simple string type for your ID, and do necessary conversions yourself wherever you need to store / transmit your ID.
For the ID Field, you should use the primitive.ObjectID provided by the bson package.
"go.mongodb.org/mongo-driver/bson/primitive"
ID primitive.ObjectID `json:"_id" bson:"_id"`
This is function that tries to update the code matching by its value
the res collection as having the code of Marque it will be compared with doc.Marque if it is the case it will be replaced by the value of the marque.
This code is working perfectly in mongoDB CLI, but as i am working with GO.
I tried to convert it into mgo as you may see below but it is not working , i did not find the foreach function in mgo , is there something to be replaced with in this case ? thanks for the help
db.res.find().forEach(function(doc){
var v = db.brands.findOne({code: doc.Marque});
if(v){
db.res.update({"Marque": doc.Marque},
{$set: {"Marque":v.value}}, {multi: true});
}
});
Here is what i tried :
result:=Results{}
pipe:=res.find(bson.M{}).Iter()
for pipe.Next(&result) {
brands:=brands.findOne({code: doc.Marque});
if(v){
pipe.update({"Marque": doc.Marque},
{$set: {"Marque": v.value}}, {multi: true});
}
}
Visit the mgo Godoc may help you understand how it works.
Second, exported types/functions in Golang are begin with a capital letter. So res.find, brands.findOne, ... should be res.Find, brands.FineOne respectively, if such functions exist.
// let's say you have a type like this
type myResult struct {
ID bson.ObjectId `bson:"_id"`
Marque string `bson:"Marque"`
// other fields...
}
// and another type like this
type myCode struct {
Code string `bson:"code"`
// other fields...
}
res := db.C("res")
brands := db.C("brands")
result := myResult{}
// iterate all documents
iter := res.Find(nil).Iter()
for iter.Next(&result) {
var v myCode
err := brands.Find(bson.M{"code": result.Marque}).One(&v)
if err != nil {
// maybe not found or other reason,
// it is recommend to have additional check
continue
}
query := bson.M{"_id": result.ID}
update := bson.M{"Marque": v.value}
if err = res.Update(query, update); err != nil {
// handle error
}
}
if err := iter.Close(); err != nil {
fmt.Println(err)
}
Now I'm doing:
sess := mongodb.DB("mybase").C("mycollection")
var users []struct {
Username string `bson:"username"`
}
err = sess.Find(nil).Select(bson.M{"username": 1, "_id": 0}).All(&users)
if err != nil {
fmt.Println(err)
}
var myUsers []string
for _, user := range users{
myUsers = append(myUsers, user.Username)
}
Is there a more effective way to get slice with usernames from Find (or another search function) directly, without struct and range loop?
The result of a MongoDB find() is always a list of documents. So if you want a list of values, you have to convert it manually just as you did.
Using a custom type (derived from string)
Also note that if you would create your own type (derived from string), you could override its unmarshaling logic, and "extract" just the username from the document.
This is how it could look like:
type Username string
func (u *Username) SetBSON(raw bson.Raw) (err error) {
doc := bson.M{}
if err = raw.Unmarshal(&doc); err != nil {
return
}
*u = Username(doc["username"].(string))
return
}
And then querying the usernames into a slice:
c := mongodb.DB("mybase").C("mycollection") // Obtain collection
var uns []Username
err = c.Find(nil).Select(bson.M{"username": 1, "_id": 0}).All(&uns)
if err != nil {
fmt.Println(err)
}
fmt.Println(uns)
Note that []Username is not the same as []string, so this may or may not be sufficient to you. Should you need a user name as a value of string instead of Username when processing the result, you can simply convert a Username to string.
Using Query.Iter()
Another way to avoid the slice copying would be to call Query.Iter(), iterate over the results and extract and store the username manually, similarly how the above custom unmarshaling logic does.
This is how it could look like:
var uns []string
it := c.Find(nil).Select(bson.M{"username": 1, "_id": 0}).Iter()
defer it.Close()
for doc := (bson.M{}); it.Next(&doc); {
uns = append(uns, doc["username"].(string))
}
if err := it.Err(); err != nil {
fmt.Println(err)
}
fmt.Println(uns)
I don't see what could be more effective than a simple range loop with appends. Without all the Mongo stuff your code basically is this and that's exactly how I would do this.
package main
import (
"fmt"
)
type User struct {
Username string
}
func main() {
var users []User
users = append(users, User{"John"}, User{"Jane"}, User{"Jim"}, User{"Jean"})
fmt.Println(users)
// Interesting part starts here.
var myUsers []string
for _, user := range users {
myUsers = append(myUsers, user.Username)
}
// Interesting part ends here.
fmt.Println(myUsers)
}
https://play.golang.com/p/qCwENmemn-R
I have a MongoDB collection with an example document like this:
What I want to do (as you can see from the actual code) is to update a role field in members.x.role where members.x.id equals given ID (ID is UUID so it's unique; this part of code works without problem) and then I want to return that members.x. But the problem is that it always returns first member instead of the one that has been just updated. I've tried some methods of mgo and found Distinct() be closest to my expectations, but that doesn't work as I want.
My question is how can I return member embedded document with specified ID?
I've already looked on this and this but it didn't help me.
func (r MongoRepository) UpdateMemberRole(id string, role int8) (*Member, error) {
memberQuery := &bson.M{"members": &bson.M{"$elemMatch": &bson.M{"id": id}}}
change := &bson.M{"members.$.role": role}
err := r.db.C("groups").Update(memberQuery, &bson.M{"$set": &change})
if err == mgo.ErrNotFound {
return nil, fmt.Errorf("member with ID '%s' does not exist", id)
}
// FIXME: Retrieve this member from query below. THIS ALWAYS RETURNS FIRST MEMBER!!!
var member []Member
r.db.C("groups").Find(&bson.M{"members.id": id}).Distinct("members.0", &member)
return &member[0], nil
}
I found a workaround, it's not stricte Mongo query that is returning this embedded document, but this code is IMO more clear and understandable than some fancy Mongo query that fetches whole document anyway.
func (r MongoRepository) UpdateMemberRole(id string, role int8) (*Member, error) {
change := mgo.Change{
Update: bson.M{"$set": bson.M{"members.$.role": role}},
ReturnNew: true,
}
var updatedGroup Group
_, err := r.db.C("groups").Find(bson.M{"members": bson.M{"$elemMatch": bson.M{"id": id}}}).Apply(change, &updatedGroup)
if err == mgo.ErrNotFound {
return nil, fmt.Errorf("member with ID '%s' does not exist", id)
} else if err != nil {
return nil, err
}
for _, member := range updatedGroup.Members {
if member.Id == id {
return &member, nil
}
}
return nil, fmt.Errorf("weird error, Id cannot be found")
}
I'm trying to fetch data from mongodb in golang using the gopkg.in/mgo.v2 driver, the format of the data is not fixed , as in few rows will be containing some fields which other rows might not.
here is the code for the same
session, err := mgo.Dial("mongodb://root:root#localhost:27017/admin")
db := session.DB("test")
fmt.Println(reflect.TypeOf(db))
CheckError(err,"errpor")
result := make(map[string]string)
//query := make(map[string]string)
//query["_id"] = "3434"
err1 := db.C("mycollection").Find(nil).One(&result)
CheckError(err1,"error")
for k := range result {
fmt.Println(k)
}
Now the data contained in the collection is { "_id" : "3434", "0" : 1 }, however the for loop gives the output as _id , shouldn't there be two keys '_id' and '0' ? or am I doing something wrong here.
oh I found the solution
the "result" variable should be of type bson.M and then you can typecast accordingly as you go deep into the nesting structure.
Give a try with the following piece of code. This will help you fetching matching records from the Database using BSON Object.
Do not forget to rename the Database name and Collection name of your MongoDB in the below code. Also needs to change the query parameter accordingly.
Happy Coding...
package main
import (
"context"
"fmt"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// This is a user defined method to close resourses.
// This method closes mongoDB connection and cancel context.
func close(client *mongo.Client, ctx context.Context, cancel context.CancelFunc) {
defer cancel()
defer func() {
if err := client.Disconnect(ctx); err != nil {
panic(err)
}
}()
}
// This is a user defined method that returns
// a mongo.Client, context.Context,
// context.CancelFunc and error.
// mongo.Client will be used for further database
// operation. context.Context will be used set
// deadlines for process. context.CancelFunc will
// be used to cancel context and resourse
// assositated with it.
func connect(uri string) (*mongo.Client, context.Context, context.CancelFunc, error) {
ctx, cancel := context.WithTimeout(context.Background(),
30*time.Second)
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
return client, ctx, cancel, err
}
// query is user defined method used to query MongoDB,
// that accepts mongo.client,context, database name,
// collection name, a query and field.
// datbase name and collection name is of type
// string. query is of type interface.
// field is of type interface, which limts
// the field being returned.
// query method returns a cursor and error.
func query(client *mongo.Client, ctx context.Context, dataBase, col string, query, field interface{}) (result *mongo.Cursor, err error) {
// select database and collection.
collection := client.Database(dataBase).Collection(col)
// collection has an method Find,
// that returns a mongo.cursor
// based on query and field.
result, err = collection.Find(ctx, query,
options.Find().SetProjection(field))
return
}
func main() {
// Get Client, Context, CalcelFunc and err from connect method.
client, ctx, cancel, err := connect("mongodb://localhost:27017")
if err != nil {
panic(err)
}
// Free the resource when mainn dunction is returned
defer close(client, ctx, cancel)
// create a filter an option of type interface,
// that stores bjson objects.
var filter, option interface{}
// filter gets all document,
// with maths field greater that 70
filter = bson.D{
{"_id", bson.D{{"$eq", 3434}}},
}
// option remove id field from all documents
option = bson.D{{"_id", 0}}
// call the query method with client, context,
// database name, collection name, filter and option
// This method returns momngo.cursor and error if any.
cursor, err := query(client, ctx, "YourDataBaseName",
"YourCollectioName", filter, option)
// handle the errors.
if err != nil {
panic(err)
}
var results []bson.D
// to get bson object from cursor,
// returns error if any.
if err := cursor.All(ctx, &results); err != nil {
// handle the error
panic(err)
}
// printing the result of query.
fmt.Println("Query Reult")
for _, doc := range results {
fmt.Println(doc)
}
}