MongoDB find query with combination of $or and $and in golang - mongodb

I want to get rows where:
{repair field has "ac" OR {repair is "tv" and phone field in range 1091-1100}}
I am trying the following query:
type M map[string]interface{}
conditions := M{"name": M{"$regex": "me"},
"$or": []M{M{"repair": M{"$eq": "ac"}},
"$and": []M{M{"repair": M{"$eq": "tv"}}, M{"phone": M{"$gte": 1091, "$lte": 1100}}}}}
fmt.Println(conditions)
err = c.Find(conditions).Sort("phone").Limit(20).All(&j)
However, I am getting a compile error:
index must be non-negative integer constant
cannot use []M literal (type []M) as type M in array or slice literal.

You're missing one M{ before "$and" and after you add that don't forget to add another closing brace }.
the good and the bad for comparison.

I don't know what driver your using, but I would probably do something like this..
package main
import (
"log"
"time"
mgo "gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
const (
databaseString = ""
)
var db DataStore
type DataStore struct {
Session *mgo.Session
}
// database
func (ds *DataStore) ConnectToDB() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{"localhost:27017"},
Timeout: 1 * time.Hour,
}
sess, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
sess.Refresh()
panic(err)
}
sess.SetMode(mgo.Monotonic, true)
db.Session = sess
}
// J is the expected mongo return object
type J struct {
ID bson.ObjectId `bson:"_id,omitempty" json:"_id"`
// example data below
Status string `bson:"status" json:"status"`
}
func init() {
db.ConnectToDB()
}
func main() {
colectionString := ""
// probably best if this was a mapped mgo struct see above
// var j bson.M
var j J
// your orignal code
// I don't know why you are using $eq couldn't you just do bson.M{"repair":"ac"}, and bson.M{"repair":"tv"}
conditions := bson.M{"name": bson.M{"$regex": "me"},
"$or": []bson.M{
bson.M{"repair": bson.M{"$eq": "ac"}},
},
"$and": []bson.M{
bson.M{"repair": bson.M{"$eq": "tv"}},
bson.M{"phone": bson.M{"$gte": 1091, "$lte": 1100}},
}}
err := db.Session.DB(databaseString).C(colectionString).Find(conditions).Sort("phone").Limit(20).All(&j)
if err != nil {
log.Fatal(err)
}
}
I would probably also end up creating a separate package for the mongo connection stuff so that I can write wrapper functions around calls,
func FindItem(db *mgo.Session, id string) (data, error) {
defer sess.Close()
var res data //some data type struct
err := sess.DB("my db").C("my collection").Find(bson.M{"user": someID}).One(&data)
return data, err
}
then I'd be able to do things like this, which allow for concurrency
res, err := packagemain.FindItem(sess.Copy(), someID)
and your original code your you where missing } and ,. I suggest you use go vet or an ide which vets the code for you. Also, mgo is the mongo driver you probably what you want to be using if you're not using it already.

Related

What is the bson syntax for $set in UpdateOne for the official mongo-go-driver

I am trying to get some familiarity with the official mongo-go-driver and the right syntax for UpdateOne.
My simplest full example follows:
(NOTE: in order to use this code you will need to substitute in your own user and server names as well as export the login password to the environment as MONGO_PW):
package main
import (
"context"
"fmt"
"os"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type DB struct {
User string
Server string
Database string
Collection string
Client *mongo.Client
Ctx context.Context
}
var db = DB{
User: <username>,
Server: <server_IP>,
Database: "test",
Collection: "movies",
Ctx: context.TODO(),
}
type Movie struct {
ID primitive.ObjectID `bson:"_id" json:"id"`
Name string `bson:"name" json:"name"`
Description string `bson:"description" json:"description"`
}
func main() {
if err := db.Connect(); err != nil {
fmt.Println("error: unable to connect")
os.Exit(1)
}
fmt.Println("connected")
// The code assumes the original entry for dunkirk is the following
// {"Name":"dunkirk", "Description":"a world war 2 movie"}
updatedMovie := Movie{
Name: "dunkirk",
Description: "movie about the british evacuation in WWII",
}
res, err := db.UpdateByName(updatedMovie)
if err != nil {
fmt.Println("error updating movie:", err)
os.Exit(1)
}
if res.MatchedCount < 1 {
fmt.Println("error: update did not match any documents")
os.Exit(1)
}
}
// UpdateByName changes the description for a movie identified by its name
func (db *DB) UpdateByName(movie Movie) (*mongo.UpdateResult, error) {
filter := bson.D{{"name", movie.Name}}
res, err := db.Client.Database(db.Database).Collection(db.Collection).UpdateOne(
db.Ctx,
filter,
movie,
)
if err != nil {
return nil, err
}
return res, nil
}
// Connect assumes that the database password is stored in the
// environment variable MONGO_PW
func (db *DB) Connect() error {
pw, ok := os.LookupEnv("MONGO_PW")
if !ok {
fmt.Println("error: unable to find MONGO_PW in the environment")
os.Exit(1)
}
mongoURI := fmt.Sprintf("mongodb+srv://%s:%s#%s", db.User, pw, db.Server)
// Set client options and verify connection
clientOptions := options.Client().ApplyURI(mongoURI)
client, err := mongo.Connect(db.Ctx, clientOptions)
if err != nil {
return err
}
err = client.Ping(db.Ctx, nil)
if err != nil {
return err
}
db.Client = client
return nil
}
The function signature for UpdateOne from the package docs is:
func (coll *Collection) UpdateOne(ctx context.Context, filter interface{},
update interface{}, opts ...*options.UpdateOptions) (*UpdateResult, error)
So I am clearly making some sort of mistake in creating the update interface{} argument to the function because I am presented with this error
error updating movie: update document must contain key beginning with '$'
The most popular answer here shows that I need to use a document sort of like this
{ $set: {"Name" : "The Matrix", "Decription" "Neo and Trinity kick butt" } }
but taken verbatim this will not compile in the mongo-go-driver.
I think I need some form of a bson document to comply with the Go syntax. What is the best and/or most efficient syntax to create this bson document for the update?
After playing around with this for a little while longer I was able to solve the problem after A LOT OF TRIAL AND ERROR using the mongodb bson package by changing the UpdateByName function in my code above as follows:
// UpdateByName changes the description for a movie identified by its name
func (db *DB) UpdateByName(movie Movie) (*mongo.UpdateResult, error) {
filter := bson.D{{"name", movie.Name}}
update := bson.D{{"$set",
bson.D{
{"description", movie.Description},
},
}}
res, err := db.Client.Database(db.Database).Collection(db.Collection).UpdateOne(
db.Ctx,
filter,
update,
)
if err != nil {
return nil, err
}
return res, nil
}
Note the use of bson.D{{$"set", .... It is unfortunate the way MongoDB has implemented the bson package this syntax still does not pass the go-vet. If anyone has a comment to fix the lint conflict below it would be appreciated.
go.mongodb.org/mongo-driver/bson/primitive.E composite literal uses unkeyed fields
In many cases you can replace construction
filter := bson.D{{"name", movie.Name}}
with
filter := bson.M{"name": movie.Name}
if arguments order dowsn't matter

converting MongoDB function foreach into mgo (Golang) function

This is function that tries to update the code matching by its value
the res collection as having the code of Marque it will be compared with doc.Marque if it is the case it will be replaced by the value of the marque.
This code is working perfectly in mongoDB CLI, but as i am working with GO.
I tried to convert it into mgo as you may see below but it is not working , i did not find the foreach function in mgo , is there something to be replaced with in this case ? thanks for the help
db.res.find().forEach(function(doc){
var v = db.brands.findOne({code: doc.Marque});
if(v){
db.res.update({"Marque": doc.Marque},
{$set: {"Marque":v.value}}, {multi: true});
}
});
Here is what i tried :
result:=Results{}
pipe:=res.find(bson.M{}).Iter()
for pipe.Next(&result) {
brands:=brands.findOne({code: doc.Marque});
if(v){
pipe.update({"Marque": doc.Marque},
{$set: {"Marque": v.value}}, {multi: true});
}
}
Visit the mgo Godoc may help you understand how it works.
Second, exported types/functions in Golang are begin with a capital letter. So res.find, brands.findOne, ... should be res.Find, brands.FineOne respectively, if such functions exist.
// let's say you have a type like this
type myResult struct {
ID bson.ObjectId `bson:"_id"`
Marque string `bson:"Marque"`
// other fields...
}
// and another type like this
type myCode struct {
Code string `bson:"code"`
// other fields...
}
res := db.C("res")
brands := db.C("brands")
result := myResult{}
// iterate all documents
iter := res.Find(nil).Iter()
for iter.Next(&result) {
var v myCode
err := brands.Find(bson.M{"code": result.Marque}).One(&v)
if err != nil {
// maybe not found or other reason,
// it is recommend to have additional check
continue
}
query := bson.M{"_id": result.ID}
update := bson.M{"Marque": v.value}
if err = res.Update(query, update); err != nil {
// handle error
}
}
if err := iter.Close(); err != nil {
fmt.Println(err)
}

mgo - query performance seems consistently slow (500-650ms)

My data layer uses Mongo aggregation a decent amount, and on average, queries are taking 500-650ms to return. I am using mgo.
A sample query function is shown below which represents what most of my queries look like.
func (r userRepo) GetUserByID(id string) (User, error) {
info, err := db.Info()
if err != nil {
log.Fatal(err)
}
session, err := mgo.Dial(info.ConnectionString())
if err != nil {
log.Fatal(err)
}
defer session.Close()
var user User
c := session.DB(info.Db()).C("users")
o1 := bson.M{"$match": bson.M{"_id": id}}
o2 := bson.M{"$project": bson.M{
"first": "$first",
"last": "$last",
"email": "$email",
"fb_id": "$fb_id",
"groups": "$groups",
"fulfillments": "$fulfillments",
"denied_requests": "$denied_requests",
"invites": "$invites",
"requests": bson.M{
"$filter": bson.M{
"input": "$requests",
"as": "item",
"cond": bson.M{
"$eq": []interface{}{"$$item.active", true},
},
},
},
}}
pipeline := []bson.M{o1, o2}
err = c.Pipe(pipeline).One(&user)
if err != nil {
return user, err
}
return user, nil
}
The user struct I have looks like the following..
type User struct {
ID string `json:"id" bson:"_id,omitempty"`
First string `json:"first" bson:"first"`
Last string `json:"last" bson:"last"`
Email string `json:"email" bson:"email"`
FacebookID string `json:"facebook_id" bson:"fb_id,omitempty"`
Groups []UserGroup `json:"groups" bson:"groups"`
Requests []Request `json:"requests" bson:"requests"`
Fulfillments []Fulfillment `json:"fulfillments" bson:"fulfillments"`
Invites []GroupInvite `json:"invites" bson:"invites"`
DeniedRequests []string `json:"denied_requests" bson:"denied_requests"`
}
Based on what I have provided, is there anything obvious that would suggest why my queries are averaging 500-650ms?
I know that I am probably swallowing a bit of a performance hit by using aggregation pipeline, but I wouldn't expect it to be this bad.
.. is there anything obvious that would suggest why my queriers are averaging 500-650ms?
Yes, there is. You are calling mgo.Dial() before executing each query. mgo.Dial() has to connect to the MongoDB server every time, which you close right after the query. The connection may very likely take hundreds of milliseconds to estabilish, including authentication, allocating resources (both at server and client side), etc. This is very wasteful.
This method is generally called just once for a given cluster. Further sessions to the same cluster are then established using the New or Copy methods on the obtained session. This will make them share the underlying cluster, and manage the pool of connections appropriately.
Create a global session variable, connect on startup once (using e.g. a package init() function), and use that session (or a copy / clone of it, obtained by Session.Copy() or Session.Clone()).
For example:
var session *mgo.Session
var info *db.Inf // Use your type here
func init() {
var err error
if info, err = db.Info(); err != nil {
log.Fatal(err)
}
if session, err = mgo.Dial(info.ConnectionString()); err != nil {
log.Fatal(err)
}
}
func (r userRepo) GetUserByID(id string) (User, error) {
sess := session.Clone()
defer sess.Close()
// Now we use sess to execute the query:
var user User
c := sess.DB(info.Db()).C("users")
// Rest of the method is unchanged...
}

Querying mongodb from golang using the _id stored in an array

So here is my question. I have an array which are stored the _ids of mongodbs objects. Whats the right way to retrieve them all in one query using the mgo and bson package?
So if the array is like that: ids:=["543d171c5b2c12420dd016","543d171c5b2dd016"]
How we make the query ? I tried that but I know its wrong.
query := bson.M{"_id": bson.M{"$in": ids}}
c.Find(query).All()
Thanks in advance
If the documents are stored with string ids, then the code looks correct.
The ids look like hex encoded object ids. If the object identifiers are object ids, then you need to the convert the hex strings to object ids:
oids := make([]bson.ObjectId, len(ids))
for i := range ids {
oids[i] = bson.ObjectIdHex(ids[i])
}
query := bson.M{"_id": bson.M{"$in": oids}}
MongoDB syntax for go.mongodb.org/mongo-driver has been updated, this should work using the official driver.
oids := make([]primitive.ObjectID, len(ids))
for i := range ids {
objID, err := primitive.ObjectIDFromHex(ids[i])
if err == nil {
oids = append(oids, objID)
}
}
This is to convert back to a struct that can be used through out the app
type MongoUser struct {
ID *primitive.ObjectID `json:"id" bson:"_id"`
FirstName string `json:"first_name" bson:"firstName"`
LastName string `json:"last_name" bson:"lastName"`
Email string `json:"email" bson:"email"`
}
This is a helper method that takes your slice of ids and turns it into the object id type.
func formatObjectIdMultiple(hex []string) ([]primitive.ObjectID, error) {
var list []primitive.ObjectID
oids := make([]primitive.ObjectID, len(hex))
for _, i := range hex {
objectId, err := primitive.ObjectIDFromHex(i)
if err != nil {
return nil, err
}
oids = append(oids, objectId)
}
return list, nil
}
Here is my method for the db. Its important you use bson.M for some reason bson.D does not work with this. Also dont forget to close your cursor the defer function will close it at the end of your GetMultipleUser function.
func (mongo *Mongo) GetMultipleUser(ids []string) ([]*MongoUser, error) {
objectIDs, err := formatObjectIdMultiple(ids)
if err != nil {
return nil, err
}
query := bson.M{"_id": bson.M{"$in": objectIDs}}
coll := mongo.Con.Database("dbName").Collection("users")
cursor, err := coll.Find(context.Background(), query)
if err != nil {
return nil, err
}
defer func() {
cursor.Close(context.Background())
}()
var output []*MongoUser
for cursor.Next(context.Background()) {
var temp *MongoUser
cursor.Decode(&temp)
output = append(output, temp)
}
return output, nil
}

Golang mongodb mgo driver Upsert / UpsertId documentation

The mongodb documentation says:
The fields and values of both the and parameters if the parameter contains only update operator expressions. The update creates a base document from the equality clauses in the parameter, and then applies the update expressions from the parameter.
And the mgo documentation says:
Upsert finds a single document matching the provided selector document and modifies it according to the update document. If no document matching the selector is found, the update document is applied to the selector document and the result is inserted in the collection.
But if i do an upsert like this:
session.UpsertId(data.Code, data)
I end up with an entry which have an ObjectID generated automatically by mongodb, instead of data.Code.
this means that UpsertId expect data to be formated with update operators and you can't use a an arbitrary struct? Or what i'm missing here?
Pd. Mongo 2.4.9 mgo v2 golang go version devel +f613443bb13a
EDIT:
This is a sample of what i mean, using the sample code from Neil Lunn:
package main
import (
"fmt"
"gopkg.in/mgo.v2"
// "gopkg.in/mgo.v2/bson"
)
type Person struct {
Code string
Name string
}
func main() {
session, err := mgo.Dial("admin:admin#localhost");
if err != nil {
fmt.Println("Error: ", err)
return
// panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB("test").C("people")
var p = Person{
Code: "1234",
Name: "Bill",
}
_, err = c.UpsertId( p.Code, &p )
result := Person{}
err = c.FindId(p.Code).One(&result)
if err != nil {
fmt.Println("FindId Error: ", err)
return
// panic(err)
}
fmt.Println("Person", result)
}
I found the documentation of the MongoDB was right. The correct way to do this is to wrap the struct to insert into an update operator.
The sample code provided by Neil Lunn, would look like:
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
Code string
Name string
}
func main() {
session, err := mgo.Dial("admin:admin#localhost");
if err != nil {
fmt.Println("Error: ", err)
return
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB("test").C("people")
var p = Person{
Code: "1234",
Name: "Bill",
}
upsertdata := bson.M{ "$set": p}
info , err2 := c.UpsertId( p.Code, upsertdata )
fmt.Println("UpsertId -> ", info, err2)
result := Person{}
err = c.FindId(p.Code).One(&result)
if err != nil {
fmt.Println("FindId Error: ", err)
return
}
fmt.Println("Person", result)
}
Thank you very much for your interest and help Neil.
You seem to be talking about assigning a struct with a custom _id field here. This really comes down to how you define your struct. Here is a quick example:
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
ID string `bson:"_id"`
Name string
}
func main() {
session, err := mgo.Dial("127.0.0.1");
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB("test").C("people")
var p = Person{
ID: "1",
Name: "Bill",
}
_, err = c.UpsertId( p.ID, &p )
result := Person{}
err = c.Find(bson.M{"_id": p.ID}).One(&result)
if err != nil {
panic(err)
}
fmt.Println("Person", result)
}
So in the custom definition here I am mapping the ID field to bson _id and defining it's type as string. As shown in the example this is exactly what happens when serialized via UpsertId and then retrieved.
Now you have elaborated I'll point to the difference on the struct definition.
What I have produces this:
{ "_id": 1, "name": "Bill" }
What you have ( without the same mapping on the struct ) does this:
{ "_id": ObjectId("53cfa557e248860d16e1f7e0"), "code": 1, "name": "Bill" }
As you see, the _id given in the upsert will never match because none of your fields in the struct are mapped to _id. You need the same as I have:
type Person struct {
Code string `bson:"_id"`
Name string
}
That maps a field to the mandatory _id field, otherwise one is automatically produced for you.