Find entries via substring regex query in mongodb-go-driver - mongodb

I can't get the official go mongo driver to successfully return objects that are queried via a regex query.
I already know how to do it via the mongo shell and get my expected results.
With this example i get all entries that contain "he" in their 'text' field:
db.getCollection('test').find({"text": /he/})
same with this one:
db.getCollection('test').find({"text": {$regex: /he/, $options: ''}})
This is my current code that doesn't work:
package main
import (
"context"
"fmt"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func main() {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
defer cancel()
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://localhost:27017"))
if err != nil {
fmt.Println(err)
return
}
err = client.Connect(ctx)
if err != nil {
fmt.Println(err)
return
}
db := client.Database("test")
coll := db.Collection("test")
filter := bson.D{{"text", primitive.Regex{Pattern: "/he/", Options: ""}}}
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cur, err := coll.Find(ctx, filter)
if err != nil {
fmt.Println(err)
return
}
i := 0
for cur.Next(ctx) {
i = i + 1
}
fmt.Println("Found", i, "elements")
}
Per example in the official mongo-go-driver repository, this should work.
My current entries in the collection just contain 2 fields, the id field and an extra text field. I currently have 3 entries. that look like this:
{
"_id" : ObjectId("5c9cc7e9950198ceeefecbdd"),
"text" : "hello world"
},
{
"_id" : ObjectId("5c9cc7f6950198ceeefecbec"),
"text" : "hello"
},
{
"_id" : ObjectId("5c9cc804950198ceeefecbfa"),
"text" : "test world"
}
My expected results with the code from above, should be the first 2 entries. Instead i get an empty cursor back.
Does anybode know, what i am doing wrong?
Thanks for your help.

primitive.Regex struct accepts Pattern value without slashes, so it must be:
filter := bson.D{{"text", primitive.Regex{Pattern: "he", Options: ""}}}

Try this one
. -> matches any character
*-> matches zero of more occurrence of previous character
So below filter will give you all results which has "he" as substring
filter = bson.M{"text": primitive.Regex{Pattern: ".*" + "he" + ".*", Options: "i"}}

Related

filter query mongoldb Golang

I am trying to get a list of data that match specific queries but I am getting this error
"(AtlasError) merchant is not allowed or the syntax is incorrect, see
the Atlas documentation for more information"
func ...
var result []*model.Package
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
tokenData, err := middleware.CtxValue(ctx)
if err != nil {
return nil, err
}
orQuery := []bson.M{}
merchant := "abc"
completedQuery := bson.M{"status": "completed"}
cancelledQuery := bson.M{"status": "cancelled"}
orQuery = append(
orQuery,
cancelledQuery,
completedQuery)
limit64 := int64(limit)
page64 := int64(page)
match := bson.M{"$match": bson.M{"$nor": orQuery}}
var filterQuery primitive.M
if tokenData.Role == "admin" && merchant != nil {
filterQuery = bson.M{"merchant": bson.M{"id": merchant}}
} else {
filterQuery = bson.M{"user": bson.M{"id": tokenData.Id}}
}
paginatedData, err1 := paginate.New(r.Collection).Context(ctx).Limit(limit64).Page(page64).Aggregate(match, filterQuery)
if err1 != nil {
return nil, err1
}
...
filterQuery, which seems to contain { "merchant" : { "id" : "abc" } }, is being passed sepearately to .Aggregate(). But the aggregation framework is expecting to receive something that represents a sequence of pipeline stages. Each of these stages, outlined here in the documentation, are expected to begin with a $ character such as the $match stage.
Currently the database is attempting to process merchant as an options for the pipeline (see here and here). But such an option doesn't exist, hence the error message.
To resolve this, you should incorporate the filterQuery logic into the existing match variable/stage that you are building and passing. Alternatively you can wrap filterQuery in a different $match and then pass both of them (as a single argument) to .Aggregate().
This example in the documentation shows they build multiple stages and then submit them together to .Aggregate() via mongo.Pipeline{...}:
// create the stages
matchStage := bson.D{{"$match", bson.D{{"toppings", "milk foam"}}}}
unsetStage := bson.D{{"$unset", bson.A{"_id", "category"}}}
sortStage := bson.D{{"$sort", bson.D{
{"price", 1},
{"toppings", 1}},
}}
limitStage := bson.D{{"$limit", 2}}
// pass the stage into a pipeline
// pass the pipeline as the second paramter in the Aggregate() method
cursor, err := coll.Aggregate(context.TODO(), mongo.Pipeline{matchStage, unsetStage, sortStage, limitStage})

mongodb getting 10_000 rows at a time

I'm trying to get 10000 document at a time in mongodb, but i got :
Information :
Driver https://github.com/mongodb/mongo-go-driver
opt.SetBatchSize(15_000)
opt.SetAllowPartialResults(false)
index on timestamp
Code :
package main
import (
"context"
"fmt"
"net/http"
"os"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
var database *mongo.Database
func main() {
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://20.20.20.43:27017"))
if err != nil {
panic(err)
}
database = client.Database("chat_data")
chatText := make([]chat, 0)
now := time.Now().Unix()
ctx, _ = context.WithTimeout(context.Background(), 30*time.Second)
// mongodb batch option
opt := options.Find()
opt.SetBatchSize(15_000)
opt.SetAllowPartialResults(false)
// mongodb filter
filter := bson.M{"timestamp": bson.M{"$gte": now - 108000}}
cur, err := database.Collection("chat").Find(ctx, filter, opt)
if err != nil {
// fmt.Fprint(w, err)
fmt.Println(err)
return
}
defer cur.Close(ctx)
for cur.Next(ctx) {
var result chat
err := cur.Decode(&result)
if err != nil {
fmt.Println(err)
continue
}
// do something with result....
// fmt.Println(result)
chatText = append(chatText, result)
}
if err := cur.Err(); err != nil {
// fmt.Fprint(w, cur.Err())
fmt.Println(err)
return
}
fmt.Println("done")
fmt.Println(len(chatText))
}
can i achieve this with mongodb & go driver ?, 30 second timeout are always reached
Edit 1
i try in python (with pymongo) it's only need 0m2.159s to query 36k doc with that filter
Try 7000, if it works try 12000, if it doesn't work try 4000, etc.
Make note of how long these requests take to figure out if the execution time is proportional to batch size.
You are querying on just the timestamp field. If you create an index on that collection with the timestamp field first, you should get faster results, and get a free sort in the process.

How to get the distinct value of a collection

I am trying the use of mongodb and Go and I cannot get the distinct values of the field in a collection.
This is my code:
import (
"context"
"fmt"
"log"
"time"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type House struct {
Ciudad string
}
func main() {
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://localhost:27017"))
if err != nil {
log.Fatal(err)
}
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
err = client.Connect(ctx)
collection := client.Database("test").Collection("houses")
var house repository.House
fmt.Println(collection.Distinct(ctx, "City", &house))
}
After execute this always I am geting an empty array. Any idea that is wrong in this code?
Replace the line
fmt.Println(collection.Distinct(ctx, "City", &house))
With
fmt.Println(collection.Distinct(ctx, "City", bson.D{{}}))
The third parameter, filter, is a BSON document, https://godoc.org/go.mongodb.org/mongo-driver/mongo#Collection.Distinct. Note that Distinct() returns two values, ([]interface, error).

What is the bson syntax for $set in UpdateOne for the official mongo-go-driver

I am trying to get some familiarity with the official mongo-go-driver and the right syntax for UpdateOne.
My simplest full example follows:
(NOTE: in order to use this code you will need to substitute in your own user and server names as well as export the login password to the environment as MONGO_PW):
package main
import (
"context"
"fmt"
"os"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type DB struct {
User string
Server string
Database string
Collection string
Client *mongo.Client
Ctx context.Context
}
var db = DB{
User: <username>,
Server: <server_IP>,
Database: "test",
Collection: "movies",
Ctx: context.TODO(),
}
type Movie struct {
ID primitive.ObjectID `bson:"_id" json:"id"`
Name string `bson:"name" json:"name"`
Description string `bson:"description" json:"description"`
}
func main() {
if err := db.Connect(); err != nil {
fmt.Println("error: unable to connect")
os.Exit(1)
}
fmt.Println("connected")
// The code assumes the original entry for dunkirk is the following
// {"Name":"dunkirk", "Description":"a world war 2 movie"}
updatedMovie := Movie{
Name: "dunkirk",
Description: "movie about the british evacuation in WWII",
}
res, err := db.UpdateByName(updatedMovie)
if err != nil {
fmt.Println("error updating movie:", err)
os.Exit(1)
}
if res.MatchedCount < 1 {
fmt.Println("error: update did not match any documents")
os.Exit(1)
}
}
// UpdateByName changes the description for a movie identified by its name
func (db *DB) UpdateByName(movie Movie) (*mongo.UpdateResult, error) {
filter := bson.D{{"name", movie.Name}}
res, err := db.Client.Database(db.Database).Collection(db.Collection).UpdateOne(
db.Ctx,
filter,
movie,
)
if err != nil {
return nil, err
}
return res, nil
}
// Connect assumes that the database password is stored in the
// environment variable MONGO_PW
func (db *DB) Connect() error {
pw, ok := os.LookupEnv("MONGO_PW")
if !ok {
fmt.Println("error: unable to find MONGO_PW in the environment")
os.Exit(1)
}
mongoURI := fmt.Sprintf("mongodb+srv://%s:%s#%s", db.User, pw, db.Server)
// Set client options and verify connection
clientOptions := options.Client().ApplyURI(mongoURI)
client, err := mongo.Connect(db.Ctx, clientOptions)
if err != nil {
return err
}
err = client.Ping(db.Ctx, nil)
if err != nil {
return err
}
db.Client = client
return nil
}
The function signature for UpdateOne from the package docs is:
func (coll *Collection) UpdateOne(ctx context.Context, filter interface{},
update interface{}, opts ...*options.UpdateOptions) (*UpdateResult, error)
So I am clearly making some sort of mistake in creating the update interface{} argument to the function because I am presented with this error
error updating movie: update document must contain key beginning with '$'
The most popular answer here shows that I need to use a document sort of like this
{ $set: {"Name" : "The Matrix", "Decription" "Neo and Trinity kick butt" } }
but taken verbatim this will not compile in the mongo-go-driver.
I think I need some form of a bson document to comply with the Go syntax. What is the best and/or most efficient syntax to create this bson document for the update?
After playing around with this for a little while longer I was able to solve the problem after A LOT OF TRIAL AND ERROR using the mongodb bson package by changing the UpdateByName function in my code above as follows:
// UpdateByName changes the description for a movie identified by its name
func (db *DB) UpdateByName(movie Movie) (*mongo.UpdateResult, error) {
filter := bson.D{{"name", movie.Name}}
update := bson.D{{"$set",
bson.D{
{"description", movie.Description},
},
}}
res, err := db.Client.Database(db.Database).Collection(db.Collection).UpdateOne(
db.Ctx,
filter,
update,
)
if err != nil {
return nil, err
}
return res, nil
}
Note the use of bson.D{{$"set", .... It is unfortunate the way MongoDB has implemented the bson package this syntax still does not pass the go-vet. If anyone has a comment to fix the lint conflict below it would be appreciated.
go.mongodb.org/mongo-driver/bson/primitive.E composite literal uses unkeyed fields
In many cases you can replace construction
filter := bson.D{{"name", movie.Name}}
with
filter := bson.M{"name": movie.Name}
if arguments order dowsn't matter

Golang mongodb mgo driver Upsert / UpsertId documentation

The mongodb documentation says:
The fields and values of both the and parameters if the parameter contains only update operator expressions. The update creates a base document from the equality clauses in the parameter, and then applies the update expressions from the parameter.
And the mgo documentation says:
Upsert finds a single document matching the provided selector document and modifies it according to the update document. If no document matching the selector is found, the update document is applied to the selector document and the result is inserted in the collection.
But if i do an upsert like this:
session.UpsertId(data.Code, data)
I end up with an entry which have an ObjectID generated automatically by mongodb, instead of data.Code.
this means that UpsertId expect data to be formated with update operators and you can't use a an arbitrary struct? Or what i'm missing here?
Pd. Mongo 2.4.9 mgo v2 golang go version devel +f613443bb13a
EDIT:
This is a sample of what i mean, using the sample code from Neil Lunn:
package main
import (
"fmt"
"gopkg.in/mgo.v2"
// "gopkg.in/mgo.v2/bson"
)
type Person struct {
Code string
Name string
}
func main() {
session, err := mgo.Dial("admin:admin#localhost");
if err != nil {
fmt.Println("Error: ", err)
return
// panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB("test").C("people")
var p = Person{
Code: "1234",
Name: "Bill",
}
_, err = c.UpsertId( p.Code, &p )
result := Person{}
err = c.FindId(p.Code).One(&result)
if err != nil {
fmt.Println("FindId Error: ", err)
return
// panic(err)
}
fmt.Println("Person", result)
}
I found the documentation of the MongoDB was right. The correct way to do this is to wrap the struct to insert into an update operator.
The sample code provided by Neil Lunn, would look like:
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
Code string
Name string
}
func main() {
session, err := mgo.Dial("admin:admin#localhost");
if err != nil {
fmt.Println("Error: ", err)
return
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB("test").C("people")
var p = Person{
Code: "1234",
Name: "Bill",
}
upsertdata := bson.M{ "$set": p}
info , err2 := c.UpsertId( p.Code, upsertdata )
fmt.Println("UpsertId -> ", info, err2)
result := Person{}
err = c.FindId(p.Code).One(&result)
if err != nil {
fmt.Println("FindId Error: ", err)
return
}
fmt.Println("Person", result)
}
Thank you very much for your interest and help Neil.
You seem to be talking about assigning a struct with a custom _id field here. This really comes down to how you define your struct. Here is a quick example:
package main
import (
"fmt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Person struct {
ID string `bson:"_id"`
Name string
}
func main() {
session, err := mgo.Dial("127.0.0.1");
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
c := session.DB("test").C("people")
var p = Person{
ID: "1",
Name: "Bill",
}
_, err = c.UpsertId( p.ID, &p )
result := Person{}
err = c.Find(bson.M{"_id": p.ID}).One(&result)
if err != nil {
panic(err)
}
fmt.Println("Person", result)
}
So in the custom definition here I am mapping the ID field to bson _id and defining it's type as string. As shown in the example this is exactly what happens when serialized via UpsertId and then retrieved.
Now you have elaborated I'll point to the difference on the struct definition.
What I have produces this:
{ "_id": 1, "name": "Bill" }
What you have ( without the same mapping on the struct ) does this:
{ "_id": ObjectId("53cfa557e248860d16e1f7e0"), "code": 1, "name": "Bill" }
As you see, the _id given in the upsert will never match because none of your fields in the struct are mapped to _id. You need the same as I have:
type Person struct {
Code string `bson:"_id"`
Name string
}
That maps a field to the mandatory _id field, otherwise one is automatically produced for you.