I need pagination for Go and MongoDB and I'm highly frustrated with this problem since I cant achieve pagination since weeks now! I dont want to use "Skip" as it is too much resources consuming.
Here is the code:
var limit int64 = 20
var offset int64 = 40
findOptions0 := options.Find()
findOptions0.SetLimit(offset)
findOptions0.SetSort(bson.M{"_id": 1})
findOptions := options.Find()
findOptions.SetLimit(limit)
findOptions.SetSort(bson.M{"_id": 1})
cursor0, err32 := collection.Find(context.Background(), bson.M{}, findOptions)
if err32 != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Unknown1 internal error: %v", err32))
}
for cursor0.Next(context.Background()) {
err := cursor0.Decode(&data0)
if err != nil {
return status.Errorf(codes.Unavailable, fmt.Sprintf("Could not decode data of cursor0: %v", err))
}
}
err := cursor0.Decode(&data0)
if err != nil {
return status.Errorf(codes.Unavailable, fmt.Sprintf("Could not decode data of cursor0: %v", err))
}
cursor, err34 := collection.Find(context.Background(), bson.M{"_id": bson.M{"$gt": data0.ID}}, findOptions)
if err34 != nil {
log.Printf("lastID: ", data0.ID)
return status.Errorf(codes.Internal, fmt.Sprintf("Unknown2 internal error: %v", err34))
}
for cursor.Next(context.Background()) {
log.Printf("result: ", cursor)
// Decode the data at the current pointer and write it to data
err := cursor.Decode(data)
// check error
if err != nil {
return status.Errorf(codes.Unavailable, fmt.Sprintf("Could not decode data: %v", err))
}
log.Printf("End of the for loop ")
Related
I'm trying to build a Documentdb Prometheus exporter. I'm trying to get the IndexStats to identify the unused Indices. But I'm getting ops value as 0 even though the ops value is not 0. Can you please help me identify the error?
This the Go code for the same.
databases, err := client.ListDatabaseNames(ctx, bson.D{})
if err != nil{
return make(map[string]map[string]interface{}), err
}
var indexMap map[string]map[string]interface{} = make(map[string]map[string]interface{})
for _, database := range databases {
collections, err := client.Database(database).ListCollectionNames(ctx, bson.D{})
if err != nil {
return make(map[string]map[string]interface{}), err
}
for _, collectionName := range collections {
collection := client.Database(database).Collection(collectionName)
indexStats := bson.D{{Key: "$indexStats", Value: bson.M{}}}
cursor, err := collection.Aggregate(ctx, mongo.Pipeline{indexStats})
if err != nil {
return make(map[string]map[string]interface{}), err
}
var indices []bson.M
err = cursor.All(ctx, &indices)
if err != nil {
return make(map[string]map[string]interface{}), err
}
err = cursor.Close(ctx)
if err != nil {
return make(map[string]map[string]interface{}), err
}
fmt.Print(database + " " + collectionName + " ")
fmt.Println(indices)
indicesBytes, err := json.Marshal(indices)
if err != nil {
return make(map[string]map[string]interface{}), err
}
var indicesJson []interface{}
err = json.Unmarshal(indicesBytes, &indicesJson)
if err != nil {
return make(map[string]map[string]interface{}), err
}
if _, ok := indexMap[database]; !ok {
indexMap[database] = make(map[string]interface{})
}
indexMap[database][collectionName] = indicesJson
}
}
May I know the reason and the fix for this issue?
Thanks in Advance :)
The error
(*errors.errorString) *{s: "pq: unexpected DataRow in simple query execution"}
appears after the line with the commentary. Didn't find any solution online. Since stackoverflow asks for more details, this is an update query that is supposed to update a todo and a list of subtasks in the database. The exact error is in the question topic. I post the complete code for the function that returns the error.
func (t *TodoTable) UpdateTodo(ctx context.Context, todo *Todo, t_id int) error {
tx, err := t.sqlxdb.BeginTxx(ctx, &sql.TxOptions{})
if err != nil {
return err
}
rollback_err := func(err error) error {
if err2 := tx.Rollback(); err2 != nil {
return fmt.Errorf("%v; %v", err, err2)
}
return err
}
row := tx.QueryRowxContext(ctx, "UPDATE todos SET todo_name=$1, deadline=$2, updated_at=$3 WHERE todo_id=$4 returning todo_id", todo.TodoName, todo.Deadline, todo.UpdatedAt, t_id)
if row.Err() != nil {
return rollback_err(err)
}
var subs_ids []int
// Getting subs ids from database
query := fmt.Sprintf("SELECT sub_id FROM subs WHERE todo_id=%d", t_id)
// THE ERROR COMES AFTER EXECUTING THE LINE BELOW
rows, err := tx.Query(query)
if err != nil {
rollback_err(err)
}
if rows != nil {
for rows.Next() {
var sub_id int
err = rows.Scan(&sub_id)
if err != nil {
rollback_err(err)
}
subs_ids = append(subs_ids, sub_id)
}
if err := tx.Commit(); err != nil {
return rollback_err(err)
}
}
// Updating subs
for i, sub := range todo.Subs {
_, err = tx.ExecContext(ctx, fmt.Sprintf("UPDATE subs SET sub_name='%s' WHERE sub_id=%d", sub.Sub_name, subs_ids[i]))
if err != nil {
return rollback_err(err)
}
}
return nil
}
Currently I am able to return all my products from the collection.
I however want to be able to return products that come after a specific product ID (which would be the last one on the client side so they could load more)
Current way (return all)
query := bson.M{}
var product ReturnedProdcut
var products []ReturnedProduct
cur, err := mg.Db.Collection("products").Find(c.Request().Context(), query)
if err != nil {
fmt.Println(err)
}
for cur.Next(c.Request().Context()) {
err := cur.Decode(&product)
if err != nil {
fmt.Println(err)
}
products = append(products, product)
}
// return products list in JSON format
return c.JSON(http.StatusOK, products)
New Way Attempt(return based on page)
afterID := c.QueryParam("afterID")
if afterID == "" {
// get from start of collection based on latest date
}
// get 10 products after this ID, if no ID then get from start
query := bson.M{}
var product ReturnedProduct
var products []Returnedproduct
//.find(afterId).limit(10) - something like this?
cur, err := mg.Db.Collection("products").Find(c.Request().Context(), query)
if err != nil {
fmt.Println(err)
}
for cur.Next(c.Request().Context()) {
err := cur.Decode(&product)
if err != nil {
fmt.Println(err)
}
products = append(products, product)
}
// return products list in JSON format
return c.JSON(http.StatusOK, products)
The official MongoDB Go driver also has a *FindOptions optional parameter you could also explore.
pageOptions := options.Find()
pageOptions.SetSkip(int64(page)) //0-i
pageOptions.SetLimit(int64(limit)) // number of records to return
cur, err := userCollection.Find(c.Request().Context(), bson.D{{}}, pageOptions)
if err != nil {
// handle error
}
defer cur.Close(ctx)
var products []Returnedproduct
for cur.Next(c.Request().Context()) {
var product Returnedproduct
if err := cur.Decode(&product); err != nil {
// handle error
}
products = append(products, &product)
}
if err := cur.Err(); err != nil {
// handle error
}
You may construct a query where _id is greater than afterID, in which case you should also specify sorting by _id. For sorting and for setting a limit, you may use options.FindOptions.
You also should use Cursor.All() to decode all results and not one-by-one.
This is how it could look like:
query := bson.M{"_id": bson.M{"$gt": afterID}}
opts := options.Find().
SetSort(bson.M{"_id": 1}).
SetLimit(10)
ctx := c.Request().Context()
curs, err := mg.Db.Collection("products").Find(ctx, query, opts)
if err != nil {
// Handle error
}
var products []Returnedproduct
if err = curs.All(ctx, &products); err != nil {
// Handle error
}
return c.JSON(http.StatusOK, products)
I create an api using golang, i would like to create some functionnal test, for that i create an interface to abstract my database. But for that i need to be able to convert the cursor to an array without knowing the type.
func (self *KeyController) GetKey(c echo.Context) (err error) {
var res []dto.Key
err = db.Keys.Find(bson.M{}, 10, 0, &res)
if err != nil {
fmt.Println(err)
return c.String(http.StatusInternalServerError, "internal error")
}
c.JSON(http.StatusOK, res)
return
}
//THE FIND FUNCTION ON THE DB PACKAGE
func (s MongoCollection) Find(filter bson.M, limit int, offset int, res interface{}) (err error) {
ctx := context.Background()
var cursor *mongo.Cursor
l := int64(limit)
o := int64(offset)
objectType := reflect.TypeOf(res).Elem()
cursor, err = s.c.Find(ctx, filter, &options.FindOptions{
Limit: &l,
Skip: &o,
})
if err != nil {
return
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
result := reflect.New(objectType).Interface()
err := cursor.Decode(&result)
if err != nil {
panic(err)
}
res = append(res.([]interface{}), result)
}
return
}
Does someone have an idea?
You can call directly the "All" method:
ctx := context.Background()
err = cursor.All(ctx, res)
if err != nil {
fmt.Println(err.Error())
}
For reference:
https://godoc.org/go.mongodb.org/mongo-driver/mongo#Cursor.All
i think you want to encapsulate the Find method for mongo query.
Using the reflect package i have improved your code by adding an additional parameter that serves as a template to instantiate new instances of slice items.
func (m *MongoDbModel) FindAll(database string, colname string, obj interface{}, parameter map[string]interface{}) ([]interface{}, error) {
var list = make([]interface{}, 0)
collection, err := m.Client.Database(database).Collection(colname).Clone()
objectType := reflect.TypeOf(obj).Elem()
fmt.Println("objectype", objectType)
if err != nil {
log.Println(err)
return nil, err
}
filter := bson.M{}
filter["$and"] = []bson.M{}
for key, value := range parameter {
filter["$and"] = append(filter["$and"].([]bson.M), bson.M{key: value})
}
cur, err := collection.Find(context.Background(), filter)
if err != nil {
log.Fatal(err)
}
defer cur.Close(context.Background())
for cur.Next(context.Background()) {
result := reflect.New(objectType).Interface()
err := cur.Decode(result)
if err != nil {
log.Println(err)
return nil, err
}
list = append(list, result)
}
if err := cur.Err(); err != nil {
return nil, err
}
return list, nil
}
The difference is that FindAll method returns []interface{}, where err := cur.Decode(result) directly consumes a pointer like the result variable.
I am relatively new to Golang and don't fully understand streams. I have a function (circuit breaker function) that is making Rest calls. I have it working but it is only streaming the "responseBody" back. I would actually like to stream back the entire request of stream back both the Body and the Header together.
When I try to use a similar approach on the "header" then I get an error that the header is not streamable.
Is there a good/best way to accomplish this? Thank you. Below is my function.
func CallWithRetries(req *http.Request, output chan []byte) error {
r := retrier.New(retrier.ConstantBackoff(RETRIES, 100 * time.Millisecond), nil)
attempt := 0
err := r.Run(func() error {
attempt++
resp, err := Client.Do(req)
if err == nil && resp.StatusCode < 299 {
responseBody, err := ioutil.ReadAll(resp.Body)
if err == nil {
output <- responseBody
return err
}
return err
} else if err == nil {
customLogger.LogDebug("Status code was: " , transactionId)
err = fmt.Errorf("Status was %v", resp.StatusCode)
}
return err
})
return err
}
You are looking for the httputil.DumpResponse function.
The code might be changed to something similar to
func CallWithRetries(req *http.Request, output chan []byte) error {
r := retrier.New(retrier.ConstantBackoff(RETRIES, 100*time.Millisecond), nil)
attempt := 0
err := r.Run(func() error {
attempt++
resp, err := Client.Do(req)
if err == nil && resp.StatusCode < 299 {
dump, err := httputil.DumpResponse(resp, true)
if err == nil {
output <- dump
return err
}
return err
} else if err == nil {
customLogger.LogDebug("Status code was: ", transactionId)
err = fmt.Errorf("Status was %v", resp.StatusCode)
}
return err
})
return err
}
Side notes,
you might want to consider to close the response body as mentioned in the documentation https://golang.org/pkg/net/http/#Client.Get
Looks likes the err variable is shadowed, this should be modified to avoid any surprises.
This version of the code attempts to return errors early, and to close the response body. It was not tested, only written on the fly, to use with care.
func CallWithRetries(req *http.Request, output chan []byte) error {
r := retrier.New(retrier.ConstantBackoff(RETRIES, 100*time.Millisecond), nil)
attempt := 0
return r.Run(func() error {
attempt++
var resp *http.Response
{
r, err := Client.Do(req)
if err != nil {
return err
}
defer r.Body.Close()
if resp.StatusCode > 299 {
customLogger.LogDebug("Status code was: ", transactionId)
return fmt.Errorf("Status was %v", resp.StatusCode)
}
resp = r
}
var out []byte
{
x, err := httputil.DumpResponse(resp, true)
if err != nil {
return err
}
out = x
}
output <- out
return nil
})
}