MongoDB driver pagination - mongodb

Currently I am able to return all my products from the collection.
I however want to be able to return products that come after a specific product ID (which would be the last one on the client side so they could load more)
Current way (return all)
query := bson.M{}
var product ReturnedProdcut
var products []ReturnedProduct
cur, err := mg.Db.Collection("products").Find(c.Request().Context(), query)
if err != nil {
fmt.Println(err)
}
for cur.Next(c.Request().Context()) {
err := cur.Decode(&product)
if err != nil {
fmt.Println(err)
}
products = append(products, product)
}
// return products list in JSON format
return c.JSON(http.StatusOK, products)
New Way Attempt(return based on page)
afterID := c.QueryParam("afterID")
if afterID == "" {
// get from start of collection based on latest date
}
// get 10 products after this ID, if no ID then get from start
query := bson.M{}
var product ReturnedProduct
var products []Returnedproduct
//.find(afterId).limit(10) - something like this?
cur, err := mg.Db.Collection("products").Find(c.Request().Context(), query)
if err != nil {
fmt.Println(err)
}
for cur.Next(c.Request().Context()) {
err := cur.Decode(&product)
if err != nil {
fmt.Println(err)
}
products = append(products, product)
}
// return products list in JSON format
return c.JSON(http.StatusOK, products)

The official MongoDB Go driver also has a *FindOptions optional parameter you could also explore.
pageOptions := options.Find()
pageOptions.SetSkip(int64(page)) //0-i
pageOptions.SetLimit(int64(limit)) // number of records to return
cur, err := userCollection.Find(c.Request().Context(), bson.D{{}}, pageOptions)
if err != nil {
// handle error
}
defer cur.Close(ctx)
var products []Returnedproduct
for cur.Next(c.Request().Context()) {
var product Returnedproduct
if err := cur.Decode(&product); err != nil {
// handle error
}
products = append(products, &product)
}
if err := cur.Err(); err != nil {
// handle error
}

You may construct a query where _id is greater than afterID, in which case you should also specify sorting by _id. For sorting and for setting a limit, you may use options.FindOptions.
You also should use Cursor.All() to decode all results and not one-by-one.
This is how it could look like:
query := bson.M{"_id": bson.M{"$gt": afterID}}
opts := options.Find().
SetSort(bson.M{"_id": 1}).
SetLimit(10)
ctx := c.Request().Context()
curs, err := mg.Db.Collection("products").Find(ctx, query, opts)
if err != nil {
// Handle error
}
var products []Returnedproduct
if err = curs.All(ctx, &products); err != nil {
// Handle error
}
return c.JSON(http.StatusOK, products)

Related

Can i use Query() func twice in pgx Golang lib?

I use query to Find All DB data, and one of tables need to be array, so i integrated the loop.
EveryTime errors is conn is busy.
`
for rows.Next() {
var ord Order
err = rows.Scan(
and after one more Query() which is uses to find all Item with ord.UID. So the question is, what is wrong in my code? And how to use this. Here is my FindAll Func:
rows, err := r.client.Query(ctx, q)
if err != nil {
return nil, err
}
defer rows.Close()
orders := make([]Order, 0)
for rows.Next() {
var ord Order
err = rows.Scan(
&ord.OrderUID,
&ord.TrackNumber,
&ord.Entry,
&ord.Delivery.Name,
&ord.Delivery.Phone,
&ord.Delivery.Zip,
&ord.Delivery.City,
&ord.Delivery.Address,
&ord.Delivery.Region,
&ord.Delivery.Email,
&ord.Payment.Transaction,
&ord.Payment.RequestID,
&ord.Payment.Currency,
&ord.Payment.Provider,
&ord.Payment.Amount,
&ord.Payment.PaymentDT,
&ord.Payment.Bank,
&ord.Payment.DeliveryCost,
&ord.Payment.GoodsTotal,
&ord.Payment.CustomFee,
&ord.Locale,
&ord.InternalSignature,
&ord.CustomerID,
&ord.DeliveryService,
&ord.ShardKey,
&ord.SmID,
&ord.DateCreated,
&ord.OofShard,
)
if err != nil {
return nil, err
}
iq := ` ... `
itemRows, err := r.client.Query(ctx, iq, ord.OrderUID)
if err != nil {
return nil, err
}
items := make([]item.Item, 0)
for itemRows.Next() {
var item item.Item
err = itemRows.Scan(
&item.ID,
&item.ChrtID,
&item.TrackNumber,
&item.Price,
&item.Rid,
&item.Name,
&item.Sale,
&item.Size,
&item.TotalPrice,
&item.NmID,
&item.Brand,
&item.Status,
)
if err != nil {
return nil, err
}
items = append(items, item)
}
ord.Items = items
orders = append(orders, ord)
}
`
I tried to rows.Close, but then i Can't use row anymore.

Receiving error(*errors.errorString) *{s: "pq: unexpected DataRow in simple query execution"}

The error
(*errors.errorString) *{s: "pq: unexpected DataRow in simple query execution"}
appears after the line with the commentary. Didn't find any solution online. Since stackoverflow asks for more details, this is an update query that is supposed to update a todo and a list of subtasks in the database. The exact error is in the question topic. I post the complete code for the function that returns the error.
func (t *TodoTable) UpdateTodo(ctx context.Context, todo *Todo, t_id int) error {
tx, err := t.sqlxdb.BeginTxx(ctx, &sql.TxOptions{})
if err != nil {
return err
}
rollback_err := func(err error) error {
if err2 := tx.Rollback(); err2 != nil {
return fmt.Errorf("%v; %v", err, err2)
}
return err
}
row := tx.QueryRowxContext(ctx, "UPDATE todos SET todo_name=$1, deadline=$2, updated_at=$3 WHERE todo_id=$4 returning todo_id", todo.TodoName, todo.Deadline, todo.UpdatedAt, t_id)
if row.Err() != nil {
return rollback_err(err)
}
var subs_ids []int
// Getting subs ids from database
query := fmt.Sprintf("SELECT sub_id FROM subs WHERE todo_id=%d", t_id)
// THE ERROR COMES AFTER EXECUTING THE LINE BELOW
rows, err := tx.Query(query)
if err != nil {
rollback_err(err)
}
if rows != nil {
for rows.Next() {
var sub_id int
err = rows.Scan(&sub_id)
if err != nil {
rollback_err(err)
}
subs_ids = append(subs_ids, sub_id)
}
if err := tx.Commit(); err != nil {
return rollback_err(err)
}
}
// Updating subs
for i, sub := range todo.Subs {
_, err = tx.ExecContext(ctx, fmt.Sprintf("UPDATE subs SET sub_name='%s' WHERE sub_id=%d", sub.Sub_name, subs_ids[i]))
if err != nil {
return rollback_err(err)
}
}
return nil
}

How to do pagination with MongoDB

I need pagination for Go and MongoDB and I'm highly frustrated with this problem since I cant achieve pagination since weeks now! I dont want to use "Skip" as it is too much resources consuming.
Here is the code:
var limit int64 = 20
var offset int64 = 40
findOptions0 := options.Find()
findOptions0.SetLimit(offset)
findOptions0.SetSort(bson.M{"_id": 1})
findOptions := options.Find()
findOptions.SetLimit(limit)
findOptions.SetSort(bson.M{"_id": 1})
cursor0, err32 := collection.Find(context.Background(), bson.M{}, findOptions)
if err32 != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Unknown1 internal error: %v", err32))
}
for cursor0.Next(context.Background()) {
err := cursor0.Decode(&data0)
if err != nil {
return status.Errorf(codes.Unavailable, fmt.Sprintf("Could not decode data of cursor0: %v", err))
}
}
err := cursor0.Decode(&data0)
if err != nil {
return status.Errorf(codes.Unavailable, fmt.Sprintf("Could not decode data of cursor0: %v", err))
}
cursor, err34 := collection.Find(context.Background(), bson.M{"_id": bson.M{"$gt": data0.ID}}, findOptions)
if err34 != nil {
log.Printf("lastID: ", data0.ID)
return status.Errorf(codes.Internal, fmt.Sprintf("Unknown2 internal error: %v", err34))
}
for cursor.Next(context.Background()) {
log.Printf("result: ", cursor)
// Decode the data at the current pointer and write it to data
err := cursor.Decode(data)
// check error
if err != nil {
return status.Errorf(codes.Unavailable, fmt.Sprintf("Could not decode data: %v", err))
}
log.Printf("End of the for loop ")

Passing in dynamic struct into function in golang

I having a couple of structs, Products and Categories. I have 2 functions listed below that have identical logic just different structs that are being used and returned. Is there anyway I can abstract out the struct data types and use the same logic in a single function called GetObjects?
func GetCategories(collection *mongo.Collection) []Category {
ctx := context.Background()
cats := []Category{}
cur, err := collection.Find(ctx, bson.M{})
if err != nil {
log.Fatal("Error: ", err)
}
for cur.Next(context.TODO()) {
var cat Category
err = cur.Decode(&cat)
if err != nil {
log.Fatal(err)
}
cats = append(cats, cat)
}
return cats
}
func GetProducts(collection *mongo.Collection) []Product {
ctx := context.Background()
prods := []Product{}
cur, err := collection.Find(ctx, bson.M{})
if err != nil {
log.Fatal("Error: ", err)
}
for cur.Next(context.TODO()) {
var prod Product
err = cur.Decode(&prod)
if err != nil {
log.Fatal(err)
}
prods = append(prods, prod)
}
return prods
}
You could create a generalized GetObjs() if you would pass in the destination where you want the results to be loaded.
Something like:
func GetObjs(c *mongo.Collection, dst interface{})
And callers are responsible to pass a ready slice or a pointer to a slice variable where results will be stored.
Also note that context.Context should be passed and not created arbitrarily inside the function:
func GetObjs(ctx context.Context, c *mongo.Collection, dst interface{})
Also, errors should be returned and not "swallowed", so if one occurs, it can be dealt with appropriately at the caller:
func GetObjs(ctx context.Context, c *mongo.Collection, dst interface{}) error
Also, if you need all results, you don't need to iterate over them and decode all one-by-one. Just use Cursor.All().
This is how the "improved" GetObjs() could look like:
func GetObjs(ctx context.Context, c *mongo.Collection, dst interface{}) error {
cur, err := c.Find(ctx, bson.M{})
if err != nil {
return err
}
return cur.All(ctx, dst)
}
(Although this became quite simple, not sure it warrants its own existence.)
And this is how you could use it:
ctx := ... // Obtain context, e.g. from the request: r.Context()
c := ... // Collection you want to query from
var cats []Category
if err := GetObjs(ctx, c, &cats); err != nil {
// Handle error
return
}
// Use cats
var prods []Product
if err := GetObjs(ctx, c, &prods); err != nil {
// Handle error
return
}
// Use prods

Deserialize cursor into array with mongo-go-driver and interface

I create an api using golang, i would like to create some functionnal test, for that i create an interface to abstract my database. But for that i need to be able to convert the cursor to an array without knowing the type.
func (self *KeyController) GetKey(c echo.Context) (err error) {
var res []dto.Key
err = db.Keys.Find(bson.M{}, 10, 0, &res)
if err != nil {
fmt.Println(err)
return c.String(http.StatusInternalServerError, "internal error")
}
c.JSON(http.StatusOK, res)
return
}
//THE FIND FUNCTION ON THE DB PACKAGE
func (s MongoCollection) Find(filter bson.M, limit int, offset int, res interface{}) (err error) {
ctx := context.Background()
var cursor *mongo.Cursor
l := int64(limit)
o := int64(offset)
objectType := reflect.TypeOf(res).Elem()
cursor, err = s.c.Find(ctx, filter, &options.FindOptions{
Limit: &l,
Skip: &o,
})
if err != nil {
return
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
result := reflect.New(objectType).Interface()
err := cursor.Decode(&result)
if err != nil {
panic(err)
}
res = append(res.([]interface{}), result)
}
return
}
Does someone have an idea?
You can call directly the "All" method:
ctx := context.Background()
err = cursor.All(ctx, res)
if err != nil {
fmt.Println(err.Error())
}
For reference:
https://godoc.org/go.mongodb.org/mongo-driver/mongo#Cursor.All
i think you want to encapsulate the Find method for mongo query.
Using the reflect package i have improved your code by adding an additional parameter that serves as a template to instantiate new instances of slice items.
func (m *MongoDbModel) FindAll(database string, colname string, obj interface{}, parameter map[string]interface{}) ([]interface{}, error) {
var list = make([]interface{}, 0)
collection, err := m.Client.Database(database).Collection(colname).Clone()
objectType := reflect.TypeOf(obj).Elem()
fmt.Println("objectype", objectType)
if err != nil {
log.Println(err)
return nil, err
}
filter := bson.M{}
filter["$and"] = []bson.M{}
for key, value := range parameter {
filter["$and"] = append(filter["$and"].([]bson.M), bson.M{key: value})
}
cur, err := collection.Find(context.Background(), filter)
if err != nil {
log.Fatal(err)
}
defer cur.Close(context.Background())
for cur.Next(context.Background()) {
result := reflect.New(objectType).Interface()
err := cur.Decode(result)
if err != nil {
log.Println(err)
return nil, err
}
list = append(list, result)
}
if err := cur.Err(); err != nil {
return nil, err
}
return list, nil
}
The difference is that FindAll method returns []interface{}, where err := cur.Decode(result) directly consumes a pointer like the result variable.