In golang..
I get list mongodb database name below..
filter := bson.D{{}}
dbs, _ := client.ListDatabaseNames(context.TODO(), filter)
fmt.Printf("%+v\n", dbs)
But, I want to get list collections name.
How about ListCollectionNames?
Here's an example from the documentation. I added some placeholder lines that you need to replace by your client connection code and getting a database:
package main
import (
"context"
"fmt"
"log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
)
func main() {
// Placeholder:
//
// Connect to MongoDB, handle error
client, err := mongo.connect(....)
if err != nil {
log.Fatal(err)
}
// Obtain the DB, by name. db will have the type
// *mongo.Database
db := client.Database("name-of-your-DB")
// use a filter to only select capped collections
result, err := db.ListCollectionNames(
context.TODO(),
bson.D{{"options.capped", true}})
if err != nil {
log.Fatal(err)
}
for _, coll := range result {
fmt.Println(coll)
}
}
Related
I'm trying to get 10000 document at a time in mongodb, but i got :
Information :
Driver https://github.com/mongodb/mongo-go-driver
opt.SetBatchSize(15_000)
opt.SetAllowPartialResults(false)
index on timestamp
Code :
package main
import (
"context"
"fmt"
"net/http"
"os"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
var database *mongo.Database
func main() {
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://20.20.20.43:27017"))
if err != nil {
panic(err)
}
database = client.Database("chat_data")
chatText := make([]chat, 0)
now := time.Now().Unix()
ctx, _ = context.WithTimeout(context.Background(), 30*time.Second)
// mongodb batch option
opt := options.Find()
opt.SetBatchSize(15_000)
opt.SetAllowPartialResults(false)
// mongodb filter
filter := bson.M{"timestamp": bson.M{"$gte": now - 108000}}
cur, err := database.Collection("chat").Find(ctx, filter, opt)
if err != nil {
// fmt.Fprint(w, err)
fmt.Println(err)
return
}
defer cur.Close(ctx)
for cur.Next(ctx) {
var result chat
err := cur.Decode(&result)
if err != nil {
fmt.Println(err)
continue
}
// do something with result....
// fmt.Println(result)
chatText = append(chatText, result)
}
if err := cur.Err(); err != nil {
// fmt.Fprint(w, cur.Err())
fmt.Println(err)
return
}
fmt.Println("done")
fmt.Println(len(chatText))
}
can i achieve this with mongodb & go driver ?, 30 second timeout are always reached
Edit 1
i try in python (with pymongo) it's only need 0m2.159s to query 36k doc with that filter
Try 7000, if it works try 12000, if it doesn't work try 4000, etc.
Make note of how long these requests take to figure out if the execution time is proportional to batch size.
You are querying on just the timestamp field. If you create an index on that collection with the timestamp field first, you should get faster results, and get a free sort in the process.
Question
How can I list databases only with the given prefix (prefix_)?
Example:
package main
import (
"context"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"log"
)
type foo struct {
Value string
}
func main() {
clientOptions := options.Client().ApplyURI("mongodb://10.0.12.76:27018")
client, err := mongo.Connect(context.TODO(), clientOptions)
if err != nil {
log.Fatal(err)
}
db := [3]string{"prefix_foo", "prefix_bar", "bar"}
for _, element := range db {
_, err := client.Database(element).Collection("placeholder").InsertOne(context.TODO(), foo{"sth"})
if err != nil {
log.Fatal(err)
}
}
filter := bson.D{{}}
dbs, err := client.ListDatabaseNames(context.TODO(), filter)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%+v\n", dbs)
}
Output:
[admin bar config local prefix_bar prefix_foo]
Expected output:
[prefix_bar prefix_foo]
Bonus:
It is possible to create a database without defining new struct in my case foo?
My goal is to run a query on databases only with a prefix, so maybe better solution exists than listing dbs and then run a query on each database?
Simply filter by the name property, which denotes the database name. And to list databases starting with a given prefix, you may use a regexp being ^prefix_:
filter := bson.M{"name": primitive.Regex{Pattern: "^prefix_"}}
Other filter options are listed on the listDatabases command page:
You can specify a condition on any of the fields in the output of listDatabases:
name
sizeOnDisk
empty
shards
And you may use an empty bson.M{} to insert an empty document (_id will be added of course).
Question
How can I list databases only with the given prefix (prefix_)?
Example:
package main
import (
"context"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"log"
)
type foo struct {
Value string
}
func main() {
clientOptions := options.Client().ApplyURI("mongodb://10.0.12.76:27018")
client, err := mongo.Connect(context.TODO(), clientOptions)
if err != nil {
log.Fatal(err)
}
db := [3]string{"prefix_foo", "prefix_bar", "bar"}
for _, element := range db {
_, err := client.Database(element).Collection("placeholder").InsertOne(context.TODO(), foo{"sth"})
if err != nil {
log.Fatal(err)
}
}
filter := bson.D{{}}
dbs, err := client.ListDatabaseNames(context.TODO(), filter)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%+v\n", dbs)
}
Output:
[admin bar config local prefix_bar prefix_foo]
Expected output:
[prefix_bar prefix_foo]
Bonus:
It is possible to create a database without defining new struct in my case foo?
My goal is to run a query on databases only with a prefix, so maybe better solution exists than listing dbs and then run a query on each database?
Simply filter by the name property, which denotes the database name. And to list databases starting with a given prefix, you may use a regexp being ^prefix_:
filter := bson.M{"name": primitive.Regex{Pattern: "^prefix_"}}
Other filter options are listed on the listDatabases command page:
You can specify a condition on any of the fields in the output of listDatabases:
name
sizeOnDisk
empty
shards
And you may use an empty bson.M{} to insert an empty document (_id will be added of course).
I am trying to get some familiarity with the official mongo-go-driver and the right syntax for UpdateOne.
My simplest full example follows:
(NOTE: in order to use this code you will need to substitute in your own user and server names as well as export the login password to the environment as MONGO_PW):
package main
import (
"context"
"fmt"
"os"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type DB struct {
User string
Server string
Database string
Collection string
Client *mongo.Client
Ctx context.Context
}
var db = DB{
User: <username>,
Server: <server_IP>,
Database: "test",
Collection: "movies",
Ctx: context.TODO(),
}
type Movie struct {
ID primitive.ObjectID `bson:"_id" json:"id"`
Name string `bson:"name" json:"name"`
Description string `bson:"description" json:"description"`
}
func main() {
if err := db.Connect(); err != nil {
fmt.Println("error: unable to connect")
os.Exit(1)
}
fmt.Println("connected")
// The code assumes the original entry for dunkirk is the following
// {"Name":"dunkirk", "Description":"a world war 2 movie"}
updatedMovie := Movie{
Name: "dunkirk",
Description: "movie about the british evacuation in WWII",
}
res, err := db.UpdateByName(updatedMovie)
if err != nil {
fmt.Println("error updating movie:", err)
os.Exit(1)
}
if res.MatchedCount < 1 {
fmt.Println("error: update did not match any documents")
os.Exit(1)
}
}
// UpdateByName changes the description for a movie identified by its name
func (db *DB) UpdateByName(movie Movie) (*mongo.UpdateResult, error) {
filter := bson.D{{"name", movie.Name}}
res, err := db.Client.Database(db.Database).Collection(db.Collection).UpdateOne(
db.Ctx,
filter,
movie,
)
if err != nil {
return nil, err
}
return res, nil
}
// Connect assumes that the database password is stored in the
// environment variable MONGO_PW
func (db *DB) Connect() error {
pw, ok := os.LookupEnv("MONGO_PW")
if !ok {
fmt.Println("error: unable to find MONGO_PW in the environment")
os.Exit(1)
}
mongoURI := fmt.Sprintf("mongodb+srv://%s:%s#%s", db.User, pw, db.Server)
// Set client options and verify connection
clientOptions := options.Client().ApplyURI(mongoURI)
client, err := mongo.Connect(db.Ctx, clientOptions)
if err != nil {
return err
}
err = client.Ping(db.Ctx, nil)
if err != nil {
return err
}
db.Client = client
return nil
}
The function signature for UpdateOne from the package docs is:
func (coll *Collection) UpdateOne(ctx context.Context, filter interface{},
update interface{}, opts ...*options.UpdateOptions) (*UpdateResult, error)
So I am clearly making some sort of mistake in creating the update interface{} argument to the function because I am presented with this error
error updating movie: update document must contain key beginning with '$'
The most popular answer here shows that I need to use a document sort of like this
{ $set: {"Name" : "The Matrix", "Decription" "Neo and Trinity kick butt" } }
but taken verbatim this will not compile in the mongo-go-driver.
I think I need some form of a bson document to comply with the Go syntax. What is the best and/or most efficient syntax to create this bson document for the update?
After playing around with this for a little while longer I was able to solve the problem after A LOT OF TRIAL AND ERROR using the mongodb bson package by changing the UpdateByName function in my code above as follows:
// UpdateByName changes the description for a movie identified by its name
func (db *DB) UpdateByName(movie Movie) (*mongo.UpdateResult, error) {
filter := bson.D{{"name", movie.Name}}
update := bson.D{{"$set",
bson.D{
{"description", movie.Description},
},
}}
res, err := db.Client.Database(db.Database).Collection(db.Collection).UpdateOne(
db.Ctx,
filter,
update,
)
if err != nil {
return nil, err
}
return res, nil
}
Note the use of bson.D{{$"set", .... It is unfortunate the way MongoDB has implemented the bson package this syntax still does not pass the go-vet. If anyone has a comment to fix the lint conflict below it would be appreciated.
go.mongodb.org/mongo-driver/bson/primitive.E composite literal uses unkeyed fields
In many cases you can replace construction
filter := bson.D{{"name", movie.Name}}
with
filter := bson.M{"name": movie.Name}
if arguments order dowsn't matter
I'm trying to fetch data from mongodb in golang using the gopkg.in/mgo.v2 driver, the format of the data is not fixed , as in few rows will be containing some fields which other rows might not.
here is the code for the same
session, err := mgo.Dial("mongodb://root:root#localhost:27017/admin")
db := session.DB("test")
fmt.Println(reflect.TypeOf(db))
CheckError(err,"errpor")
result := make(map[string]string)
//query := make(map[string]string)
//query["_id"] = "3434"
err1 := db.C("mycollection").Find(nil).One(&result)
CheckError(err1,"error")
for k := range result {
fmt.Println(k)
}
Now the data contained in the collection is { "_id" : "3434", "0" : 1 }, however the for loop gives the output as _id , shouldn't there be two keys '_id' and '0' ? or am I doing something wrong here.
oh I found the solution
the "result" variable should be of type bson.M and then you can typecast accordingly as you go deep into the nesting structure.
Give a try with the following piece of code. This will help you fetching matching records from the Database using BSON Object.
Do not forget to rename the Database name and Collection name of your MongoDB in the below code. Also needs to change the query parameter accordingly.
Happy Coding...
package main
import (
"context"
"fmt"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// This is a user defined method to close resourses.
// This method closes mongoDB connection and cancel context.
func close(client *mongo.Client, ctx context.Context, cancel context.CancelFunc) {
defer cancel()
defer func() {
if err := client.Disconnect(ctx); err != nil {
panic(err)
}
}()
}
// This is a user defined method that returns
// a mongo.Client, context.Context,
// context.CancelFunc and error.
// mongo.Client will be used for further database
// operation. context.Context will be used set
// deadlines for process. context.CancelFunc will
// be used to cancel context and resourse
// assositated with it.
func connect(uri string) (*mongo.Client, context.Context, context.CancelFunc, error) {
ctx, cancel := context.WithTimeout(context.Background(),
30*time.Second)
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
return client, ctx, cancel, err
}
// query is user defined method used to query MongoDB,
// that accepts mongo.client,context, database name,
// collection name, a query and field.
// datbase name and collection name is of type
// string. query is of type interface.
// field is of type interface, which limts
// the field being returned.
// query method returns a cursor and error.
func query(client *mongo.Client, ctx context.Context, dataBase, col string, query, field interface{}) (result *mongo.Cursor, err error) {
// select database and collection.
collection := client.Database(dataBase).Collection(col)
// collection has an method Find,
// that returns a mongo.cursor
// based on query and field.
result, err = collection.Find(ctx, query,
options.Find().SetProjection(field))
return
}
func main() {
// Get Client, Context, CalcelFunc and err from connect method.
client, ctx, cancel, err := connect("mongodb://localhost:27017")
if err != nil {
panic(err)
}
// Free the resource when mainn dunction is returned
defer close(client, ctx, cancel)
// create a filter an option of type interface,
// that stores bjson objects.
var filter, option interface{}
// filter gets all document,
// with maths field greater that 70
filter = bson.D{
{"_id", bson.D{{"$eq", 3434}}},
}
// option remove id field from all documents
option = bson.D{{"_id", 0}}
// call the query method with client, context,
// database name, collection name, filter and option
// This method returns momngo.cursor and error if any.
cursor, err := query(client, ctx, "YourDataBaseName",
"YourCollectioName", filter, option)
// handle the errors.
if err != nil {
panic(err)
}
var results []bson.D
// to get bson object from cursor,
// returns error if any.
if err := cursor.All(ctx, &results); err != nil {
// handle the error
panic(err)
}
// printing the result of query.
fmt.Println("Query Reult")
for _, doc := range results {
fmt.Println(doc)
}
}