I am attempting to export a full SQL dump of one of our Cloud SQL Postgres instances to Google Cloud Storage so we can have more frequent backups using the google-api-go-client (https://github.com/googleapis/google-api-go-client)
Not matter what I configure, I keep getting this error: panic: googleapi: Error 403: The client is not authorized to make this request., notAuthorized from sqladminService.Instances.Export.
I have a service account configured with the following permissions:
Cloud SQL Admin
Storage Admin
Compute Storage Admin (for something else)
The bucket I am exporting to inherits permissions from the Storage Admin role.
Code:
./k8s/sql.go
package gcp
import (
"fmt"
"time"
"golang.org/x/net/context"
"golang.org/x/oauth2/google"
sqladmin "google.golang.org/api/sqladmin/v1beta4"
)
type SQLService interface {
Test(project string) error
}
type sqlService struct {
context context.Context
sqladminService *sqladmin.Service
}
func NewSQLService(serviceAccountJSON []byte) (SQLService, error) {
context := context.Background()
jwtCfg, err := google.JWTConfigFromJSON(serviceAccountJSON, sqladmin.SqlserviceAdminScope, sqladmin.CloudPlatformScope)
if err != nil {
return sqlService{}, err
}
httpClient := jwtCfg.Client(context)
sqladminService, err := sqladmin.New(httpClient)
if err != nil {
return sqlService{}, err
}
return sqlService{
context: context,
sqladminService: sqladminService,
}, nil
}
func (s sqlService) Test(project string) error {
instance := "REGION:INSTANCE_NAME
storageURI := fmt.Sprintf("gs://BUCKET/FILE-%s.sql.gz", time.Now().Format(time.RFC3339))
databases := []string{"DATABASE"}
req := &sqladmin.InstancesExportRequest{
ExportContext: &sqladmin.ExportContext{
Uri: storageURI,
Databases: databases,
},
}
_resp, err := s.sqladminService.Instances.Export(project, instance, req).Context(s.context).Do()
if err != nil {
return err
}
return nil
}
Test code:
func Test(cfg config.Config) {
sql, err := gcp.NewSQLService(cfg.GCPServiceAccountEncodedCreds)
if err != nil {
panic(err)
}
err = sql.Test(cfg.Project)
if err != nil {
panic(err)
}
}
Any help would be appreciated
The documentation for InstancesExport shows that the required parameters are the "projectId" and the "instanceId". You have declared instance as "REGION:INSTANCE_NAME" - but what you really want is "INSTANCE_NAME".
You aren't authorized to view that instance (because in this case, it doesn't exist).
Related
Hi i am trying to get cluster resource details using Application Default Credentials (ADC).
package main
import (
"context"
"encoding/base64"
"flag"
"fmt"
"log"
"google.golang.org/api/container/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // register GCP auth provider
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/clientcmd/api"
)
var fProjectId = flag.String("projectId", "", "specify a project id to examine")
func main() {
flag.Parse()
if *fProjectId == "" {
log.Fatal("must specific -projectId")
}
if err := run(context.Background(), *fProjectId); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, projectId string) error {
kubeConfig, err := getK8sClusterConfigs(ctx, projectId)
if err != nil {
return err
}
// Just list all the namespaces found in the project to test the API.
for clusterName := range kubeConfig.Clusters {
cfg, err := clientcmd.NewNonInteractiveClientConfig(*kubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig()
if err != nil {
return fmt.Errorf("failed to create Kubernetes configuration cluster=%s: %w", clusterName, err)
}
k8s, err := kubernetes.NewForConfig(cfg)
if err != nil {
return fmt.Errorf("failed to create Kubernetes client cluster=%s: %w", clusterName, err)
}
ns, err := k8s.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
if err != nil {
return fmt.Errorf("failed to list namespaces cluster=%s: %w", clusterName, err)
}
log.Printf("Namespaces found in cluster=%s", clusterName)
for _, item := range ns.Items {
log.Println(item.Name)
}
}
return nil
}
func getK8sClusterConfigs(ctx context.Context, projectId string) (*api.Config, error) {
svc, err := container.NewService(ctx)
if err != nil {
return nil, fmt.Errorf("container.NewService: %w", err)
}
// Basic config structure
ret := api.Config{
APIVersion: "v1",
Kind: "Config",
Clusters: map[string]*api.Cluster{}, // Clusters is a map of referencable names to cluster configs
AuthInfos: map[string]*api.AuthInfo{}, // AuthInfos is a map of referencable names to user configs
Contexts: map[string]*api.Context{}, // Contexts is a map of referencable names to context configs
}
// Ask Google for a list of all kube clusters in the given project.
resp, err := svc.Projects.Zones.Clusters.List(projectId, "-").Context(ctx).Do()
if err != nil {
return nil, fmt.Errorf("clusters list project=%s: %w", projectId, err)
}
for _, f := range resp.Clusters {
name := fmt.Sprintf("gke_%s_%s_%s", projectId, f.Zone, f.Name)
cert, err := base64.StdEncoding.DecodeString(f.MasterAuth.ClusterCaCertificate)
if err != nil {
return nil, fmt.Errorf("invalid certificate cluster=%s cert=%s: %w", name, f.MasterAuth.ClusterCaCertificate, err)
}
// example: gke_my-project_us-central1-b_cluster-1 => https://XX.XX.XX.XX
ret.Clusters[name] = &api.Cluster{
CertificateAuthorityData: cert,
Server: "https://" + f.Endpoint,
}
// Just reuse the context name as an auth name.
ret.Contexts[name] = &api.Context{
Cluster: name,
AuthInfo: name,
}
// GCP specific configation; use cloud platform scope.
ret.AuthInfos[name] = &api.AuthInfo{
AuthProvider: &api.AuthProviderConfig{
Name: "gcp",
Config: map[string]string{
"scopes": "https://www.googleapis.com/auth/cloud-platform",
},
},
}
}
return &ret, nil
}
It is giving me the error :
go run main.go -projectId=<Project-id>
2023/01/23 12:13:47 failed to create Kubernetes client cluster=<cluster-name>: The gcp auth plugin has been removed.
Please use the "gke-gcloud-auth-plugin" kubectl/client-go credential plugin instead.
See https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke for further details
exit status 1
It will be helpful for if you guys suggest me how to solve this issue.
Since k8s 1.26 you need to install GKE auth plugin separately. So, depending on your operating system install google-cloud-sdk-gke-gcloud-auth-plugin and it should work. It's pretty well described here:
https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke
After downgrading the dependencies it is working fine
go get k8s.io/client-go/tools/clientcmd#v0.25.5
go get k8s.io/cloud-provider-gcp/pkg/clientauthplugin/gcp#bb1acae5826dc877953d48
i really need some help to test the behaviour of my API, let me explain the context :
This API is written in go generated by swagger using the repository patern. In case of creating a new object i need to check if the name is not already exists in the database so i call repository's function to get Applications by name in the database (see below)
func (a *mongoApplicationRepository) GetApplicationByName(name string, ctx context.Context) (*models.Application, error) {
_, span := otel.Tracer("GetApplicationByName").Start(ctx, "GetApplicationByName")
defer span.End()
var application models.Application
err := a.mgConn.Collection("applications").
FindOne(context.TODO(), bson.M{"name": name}).Decode(&application)
if err != nil {
if err == mongo.ErrNoDocuments {
return &application, nil
}
return nil, err
}
return &application, nil
}
This function is called by the api handler that you can see bellow
type ApplicationCreateHandler struct {
API *operations.KubeesAPI
repo repo.ApplicationRepository
statsRepo repo.StatsRepository
}
func NewApplicationCreateHandler(API *operations.KubeesAPI, repo repo.ApplicationRepository, statsRepo repo.StatsRepository) ApplicationCreateHandler {
return ApplicationCreateHandler{
API: API,
repo: repo,
statsRepo: statsRepo,
}
}
// Handle is the HTTP handler for application creation
func (h *ApplicationCreateHandler) Handle(params application.AppCreateParams, principal *models.Principal) middleware.Responder {
traceName := "application-create"
ctx, span := otel.GetTracerProvider().Tracer(traceName).Start(context.TODO(), traceName)
defer span.End()
if params.Data == nil {
err := "unable to validate input"
logger().Errorln(err)
return application.NewAppCreateBadRequest().WithPayload(&models.APIResponse{
Msg: &err,
})
}
if err := params.Data.Validate(h.API.Formats()); err != nil {
err := fmt.Sprintf("unable to validate input: %v", err)
logger().Errorln(err)
return application.NewAppCreateBadRequest().WithPayload(&models.APIResponse{
Msg: &err,
})
}
app := models.Application{
ID: uuid.NewString(),
Name: params.Data.Name,
Selector: params.Data.Selector,
}
c, err := h.repo.GetApplicationByName(*params.Data.Name, ctx)
if err != nil {
err := fmt.Sprintf("Unable to get Application: %v", err)
logger().Errorln(err)
return application.NewAppCreateInternalServerError().WithPayload(&models.APIResponse{
Msg: &err,
})
}
if c.ID != "" {
r := "name already taken"
logger().Errorln(r)
return application.NewAppCreateInternalServerError().WithPayload(&models.APIResponse{
Msg: &r,
})
}
appDomain, err := app.ToDomainModel()
if err != nil {
err := fmt.Sprintf("Unable to convert application to domain model: %v", err)
logger().Errorln(err)
return application.NewAppCreateInternalServerError().WithPayload(&models.APIResponse{
Msg: &err,
})
}
err = h.repo.CreateApplication(appDomain, ctx)
if err != nil {
err := fmt.Sprintf("Unable to create Application: %v", err)
logger().Errorln(err)
return application.NewAppCreateInternalServerError().WithPayload(&models.APIResponse{
Msg: &err,
})
}
if err := h.statsRepo.UpdateApplicationsHistory(ctx); err != nil {
err := fmt.Sprintf("unable to update applications history: %v", err)
logger().Errorln(err)
return application.NewAppCreateInternalServerError().WithPayload(&models.APIResponse{
Msg: &err,
})
}
return application.NewAppCreateCreated()
}
I just want to check this behaviour and test in case we have already an application called the same as the new created one, which would raise an error and not in opposite case. But i don't know what to mock because the repository have some external dependencies
I would strongly suggest to make sure your ApplicationCreateHandler does only rely on interfaces. When that's the case you can mock the interfaces by writing a mock class, or generating on. You can test the ApplicationCreateHandlerby creating a new instance, and passing the mocked dependencies to it. Then execute theHandle()` method and ensure all dependencies are called.
I would strongly recommend generting your mock-classes. When you write your mocks yourself it's possible to introduce bugs in your test code. You wouldn't want that, right?
One great example of a mocking library is mockgen. You can easily instal it by following their github-readme.
Steps
Create interfaces for your dependencies.
It is best define the interface on the side that relies on the interface. So in you case that would be in the ApplicationCreateHandler file. The interfaces should reflect the needs of the handler.
type (
ApplicationRepo interface {
GetApplicationByName(name string, ctx context.Context)
CreateApplication(domain Domain, ctx context.Context)
}
// other dependencies here...
)
Writing a unit test
Create a new unit test
// youfile_test.go
func TestApplicationHandler(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockRepo := NewMockRepository(ctrl)
mockRepo.EXPECT().
Handle(gomock.any(), gomock.any()). // name, ctx
Return(nil, nil). // c, err
Once()
// other mocks...
// mockApi
// mockStatsRepo
h := NewApplicationHandler(mockApi, mockStats, mockRepo)
h.Handle(/*your test input*/)
// do checks
}
The test will throw an error if the mockRepo wasn't called exactly once.
I am expected to have 80% test coverage even for pushing the basic project structure. I am a bit confused how do I write unit tests for the following code to Connect to postgres db and ping postgres for health check. Can someone help me please.
var postgres *sql.DB
// ConnectToPostgres func to connect to postgres
func ConnectToPostgres(connStr string) (*sql.DB, error) {
db, err := sql.Open("postgres", connStr)
if err != nil {
log.Println("postgres-client ", err)
return nil, err
}
postgres = db
return db, nil
}
// PostgresHealthCheck to ping database and check for errors
func PostgresHealthCheck() error {
if err := postgres.Ping(); err != nil {
return err
}
return nil
}
type PostgresRepo struct {
db *sql.DB
}
// NewPostgresRepo constructor
func NewPostgresRepo(database *sql.DB) *PostgresRepo {
return &PostgresRepo{
db: database,
}
}
You need to use this : https://github.com/DATA-DOG/go-sqlmock
Its very easy to use. Here is an example where a controller is getting tested using a mocked SQL :
Implementation
func (up UserProvider) GetUsers() ([]models.User, error) {
var users = make([]models.User, 0, 10)
rows, err := up.DatabaseProvider.Query("SELECT firstname, lastname, email, age FROM Users;")
if err != nil {
return nil, err
}
for rows.Next() {
var u models.User = models.User{}
err := rows.Scan(&u.Name, &u.Lastname, &u.Email, &u.Age)
if err != nil {
return nil, err
}
users = append(users, u)
}
if err := rows.Err(); err != nil {
return nil, err
}
return users, nil
}
Test
func TestGetUsersOk(t *testing.T) {
db, mock := NewMock()
mock.ExpectQuery("SELECT firstname, lastname, email, age FROM Users;").
WillReturnRows(sqlmock.NewRows([]string{"firstname", "lastname", "email", "age"}).
AddRow("pepe", "guerra", "pepe#gmail.com", 34))
subject := UserProvider{
DatabaseProvider: repositories.NewMockDBProvider(db, nil),
}
resp, err := subject.GetUsers()
assert.Nil(t, err)
assert.NotNil(t, resp)
assert.Equal(t, 1, len(resp))
}
func NewMock() (*sql.DB, sqlmock.Sqlmock) {
db, mock, err := sqlmock.New()
if err != nil {
log.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
}
return db, mock
}
I find that writing tests against a live database makes for more high quality tests. The challenge with Postgres is that there's no good in-memory fake that you can substitute in.
What I came up with is standing up the postgres Docker container and creating temporary databases in there. The PostgresContainer type in the github.com/bitcomplete/sqltestutil package does exactly this:
# Postgres version is "12"
pg, _ := sqltestutil.StartPostgresContainer(context.Background(), "12")
defer pg.Shutdown(ctx)
db, err := sql.Open("postgres", pg.ConnectionString())
// ... execute SQL
Per the docs, it's a good idea to set up your tests so that the container is only started once, as it can take a few seconds to start up (more if the image needs to be downloaded). It suggests some approaches for mitigating that problem.
I have mongo capped collection and a simple API, written on Go. I built and run it. When I try to sent Get request or simply go localhost:8000/logger in browser - my process closes. Debug shows this happens, while executing "find" in collection. It produces error "client is disconnected". Collection has 1 document, and debug shows it is connected with my helper.
Go version 1.13
My code:
func main() {
r := mux.NewRouter()
r.HandleFunc("/logger", getDocs).Methods("GET")
r.HandleFunc("/logger", createDoc).Methods("POST")
log.Fatal(http.ListenAndServe("localhost:8000", r))
}
func getDocs(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
var docs []models.Logger
//Connection mongoDB with helper class
collection := helper.ConnectDB()
cur, err := collection.Find(context.TODO(), bson.M{})
if err != nil {
helper.GetError(err, w)
return
}
defer cur.Close(context.TODO())
for cur.Next(context.TODO()) {
var doc models.Logger
err := cur.Decode(&doc)
if err != nil {
log.Fatal(err)
}
docs = append(docs, doc)
}
if err := cur.Err(); err != nil {
log.Fatal(err)
}
json.NewEncoder(w).Encode(docs)
}
func ConnectDB() *mongo.Collection {
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://127.0.0.1:27017"))
if err != nil {
log.Fatal(err)
}
fmt.Println("Connected to MongoDB!")
logCollection := client.Database("local").Collection("loggerCollection")
return logCollection
}
According to the documentation, the call to mongo.NewClient doesn't ensure that you can connect the Mongo server. You should first call mongo.Client.Ping() to verify if you can connect to the database or not.
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://127.0.0.1:27017"))
if err != nil {
log.Fatal(err)
}
if err := client.Ping(context.TODO(), readpref.Primary()); err != nil {
// Can't connect to Mongo server
log.Fatal(err)
}
There could be several reasons behind failing to connect, the most obvious one is incorrect setup of ports. Is your mongodb server up and listening on port 27017? Is there any change you're running mongodb with Docker and it's not forwarding to the correct port?
I faced similar issue , read #Jay answer it definitely helped , as I checked my MongoDB was running using "MongoDB Compass" , then I changed the location of my insert statement , previously I was calling before the call of "context.WithTimeout". Below is working code.
package main
import (
"context"
"log"
"time"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type Book struct {
Name string `json:"name,omitempty"`
PublisherID string `json:"publisherid,omitempty"`
Cost string `json:"cost,omitempty"`
StartTime string `json:"starttime,omitempty"`
EndTime string `json:"endtime,omitempty"`
}
func main() {
client, err := mongo.NewClient(options.Client().ApplyURI("mongodb://localhost:27017"))
if err != nil {
log.Fatal(err)
}
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
err = client.Connect(ctx)
if err != nil {
log.Fatal(err)
}
defer client.Disconnect(ctx)
testCollection := client.Database("BooksCollection").Collection("BooksRead")
inserRes, err := testCollection.InsertOne(context.TODO(), Book{Name: "Harry Potter", PublisherID: "IBN123", Cost: "1232", StartTime: "2013-10-01T01:11:18.965Z", EndTime: "2013-10-01T01:11:18.965Z"})
log.Println("InsertResponse : ", inserRes)
log.Println("Error : ", err)
}
I can see document inserted in console as well as in "MongoDB Comapass."
In heiper function "ConnectDB" after "NewClient" I must use "client.Connect(context.TODO())"
before any other use of client
Context
Currently I have a REST API that manages customer's data in a db. I'm using the following stack:
Go 1.13
github.com/jinzhu/gorm v1.9.1
Postgres 11
I have the following connection settings.
// NewConnection ...
func NewConnection() (*gorm.DB, error) {
config := getConfig()
connStr := "host=xx.xx.xx port=5432 user=chavista-hatter dbname=my-db password=abc sslmode=verify-ca sslrootcert=/path/to/rcert sslcert=/path/to/cert sslkey=/path/to/key connect_timeout=0"
db, err := gorm.Open("postgres", conn)
if err != nil {
return nil, err
}
db.DB().SetMaxOpenConns(25)
db.DB().SetMaxIdleConns(25)
db.DB().SetConnMaxLifetime(5 * time.Minute)
db.SingularTable(true)
if config.LogQueries {
db = db.Debug()
}
return db, nil
}
I get a connection in the main class and inject that connection into a repository class that executes the queries through Gorm (ORM)
Main class
db, err := database.NewConnection()
if err != nil {
panic(fmt.Sprintf("failed to connect database --> %v", err))
}
fmt.Println("database connection established successfully")
defer db.Close()
customerRepo := customer.NewRepository(db)
Repository class
type repository struct {
db *gorm.DB
}
//NewRepository
func NewRepository(db *gorm.DB) Repository {
return &repository{
db: db,
}
}
func (r *repository) Register(customer *models.Customer) (string, error) {
err := r.db.Create(&customer).Error
if err != nil {
return "", err
}
return customer.key, nil
}
Problem
I sending over 500k request (INSERTS) to my db which have 512 connections available and after a few minutes the following error starts to come up repeatedly in postgres log:
unexpected EOF on client connection with an open transaction
could not receive data from client: Connection reset by peer
Any help?
How are you using GetConnection in your code? It's creating a new connection pool every time it's called - ideally you'd only want to call it once, and pass that single connection around wherever it's used.
I would try changing it to this:
var db *gorm.DB
func NewConnection() (*gorm.DB, error) {
if db != nil {
return db, nil
}
config := getConfig()
connStr := "host=xx.xx.xx port=5432 user=chavista-hatter dbname=my-db password=abc sslmode=verify-ca sslrootcert=/path/to/rcert sslcert=/path/to/cert sslkey=/path/to/key connect_timeout=0"
var err error
db, err = gorm.Open("postgres", conn)
if err != nil {
return nil, err
}
db.DB().SetMaxOpenConns(25)
db.DB().SetMaxIdleConns(25)
db.DB().SetConnMaxLifetime(5 * time.Minute)
db.SingularTable(true)
if config.LogQueries {
db = db.Debug()
}
return db, nil
}
and see if it solves the issue.