fetching data and upsert into database - postgresql

I want to get data from api and then doing query into specific tables using upsert, for example :
Repository
// get data from api
response, resErr := http.Get("look at json that want to fetch")
if resErr != nil {
fmt.Print(resErr.Error())
return resErr
}
// Read response from the API
defer response.Body.Close()
responseOdoo, err := ioutil.ReadAll(response.Body)
if err != nil {
log.Fatal(err)
}
//unmarshaling data to struct
var responseObject []models.OdooResponse
json.Unmarshal(responseOdoo, &responseObject)
here I want to perform an upsert to an existing table, but I don't know how to combine the data that has been retrieved and then fill it into the query
// get data from table and then upsert into column
qapi := `
insert into services_new
(code, service_id, service_name, service_category_id, service_category, service_type_id, service_type , price, request_type, status, uom_code, created_at , created_by, created_by_name, updated_at, updated_by, updated_by_name)
values
($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NOW(),$12,$13,NOW(),$12,$13)
on conflict (service_id, service_category_id, service_type_id)
do update set service_name =excluded.service_name, service_category =excluded.service_category, service_type =excluded.service_type, price =excluded.price, request_type =excluded.request_type, status =excluded.status, uom_code =exluded.uom_code, created_at =excluded.created_at, created_by =excluded.created_by, created_by_name =excluded.created_by_name;
`
_, sqlErr := tx.ExecContext(ctx, qapi, request.Code, request.ServiceID, request.ServiceName, request.ServiceCategoryID, request.ServiceCategory, request.ServiceTypeID, request.ServiceType, request.Price, request.RequestType, request.Status, request.UomCode, helper.Fullname, helper.Email)
// checking query upsert services_new
if sqlErr != nil {
tx.Rollback()
log.Println("sql Error on Repository Upsert on ODOO Services", sqlErr)
return sqlErr
}
txErr = tx.Commit()
if txErr != nil {
return txErr
}
return nil
until here, is the flow correct?
JSON That want to fetch
{
"service_id": 1129,
"service_name": "Adobe Illustrator",
"service_category_id": 28,
"service_category_name": "License Software",
"service_type_id": 25,
"service_type_name": "Software",
"create_date": "2020-03-09 03:47:44"
},
struct models
type UpsertFromOdooServices struct {
Code string `json:"code"`
ServiceID uint `json:"service_id"`
ServiceName string `json:"service_name"`
ServiceCategoryID uint `json:"service_category_id"`
ServiceCategory uint `json:"service_category"`
ServiceTypeID uint `json:"service_type_id"`
ServiceType string `json:"service_type"`
Price float64 `json:"price"`
RequestType string `json:"request_type"`
Status string `json:"status"`
UomCode string `json:"uom_code"`
CreatedAt time.Time `json:"created_at"`
CreatedBy string `json:"created_by"`
CreatedByName string `json:"created_by_name"`
UpdatedAt time.Time `json:"updated_at"`
UpdatedBy string `json:"updated_by"`
UpdatedByName string `json:"updated_by_name"`
}
type OdooResponse struct {
ServiceID uint `json:"service_id"`
ServiceName string `json:"service_name"`
ServiceCategoryID uint `json:"service_category_id"`
ServiceCatrgoryName string `json:"service_category_name"`
ServiceTypeID uint `json:"service_type_id"`
ServiceTypeName string `json:"service_type_name"`
Response []UpsertFromOdooServices
}

Related

Best practices to perform a JOIN with SQLX, Squirrel

I have this model.
// Incident is a security incident.
type Incident struct {
ID string `json:"id" bson:"_id"`
Title string `json:"title" bson:"title"`
Description string `json:"description" bson:"description"`
Issue *Issue `json:"issue" bson:"issue"`
CreatedAt time.Time `json:"created_at" bson:"created_at"`
ModifiedAt time.Time `json:"modified_at" bson:"modified_at"`
}
// Issue is a security issue.
type Issue struct {
ID string `json:"id" bson:"_id"`
Title string `json:"title" bson:"title"`
IncidentID string `json:"incident_id"`
Description string `json:"description" bson:"description"`
}
Each Incident has one Issue.
When I insert an Incident into Postgres, I only add the issue_id.
type incident struct {
ID string `db:"id"`
CustomerID string `db:"customer_id"`
InternalID string `db:"internal_id"`
Title string `db:"title"`
IssueID string `db:"issue_id"`
Description string `db:"description"`
CreatedAt time.Time `db:"created_at"`
ModifiedAt time.Time `db:"modified_at"`
}
func toIncident(model *models.Incident) (*incident, error) {
// Create the SQL
incident := &sqlIncident{
ID: model.ID,
CustomerID: model.CustomerID,
InternalID: model.InternalID,
Title: model.Title,
Description: model.Description,
IssueID: "",
CreatedAt: model.CreatedAt,
ModifiedAt: model.ModifiedAt,
}
// Add the issue ID
if model.Issue != nil {
incident.IssueID = model.Issue.ID
}
return incident, nil
}
I would like to be able to do the reverse operation with a JOIN on the Issue when I get() or list() incidents.
I am using "github.com/Masterminds/squirrel" and "github.com/jmoiron/sqlx".
This code below would be the code to get an Incident.
// Prepare query
query := squirrel.Select(*).
From(r.GetTableName()).
PlaceholderFormat(squirrel.Dollar).
Join("????????")
// Build the SQL query
q, args, err := query.ToSql()
if err != nil {
return fmt.Errorf("postgresql: unable to build query: %w", err)
}
// Get the session repo
session := r.GetSession().(*sqlx.DB)
// Prepare the statement
stmt, err := session.PreparexContext(ctx, q)
if err != nil {
return fmt.Errorf("postgresql: unable to prepapre query: %w", err)
}
// Do the query
err = stmt.SelectContext(ctx, result, args...)
if err == sql.ErrNoRows {
return repositories.ErrNoResult
} else if err != nil {
return fmt.Errorf("postgresql: unable to execute query: %w", err)
}
How can I perform this correctly please ?
I am feeling that I am doing this wrong with the issue_id field that seems to be useless, and that I should add the definition of the Issue in my incident SQL structure, something like this below.
type incident struct {
ID string `db:"id"`
CustomerID string `db:"customer_id"`
InternalID string `db:"internal_id"`
Title string `db:"title"`
Issue issue `db:"issue"`
Description string `db:"description"`
CreatedAt time.Time `db:"created_at"`
ModifiedAt time.Time `db:"modified_at"`
}
But I can't see the next steps.

How to load structs with multiple nested association

I am trying to load one to one and one to many associations for one of my object using GORM. I keep getting the following error when I am trying to run.
{
"errors": [
{
"message": "can't preload field UserProfiles for models.Renter",
"path": [
"users"
]
}
],
"data": null
}
Here is my code
type User struct {
BaseModelSoftDelete // We don't to actually delete the users, audit
Email string `gorm:"not null;unique_index"`
Password string
FirstName *string
LastName *string
Renter Renter `gorm:"auto_preload"`
Rentee Rentee `gorm:"auto_preload"`
UserProfiles []UserProfile `gorm:"association_autocreate:false;association_autoupdate:false"`
Roles []Role `gorm:"many2many:user_roles;association_autocreate:false;association_autoupdate:false"`
Permissions []Permission `gorm:"many2many:user_permissions;association_autocreate:false;association_autoupdate:false"`
}
// UserProfile saves all the related OAuth Profiles
type UserProfile struct {
BaseModelSeq
Email string `gorm:"unique_index:idx_email_provider_external_user_id"`
UserID uuid.UUID `gorm:"not null;index"`
User User `gorm:"association_autocreate:false;association_autoupdate:false"`
Provider string `gorm:"not null;index;unique_index:idx_email_provider_external_user_id;default:'DB'"` // DB means database or no ExternalUserID
ExternalUserID string `gorm:"not null;index;unique_index:idx_email_provider_external_user_id"` // User ID
Name string
FirstName string
LastName string
AvatarURL string `gorm:"size:1024"`
Description string `gorm:"size:1024"`
}
type Renter struct {
BaseModelSoftDelete // We don't to actually delete the users, audit
UserID uuid.UUID `gorm:"unique;not null;index"`
Verified bool
Properties []Property `gorm:"association_autocreate:false;association_autoupdate:false"`
Listings []Listing `gorm:"association_autocreate:false;association_autoupdate:false"`
}
type Rentee struct {
BaseModelSoftDelete // We don't to actually delete the users, audit
UserID uuid.UUID `gorm:"unique;not null;index"`
Verified bool
Bookings []Booking `gorm:"association_autocreate:false;association_autoupdate:false"`
}
then I call this function
func userList(r *queryResolver, id *string) (*gql.Users, error) {
entity := consts.GetTableName(consts.EntityNames.Users)
whereID := "id = ?"
record := &gql.Users{}
dbRecords := []*dbm.User{}
tx := r.ORM.DB.Begin().Preload(consts.EntityNames.UserProfiles)
defer tx.RollbackUnlessCommitted()
if id != nil {
tx = tx.Where(whereID, *id)
}
tx = tx.Find(&dbRecords).Count(&record.Count)
for _, dbRec := range dbRecords {
renter := dbm.Renter{}
tx = tx.Model(&dbRec).Related(&renter)
logger.Infof("%+v", dbRec.Renter)
// rentee := dbm.Rentee{}
// tx.Related(&rentee)
// logger.Info(rentee)
if rec, err := tf.DBUserToGQLUser(dbRec); err != nil {
logger.Errorfn(entity, err)
} else {
record.List = append(record.List, rec)
}
}
return record, tx.Error
}
If I get rid of tx = tx.Model(&dbRec).Related(&renter) the query runs, the profile object loads but my renter and rentee object doesn't have the data from database. And I notice it doesn't run the query SELECT * FROM "renters" WHERE "renters"."deleted_at" IS NULL AND (("user_id" = 'my-user-uuid'))
I also tried to this this:
tx = tx.Preload(consts.EntityNames.Renters).Preload(consts.EntityNames.Rentees).Preload(consts.EntityNames.UserProfiles).Find(&dbRecords).Count(&record.Count)
but get thise error: can't preload field Renters for models.User

GORM database colum with json data

I am trying to write an email service, where I want to store some data related to email into a Postgres DB using gorm. I have a field which needs to be stored as a JSON blob, which in request passed as a JSON object. When I am trying to migrate, it errors keep saying unsupported type map. When manually add the DB, then run gorm, it doesn't write the row to the database.
I have seen some example where it's using postgres.Json as field types, but I want the field loaded from the request as map[string]string.
// Email : Base with injected fields `UUID`, `CreatedAt`, `UpdatedAt`
type Email struct {
gorm.Model
Status string `grom:"type:varchar(32);not null"`
TemplateID string `grom:"type:varchar(256)"`
ToEmai string `grom:"type:varchar(256);not null"`
FromEmail string `grom:"type:varchar(256);not null"`
ToName string `grom:"type:varchar(256);not null"`
FromName string `grom:"type:varchar(256);not null"`
Subject string `grom:"type:varchar(256)"`
Message string `grom:"type:varchar"`
DynamicData *map[string]string `grom:"type:json"`
}
this is my model.
then I do a gin request:
// SendEmail : sends an email
func SendEmail(c *gin.Context) {
body, err := ioutil.ReadAll(c.Request.Body)
if err != nil {
log.Error("Error reading request body to get rate estimates")
}
var newEmail = models.Email{
Status: "PENDING",
}
jsonErr := json.Unmarshal(body, &newEmail)
if jsonErr != nil {
log.Error(jsonErr)
}
database.DB.Create(&newEmail)
defer c.Request.Body.Close()
err = newEmail.SendSendgridEmail()
if err != nil {
c.JSON(http.StatusBadRequest, err)
} else {
c.JSON(http.StatusOK, "Successfully sent email")
}
}
which then looks into this function
func (e Email) dynamicTemplateEmailBody() []byte {
newMail := mail.NewV3Mail()
emailFrom := mail.NewEmail(e.FromName, e.FromEmail)
newMail.SetFrom(emailFrom)
newMail.SetTemplateID(e.TemplateID)
p := mail.NewPersonalization()
tos := []*mail.Email{
mail.NewEmail(e.ToName, e.ToEmai),
}
p.AddTos(tos...)
if e.DynamicData != nil {
for key, value := range *e.DynamicData {
log.Infof("%s %s", key, value)
p.SetDynamicTemplateData(key, value)
}
}
newMail.AddPersonalizations(p)
return mail.GetRequestBody(newMail)
}
I would like to be able to run DB.AutoMigrate(&models.Email{}) and automatically migrate the objects, or and when I make a request to the endpoint, the row gets added to my email table.

How to access array values present in mongodb?

I want to access array values (access SpecCode) present in mongodb database from Go.
type MTopic struct {
SpecCodes []struct {
SpecCode string `json:speccode`
}
TopicCode string `json:topiccode`
TopicDesc string `json:topicdesc`
TopicBigDesc string `json:topicbigdesc`
TopicSource string `json:topicsource`
TopicSources []struct {
Topic string `json:topic`
}
CreatedBy string `json:createdby`
CreatedOn string `json:createdon`
UpdatedBy string `json:updatedby`
UpdatedOn string `json:updatedon`
}
using the following code:
func (m *TopicMaster) GetTopic(userdetails string) (list []MTopic, err error) {
collection := dbConnect7.Use("masterdata", "topic_master")
err = collection.Find(bson.M{"speccodes": userdetails}).All(&list)
return list, err
}
I have to get all values which have speccodes of userdetails in the collection topic_master. It's gin framework. This code is from models.
Just try like this
type MTopic struct {
SpecCodes []struct {
SpecCode string `json:"speccode"`
} `json:"speccodes"`
TopicCode string `json:"topiccode"`
TopicDesc string `json:"topicdesc"`
TopicBigDesc string `json:"topicbigdesc"`
TopicSource string `json:"topicsource"`
TopicSources []struct {
Topic string `json:"topic"`
}
CreatedBy string `json:"createdby"`
CreatedOn string `json:"createdon"`
UpdatedBy string `json:"updatedby"`
UpdatedOn string `json:"updatedon"`
}
and your function should be like this
func (m *TopicMaster) GetTopic(userdetails string) (list []MTopic, err error) {
collection := dbConnect7.Use("masterdata", "topic_master")
findQ := bson.M{"speccodes.speccode": userdetails}
list := make([]MTopic, 0)
if err = collection.Find(findQ).All(&list); err != nil {
err=errors.Wrapf(err, "failed to fetch topic info for user detail %s", userdetails)
return nil, err
}
return list, err
}

I'm trying to find a record by email

I have the following code.
Handler
func (authHandler *AuthHandler) Login(c *gin.Context) {
var user models.User
c.Bind(&user)
if &user == nil {
c.BindJSON(&user)
}
userObject, err := authHandler.userRepo.FindBy(
models.User{
Email: user.Email,
},
)
if err != nil {
c.JSON(401, gin.H{"_message": "User not found."})
return
}
passErr := bcrypt.CompareHashAndPassword([]byte(userObject.Password), []byte(user.Password))
if passErr != nil {
c.JSON(401, gin.H{"_message": "Password incorrect."})
return
}
token, err := services.CreateToken(userObject)
if err != nil {
c.JSON(401, gin.H{"_message": "Password incorrect."})
return
}
c.JSON(200, gin.H{"token": token, "user": gin.H{
"id": &userObject.ID,
"first_name": &userObject.FirstName,
"last_name": &userObject.LastName,
"email": &userObject.Email,
"avatar": &userObject.Avatar,
"location": &userObject.Location,
"bg_img": &userObject.BgImg,
}})
}
Model
// user.go
type User struct {
ID string `gorm:"primary_key:true"`
FirstName string `form:"first_name" json:"first_name,omitempty"`
LastName string `form:"last_name" json:"last_name,omitempty"`
Password string `form:"password" json:"password" bindind:"required"`
Email string `gorm:"type:varchar(110);unique_index" form:"email" json:"email,omitempty" binding:"required"`
Location string `form:"location" json:"location,omitempty"`
Avatar string `form:"avatar" json:"avatar,omitempty"`
BgImg string `form:"bg_img" json:"bg_img,omitempty"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
}
func (repo *UserRepo) FindBy(params User) (User, error) {
var user User
err := repo.db.First(&user, "email = ?", params.Email).Error
fmt.Println(err)
if err != nil {
return user, err
}
return user, nil
}
I've tried doing the find in several different ways with no luck. I get a 401 response every time and the fmt.Prinlnt(err) in the model shows an record not found error.
So it turns out I needed to set the DeletedAt field to a null value. Under the hood, GORM automatically checks for a DeletedAt value. I.e SELECT * FROM users WHERE email = 'someone#gmail.com' AND deleted_at IS NULL. However, my DeletedAt fields were automatically being set to a blank date, which technically isn't NULL.
I added a struct...
type NullTime struct {
time.Time
Valid bool
}
Then updated my model...
type User struct {
ID string `gorm:"primary_key:true"`
FirstName string `form:"first_name" json:"first_name,omitempty"`
LastName string `form:"last_name" json:"last_name,omitempty"`
Password string `form:"password" json:"password" bindind:"required"`
Email string `gorm:"type:varchar(110);unique_index" form:"email" json:"email,omitempty" binding:"required"`
Location string `form:"location" json:"location,omitempty"`
Avatar string `form:"avatar" json:"avatar,omitempty"`
BgImg string `form:"bg_img" json:"bg_img,omitempty"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt NullTime
}
The convention in 'gorm' is to include the base model in your structs.
// Base Model definition from gomodel.go
type Model struct {
ID uint `gorm:"primary_key"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt *time.Time
}
// Your struct
type User struct {
gorm.Model
Name string
}
As you can see, 'gorm' expected a pointer to handle the NULL value.