Post Request with PostgreSQL and json-api returns an empty body - postgresql

After a POST request, i was expecting to have a last inserted record marshalled into json, but instead returns an empty body. What am i not doing well?
package models
import (
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"net/http"
"strconv"
"github.com/go-chi/chi"
"github.com/google/jsonapi"
"github.com/thedevsaddam/renderer"
"github.com/xo/dburl"
)
var rnd = renderer.New()
var flagVerbose = flag.Bool("v", false, "verbose")
var FlagURL = flag.String("url", "postgres://postgres:#127.0.0.1/sweb", "url")
// Page represents a row from 'public.pages'.
type Page struct {
Tag string `jsonapi:"attr,tag"` // tag
Body string `jsonapi:"attr,body"` // body
Slug string `jsonapi:"attr,slug"` // slug
Title string `jsonapi:"attr,title"` // title
ID int `jsonapi:"primary,pages"` // id
Link string `jsonapi:"attr,link"` // link
// xo fields
_exists, _deleted bool
}
func (page Page) JSONAPILinks() *jsonapi.Links {
return &jsonapi.Links{
"self": fmt.Sprintf("https://%d", page.ID),
}
}
I believe this is the culprit. After inserting a record, it should return the last inserted record as specified.
func (p *Page) PInsert(db XODB) (*Page, error) {
var err error
// if already exist, bail
if p._exists {
return p, errors.New("insert failed: already exists")
}
// sql insert query, primary key provided by sequence
const sqlstr = `INSERT INTO public.pages (` +
`tag, body, slug, title` +
`) VALUES (` +
`$1, $2, $3, $4` +
`) RETURNING id, tag, body, title`
// run query
XOLog(sqlstr, p.Tag, p.Body, p.Slug, p.Title)
err = db.QueryRow(sqlstr, p.Tag, p.Body, p.Slug, p.Title).Scan(&p.ID, &p.Tag, &p.Body, &p.Title)
if err != nil {
return p, err
}
// set existence
p._exists = true
return p, nil
}
Update updates the Page in the database and return last inserted records.
The same should apply for the Update function
func (p *Page) Update(db XODB) (*Page, error) {
var err error
// if doesn't exist, bail
if !p._exists {
return p, errors.New("update failed: does not exist")
}
// if deleted, bail
if p._deleted {
return p, errors.New("update failed: marked for deletion")
}
// sql query
const sqlstr = `UPDATE public.pages SET (` +
`tag, body, slug, title` +
`) = ( ` +
`$1, $2, $3, $4` +
`) WHERE id = $5`
// run query
XOLog(sqlstr, p.Tag, p.Body, p.Slug, p.Title, p.ID)
_, err = db.Exec(sqlstr, p.Tag, p.Body, p.Slug, p.Title, p.ID)
return p, err
}
func (p *Page) PSave(db XODB) (*Page, error) {
if p.Exists() {
return p.Update(db)
}
return p.PInsert(db)
}
func NewPage(w http.ResponseWriter, r *http.Request) {
db, err := dburl.Open(*FlagURL)
defer db.Close()
if err != nil {
log.Fatal(err)
}
var page Page
//page := new(Page)
if err := jsonapi.UnmarshalPayload(r.Body, &page); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
p, err := page.PSave(db)
if err != nil {
fmt.Println(err)
if err := jsonapi.MarshalPayload(w, p); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
fmt.Println(err)
}
}
w.Header().Set("Content-Type", jsonapi.MediaType)
w.WriteHeader(http.StatusCreated)
}
This is the last function i believe the issue is happening from. the last inserted record supposed to be marshalled into json.

Your last section of code contains a number of mistakes. The relevant section (without the useless and obfuscating Printlns) is:
p, err := page.PSave(db)
if err != nil {
if err := jsonapi.MarshalPayload(w, p); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
w.Header().Set("Content-Type", jsonapi.MediaType)
w.WriteHeader(http.StatusCreated)
And the primary mistake is that json.MarshalPayload is only called when err != nil. In other words, you only serialize the page if you failed to save it.
The secondary mistake is that jsonapi.MarshalPayload will call Write on the http.ResponseWriter. This turns all subsequent calls to Header().Set and WriteHeader into no-ops.
More correct code would look like this.
// 1. Save the page in the database, bail on error
p, err := page.PSave(db)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// 2. Marshal the page into an intermediate buffer, bail on error
var buf bytes.Buffer
if err := jsonapi.MarshalPayload(&buf, p); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// 3. Write the entire response; failures to write the intermediate buffer
// cannot be communicated over HTTP
w.Header().Set("Content-Type", jsonapi.MediaType)
w.WriteHeader(http.StatusCreated)
if _, err := buf.WriteTo(w); err != nil {
log.Printf("failed to write response: %v", err)
return
}

Related

Receiving error(*errors.errorString) *{s: "pq: unexpected DataRow in simple query execution"}

The error
(*errors.errorString) *{s: "pq: unexpected DataRow in simple query execution"}
appears after the line with the commentary. Didn't find any solution online. Since stackoverflow asks for more details, this is an update query that is supposed to update a todo and a list of subtasks in the database. The exact error is in the question topic. I post the complete code for the function that returns the error.
func (t *TodoTable) UpdateTodo(ctx context.Context, todo *Todo, t_id int) error {
tx, err := t.sqlxdb.BeginTxx(ctx, &sql.TxOptions{})
if err != nil {
return err
}
rollback_err := func(err error) error {
if err2 := tx.Rollback(); err2 != nil {
return fmt.Errorf("%v; %v", err, err2)
}
return err
}
row := tx.QueryRowxContext(ctx, "UPDATE todos SET todo_name=$1, deadline=$2, updated_at=$3 WHERE todo_id=$4 returning todo_id", todo.TodoName, todo.Deadline, todo.UpdatedAt, t_id)
if row.Err() != nil {
return rollback_err(err)
}
var subs_ids []int
// Getting subs ids from database
query := fmt.Sprintf("SELECT sub_id FROM subs WHERE todo_id=%d", t_id)
// THE ERROR COMES AFTER EXECUTING THE LINE BELOW
rows, err := tx.Query(query)
if err != nil {
rollback_err(err)
}
if rows != nil {
for rows.Next() {
var sub_id int
err = rows.Scan(&sub_id)
if err != nil {
rollback_err(err)
}
subs_ids = append(subs_ids, sub_id)
}
if err := tx.Commit(); err != nil {
return rollback_err(err)
}
}
// Updating subs
for i, sub := range todo.Subs {
_, err = tx.ExecContext(ctx, fmt.Sprintf("UPDATE subs SET sub_name='%s' WHERE sub_id=%d", sub.Sub_name, subs_ids[i]))
if err != nil {
return rollback_err(err)
}
}
return nil
}

Efficient transaction wrapper function with mongodb go driver

I'm currently integrating the transaction logic into my go+mongodb api.
I already created this example endpoint. It allows you to retrieve a user document and send it back to the client with json encoding.
func GetUser(w http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["user-id"]
objectId, _ := primitive.ObjectIDFromHex(id)
user, err := UserById(objectId)
if err != nil {
// ...
}
// do some stuff with the user, whatever
// eventually send it back
json.NewEncoder(w).Encode(user)
}
func UserById(id primitive.ObjectID) (models.StoredUser, error) {
session, err := mongoClient.StartSession()
if err != nil {
return models.StoredUser{}, fmt.Errorf("failed starting session for transaction | %s", err.Error())
}
defer session.EndSession(context.TODO())
callback := func(ctx mongo.SessionContext) (any, error) {
res := usersCollection.FindOne(
ctx,
bson.M{
"_id": id,
},
)
if res.Err() != nil {
return models.StoredUser{}, fmt.Errorf("failed querying db | %s", res.Err())
}
return res, nil
}
result, err := session.WithTransaction(context.TODO(), callback)
if err != nil {
return models.StoredUser{}, fmt.Errorf("failed executing transaction | %s", err.Error())
}
asserted := result.(*mongo.SingleResult)
var ret models.StoredUser
if err := asserted.Decode(&ret); err != nil {
return models.StoredUser{}, fmt.Errorf("failed parsing user data in struct | %s", err.Error())
}
return ret, nil
}
Here are the big steps :
Parse the request content to get the user id
Create a session to perform the transaction
Declare the callback function using the id argument
Call the callback function from a transaction
Get back the *mongo.SingleResult as an interface{} and parsing it back to its original type
Decode the bson document contained in the *mongo.SingleResult to put it in the return struct
This function works, but is very verbose. The code is very duplicated.
I wonder if there is a way of not repeating the same code for each function I wanna make. My previous wrapper function attempts didn't lead to anything, as I actually need the variables where they are now at each call.
Still, the steps 2 and 5 especially seem very redundant and inefficient.
Anyone got any idea ?
Ok I found the following :
func Transaction(callback func(ctx mongo.SessionContext) (any, error)) (any, error) {
session, err := mongoClient.StartSession()
if err != nil {
return nil, fmt.Errorf("failed creating session | %s", err.Error())
}
defer session.EndSession(context.TODO())
res, err := session.WithTransaction(ctx, callback)
if err != nil {
return nil, fmt.Errorf("failed executing transaction | %s", err.Error())
}
return res, nil
}
Let's say i want then to fetch user objects :
func GetUsers() ([]models.User, error) {
callback := func(ctx mongo.SessionContext) (any, error) {
res, err := usersCollection.Find(ctx, bson.M{})
if err != nil {
return nil, fmt.Errorf("failed querying users collection | %s", err.Error())
}
var ret []models.User
if err := res.All(context.TODO(), &ret); err != nil {
return nil, fmt.Errorf("failed parsing results in struct | %s", err.Error())
}
return ret, nil
}
result, err := Transaction(callback)
if err != nil {
return []models.User{}, fmt.Errorf("failed executing transaction | %s", err.Error())
}
classes, _ := result.([]models.StoredClass)
return classes, nil
}

Is there a way with GRPC to notify the stream of CRUD operations to give realtime updates to the client?

I am new to GRPC and I am trying to implement a basic CRUD + listing. I use unary rpc's for the CRUD and a server stream for the listing. What I would like to do however is update the client whenever someone changes a record in the database that you are listing.
So for example user A is listing 10 companies. And user B is updating one of those companies. I want user A's client to be updated once the update rpc is called.
This is what I have for now
func RegisterCompanyServer(l hclog.Logger, gs *grpc.Server) {
r := postgres.NewPostgresCompanyRepository()
cs := NewCompanyServer(l, r)
pb.RegisterCompanyServiceServer(gs, cs)
}
type CompanyServer struct {
logger hclog.Logger
repo repo.CompanyRepository
pb.UnimplementedCompanyServiceServer
}
func NewCompanyServer(l hclog.Logger, r repo.CompanyRepository) *CompanyServer {
return &CompanyServer{
logger: l,
repo: r,
}
}
func (c *CompanyServer) ListCompany(req *pb.CompanyListRequest, stream pb.CompanyService_ListCompanyServer) error {
//Somehow listen to CreateCompany() and update the client
companies, err := c.repo.List(req.Query)
if err != nil {
return err
}
for _, c := range companies {
bytes, err := json.Marshal(c)
if err != nil {
return err
}
out := &pb.Company{}
if err = jsonEnc.Unmarshal(bytes, out); err != nil {
return err
}
res := &pb.CompanyListResponse{
Company: out,
}
err = stream.Send(res)
if err != nil {
return err
}
}
return nil
}
func (c *CompanyServer) CreateCompany(context context.Context, req *pb.CompanyCreateRequest) (*pb.CompanyCreateResponse, error) {
input := req.GetCompany()
if input == nil {
return nil, errors.New("Parsing Error")
}
bytes, err := jsonEnc.Marshal(input)
if err != nil {
return nil, err
}
company := &myCompany.Company{}
if err = json.Unmarshal(bytes, company); err != nil {
return nil, err
}
result, err := c.repo.Create(company)
if err != nil {
return nil, err
}
res := &pb.CompanyCreateResponse{
Id: result,
}
//Somehow notify the stream that a company was created
return res, nil
}
Is this even feasable with GRPC? What techniques are out there to do this? I am currently working with a postgresql database.

Need the ID of the current row inserted within a transaction

Within a transaction I'm inserting a row.
How can I access and return the ID of the inserted row.
As you can see in the code below(See under comment // Return last Inserted ID.) I tried to use the LastInsertedId() function, but it gives me an error back.
Btw, I'm using Postgres.
What am I missing here?
Thx!
/**
* Creates order and returns its ID.
*/
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
ret, err := stmt.Exec(order.CustomerID)
// Rollback.
if err != nil {
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Return last Inserted ID.
lastID, err := ret.LastInsertId()
if err != nil {
orderID = 0
} else {
orderID = int(lastID)
}
// Commit.
tx.Commit()
return orderID
} // createOrder
Here is a working solution for now, further improvement is welcomed.
/**
* Creates order and returns its ID.
*/
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
_, err = stmt.Exec(order.CustomerID)
// Rollback.
if err != nil {
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Return last Inserted ID.
//lastID, err := ret.LastInsertId()
err = stmt.QueryRow(order.CustomerID).Scan(&orderID)
if err != nil {
orderID = 0
}
// Commit.
tx.Commit()
return orderID
} // createOrder
This happens because the postgresql driver you are using for go doesn't supports the LastInsertedId() function. You didn't say which driver you are using but I have had this issue working with github.com/lib/pq.
The answer to this is to use QueryRow insted of Exec in your original example. Just make sure you are using RETURNING ID on your query and treat it as if it was a select.
Here is an example (I didn't test this and I might be missing something but you get the idea):
func createOrder(w http.ResponseWriter, r *http.Request) (orderID int) {
// Begin.
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// Db query.
sqlQuery := `INSERT INTO ORDER_CUSTOMER
(CUSTOMER_ID)
VALUES ($1)
RETURNING ID`
// Prepare.
stmt, err := tx.Prepare(sqlQuery)
if err != nil {
log.Fatal(err)
return
}
// Defer Close.
defer stmt.Close()
customerEmail := validateSession(r)
ID := getIDFromCustomer(customerEmail)
order := order{}
order.CustomerID = ID
// Exec.
var orderID int // Or whatever type you are using
err := stmt.QueryRow(order.CustomerID).Scan(&orderID)
// Rollback.
if err != nil {
//if something goes wrong set the orderID to 0 as in your original code
orderID = 0
tx.Rollback()
e := errors.New(err.Error())
msg.Warning = e.Error()
tpl.ExecuteTemplate(w, "menu.gohtml", msg)
return
}
// Commit.
tx.Commit()
return orderID
} // createOrder

Google Analytics resumable upload failing

I'm attempting to do a resumable upload as described here.
When I perform the upload, I receive a response status of 400 Bad Request, and a response body of:
{
"error": {
"errors": [
{
"domain": "global",
"reason": "badContent",
"message": "Unsupported content with type: application/octet-stream"
}
],
"code": 400,
"message": "Unsupported content with type: application/octet-stream"
}
}
The script I'm using to perform the upload is in Go, here:
package main
import(
"bytes"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
func main(){
// config
accessToken := "a valid token"
acctId := "a valid account id"
webPropertyId := "a valid web property id"
customDataSourceId := "a valid custom data source id"
path := "/a/path/to/a/validly/formatted/file.csv"
params := map[string]string{
"title": "My Document",
"author": "Becca Petrin",
"description": "Riveting stuff",
}
paramName := "file"
url := fmt.Sprintf("https://www.googleapis.com/upload/analytics/v3/management/accounts/%s/webproperties/%s/customDataSources/%s/uploads?uploadType=resumable", acctId, webPropertyId, customDataSourceId)
// create the body
file, err := os.Open(path)
if err != nil {
fmt.Println("Err opening file:", err.Error())
return
}
defer file.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(paramName, filepath.Base(path))
if err != nil {
fmt.Println("Err creating form file:", err.Error())
return
}
_, err = io.Copy(part, file)
if err != nil {
fmt.Println("Err copying:", err.Error())
return
}
for k, v := range params {
_ = writer.WriteField(k, v)
}
if err := writer.Close(); err != nil {
fmt.Println("Err closing writer:", err.Error())
return
}
req, err := http.NewRequest("POST", url, body)
if err != nil {
fmt.Printf("Err creating request:", err.Error())
return
}
// add authorization
req.Header.Set("Authorization", "Bearer "+accessToken)
// add headers
// no multipart headers work, and "application/octet-stream"" doesn't work
// uncommenting and using "text/plain" results in a 200 without the expected response body
//req.Header.Add("Content-Type", "text/plain")
// execute request
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println("Err doing request:", err.Error())
return
}
fmt.Println("Response status:", resp.Status)
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println("Err reading resp body:", err.Error())
return
}
fmt.Printf("Response body: %s", b)
}
As noted in the comments, I receive the unsupported content type response if I don't include a Content-Type header. I also receive it if I use any multipart content types, or application/octet-stream. If I use text/plain, I receive a 200, but I don't receive the expected body.
What am I doing wrong? Thanks in advance!
I decided to take an alternate approach to this by trying through the Google Go client library, and I got it working with this snippet:
package main
import (
"errors"
"fmt"
"os"
"time"
google "google.golang.org/api/analytics/v3"
"golang.org/x/oauth2"
)
var (
accessToken = "a valid token"
acctId = "a valid account ID"
webPropertyId = "a valid web property ID"
customDataSourceId = "a valid custom data source ID"
filePath = "/path/to/file.csv"
)
func main(){
ctx1 := NewCallContext()
tokenSource := TokenSource{}
httpClient := oauth2.NewClient(ctx1, &tokenSource)
service, err := google.New(httpClient)
if err != nil {
fmt.Println("Err making client:", err.Error())
return
}
r, err := NewFileReader(filePath)
if err != nil {
fmt.Println("Err making reader:", err.Error())
return
}
ctx2 := NewCallContext()
call := service.Management.Uploads.UploadData(acctId, webPropertyId, customDataSourceId)
call = call.ResumableMedia(ctx2, r, 10, "application/octet-stream")
upload, err := call.Do()
if err != nil {
fmt.Println("Err doing call: %v", err)
return
}
fmt.Printf("%s", upload)
}
// http://golang.org/pkg/io/#ReaderAt
func NewFileReader(pathToFile string) (*FileReader, error) {
f, err := os.Open(pathToFile)
if err != nil {
return nil, err
}
return &FileReader{f}, nil
}
type FileReader struct {
f *os.File
}
func (s *FileReader) ReadAt(p []byte, off int64) (int, error) {
return s.f.ReadAt(p, off)
}
// https://godoc.org/golang.org/x/net/context#Context
func NewCallContext() *CallContext {
c := make(<-chan struct{})
return &CallContext{c}
}
type CallContext struct {
doneChan <-chan struct{}
}
func (s *CallContext) Deadline() (time.Time, bool) {
return time.Now().Add(time.Duration(10) * time.Second), true
}
func (s *CallContext) Done() <-chan struct{} {
return s.doneChan
}
func (s *CallContext) Err() error {
select {
case <- s.doneChan:
return errors.New("Done")
default:
}
return nil
}
func (s *CallContext) Value(key interface{}) interface{} {
return nil
}
// satisfies the oauth2 tokensource interface
type TokenSource struct {}
func (t *TokenSource) Token() (*oauth2.Token, error) {
return &oauth2.Token{AccessToken:accessToken}, nil
}