Consuming Paginated REST API - rest

Question

I am trying to consume an API that on each successful request has a json key like the following:
{
"hasNextPage": true,
"endCursor": "some_guid_present_here"
}
The way the API works (I have used it many times in Python but am trying with Go for separate use case) is to basically check if there is a next page, then use the appropriate cursor to continue to paginate.
However, every time I use this structure, it will sometimes keep looping even though the response.Paginations.HasNextPage will be false.
I am trying to understand if it is my structure of consuming a paginated API like this that is causing this or else.
Also, I have say 4-5 requests to start off with, which I sent separately via goroutines. I am not sure if this is causing an issue but I've attached that as well in etl.go.
The main request making structs are in api.go.
I've confirmed that I do receive responses and they are unmarshaling properly, but am trying to wrangle this uncertain behavior.
api.go
package models
import (
"encoding/json"
"io/ioutil"
"net/http"
"fmt"
)
type Request struct {
Url string
ApiKey string
}
type Response struct {
...some fields...
Paginations Pagination `json:"pagination"`
}
type Pagination struct {
EndCursor string `json:"endCursor"`
HasNextPage bool `json:"hasNextPage"`
}
func (request *Request) Get() ([]Response, error) {
var responses []Response
var response Response
// Set up new request
req, err := http.NewRequest("GET", request.Url, nil)
if err != nil {
fmt.Println("Error creating request...")
return responses, err
}
// Add request headers
req.Header = http.Header{
"accept": {"application/json"},
"authorization": {"Bearer " + request.ApiKey},
}
// Get our initial response from the API and capture status code
resp, _ := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
// Read the response body and Unmarshal into struct
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// This field will be in the response payload
// It is verified to be of type bool (not string)
fmt.Printf("Has Next Page? %t\n", resp.Paginations.HasNextPage)
// Append response to our slice of responses
responses = append(responses, response)
// If paginations are available, continue to loop through until all paginations are exhausted
for response.Paginations.HasNextPage == true {
req, err := http.NewRequest("GET", request.Url, nil)
if err != nil {
return responses, err
}
// Append "after" cursor to query in order to redirect to paginated response
qry := req.URL.Query()
qry.Set("after", response.Paginations.EndCursor)
req.URL.RawQuery = qry.Encode()
fmt.Println("Paginated request query: ", req.URL.String())
// Make request
resp, err := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
fmt.Printf("Status Code: %d\n", response.Status)
// Read response and deserialize it
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
fmt.Println("Pagination Cursor: ", response.Paginations.EndCursor)
fmt.Printf("Has Next Page? %t\n", response.Paginations.HasNextPage)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// Append response to our slice of responses
responses = append(responses, response)
}
return responses, nil
}
etl.go
package models
import (
"fmt"
"sync"
)
type Etl struct {
Requests []Request
}
func (etl *Etl) Extract() [][]Response {
var wg sync.WaitGroup
ch := make(chan []Response)
defer close(ch)
for _, req := range etl.Requests {
wg.Add(1) // Increment
fmt.Println("Incremented wait group")
go func(i Request) {
defer wg.Done() // Decrement
resp, err := req.Get()
if err != nil {
fmt.Println(err)
}
ch <- resp
fmt.Println("Decremented wait group")
}(req)
}
results := make([][]Response, len(etl.Requests))
for i, _ := range results {
results[i] = <-ch
//fmt.Println(results[i])
}
wg.Wait()
return nil
}

Answer 1

I believe I found the issue. In my pagination loop for response.Paginations.HasNextPage == true I was creating a new request object (http.NewRequest) on each iteration which did not have headers added from the prior (initial request).
This caused a 401 unauthorized error to return and continuous querying of the API since it was not receiving a new response.Paginations.HasNextPage value.
My solution was to simply alter the for loop a bit like so:
package models
import (
"encoding/json"
"io/ioutil"
"net/http"
"fmt"
)
type Request struct {
Url string
ApiKey string
}
type Response struct {
...some fields...
Paginations Pagination `json:"pagination"`
}
type Pagination struct {
EndCursor string `json:"endCursor"`
HasNextPage bool `json:"hasNextPage"`
}
func (request *Request) Get() ([]Response, error) {
var responses []Response
var response Response
// Set up new request
req, err := http.NewRequest("GET", request.Url, nil)
if err != nil {
fmt.Println("Error creating request...")
return responses, err
}
// Add request headers
req.Header = http.Header{
"accept": {"application/json"},
"authorization": {"Bearer " + request.ApiKey},
}
// Get our initial response from the API and capture status code
resp, _ := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
// Read the response body and Unmarshal into struct
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// This field will be in the response payload
// It is verified to be of type bool (not string)
fmt.Printf("Has Next Page? %t\n", resp.Paginations.HasNextPage)
// Append response to our slice of responses
responses = append(responses, response)
// If paginations are available, continue to loop through until all paginations are exhausted
for response.Paginations.HasNextPage == true {
// Append "after" cursor to query in order to redirect to paginated response
qry := req.URL.Query()
qry.Set("after", response.Paginations.EndCursor)
req.URL.RawQuery = qry.Encode()
fmt.Println("Paginated request query: ", req.URL.String())
// Make request
resp, err := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
fmt.Printf("Status Code: %d\n", response.Status)
// Read response and deserialize it
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
fmt.Println("Pagination Cursor: ", response.Paginations.EndCursor)
fmt.Printf("Has Next Page? %t\n", response.Paginations.HasNextPage)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// Append response to our slice of responses
responses = append(responses, response)
}
return responses, nil
}