I'm trying to use Elixir to access Azure Storage Services via their REST API but I'm having difficulty getting the Authentication Header to work. I am able to connect if I use the ex_azure package (wrapper for erlazure) but not when I try to build the request and use HTTPoison.
Most Recent Error Messages
<?xml version=\"1.0\" encoding=\"utf-8\"?>
<Error>
<Code>AuthenticationFailed</Code>
<Message>Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.\nRequestId:00000000-0000-0000-0000-000000000000\nTime:2017-08-02T21:46:08.6488342Z</Message>
<AuthenticationErrorDetail>The MAC signature found in the HTTP request '<signature>' is not the same as any computed signature. Server used following string to sign: 'GET\n\n\nWed, 02 Aug 2017 21:46:08
GMT\nx-ms-date-h:Wed, 02 Aug 2017 21:46:08 GMT\nx-ms-version-h:2017-05-10\n/storage_name/container_name?comp=list'.</AuthenticationErrorDetail>
</Error>
After 1st Edit
<?xml version=\"1.0\" encoding=\"utf-8\"?>
<Error>
<Code>AuthenticationFailed</Code>
<Message>Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.\nRequestId:00000000-0000-0000-0000-000000000000\nTime:2017-08-03T03:03:57.1385277Z</Message>
<AuthenticationErrorDetail>The MAC signature found in the HTTP request '<signature>' is not the same as any computed signature. Server used following string to sign: 'GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:Thu, 03 Aug
2017 03:03:57 GMT\nx-ms-version:2017-04-17\n/storage_name/container_name\ncomp:list\nrestype:container'.</AuthenticationErrorDetail>
</Error>
Dependencies
# mix.exs
defp deps do
{:httpoison, "~> 0.12"}
{:timex, "~> 3.1"}
end
Code
Am I formatting the Authentication Header (string_to_sign) right?
Am I using encode/decode right?
Am I adding headers correctly to HTTPoison?
Should I be using something else for REST actions instead of HTTPoison?
# account credentials
storage_name = "storage_name"
container_name = "container_name"
storage_key = "storage_key"
storage_service_version = "2017-04-17" # fixed version
request_date =
Timex.now
|> Timex.format!("{RFC1123}") # Wed, 02 Aug 2017 00:52:10 +0000
|> String.replace("+0000", "GMT") # Wed, 02 Aug 2017 00:52:10 GMT
# set canonicalized headers
x_ms_date = "x-ms-date:#{request_date}"
x_ms_version = "x-ms-version:#{storage_service_version}"
# assign values for string_to_sign
verb = "GET\n"
content_encoding = "\n"
content_language = "\n"
content_length = "\n"
content_md5 = "\n"
content_type = "\n"
date = "\n"
if_modified_since = "\n"
if_match = "\n"
if_none_match = "\n"
if_unmodified_since = "\n"
range = "\n"
canonicalized_headers = "#{x_ms_date}\n#{x_ms_version}\n"
canonicalized_resource = "/#{storage_name}/#{container_name}\ncomp:list\nrestype:container" # removed timeout. removed space
# concat string_to_sign
string_to_sign =
verb <>
content_encoding <>
content_language <>
content_length <>
content_md5 <>
content_type <>
date <>
if_modified_since <>
if_match <>
if_none_match <>
if_unmodified_since <>
range <>
canonicalized_headers <>
canonicalized_resource
# decode storage_key
{:ok, decoded_key} =
storage_key
|> Base.decode64
# sign and encode string_to_sign
signature =
:crypto.hmac(:sha256, decoded_key, string_to_sign)
|> Base.encode64
# build authorization header
authorization_header = "SharedKey #{storage_name}:#{signature}"
# build request and use HTTPoison
url = "https://storage_name.blob.core.windows.net/container_name?restype=container&comp=list"
headers = [ # "Date": request_date,
"x-ms-date": request_date, # fixed typo
"x-ms-version": storage_service_version, # fixed typo
# "Accept": "application/json",
"Authorization": authorization_header]
options = [ssl: [{:versions, [:'tlsv1.2']}], recv_timeout: 500]
HTTPoison.get(url, headers, options)
Notes
Some sources I used/tried...
Authentication for the Azure Storage Services
The MAC signature found in the HTTP request is not the same as any computed signature azure integration using php
How to access rest azure blob using cURL
Accessing Azure blob storage using bash, curl
A few issues I noticed:
You included Date request header in your request but it is not included in your string_to_sign. Either include this header in your string_to_sign or remove this header from request headers.
You included timeout:30 in your canonicalized_resource but it is not included in your request URL. Again, either add timeout=30 in your request querystring or remove timeout:30 from canonicalized_resource.
I have not used Elixir as such so I don't know how request headers work there, but you're naming your request headers as x-ms-date-h and x-ms-version-h. Shouldn't they be x-ms-date and x-ms-version respectively?
Related
I'm writing an HTTP server and client in python. When I run my scripts for client and server in terminal everything works fine. However, when I go to my browser and type "localhost:12000" in the searchbar, I get an error saying "The page isn't working. localhost didn't send any data. ERR_EMPTY_RESPONSE". What I expect to see instead, is the content of the html file contained in the response message.
This is the code for my HTTP client.
from socket import *
clientSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 18000
clientSocket.connect(("localhost",serverPort))
request = "GET www.somepage/index.html HTTP/1.0\r\nHost: www.somepage.com\r\nConnection: close\r\nUser-Agent: Chrome/86.0.4240.183\r\nAccept: text/html, application/xhtml+xml\r\nAccept-Language: it-IT, en-US\r\nAccept-Encoding: gzip, deflate\r\nAccept-Charset: ISO-8859-1, utf-8\r\n"
print(request)
clientSocket.send(request.encode())
response = (clientSocket.recv(1024)).decode()
print(response)
clientSocket.close()
This is the code for my server.
from socket import *
from datetime import date
from time import gmtime, strftime
import calendar
serverSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 12000
serverSocket.bind(("localhost",serverPort))
serverSocket.listen(1)
current_date = calendar.day_abbr[date.today().weekday()]+", "+date.today().strftime("%d %b %Y")+strftime(" %H:%M:%S", gmtime())+ " GMT"
while True:
connection , addr = serverSocket.accept()
request = (connection.recv(1024)).decode()
request = request.split()
method = request[0]
URL = request[1]
version = request[2]
if method == "GET" and URL == "www.somepage/index.html" and version == "HTTP/1.0":
response = "HTTP/1.0 200 OK\r\nConnection: close\r\nDate: {}\r\nServer: Apache\r\nLast-Modified: Tue, 10 Nov 2020, 6:31:00 GMT\r\nContent-Length: 72 bytes\r\nContent-Type: text/html\r\n<html>\r\n<title>PAGE TITLE</title>\r\n<body>\rThis is the body\r\n</body></html>".format(current_date)
connection.send(response.encode())
connection.close()
So the server is checking if the request line is correct and then sending the HTTP response, which I do see in the terminal, but when I try in the browser I get an error instead. I've also tried checking Wireshark and I do see the HTTP messages there, so I don't understand why my browsers says no data has been sent.
Thank you all for your help.
Edit:
I couldn't post my code in the comment so I'll try here. What I'm trying to do is create an HTTP client and server than don't implement the entire HTTP protocol, but just a few request methods and a few replies. For now I was starting with the GET method and the 200 OK reply.
This is the code for my client. I have added an extra \r\n at the end of the header in the request.
from socket import *
clientSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 12000
clientSocket.connect(("localhost",serverPort))
request = "GET /index.html HTTP/1.0 \r\nHost: www.somepage.com\r\nConnection: close\r\nUser-Agent: Chrome/86.0.4240.183\r\nAccept: text/html, application/xhtml+xml\r\nAccept-Language: it-IT, en-US\r\nAccept-Encoding: gzip, deflate\r\nAccept-Charset: ISO-8859-1, utf-8\r\n\r\n"
print(request)
clientSocket.send(request.encode())
response = (clientSocket.recv(1024)).decode()
print(response)
clientSocket.close()
This is the code for my server, with an added \r\n at the end of header as well.
from socket import *
from datetime import date
from time import gmtime, strftime
import calendar
serverSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 12000
serverSocket.bind(("localhost",serverPort))
serverSocket.listen(1)
current_date = calendar.day_abbr[date.today().weekday()]+", "+date.today().strftime("%d %b %Y")+strftime(" %H:%M:%S", gmtime())+ " GMT"
while True:
connection , addr = serverSocket.accept()
request = (connection.recv(1024)).decode()
request = request.split()
method = request[0]
URI = request[1]
version = request[2]
host = request[4]
if method == "GET" and URI == "/index.html" and version == "HTTP/1.0" and host == "www.somepage.com":
response = "HTTP/1.0 200 OK \r\nConnection: close\r\nDate: {}\r\nServer: Apache\r\nLast-Modified: Tue, 10 Nov 2020, 6:31:00 GMT\r\nContent-Length: 83 bytes\r\nContent-Type: text/html\r\n\r\n<html>\r\n<title>PAGE TITLE</title>\r\n<body>\rThis is the body\r\n</body></html>".format(current_date)
connection.send(response.encode())
connection.close()
I've studied the standard and I'm trying to write my code according to the specifications. What I see in my browser is this error:
error
I've also noticed that if I change my server code to this:
from socket import *
from datetime import date
from time import gmtime, strftime
import calendar
serverSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 12000
serverSocket.bind(("localhost",serverPort))
serverSocket.listen(1)
current_date = calendar.day_abbr[date.today().weekday()]+", "+date.today().strftime("%d %b %Y")+strftime(" %H:%M:%S", gmtime())+ " GMT"
while True:
connection , addr = serverSocket.accept()
request = (connection.recv(1024)).decode()
request = request.split()
method = request[0]
URI = request[1]
version = request[2]
host = request[4]
if "GET" in request:
response = "HTTP/1.0 200 OK \r\nConnection: close\r\nDate: {}\r\nServer: Apache\r\nLast-Modified: Tue, 10 Nov 2020, 6:31:00 GMT\r\nContent-Length: 83 bytes\r\nContent-Type: text/html\r\n\r\n<html>\r\n<title>PAGE TITLE</title>\r\n<body>\rThis is the body\r\n</body></html>".format(current_date)
connection.send(response.encode())
connection.close()
where basically the only difference is the way the if statement is written, then my browser will display correctly the html, that is, I see this:
page
So it seems the problem lies in the syntax I used for my python code, and not the way the standard is implemented?
Thank you again so very much for your help.
I couldn't post my code in the comment so I'll try here. What I'm trying to do is create an HTTP client and server than don't implement the entire HTTP protocol, but just a few request methods and a few replies. For now I was starting with the GET method and the 200 OK reply.
This is the code for my client. I have added an extra \r\n at the end of the header in the request.
from socket import *
clientSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 12000
clientSocket.connect(("localhost",serverPort))
request = "GET /index.html HTTP/1.0 \r\nHost: www.somepage.com\r\nConnection: close\r\nUser-Agent: Chrome/86.0.4240.183\r\nAccept: text/html, application/xhtml+xml\r\nAccept-Language: it-IT, en-US\r\nAccept-Encoding: gzip, deflate\r\nAccept-Charset: ISO-8859-1, utf-8\r\n\r\n"
print(request)
clientSocket.send(request.encode())
response = (clientSocket.recv(1024)).decode()
print(response)
clientSocket.close()
This is the code for my server, with an added \r\n at the end of header as well.
from socket import *
from datetime import date
from time import gmtime, strftime
import calendar
serverSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 12000
serverSocket.bind(("localhost",serverPort))
serverSocket.listen(1)
current_date = calendar.day_abbr[date.today().weekday()]+", "+date.today().strftime("%d %b %Y")+strftime(" %H:%M:%S", gmtime())+ " GMT"
while True:
connection , addr = serverSocket.accept()
request = (connection.recv(1024)).decode()
request = request.split()
method = request[0]
URI = request[1]
version = request[2]
host = request[4]
if method == "GET" and URI == "/index.html" and version == "HTTP/1.0" and host == "www.somepage.com":
response = "HTTP/1.0 200 OK \r\nConnection: close\r\nDate: {}\r\nServer: Apache\r\nLast-Modified: Tue, 10 Nov 2020, 6:31:00 GMT\r\nContent-Length: 83 bytes\r\nContent-Type: text/html\r\n\r\n<html>\r\n<title>PAGE TITLE</title>\r\n<body>\rThis is the body\r\n</body></html>".format(current_date)
connection.send(response.encode())
connection.close()
I've studied the standard and I'm trying to write my code according to the specifications. What I see in my browser is this error:
error
I've also noticed that if I change my server code to this:
from socket import *
from datetime import date
from time import gmtime, strftime
import calendar
serverSocket = socket(AF_INET,SOCK_STREAM)
serverPort = 12000
serverSocket.bind(("localhost",serverPort))
serverSocket.listen(1)
current_date = calendar.day_abbr[date.today().weekday()]+", "+date.today().strftime("%d %b %Y")+strftime(" %H:%M:%S", gmtime())+ " GMT"
while True:
connection , addr = serverSocket.accept()
request = (connection.recv(1024)).decode()
request = request.split()
method = request[0]
URI = request[1]
version = request[2]
host = request[4]
if "GET" in request:
response = "HTTP/1.0 200 OK \r\nConnection: close\r\nDate: {}\r\nServer: Apache\r\nLast-Modified: Tue, 10 Nov 2020, 6:31:00 GMT\r\nContent-Length: 83 bytes\r\nContent-Type: text/html\r\n\r\n<html>\r\n<title>PAGE TITLE</title>\r\n<body>\rThis is the body\r\n</body></html>".format(current_date)
connection.send(response.encode())
connection.close()
where basically the only difference is the way the if statement is written, then my browser will display correctly the html, that is, I see this:
page
So it seems the problem lies in the syntax I used for my python code, and not the way the standard is implemented?
Thank you again so very much for your help.
request = "GET www.somepage/index.html HTTP/1.0\r\nHost: www.somepage.com\r\nConnection: close\r\nUser-Agent: Chrome/86.0.4240.183\r\nAccept: text/html, application/xhtml+xml\r\nAccept-Language: it-IT, en-US\r\nAccept-Encoding: gzip, deflate\r\nAccept-Charset: ISO-8859-1, utf-8\r\n"
This is not a valid HTTP request. First, it should only contain the path /index.html and not domain/path as you currently do. It is also missing the final \r\n at the end which signals the end of the HTTP header.
In the same way the expectations of the server wrong too, which explains why it causes problems when faced with a client correctly implementing HTTP (the browser). Additionally the HTTP response is also missing the final \r\n after the HTTP header and the Content-length: 72 does not match the actual length of the content.
Please don't implement HTTP by (wrongly) second-guessing how it works. There is an actual standard for this and implementations are expected to follow this standard.
After the edit the code looks like this:
request = request.split()
...
version = request[2]
host = request[4]
if method == "GET" and URI == "/index.html" and version == "HTTP/1.0" and host == "www.somepage.com":
... send response ...
There are multiple problems here: the first one is that the browser will not use HTTP/1.0 as version but HTTP/1.1.
The next problem is that the domain might not be in the variable host since it is might not be in request[4]. It is blindly assumed that the Host header is in the second line of the request since it is implemented like this in the client. But the HTTP standard does in now way require this. And while it might be the case with some clients it is not the case with others. Instead of blindly assuming that something is in a specific place in the HTTP header the header should actually be parsed properly to extract the Host header.
I used this tutorial and created "put" endpoint successfully.
https://sanderknape.com/2017/10/creating-a-serverless-api-using-aws-api-gateway-and-dynamodb/
When I follow this advice, I get authroization required error..
Using your favorite REST client, try to PUT an item into DynamoDB
using your API Gateway URL.
python is my favorite client:
import requests
api_url = "https://0pg2858koj.execute-api.us-east-1.amazonaws.com/tds"
PARAMS = {"name": "test", "favorite_movie":"asdsf"}
r = requests.put(url=api_url, params=PARAMS)
the response is 403
My test from console is successful, but not able to put a record from python.
The first step you can take to resolve the problem is to investigate the information returned by AWS in the 403 response. It will provide a header, x-amzn-ErrorType and error message with information about the concrete error. You can test it with curl in verbose mode (-v) or with your Python code. Please, review the relevant documentation to obtain a detailed enumeration of all the possible error reasons.
In any case, looking at your code, it is very likely that you did not provide the necessary authentication or authorization information to AWS.
The kind of information that you must provide depends on which mechanism you configured to access your REST API in API Gateway.
If, for instance, you configured IAM based authentication, you need to set up your Python code to generate an Authorization header with an AWS Signature derived from your user access key ID and associated secret key. The AWS documentation provides an example of use with Postman.
The AWS documentation also provides several examples of how to use python and requests to perform this kind of authorization.
Consider, for instance, this example for posting information to DynamoDB:
# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of the
# License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
# OF ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
# AWS Version 4 signing example
# DynamoDB API (CreateTable)
# See: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
# This version makes a POST request and passes request parameters
# in the body (payload) of the request. Auth information is passed in
# an Authorization header.
import sys, os, base64, datetime, hashlib, hmac
import requests # pip install requests
# ************* REQUEST VALUES *************
method = 'POST'
service = 'dynamodb'
host = 'dynamodb.us-west-2.amazonaws.com'
region = 'us-west-2'
endpoint = 'https://dynamodb.us-west-2.amazonaws.com/'
# POST requests use a content type header. For DynamoDB,
# the content is JSON.
content_type = 'application/x-amz-json-1.0'
# DynamoDB requires an x-amz-target header that has this format:
# DynamoDB_<API version>.<operationName>
amz_target = 'DynamoDB_20120810.CreateTable'
# Request parameters for CreateTable--passed in a JSON block.
request_parameters = '{'
request_parameters += '"KeySchema": [{"KeyType": "HASH","AttributeName": "Id"}],'
request_parameters += '"TableName": "TestTable","AttributeDefinitions": [{"AttributeName": "Id","AttributeType": "S"}],'
request_parameters += '"ProvisionedThroughput": {"WriteCapacityUnits": 5,"ReadCapacityUnits": 5}'
request_parameters += '}'
# Key derivation functions. See:
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
def sign(key, msg):
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
def getSignatureKey(key, date_stamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
# Read AWS access key from env. variables or configuration file. Best practice is NOT
# to embed credentials in code.
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if access_key is None or secret_key is None:
print('No access key is available.')
sys.exit()
# Create a date for headers and the credential string
t = datetime.datetime.utcnow()
amz_date = t.strftime('%Y%m%dT%H%M%SZ')
date_stamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
# ************* TASK 1: CREATE A CANONICAL REQUEST *************
# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
# Step 1 is to define the verb (GET, POST, etc.)--already done.
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = '/'
## Step 3: Create the canonical query string. In this example, request
# parameters are passed in the body of the request and the query string
# is blank.
canonical_querystring = ''
# Step 4: Create the canonical headers. Header names must be trimmed
# and lowercase, and sorted in code point order from low to high.
# Note that there is a trailing \n.
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-date:' + amz_date + '\n' + 'x-amz-target:' + amz_target + '\n'
# Step 5: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers include those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
# For DynamoDB, content-type and x-amz-target are also required.
signed_headers = 'content-type;host;x-amz-date;x-amz-target'
# Step 6: Create payload hash. In this example, the payload (body of
# the request) contains the request parameters.
payload_hash = hashlib.sha256(request_parameters.encode('utf-8')).hexdigest()
# Step 7: Combine elements to create canonical request
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
# ************* TASK 2: CREATE THE STRING TO SIGN*************
# Match the algorithm to the hashing algorithm you use, either SHA-1 or
# SHA-256 (recommended)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = date_stamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
# ************* TASK 3: CALCULATE THE SIGNATURE *************
# Create the signing key using the function defined above.
signing_key = getSignatureKey(secret_key, date_stamp, region, service)
# Sign the string_to_sign using the signing_key
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
# Put the signature information in a header named Authorization.
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
# For DynamoDB, the request can include any headers, but MUST include "host", "x-amz-date",
# "x-amz-target", "content-type", and "Authorization". Except for the authorization
# header, the headers must be included in the canonical_headers and signed_headers values, as
# noted earlier. Order here is not significant.
# # Python note: The 'host' header is added automatically by the Python 'requests' library.
headers = {'Content-Type':content_type,
'X-Amz-Date':amz_date,
'X-Amz-Target':amz_target,
'Authorization':authorization_header}
# ************* SEND THE REQUEST *************
print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
print('Request URL = ' + endpoint)
r = requests.post(endpoint, data=request_parameters, headers=headers)
print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
print('Response code: %d\n' % r.status_code)
print(r.text)
I think it could be easily adapted to your needs.
In the console, everything works fine because when you invoke your REST endpoints in API Gateway, you are connected to a user who is already authenticated and authorized to access these REST endpoints.
No matter what value I enter as my request's "Content-Type", the outgoing request I send out seems to replace it with "application/x-www-form-urlencoded". The application I'm trying to hit expects "application/json". My code, basically, is below.
{-# LANGUAGE OverloadedStrings #-}
import Network.Wreq
...
submissionResources = ["https://widgets.example.com/v2/widgets/request"]
sendWidgetToEndpoint submissionResources workingToken key widgetArray = do
let opts = defaults & header "Content-Type" .~ ["application/json"]
& header "apikey" .~ [key]
& header "Authorization" .~ [workingToken]
endPointURL = head submissionResources
widgetId = widgetArray !! 0
numberOfWidgets = widgetArray !! 1
widgetText = widgetArray !! 2
submissionResult <- postWith opts endPointURL [ "widgetId" := widgetId
, "numWidgets" := numberOfWidgets
, "widgetText" := widgetText
]
return submissionResult
My problem is that I keep getting back Status {statusCode = 415, statusMessage = "Unsupported Media Type"} from this endpoint, and I'm confident this is because the request I'm sending appears to be overriding "Content-Type" in my header. I have tried using "application/json" and "text/plain" but the response I get back always indicates to me that all the headers I sent over look as expected except for Content-Type which invariably has become "application/x-www-form-urlencoded".
How can I ensure wreq keeps 'Content-Type: application/json' in my requests header?
EDIT: I'm determining what headers were in my original request by what the API server tells me in its response back to me.
The type of the last argument to postWith in your snippet is [FormParam], and that type is what forces the Content-Type to be urlencoded.
To send JSON, send something of type Value or Encoding (from Data.Aeson).
import Data.Aeson (pairs, (.=))
...
-- also remove the "Content-Type" field from opts
submissionResult <- postWith opts endpointURL $ pairs
( "widgetId" .= widgetId <>
"numWidgets" .= numberOfWidgets <>
"widgetText" .= widgetText )
...
The Content-Type is set by the payload you pass to postWith, via the Postable instance. If you want to use yet another Content-Type header, define your own type with a Postable instance where you set an appropriate Content-Type. You can also choose to not set any Content-Type in the Postable instance, so you can set it via the options instead.
I want to fetch all customer accounts from Zuora. Apart from Exports REST API, Is there any API available to fetch all accounts in a paginated list?
This is the format I used to fetch revenue invoices, use this code and change the endpoint
import pandas as pd
# Set the sleep time to 10 seconds
sleep = 10
# Zuora OAUTH token URL
token_url = "https://rest.apisandbox.zuora.com/oauth/token"
# URL for the DataQuery
query_url = "https://rest.apisandbox.zuora.com/query/jobs"
# OAUTH client_id & client_secret
client_id = 'your client id'
client_secret = 'your client secret'
# Set the grant type to client credential
token_data = {'grant_type': 'client_credentials'}
# Send the POST request for the OAUTH token
access_token_resp = requests.post(token_url, data=token_data,
auth=(client_id, client_secret))
# Print the OAUTH token respose text
#print access_token_resp.text
# Parse the tokens as json data from the repsonse
tokens = access_token_resp.json()
#print "access token: " + tokens['access_token']
# Use the access token in future API calls & Add to the headers
query_job_headers = {'Content-Type':'application/json',
'Authorization': 'Bearer ' + tokens['access_token']}
# JSON Data for our DataQuery
json_data = {
"query": "select * from revenuescheduleiteminvoiceitem",
"outputFormat": "JSON",
"compression": "NONE",
"retries": 3,
"output": {
"target": "s3"
}
}
# Parse the JSON output
data = json.dumps(json_data)
# Send the POST request for the dataquery
query_job_resp = requests.post(query_url, data=data,
headers=query_job_headers)
# Print the respose text
#print query_job_resp.text
# Check the Job Status
# 1) Parse the Query Job Response JSON data
query_job = query_job_resp.json()
# 2) Create the Job URL with the id from the response
query_job_url = query_url+'/'+query_job["data"]["id"]
# 3) Send the GETrequest to check on the status of the query
query_status_resp = requests.get(query_job_url, headers = query_job_headers)
#print query_status_resp.text
# Parse the status from teh response
query_status = query_status_resp.json()["data"]["queryStatus"]
#print ('query status:'+query_status)
# Loop until the status == completed
# Exit if there is an error
while (query_status != 'completed'):
time.sleep(sleep)
query_status_resp = requests.get(query_job_url, headers = query_job_headers)
#print query_status_resp.text
query_status = query_status_resp.json()["data"]["queryStatus"]
if (query_status == 'failed'):
print ("query: "+query_status_resp.json()["data"]["query"]+' Failed!\n')
exit(1)
# Query Job has completed
#print ('query status:'+query_status)
# Get the File URL
file_url = query_status_resp.json()["data"]["dataFile"]
print (file_url)```
If you don't want to use Data Query or any queue-based solution like that, use Zoql instead.
Note! You need to know all fields from the Account object you need, the asterisk (select *) doesn't work here:
select Id, ParentId, AccountNumber, Name from Account
You may also add custom fields into your selection. You will get up to 200 records per page.
I am planning to use S3 Cloudstorage in IBM Bluemix but then one strange thing I found is that there is no way to add the custom META-DATA to the objects which are stored in S3 bucket.
Is there a way I can add custom Meta-Data to the objects and if yes then can you please advise on how we can add it and access it.?
Thanks for pointing out a hole in the documentation!
Custom metadata is defined by passing a x-amz-meta-{key} header with a {value} value. As an example request:
PUT /{bucket-name}/{object-name} HTTP/1.1
Authorization: {authorization-string}
x-amz-meta-foo: bar
x-amz-date: 20160825T183001Z
x-amz-content-sha256:{hashed-body}
Content-Type: text/plain; charset=utf-8
Host: s3-api.us-geo.objectstorage.softlayer.net
Content-Length: 18
{
"foo": "bar"
}
A HEAD request to check the metadata would look like:
HEAD /{bucket-name}/{object-name} HTTP/1.1
Authorization: {authorization-string}
x-amz-date: 20160825T183244Z
Host: s3-api.us-geo.objectstorage.softlayer.net
And respond with:
HTTP/1.1 200 OK
Date: Thu, 25 Aug 2016 18:32:44 GMT
X-Clv-Request-Id: da214d69-1999-4461-a130-81ba33c484a6
Accept-Ranges: bytes
Server: Cleversafe/3.9.1.102
X-Clv-S3-Version: 2.5
ETag: {MD5-hash}
Content-Type: text/plain; charset=UTF-8
x-amz-meta-foo: bar
Last-Modified: Thu, 25 Aug 2016 17:49:06 GMT
Content-Length: 18
Using the CLI, the syntax would be:
$ aws --endpoint-url=https://{endpoint} s3 cp ~/new-file s3://bucket-1/ --metadata foo=bar
Hope that helps!
This is possible. I am using this every day . Adding meta data then sending the meta data in database by doing cron calls.
Here is a small example of python script to create/add/change metadata for a a list object :
import sys
import os
import boto3
import pprint
from boto3 import client
from botocore.utils import fix_s3_host
param_1= YOUR_ACCESS_KEY
param_2= YOUR_SECRETE_KEY
param_3= YOUR_END_POINT
param_4= YOUR_BUCKET
#Create the S3 client
s3ressource = client(
service_name='s3',
endpoint_url= param_3,
aws_access_key_id= param_1,
aws_secret_access_key=param_2,
use_ssl=True,
)
# Building a list of object per bucket
def BuildObjectListPerBucket (variablebucket):
global listofObjectstobeanalyzed
listofObjectstobeanalyzed = []
extensions = ['.jpg','.png']
for key in s3ressource.list_objects(Bucket=variablebucket)["Contents"]:
#print (key ['Key'])
onemoreObject=key['Key']
if onemoreObject.endswith(tuple(extensions)):
listofObjectstobeanalyzed.append(onemoreObject)
else :
s3ressource.delete_object(Bucket=variablebucket,Key=onemoreObject)
return listofObjectstobeanalyzed
# for a given existing object, create metadata
def createmetdata(bucketname,objectname):
s3ressource.upload_file(objectname, bucketname, objectname, ExtraArgs={"Metadata": {"metadata1":"ImageName","metadata2":"ImagePROPERTIES" ,"metadata3":"ImageCREATIONDATE"}})
# for a given existing object, add new metadata
def ADDmetadata(bucketname,objectname):
s3_object = s3ressource.get_object(Bucket=bucketname, Key=objectname)
k = s3ressource.head_object(Bucket = bucketname, Key = objectname)
m = k["Metadata"]
m["new_metadata"] = "ImageNEWMETADATA"
s3ressource.copy_object(Bucket = bucketname, Key = objectname, CopySource = bucketname + '/' + objectname, Metadata = m, MetadataDirective='REPLACE')
# for a given existing object, update a metadata with new value
def CHANGEmetadata(bucketname,objectname):
s3_object = s3ressource.get_object(Bucket=bucketname, Key=objectname)
k = s3ressource.head_object(Bucket = bucketname, Key = objectname)
m = k["Metadata"]
m.update({'watson_visual_rec_dic':'ImageCREATIONDATEEEEEEEEEEEEEEEEEEEEEEEEEE'})
s3ressource.copy_object(Bucket = bucketname, Key = objectname, CopySource = bucketname + '/' + objectname, Metadata = m, MetadataDirective='REPLACE')
def readmetadata (bucketname,objectname):
ALLDATAOFOBJECT = s3ressource.get_object(Bucket=bucketname, Key=objectname)
ALLDATAOFOBJECTMETADATA=ALLDATAOFOBJECT['Metadata']
print ALLDATAOFOBJECTMETADATA
# create the list of object on a per bucket basis
BuildObjectListPerBucket (variablebucket)
# Call functions to see the results
for objectitem in listofObjectstobeanalyzed:
readmetadata(param_4,objectitem)
createmetdata(param_4,objectitem)
readmetadata(param_4,objectitem)
ADDmetadata(param_4,objectitem)
readmetadata(param_4,objectitem)
CHANGEmetadata(param_4,objectitem)
readmetadata(param_4,objectitem)