How to generate a RS256 JWT Token in Karate API testing - jwt

How can I generate a sha256-RSA-signed JWT token in a Karate (https://github.com/karatelabs/karate) feature file?
https://github.com/karatelabs/karate/issues/1138#issuecomment-629453412 has a nice recipee for doing such for a HMAC-SHA256 (or "HmacSHA256" in Java lingo) token, i.e. using symmetric/shared secret crypto.
But we need asymmetric crypto and the RS256 algo (see RS256 vs HS256: What's the difference? for background)...

OK, think I figured it out :-).
Big thanks to the generous souls providing all the necessary info here:
JWT generation in Karate, but with HmacSHA256: https://github.com/karatelabs/karate/issues/1138#issuecomment-629453412
Signing with sha256 RSA signature in Java: https://www.quickprogrammingtips.com/java/how-to-create-sha256-rsa-signature-using-java.html
So the following is an example Karate feature file using
an RS256 JWT token (put in the x-jwt header)
mTLS (i.e. using a client certificate for mutual TLS)
To do this one needs to make use of Karate's JavaScript and Java-interop capabilities.
This is our setup to make it work:
0 $ tree
.
├── karate-config.js
├── karate.jar
├── secrets
│   ├── client-cert-keystore.p12
│   ├── client-cert.pem
│   ├── client-cert_private-key.pem
│   ├── rsa-4096-cert.pem
│   ├── rsa-4096-private.pem
│   └── rsa-4096-public.pem
└── test.feature
1 directory, 9 files
We'll use the private key rsa-4096-private.pem (keep it secret!) of our rsa-4096-* files to create the signed token.
So the essential files for the JWT parts are
rsa-4096-private.pem for creating the JWT
rsa-4096-public.pem for verifying the token/signature, that's what the api/service/server would do with your JWT token (i.e. this file's not needed/used in our feature file). You can try verifying a resulting token with e.g. https://jwt.io/.
Sidenote: public/private key pairs can be generated with e.g. openssl.
As a bonus this example contains using a client certificate and mTLS (which httpbin probably gracefully ignores). If you don't need this you can simply strip the configure ssl... line and the client_cert_keystore_pass stuff from the karate config file and the command line.
Karate feature file:
# test.feature
Feature: Simple test
Background:
# Several helper functions for creating a RS256 signed JWT token.
# Graciously adapted from:
# JWT generation in Karate, but with HmacSHA256:
# https://github.com/karatelabs/karate/issues/1138#issuecomment-629453412
# Signing with sha256 RSA signature in Java:
# https://www.quickprogrammingtips.com/java/how-to-create-sha256-rsa-signature-using-java.html
* def b64encode_bytes =
"""
function(bytes) {
// Base64-encode `bytes`.
// Returns bytes.
var encoder = Java.type('java.util.Base64')
.getUrlEncoder()
.withoutPadding()
return new java.lang.String(encoder.encode(bytes))
}
"""
# Base64-encode `str`, encodes str to UTF-8 and base64-encodes it.
# Returns bytes.
* def b64encode_str = function(str) {return b64encode_bytes(str.getBytes("UTF-8"))}
* def strip_key_header_footer_ws =
"""
function(key_text) {
// Strip -----BEGIN ... header + footer and all newline characters.
// Returns UTF-8-encoded bytes.
// Need string object for replaceAll method.
var key_text_str = new java.lang.String(key_text)
var key_str = key_text_str
.replaceAll("-----BEGIN PRIVATE KEY-----", "")
.replaceAll("-----END PRIVATE KEY-----", "")
.replaceAll("\r", "")
.replaceAll("\n", "")
return key_str.getBytes('UTF-8')
}
"""
* def sha256rsa_sign =
"""
function(bytes, privateKey) {
var decoder = Java.type('java.util.Base64')
.getDecoder()
var PKCS8EncodedKeySpec = Java.type(
'java.security.spec.PKCS8EncodedKeySpec')
var spec = new PKCS8EncodedKeySpec(decoder.decode(privateKey))
var kf = Java.type('java.security.KeyFactory').getInstance("RSA")
var signature = Java.type('java.security.Signature')
.getInstance("SHA256withRSA")
signature.initSign(kf.generatePrivate(spec))
signature.update(bytes)
var signed = signature.sign()
return signed
}
"""
* def generate_jwt_sha256rsa =
"""
function(payload) {
// Generate JWT from given `payload` object (dict).
// Returns SHA256withRSA-signed JWT token (bytes).
var header_encoded = b64encode_str(
JSON.stringify({alg: "RS256", typ: "JWT"}))
var payload_encoded = b64encode_str(JSON.stringify(payload))
var data_to_sign = header_encoded + '.' + payload_encoded
var signature = b64encode_bytes(
sha256rsa_sign(data_to_sign.getBytes("UTF-8"), privateKey)
)
var token = data_to_sign + '.' + signature
return token
}
"""
# enable X509 client certificate authentication with PKCS12 file
* configure ssl = { keyStore: 'secrets/client-cert-keystore.p12', keyStoreType: 'pkcs12', keyStorePassword: '#(client_cert_keystore_pass)' }
# get private key for JWT generation and API key
* def privateKeyContent = read('secrets/rsa-4096-private.pem')
* def privateKey = strip_key_header_footer_ws(privateKeyContent)
# generate JWT
* def jwt = generate_jwt_sha256rsa({iss: "ExampleApp", exp: "1924902000"})
# put all needed API access credential in the header
* headers { x-jwt: '#(jwt)'}
* url 'https://httpbin.org'
Scenario Outline: get anything
Given path '/anything/<anything_id>'
When method get
Then status 200
Examples:
| anything_id |
| 1 |
Karate config file:
// karate-config.js
function fn() {
//var http_proxy = java.lang.System.getenv('http_proxy');
var client_cert_keystore_pass = java.lang.System.getenv(
'CLIENT_CERT_KEYSTORE_PASS');
// setup connection
karate.configure('connectTimeout', 5000);
karate.configure('readTimeout', 5000);
//karate.configure('proxy', http_proxy);
var config = {
client_cert_keystore_pass: client_cert_keystore_pass
};
return config;
}
As noted you won't need the client_cert_keystore_pass stuff unless you want mTLS. Also, you probably won't need the timeout configurations. I've tested behind a proxy so this also contains some additional config support for http_proxy (commented, left in for educational purposes). Adapt to your tastes.
Run it:
0 $ CLIENT_CERT_KEYSTORE_PASS="$PASSWORD" java -jar karate.jar -o /tmp/karate-out test.feature
17:34:41.614 [main] INFO com.intuit.karate - Karate version: 1.2.1.RC1
17:34:42.076 [main] DEBUG com.intuit.karate.Suite - [config] karate-config.js
17:34:43.942 [main] DEBUG com.intuit.karate - key store key count for secrets/client-cert-keystore.p12: 1
17:34:44.535 [main] DEBUG com.intuit.karate - request:
1 > GET https://httpbin.org/anything/1
1 > x-jwt: eyJhbGciO...
1 > Host: httpbin.org
1 > Connection: Keep-Alive
...
---------------------------------------------------------
feature: test.feature
scenarios: 1 | passed: 1 | failed: 0 | time: 1.7300
---------------------------------------------------------
17:34:46.577 [main] INFO com.intuit.karate.Suite - <<pass>> feature 1 of 1 (0 remaining) test.feature
Karate version: 1.2.1.RC1
======================================================
elapsed: 4.74 | threads: 1 | thread time: 1.73
features: 1 | skipped: 0 | efficiency: 0.36
scenarios: 1 | passed: 1 | failed: 0
======================================================
HTML report: (paste into browser to view) | Karate version: 1.2.1.RC1
file:///tmp/karate-out/karate-reports/karate-summary.html
===================================================================
0 $
Note that I'm by no means a Karate expert nor a JavaScript or Java programmer. So this might well not be your idiomatic Karate/JS/Java code. ;-)

Related

Varnish - how to check JWT signature using digest Vmod?

I have a DockerFile based on Varnish 7.0 alpine, I have a custom vcl file to handle JWT authentication. We pass the JWT as a Bearer in the header.
I am based on this example: https://feryn.eu/blog/validating-json-web-tokens-in-varnish/
set req.http.tmpPayload = regsub(req.http.x-token,"[^\.]+\.([^\.]+)\.[^\.]+$","\1");
set req.http.tmpHeader = regsub(req.http.x-token,"([^\.]+)\.[^\.]+\.[^\.]+","\1");
set req.http.tmpRequestSig = regsub(req.http.x-token,"^[^\.]+\.[^\.]+\.([^\.]+)$","\1");
set req.http.tmpCorrectSig = digest.base64url_nopad_hex(digest.hmac_sha256(std.fileread("/jwt/privateKey.pem"), req.http.tmpHeader + "." + req.http.tmpPayload));
std.log("req sign " + req.http.tmpRequestSig);
std.log("calc sign " + req.http.tmpCorrectSig);
if(req.http.tmpRequestSig != req.http.tmpCorrectSig) {
std.log("invalid signature match");
return(synth(403, "Invalid JWT signature"));
}
My problem is that tmpCorrectSig is empty, I don't know if I can load from a file, since my file contains new lines and other caracteres ?
For information, this Vmod is doing what I want: https://code.uplex.de/uplex-varnish/libvmod-crypto, but I can't install it on my Arm M1 pro architecture, I spent so much time trying...
Can I achieve what I want?
I have a valid solution that leverages the libvmod-crypto. The VCL supports both HS256 and RS256.
These are the commands I used to generated the certificates:
cd /etc/varnish
ssh-keygen -t rsa -b 4096 -m PEM -f jwtRS256.key
openssl rsa -in jwtRS256.key -pubout -outform PEM -out jwtRS256.key.pub
I use https://jwt.io/ to generate a token and paste in the values from my certificates to encrypt the signature.
The VCL code
This is the VCL code that will extract the JWT from the token cookie:
vcl 4.1;
import blob;
import digest;
import crypto;
import std;
sub vcl_init {
new v = crypto.verifier(sha256,std.fileread("/etc/varnish/jwtRS256.key.pub"));
}
sub vcl_recv {
call jwt;
}
sub jwt {
if(req.http.cookie ~ "^([^;]+;[ ]*)*token=[^\.]+\.[^\.]+\.[^\.]+([ ]*;[^;]+)*$") {
set req.http.x-token = ";" + req.http.Cookie;
set req.http.x-token = regsuball(req.http.x-token, "; +", ";");
set req.http.x-token = regsuball(req.http.x-token, ";(token)=","; \1=");
set req.http.x-token = regsuball(req.http.x-token, ";[^ ][^;]*", "");
set req.http.x-token = regsuball(req.http.x-token, "^[; ]+|[; ]+$", "");
set req.http.tmpHeader = regsub(req.http.x-token,"token=([^\.]+)\.[^\.]+\.[^\.]+","\1");
set req.http.tmpTyp = regsub(digest.base64url_decode(req.http.tmpHeader),{"^.*?"typ"\s*:\s*"(\w+)".*?$"},"\1");
set req.http.tmpAlg = regsub(digest.base64url_decode(req.http.tmpHeader),{"^.*?"alg"\s*:\s*"(\w+)".*?$"},"\1");
if(req.http.tmpTyp != "JWT") {
return(synth(400, "Token is not a JWT: " + req.http.tmpHeader));
}
if(req.http.tmpAlg != "HS256" && req.http.tmpAlg != "RS256") {
return(synth(400, "Token does not use a HS256 or RS256 algorithm"));
}
set req.http.tmpPayload = regsub(req.http.x-token,"token=[^\.]+\.([^\.]+)\.[^\.]+$","\1");
set req.http.tmpRequestSig = regsub(req.http.x-token,"^[^\.]+\.[^\.]+\.([^\.]+)$","\1");
if(req.http.tempAlg == "HS256") {
set req.http.tmpCorrectSig = digest.base64url_nopad_hex(digest.hmac_sha256("SlowWebSitesSuck",req.http.tmpHeader + "." + req.http.tmpPayload));
if(req.http.tmpRequestSig != req.http.tmpCorrectSig) {
return(synth(403, "Invalid HS256 JWT signature"));
}
} else {
if (! v.update(req.http.tmpHeader + "." + req.http.tmpPayload)) {
return (synth(500, "vmod_crypto error"));
}
if (! v.valid(blob.decode(decoding=BASE64URLNOPAD, encoded=req.http.tmpRequestSig))) {
return(synth(403, "Invalid RS256 JWT signature"));
}
}
set req.http.tmpPayload = digest.base64url_decode(req.http.tmpPayload);
set req.http.X-Login = regsub(req.http.tmpPayload,{"^.*?"login"\s*:\s*(\w+).*?$"},"\1");
set req.http.X-Username = regsub(req.http.tmpPayload,{"^.*?"sub"\s*:\s*"(\w+)".*?$"},"\1");
unset req.http.tmpHeader;
unset req.http.tmpTyp;
unset req.http.tmpAlg;
unset req.http.tmpPayload;
unset req.http.tmpRequestSig;
unset req.http.tmpCorrectSig;
unset req.http.tmpPayload;
}
}
Installing libvmod-crypto
libvmod-crypto is required to use RS256, which is not supported by libvmod-digest.
Unfortunately I'm getting an error when running the ./configure script:
./configure: line 12829: syntax error: unexpected newline (expecting ")")
I'll talk to the maintainer of the VMOD and see if we can figure out someway to fix this. If this is an urgent matter, I suggest you use a non-Alpine Docker container for the time being.
Firstly, the configure error was caused by a missing -dev package, see the gitlab issue (the reference is in a comment, but I think it should be more prominent).
The main issue in the original question is that digest.hmac_sha256() can not be used to verify RS256 signatures. A JWT RS256 signature is a SHA256 hash of the subject encrypted with an RSA private key, which can then be verified by decrypting with the RSA public key and checking the signature. This is what crypto.verifier(sha256, ...) does.
In this regard, Thijs' previous answer is already correct.
Yet the code which is circulating and has been referenced here it nothing I would endorse. Among other issues, a fundamental problem is that regular expressions are used to (pretend to) parse JSON, which is simply not correct.
I use a better implementation for long, but just did not get around to publishing it. So now is the time, I guess.
I have just added VCL snippets from production code for JWT parsing and validation.
The example is used like so with the jwt directory in vcl_path:
include "jwt/jwt.vcl";
include "jwt/rsa_keys.vcl";
sub vcl_recv {
jwt.set(YOUR_JWT); # replace YOUR_JWT with an actual variable/header/function
call recv_jwt_validate;
# do things with jwt_payload.extract(".scope")
}
Here, the scope claim contains the data that we are actually interested in for further processing, if you want to use other claims, just rename .scope or add another jwt_payload.expect(CLAIM, ...) and then use jwt_payload.extract(CLAIM).
This example uses some vmods, which we developed and maintain in particular with JWT in mind, though not exclusively:
crypto (use gitlab mirror for issues) for RS signatures (mostly RS256)
frozen (use gitlab mirror for issues) for JSON parsing
Additionally, we use
re2 (use gitlab mirror for issues) to efficiently split the JWT into the three parts (header, payload, signature)
and taskvar from objvar (gitlab) for proper variables.
One could do without these two vmods (re2 could be replaced by the re vmod or even regsub and taskvar with headers), but they make the code more efficient and cleaner.
blobdigest (gitlab) is not contained in the example, but can be used to validate HS signtures (e.g. HS256).

authorization for API gateway

I used this tutorial and created "put" endpoint successfully.
https://sanderknape.com/2017/10/creating-a-serverless-api-using-aws-api-gateway-and-dynamodb/
When I follow this advice, I get authroization required error..
Using your favorite REST client, try to PUT an item into DynamoDB
using your API Gateway URL.
python is my favorite client:
import requests
api_url = "https://0pg2858koj.execute-api.us-east-1.amazonaws.com/tds"
PARAMS = {"name": "test", "favorite_movie":"asdsf"}
r = requests.put(url=api_url, params=PARAMS)
the response is 403
My test from console is successful, but not able to put a record from python.
The first step you can take to resolve the problem is to investigate the information returned by AWS in the 403 response. It will provide a header, x-amzn-ErrorType and error message with information about the concrete error. You can test it with curl in verbose mode (-v) or with your Python code. Please, review the relevant documentation to obtain a detailed enumeration of all the possible error reasons.
In any case, looking at your code, it is very likely that you did not provide the necessary authentication or authorization information to AWS.
The kind of information that you must provide depends on which mechanism you configured to access your REST API in API Gateway.
If, for instance, you configured IAM based authentication, you need to set up your Python code to generate an Authorization header with an AWS Signature derived from your user access key ID and associated secret key. The AWS documentation provides an example of use with Postman.
The AWS documentation also provides several examples of how to use python and requests to perform this kind of authorization.
Consider, for instance, this example for posting information to DynamoDB:
# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of the
# License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
# OF ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
# AWS Version 4 signing example
# DynamoDB API (CreateTable)
# See: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
# This version makes a POST request and passes request parameters
# in the body (payload) of the request. Auth information is passed in
# an Authorization header.
import sys, os, base64, datetime, hashlib, hmac
import requests # pip install requests
# ************* REQUEST VALUES *************
method = 'POST'
service = 'dynamodb'
host = 'dynamodb.us-west-2.amazonaws.com'
region = 'us-west-2'
endpoint = 'https://dynamodb.us-west-2.amazonaws.com/'
# POST requests use a content type header. For DynamoDB,
# the content is JSON.
content_type = 'application/x-amz-json-1.0'
# DynamoDB requires an x-amz-target header that has this format:
# DynamoDB_<API version>.<operationName>
amz_target = 'DynamoDB_20120810.CreateTable'
# Request parameters for CreateTable--passed in a JSON block.
request_parameters = '{'
request_parameters += '"KeySchema": [{"KeyType": "HASH","AttributeName": "Id"}],'
request_parameters += '"TableName": "TestTable","AttributeDefinitions": [{"AttributeName": "Id","AttributeType": "S"}],'
request_parameters += '"ProvisionedThroughput": {"WriteCapacityUnits": 5,"ReadCapacityUnits": 5}'
request_parameters += '}'
# Key derivation functions. See:
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
def sign(key, msg):
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
def getSignatureKey(key, date_stamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
# Read AWS access key from env. variables or configuration file. Best practice is NOT
# to embed credentials in code.
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if access_key is None or secret_key is None:
print('No access key is available.')
sys.exit()
# Create a date for headers and the credential string
t = datetime.datetime.utcnow()
amz_date = t.strftime('%Y%m%dT%H%M%SZ')
date_stamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
# ************* TASK 1: CREATE A CANONICAL REQUEST *************
# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
# Step 1 is to define the verb (GET, POST, etc.)--already done.
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = '/'
## Step 3: Create the canonical query string. In this example, request
# parameters are passed in the body of the request and the query string
# is blank.
canonical_querystring = ''
# Step 4: Create the canonical headers. Header names must be trimmed
# and lowercase, and sorted in code point order from low to high.
# Note that there is a trailing \n.
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-date:' + amz_date + '\n' + 'x-amz-target:' + amz_target + '\n'
# Step 5: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers include those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
# For DynamoDB, content-type and x-amz-target are also required.
signed_headers = 'content-type;host;x-amz-date;x-amz-target'
# Step 6: Create payload hash. In this example, the payload (body of
# the request) contains the request parameters.
payload_hash = hashlib.sha256(request_parameters.encode('utf-8')).hexdigest()
# Step 7: Combine elements to create canonical request
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
# ************* TASK 2: CREATE THE STRING TO SIGN*************
# Match the algorithm to the hashing algorithm you use, either SHA-1 or
# SHA-256 (recommended)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = date_stamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
# ************* TASK 3: CALCULATE THE SIGNATURE *************
# Create the signing key using the function defined above.
signing_key = getSignatureKey(secret_key, date_stamp, region, service)
# Sign the string_to_sign using the signing_key
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
# Put the signature information in a header named Authorization.
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
# For DynamoDB, the request can include any headers, but MUST include "host", "x-amz-date",
# "x-amz-target", "content-type", and "Authorization". Except for the authorization
# header, the headers must be included in the canonical_headers and signed_headers values, as
# noted earlier. Order here is not significant.
# # Python note: The 'host' header is added automatically by the Python 'requests' library.
headers = {'Content-Type':content_type,
'X-Amz-Date':amz_date,
'X-Amz-Target':amz_target,
'Authorization':authorization_header}
# ************* SEND THE REQUEST *************
print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
print('Request URL = ' + endpoint)
r = requests.post(endpoint, data=request_parameters, headers=headers)
print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
print('Response code: %d\n' % r.status_code)
print(r.text)
I think it could be easily adapted to your needs.
In the console, everything works fine because when you invoke your REST endpoints in API Gateway, you are connected to a user who is already authenticated and authorized to access these REST endpoints.

SignatureDoesNotMatch with Google's serviceAccounts.signBlob API

I am trying to generated a signed URL to an object stored on Google Cloud Storage (GCS).
Attempt 1: try the API using the API Explorer
For this, I am trying to sign the blob/object as defined in the following:
GET
<expiration time>
/<bucket name>/<object/blob name>
I first tried Google's serviceAccounts.signBlob API as discussed in the following page:
https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/signBlob
A base64-encoded string.
Note, as mentioned in the API documentation on the above-linked page, I pass a base64 representation of the blob I want to sign, to the API.
The API's response has the following structure where it contains the signedBlob key:
{
"keyId": "...",
"signedBlob": "..."
}
then I generated a signed URL using the obtained signed blob as the following:
encoded_signedBlob = base64.b64encode(signedBlob)
signed_url = "https://storage.googleapis.com/{}/{}?" \
"GoogleAccessId={}&" \
"Expires={}&" \
"Signature={}".format(
bucket_name, blob_name,
service_account_email,
expiration,
encoded_signedBlob)
and when I paste that signed URL in the browser to download the blob, I get the following error:
<Error>
<Code>SignatureDoesNotMatch</Code>
<Message>
The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.
</Message>
<StringToSign>GET <bucket name> <blob/object name></StringToSign>
</Error>
Attempt 2: try python libraries
Then I tried to implement it in python as the following, but still getting the same error.
# -------------
# Part 1: obtain access token using the authorization flow discussed at:
# https://cloud.google.com/iam/docs/creating-short-lived-service-account-credentials
# -------------
client_service_account = "..."
access_token = build(
serviceName='iamcredentials',
version='v1',
http=http
).projects().serviceAccounts().generateAccessToken(
name="projects/{}/serviceAccounts/{}".format(
'-',
service_account_email),
body=body
).execute()["accessToken"]
credentials = AccessTokenCredentials(access_token, "MyAgent/1.0", None)
# -------------
# Part 2: sign the blob
# -------------
service = discovery.build('iam', 'v1', credentials=credentials)
name = 'projects/.../serviceAccounts/...'
encoded = base64.b64encode(blob)
sign_blob_request_body = {"bytesToSign": encoded}
request = service.projects().serviceAccounts().signBlob(name=name, body=sign_blob_request_body)
response = request.execute()
keyId = response["keyId"]
signedBlob = response["signature"]
# -------------
# Part 3: generate signed URL
# -------------
encoded_signedBlob = base64.b64encode(signedBlob)
signed_url = "https://storage.googleapis.com/{}/{}?" \
"GoogleAccessId={}&" \
"Expires={}&" \
"Signature={}".format(
bucket_name, blob_name,
service_account_email,
expiration,
encoded_signedBlob)
You could take a look to the implementation of this using the client libraries, you could give it a try, remember to include google-cloud-storage in your requirements.txt and a service account as specified in the code sample for python

How to fix unterminated short string error in Varnish VCL while configuring Fastly CDN

I am trying to setup token based authentication on Fastly CDN with Varnish VCL and using this sample VCL snippet to generate and validate JWT tokens here -
sub vcl_recv {
#FASTLY recv
if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
return(pass);
}
// Generate synth
if(req.url ~ "generate") {
error 901;
}
// Validate token
if(req.url ~ "validate") {
// Ensure token exists and parse into regex
if (req.http.X-JWT !~ "^([a-zA-Z0-9\-_]+)?\.([a-zA-Z0-9\-_]+)?\.([a-zA-Z0-9\-_]+)?$") {
// Forbidden
error 403 "Forbidden";
}
// Extract token header, payload and signature
set req.http.X-JWT-Header = re.group.1;
set req.http.X-JWT-Payload = re.group.2;
set req.http.X-JWT-Signature = digest.base64url_nopad_decode(re.group.3);
set req.http.X-JWT-Valid-Signature = digest.hmac_sha256("SupSecretStr",
req.http.X-JWT-Header "." req.http.X-JWT-Payload);
// Validate signature
if(digest.secure_is_equal(req.http.X-JWT-Signature, req.http.X-JWT-Valid-Signature)) {
// Decode payload
set req.http.X-JWT-Payload = digest.base64url_nopad_decode(req.http.X-JWT-Payload);
set req.http.X-JWT-Expires = regsub(req.http.X-JWT-Payload, {"^.*?"exp"\s*?:\s*?([0-9]+).*?$"}, "\1");
// Validate expiration
if (time.is_after(now, std.integer2time(std.atoi(req.http.X-JWT-Expires)))) {
// Unauthorized
synthetic {"{"sign":""} req.http.X-JWT-Signature {"","header":""} req.http.X-JWT-Header {"","payload":""} req.http.X-JWT-Payload {"","valid": ""} req.http.X-JWT-Valid-Signature {""}"};
return(deliver);
}
// OK
synthetic {"{"header2":""} req.http.X-JWT-Header {"","payload":""} req.http.X-JWT-Payload {"","sign":""} req.http.X-JWT-Signature {"","valid": ""} req.http.X-JWT-Valid-Signature {""}"};
return(deliver);
} else {
// Forbidden
synthetic {"{"header3":""} req.http.X-JWT-Header {"","payload":""} req.http.X-JWT-Payload {"","sign":""} req.http.X-JWT-Signature {"","valid": ""} req.http.X-JWT-Valid-Signature {""}"};
return(deliver);
}
}
return(lookup);
}
sub vcl_error {
#FASTLY error
// Generate JWT token
if (obj.status == 901) {
set obj.status = 200;
set obj.response = "OK";
set obj.http.Content-Type = "application/json";
set obj.http.X-UUID = randomstr(8, "0123456789abcdef") "-" randomstr(4, "0123456789abcdef") "-4" randomstr(3, "0123456789abcdef") "-" randomstr(1, "89ab") randomstr(3, "0123456789abcdef") "-" randomstr(12, "0123456789abcdef");
set obj.http.X-JWT-Issued = now.sec;
set obj.http.X-JWT-Expires = strftime({"%s"}, time.add(now, 3600s));
set obj.http.X-JWT-Header = digest.base64url_nopad({"{"alg":"HS256","typ":"JWT""}{"}"});
set obj.http.X-JWT-Payload = digest.base64url_nopad({"{"sub":""} obj.http.X-UUID {"","exp":"} obj.http.X-JWT-Expires {","iat":"} obj.http.X-JWT-Issued {","iss":"Fastly""}{"}"});
set obj.http.X-JWT-Signature = digest.base64url_nopad(digest.hmac_sha256("SupSecretStr", obj.http.X-JWT-Header "." obj.http.X-JWT-Payload));
set obj.http.X-JWT = obj.http.X-JWT-Header "." obj.http.X-JWT-Payload "." obj.http.X-JWT-Signature;
unset obj.http.X-UUID;
unset obj.http.X-JWT-Issued;
unset obj.http.X-JWT-Expires;
unset obj.http.X-JWT-Header;
unset obj.http.X-JWT-payload;
unset obj.http.X-JWT-Signature;
synthetic {"{"payload":""} obj.http.X-JWT-Payload {"","header":""} obj.http.X-JWT-Header {"","sign":""} obj.http.X-JWT-Signatre {"","token": ""} obj.http.X-JWT {""}"};
return(deliver);
}
// Valid token
if (obj.status == 902) {
set obj.status = 200;
set obj.response = "OK";
set obj.http.Content-Type = "application/json";
synthetic {"{ "token": ""} req.http.X-JWT {"" }"};
return(deliver);
}
}
Now, when I am trying to compile this it returns -
Syntax error: Unterminated _short-string_
at: (input Line 106 Pos 197)
synthetic {"{"sign":""} req.http.X-JWT-Signature {"","header":""} req.http.X-JWT-Header {"","payload":""} req.http.X-JWT-Payload {"","valid": ""} req.http.X-JWT-Valid-Signature {""}"};
Looks like i am not somehow escaping the values correctly here during the synthetic block.
The only reason I am trying to do add this synthetic block in the vcl_recv subroutine is because I want to test how the digest is generating the JWT token and validating it and with that I wanted to create similar JWT tokens on server side in Node.Js so i was trying to output the different intermediate parts of the token for debugging.
I am not quite familiar with the Varnish syntax and semantics but still I looked for help finding any docs regarding this schedule subroutine but found none so far.
So, can anyone help out in how to fix this and have the vcl_recv, vcl_error interpolate different intermediate values in the json response.
I tried to use some of the Node.Js based base64 url decode libraries to decode the returned token parts and was able to decode the header and payload part but the signature part I am not able to generate from the Node.Js. So, can anyone suggest what is the equivalent of base64url_nopad() in node.js or any javascript libraries ?
For the hmac_256 encryption part we are trying to use the crypto library and creating an hmac like crypto.createHmac('sha256', 'SupSecretStr').update().digest('hex'); but all base64 encode url libraries in js i think return padded urls which is why the base64 encoded part of this hmac 256 digest doesn't match the one generated with varnish
My syntax coloring tool is telling me pretty much the same thing as the error message: you screwed up your quotes :-)
Your last block {""}"}; is opening quotes ({"), immediately closing them ("}), then you are opening simple quotes " and the newline arrives before you close them.
To fix, just put a space between after the final quote of the json: {"" }"};

Using S3 Cloud Storage on IBM Bluemix

I am planning to use S3 Cloudstorage in IBM Bluemix but then one strange thing I found is that there is no way to add the custom META-DATA to the objects which are stored in S3 bucket.
Is there a way I can add custom Meta-Data to the objects and if yes then can you please advise on how we can add it and access it.?
Thanks for pointing out a hole in the documentation!
Custom metadata is defined by passing a x-amz-meta-{key} header with a {value} value. As an example request:
PUT /{bucket-name}/{object-name} HTTP/1.1
Authorization: {authorization-string}
x-amz-meta-foo: bar
x-amz-date: 20160825T183001Z
x-amz-content-sha256:{hashed-body}
Content-Type: text/plain; charset=utf-8
Host: s3-api.us-geo.objectstorage.softlayer.net
Content-Length: 18
{
"foo": "bar"
}
A HEAD request to check the metadata would look like:
HEAD /{bucket-name}/{object-name} HTTP/1.1
Authorization: {authorization-string}
x-amz-date: 20160825T183244Z
Host: s3-api.us-geo.objectstorage.softlayer.net
And respond with:
HTTP/1.1 200 OK
Date: Thu, 25 Aug 2016 18:32:44 GMT
X-Clv-Request-Id: da214d69-1999-4461-a130-81ba33c484a6
Accept-Ranges: bytes
Server: Cleversafe/3.9.1.102
X-Clv-S3-Version: 2.5
ETag: {MD5-hash}
Content-Type: text/plain; charset=UTF-8
x-amz-meta-foo: bar
Last-Modified: Thu, 25 Aug 2016 17:49:06 GMT
Content-Length: 18
Using the CLI, the syntax would be:
$ aws --endpoint-url=https://{endpoint} s3 cp ~/new-file s3://bucket-1/ --metadata foo=bar
Hope that helps!
This is possible. I am using this every day . Adding meta data then sending the meta data in database by doing cron calls.
Here is a small example of python script to create/add/change metadata for a a list object :
import sys
import os
import boto3
import pprint
from boto3 import client
from botocore.utils import fix_s3_host
param_1= YOUR_ACCESS_KEY
param_2= YOUR_SECRETE_KEY
param_3= YOUR_END_POINT
param_4= YOUR_BUCKET
#Create the S3 client
s3ressource = client(
service_name='s3',
endpoint_url= param_3,
aws_access_key_id= param_1,
aws_secret_access_key=param_2,
use_ssl=True,
)
# Building a list of object per bucket
def BuildObjectListPerBucket (variablebucket):
global listofObjectstobeanalyzed
listofObjectstobeanalyzed = []
extensions = ['.jpg','.png']
for key in s3ressource.list_objects(Bucket=variablebucket)["Contents"]:
#print (key ['Key'])
onemoreObject=key['Key']
if onemoreObject.endswith(tuple(extensions)):
listofObjectstobeanalyzed.append(onemoreObject)
else :
s3ressource.delete_object(Bucket=variablebucket,Key=onemoreObject)
return listofObjectstobeanalyzed
# for a given existing object, create metadata
def createmetdata(bucketname,objectname):
s3ressource.upload_file(objectname, bucketname, objectname, ExtraArgs={"Metadata": {"metadata1":"ImageName","metadata2":"ImagePROPERTIES" ,"metadata3":"ImageCREATIONDATE"}})
# for a given existing object, add new metadata
def ADDmetadata(bucketname,objectname):
s3_object = s3ressource.get_object(Bucket=bucketname, Key=objectname)
k = s3ressource.head_object(Bucket = bucketname, Key = objectname)
m = k["Metadata"]
m["new_metadata"] = "ImageNEWMETADATA"
s3ressource.copy_object(Bucket = bucketname, Key = objectname, CopySource = bucketname + '/' + objectname, Metadata = m, MetadataDirective='REPLACE')
# for a given existing object, update a metadata with new value
def CHANGEmetadata(bucketname,objectname):
s3_object = s3ressource.get_object(Bucket=bucketname, Key=objectname)
k = s3ressource.head_object(Bucket = bucketname, Key = objectname)
m = k["Metadata"]
m.update({'watson_visual_rec_dic':'ImageCREATIONDATEEEEEEEEEEEEEEEEEEEEEEEEEE'})
s3ressource.copy_object(Bucket = bucketname, Key = objectname, CopySource = bucketname + '/' + objectname, Metadata = m, MetadataDirective='REPLACE')
def readmetadata (bucketname,objectname):
ALLDATAOFOBJECT = s3ressource.get_object(Bucket=bucketname, Key=objectname)
ALLDATAOFOBJECTMETADATA=ALLDATAOFOBJECT['Metadata']
print ALLDATAOFOBJECTMETADATA
# create the list of object on a per bucket basis
BuildObjectListPerBucket (variablebucket)
# Call functions to see the results
for objectitem in listofObjectstobeanalyzed:
readmetadata(param_4,objectitem)
createmetdata(param_4,objectitem)
readmetadata(param_4,objectitem)
ADDmetadata(param_4,objectitem)
readmetadata(param_4,objectitem)
CHANGEmetadata(param_4,objectitem)
readmetadata(param_4,objectitem)