Cloudformation aws-glue inline command - aws-cloudformation

Question

My goal is to create a glue job via cloudformation. Problem that im dealing with is that Command property doesnt seem to support inline code (like cloudformation lamba Code property does).
My question, is there a way to create a fully functional glue job solely with cloudformation template, is there a way around uploading command file in advance (and specifying ScriptLocation) ?

Answer 1

Something like this may work (untested, so you might have to tweak). Basically you create a custom resource to facilitate uploading your glue code to S3 and then reference the custom resource to obtain the location. Below you'll find the CloudFormation template, and below that the Rubycfn code you can use to generate the template dynamically.
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "Glue Job Example Stack",
"Resources": {
"GlueCode": {
"Properties": {
"BucketName": "my-amazing-bucket-for-glue-jobs",
"Content": {
"Fn::Join": [
"\n",
[
"# some job code",
"# for glue"
]
]
},
"CreateMode": "plain-literal",
"FileName": "gluecode.sh",
"ServiceToken": {
"Fn::GetAtt": [
"InlineS3UploadFunction",
"Arn"
]
}
},
"Type": "Custom::InlineUpload"
},
"GlueJob": {
"Properties": {
"Command": {
"Name": "myglueetl",
"ScriptLocation": {
"Fn::Join": [
"",
[
"s3://my-amazing-bucket-for-glue-jobs/",
{
"Ref": "GlueCode"
}
]
]
}
},
"DefaultArguments": {
"--job-bookmark-option": "job-bookmark-enable"
},
"ExecutionProperty": {
"MaxConcurrentRuns": 2
},
"MaxRetries": 0,
"Name": "glue-job-1",
"Role": {
"Ref": "SomeRole"
}
},
"Type": "AWS::Glue::Job"
},
"InlineS3UploadFunction": {
"Properties": {
"Code": {
"ZipFile": {
"Fn::Join": [
"\n",
[
"import boto3",
"import cfnresponse",
"import hashlib",
"import json",
"import logging",
"import signal",
"import zipfile",
"",
"from urllib2 import build_opener, HTTPHandler, Request",
"",
"LOGGER = logging.getLogger()",
"LOGGER.setLevel(logging.INFO)",
"",
"def lambda_handler(event, context):",
" # Setup alarm for remaining runtime minus a second",
" try:",
" signal.alarm((context.get_remaining_time_in_millis() / 1000) - 1)",
" LOGGER.info('REQUEST RECEIVED: %s', event)",
" LOGGER.info('REQUEST RECEIVED: %s', context)",
" if event['RequestType'] == 'Create' or event['RequestType'] == 'Update':",
" LOGGER.info('Creating or updating S3 Object')",
" bucket_name = event['ResourceProperties']['BucketName']",
" file_name = event['ResourceProperties']['FileName']",
" content = event['ResourceProperties']['Content']",
" create_zip = True if event['ResourceProperties']['CreateMode'] == 'zip' else False",
" literal = True if event['ResourceProperties']['CreateMode'] == 'plain-literal' else False",
" md5_hash = hashlib.md5(content).hexdigest()",
" with open('/tmp/' + file_name, 'w') as lambda_file:",
" lambda_file.write(content)",
" lambda_file.close()",
" s3 = boto3.resource('s3')",
" if create_zip == True:",
" output_filename = file_name + '_' + md5_hash + '.zip'",
" zf = zipfile.ZipFile('/tmp/' + output_filename, mode='w')",
" try:",
" zf.write('/tmp/' + file_name, file_name)",
" finally:",
" zf.close()",
" data = open('/tmp/' + output_filename, 'rb')",
" s3.Bucket(bucket_name).put_object(Key=output_filename, Body=data)",
" else:",
" if literal == True:",
" data = open('/tmp/' + file_name, 'rb')",
" s3.Bucket(bucket_name).put_object(Key=file_name, Body=content)",
" else:",
" extension = file_name.split(\".\")[-1]",
" output_filename = \".\".join(file_name.split(\".\")[:-1]) + '_' + md5_hash + '.' + extension",
" data = open('/tmp/' + file_name, 'rb')",
" s3.Bucket(bucket_name).put_object(Key=output_filename, Body=content)",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': output_filename } )",
" elif event['RequestType'] == 'Delete':",
" LOGGER.info('DELETE!')",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': 'Resource deletion successful!'} )",
" else:",
" LOGGER.info('FAILED!')",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': 'There is no such success like failure.'} )",
" except Exception as e: #pylint: disable=W0702",
" LOGGER.info(e)",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': 'There is no such success like failure.' } )",
"",
"def timeout_handler(_signal, _frame):",
" '''Handle SIGALRM'''",
" LOGGER.info('Time exceeded')",
" raise Exception('Time exceeded')",
"",
"signal.signal(signal.SIGALRM, timeout_handler)"
]
]
}
},
"Handler": "index.lambda_handler",
"Role": {
"Fn::GetAtt": [
"LambdaExecutionRole",
"Arn"
]
},
"Runtime": "python2.7",
"Timeout": "30"
},
"Type": "AWS::Lambda::Function"
},
"LambdaExecutionRole": {
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": [
"sts:AssumeRole"
],
"Effect": "Allow",
"Principal": {
"Service": [
"lambda.amazonaws.com"
]
}
}
],
"Version": "2012-10-17"
},
"Path": "/",
"Policies": [
{
"PolicyDocument": {
"Statement": [
{
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Effect": "Allow",
"Resource": "arn:aws:logs:*:*:*"
},
{
"Action": "s3:*",
"Effect": "Allow",
"Resource": "arn:aws:s3:::*"
}
],
"Version": "2012-10-17"
},
"PolicyName": "root"
}
]
},
"Type": "AWS::IAM::Role"
}
}
}
The Rubycfn code is below. Use the https://rubycfn.com/ CloudFormation compiler or the Rubycfn gem gem install rubycfn to compile:
ENV["GLUEBUCKET"] ||= "my-amazing-bucket-for-glue-jobs"
description "Glue Job Example Stack"
resource :glue_code,
type: "Custom::InlineUpload" do |r|
r.property(:service_token) { :inline_s3_upload_function.ref(:arn) }
r.property(:bucket_name) { ENV["GLUEBUCKET"] }
r.property(:file_name) { "gluecode.sh" }
r.property(:create_mode) { "plain-literal" }
r.property(:content) do
[
"# some job code",
"# for glue"
].fnjoin("\n")
end
end
resource :glue_job,
type: "AWS::Glue::Job" do |r|
r.property(:command) do
{
"Name": "myglueetl",
"ScriptLocation": ["s3://#{ENV["GLUEBUCKET"]}/", :glue_code.ref].fnjoin
}
end
r.property(:default_arguments) do
{
"--job-bookmark-option": "job-bookmark-enable"
}
end
r.property(:execution_property) do
{
"MaxConcurrentRuns": 2
}
end
r.property(:max_retries) { 0 }
r.property(:name) { "glue-job-1" }
r.property(:role) { :some_role.ref }
end
resource :inline_s3_upload_function,
type: "AWS::Lambda::Function" do |r|
r.property(:code) do
{
"ZipFile": [
"import boto3",
"import cfnresponse",
"import hashlib",
"import json",
"import logging",
"import signal",
"import zipfile",
"",
"from urllib2 import build_opener, HTTPHandler, Request",
"",
"LOGGER = logging.getLogger()",
"LOGGER.setLevel(logging.INFO)",
"",
"def lambda_handler(event, context):",
" # Setup alarm for remaining runtime minus a second",
" try:",
" signal.alarm((context.get_remaining_time_in_millis() / 1000) - 1)",
" LOGGER.info('REQUEST RECEIVED: %s', event)",
" LOGGER.info('REQUEST RECEIVED: %s', context)",
" if event['RequestType'] == 'Create' or event['RequestType'] == 'Update':",
" LOGGER.info('Creating or updating S3 Object')",
" bucket_name = event['ResourceProperties']['BucketName']",
" file_name = event['ResourceProperties']['FileName']",
" content = event['ResourceProperties']['Content']",
" create_zip = True if event['ResourceProperties']['CreateMode'] == 'zip' else False",
" literal = True if event['ResourceProperties']['CreateMode'] == 'plain-literal' else False",
" md5_hash = hashlib.md5(content).hexdigest()",
" with open('/tmp/' + file_name, 'w') as lambda_file:",
" lambda_file.write(content)",
" lambda_file.close()",
" s3 = boto3.resource('s3')",
" if create_zip == True:",
" output_filename = file_name + '_' + md5_hash + '.zip'",
" zf = zipfile.ZipFile('/tmp/' + output_filename, mode='w')",
" try:",
" zf.write('/tmp/' + file_name, file_name)",
" finally:",
" zf.close()",
" data = open('/tmp/' + output_filename, 'rb')",
" s3.Bucket(bucket_name).put_object(Key=output_filename, Body=data)",
" else:",
" if literal == True:",
" data = open('/tmp/' + file_name, 'rb')",
" s3.Bucket(bucket_name).put_object(Key=file_name, Body=content)",
" else:",
" extension = file_name.split(\".\")[-1]",
" output_filename = \".\".join(file_name.split(\".\")[:-1]) + '_' + md5_hash + '.' + extension",
" data = open('/tmp/' + file_name, 'rb')",
" s3.Bucket(bucket_name).put_object(Key=output_filename, Body=content)",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': output_filename } )",
" elif event['RequestType'] == 'Delete':",
" LOGGER.info('DELETE!')",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': 'Resource deletion successful!'} )",
" else:",
" LOGGER.info('FAILED!')",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': 'There is no such success like failure.'} )",
" except Exception as e: #pylint: disable=W0702",
" LOGGER.info(e)",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, { 'Message': 'There is no such success like failure.' } )",
"",
"def timeout_handler(_signal, _frame):",
" '''Handle SIGALRM'''",
" LOGGER.info('Time exceeded')",
" raise Exception('Time exceeded')",
"",
"signal.signal(signal.SIGALRM, timeout_handler)"
].fnjoin("\n")
}
end
r.property(:handler) { "index.lambda_handler" }
r.property(:role) { :lambda_execution_role.ref(:arn) }
r.property(:runtime) { "python2.7" }
r.property(:timeout) { "30" }
end
resource :lambda_execution_role,
type: "AWS::IAM::Role" do |r|
r.property(:assume_role_policy_document) do
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"lambda.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
}
end
r.property(:path) { "/" }
r.property(:policies) do
[
{
"PolicyName": "root",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": %w(logs:CreateLogGroup logs:CreateLogStream logs:PutLogEvents),
"Resource": "arn:aws:logs:*:*:*"
},
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": "arn:aws:s3:::*"
}
]
}
}
]
end
end