submit function to run in parallel step on emr - pyspark

Question

I'm submitting steps to run in parallel on an emr. I'm doing this by saving python scripts with pyspark code at an s3 path. I then use the boto code below to submit the steps in parallel to the emr. The steps run the code I have saved to s3 in script files. I would like to instead create a function, like the "read_write" example function below, to perform the same operations I'm doing in the saved python scripts. That way I wouldn't have to save a python script file to s3 for every step I'm trying to run in parallel. Is there a way to submit the code I would like to run in the step to the emr with the boto code below, without having to save it in a script file to s3? I've supplied example code similar to what I'm currently using to submit steps to run in parallel below. I've also supplied code similar to what I would like to try to do, creating the "read_write" function in the same code with the boto script and submitting it as a step to run in parallel to the emr.
# test1.py
from pyspark.sql.types import *
from pyspark.context import SparkContext
from pyspark.sql import SQLContext
from pyspark import SparkConf
from pyspark.sql import SparkSession
import pyspark
# from cloudwatch_logs import (log_handler, error_manager)
import boto3
#############
#############
conf = pyspark.SparkConf()
spark = SparkSession.builder \
.appName("test1") \
.config('spark.sql.codegen.wholeStage', False) \
.getOrCreate()
sc = SparkContext.getOrCreate(conf=conf)
sqlContext = SQLContext(sc)
df=sqlContext.read.parquet("path1")
df.write.mode("overwrite").parquet("write_path1")
# test2.py
from pyspark.sql.types import *
from pyspark.context import SparkContext
from pyspark.sql import SQLContext
from pyspark import SparkConf
from pyspark.sql import SparkSession
import pyspark
# from cloudwatch_logs import (log_handler, error_manager)
import boto3
#############
#############
conf = pyspark.SparkConf()
spark = SparkSession.builder \
.appName("test2") \
.config('spark.sql.codegen.wholeStage', False) \
.getOrCreate()
sc = SparkContext.getOrCreate(conf=conf)
sqlContext = SQLContext(sc)
df=sqlContext.read.parquet("path2")
df.write.mode("overwrite").parquet("write_path2")
# script to submit multiple steps to run in parallel on emr
client = boto3.client('emr', region_name='us-west-2')
response = client.add_job_flow_steps(
JobFlowId='j-xxxxxxxx', # clusterid
Steps=[
{
'Name': 'test1',
'ActionOnFailure': 'CONTINUE',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': [
"spark-submit",
"--deploy-mode",
"cluster",
"--master",
"yarn",
"--conf",
"spark.yarn.submit.waitAppCompletion=true",
"s3a://path/test1.py"
]
}
},
{
'Name': 'test2',
'ActionOnFailure': 'CONTINUE',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': [
"spark-submit",
"--deploy-mode",
"cluster",
"--master",
"yarn",
"--conf",
"spark.yarn.submit.waitAppCompletion=true",
"s3a://path/test2.py"
]
}
}
]
)
# what I would like to do
from pyspark.sql.types import *
from pyspark.context import SparkContext
from pyspark.sql import SQLContext
from pyspark import SparkConf
from pyspark.sql import SparkSession
import pyspark
# from cloudwatch_logs import (log_handler, error_manager)
import boto3
# function to read and write dataframe
def read_write(read_path,write_path):
df=sqlContext.read.parquet(read_path)
df.write.mode("overwrite").parquet(write_path)
client = boto3.client('emr', region_name='us-west-2')
response = client.add_job_flow_steps(
JobFlowId='j-xxxxxxxx', # clusterid
Steps=[
{
'Name': 'test1',
'ActionOnFailure': 'CONTINUE',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': [
"spark-submit",
"--deploy-mode",
"cluster",
"--master",
"yarn",
"--conf",
"spark.yarn.submit.waitAppCompletion=true",
read_write(read_path='path1',write_path='write_path1')
]
}
},
{
'Name': 'test2',
'ActionOnFailure': 'CONTINUE',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': [
"spark-submit",
"--deploy-mode",
"cluster",
"--master",
"yarn",
"--conf",
"spark.yarn.submit.waitAppCompletion=true",
read_write(read_path='path2',write_path='write_path2')
]
}
}
]
)

Answer 1

Why not just turn on the concurrent steps?
StepConcurrencyLevel (integer) -- Specifies the number of steps that can be executed concurrently. The default value is 1 . The maximum value is 256 .
Check the documentation for more details.