Explode array of structs to columns in pyspark - pyspark

Question

I'd like to explode an array of structs to columns (as defined by the struct fields). E.g.
root
|-- news_style_super: array (nullable = true)
| |-- element: array (containsNull = true)
| | |-- element: struct (containsNull = true)
| | | |-- name: string (nullable = true)
| | | |-- sbox_ctr: double (nullable = true)
| | | |-- wise_ctr: double (nullable = true)
Should be transformed to
|-- name: string (nullable = true)
|-- sbox_ctr: double (nullable = true)
|-- wise_ctr: double (nullable = true)
How can I do this?

Answer 1

def get_final_dataframe(pathname, df):
cur_names = pathname.split(".")
if len(cur_names) > 1:
root_name = cur_names[0]
delimiter = "."
new_path_name = delimiter.join(cur_names[1:len(cur_names)])
for field in df.schema.fields:
if field.name == root_name:
if type(field.dataType) == ArrayType:
return get_final_dataframe(pathname, df.select(explode(root_name).alias(root_name)))
elif type(field.dataType) == StructType:
if hasColumn(df, delimiter.join(cur_names[0:2])):
return get_final_dataframe(new_path_name, df.select(delimiter.join(cur_names[0:2])))
else:
return -1, -1
else:
return -1, -1
else:
root_name = cur_names[0]
for field in df.schema.fields:
if field.name == root_name:
if type(field.dataType) == StringType:
return df, "string"
elif type(field.dataType) == LongType:
return df, "numeric"
elif type(field.dataType) == DoubleType:
return df, "numeric"
else:
return df, -1
return -1, -1
then,you can
key = "a.b.c.name"
# key = "context.content_feature.tag.name"
df2, field_type = get_final_dataframe(key, df1)