pyspark parse user_agents into new columns - pyspark

Question

I am having a problem when trying to parse user_agents. I am using this lib lib and have followed the instructions from medium.
def parse_ua(ua_string):
# parse library cannot parse None
if ua_string is None:
ua_string = ""
parsed_string = parse(ua_string)
output = [
parsed_string.device.brand,
parsed_string.device.family,
parsed_string.device.model,
parsed_string.os.family,
parsed_string.os.version_string,
parsed_string.browser.family,
parsed_string.browser.version_string,
(parsed_string.is_mobile or parsed_string.is_tablet),
parsed_string.is_bot
]
# If any of the column have None value it doesn't comply with schema
# and thus throw Null Pointer Exception
for i in range(len(output)):
if output[i] is None:
print(output[i])
output[i] = 'Unknown'
return output
ua_parser_udf = F.udf(lambda z: parse_ua(z), StructType([
StructField("device_brand", StringType(), False),
StructField("device_family", StringType(), False),
StructField("device_model", StringType(), False),
StructField("os_family", StringType(), False),
StructField("os_version", StringType(), False),
StructField("browser_family", StringType(), False),
StructField("browser_version", StringType(), False)
]))
df = df.withColumn('parsed', ua_parser_udf('meta_user_agent'))\
.select('*',
F.col('parsed.device_brand').alias('device_brand'),
F.col('parsed.device_family').alias('device_family'),
F.col('parsed.device_model').alias('device_model'),
F.col('parsed.os_family').alias('os_family'),
F.col('parsed.os_version').alias('os_version'),
F.col('parsed.browser_family').alias('browser_family'),
F.col('parsed.browser_version').alias('browser_version'),
F.col('parsed.is_mobile').alias('is_mobile'),
F.col('parsed.is_bot').alias('is_bot')
)
My problem is that, when I call the function I am getting this error
Could not serialize object: Py4JError: An error occurred while calling o517.__getstate__. Trace:
py4j.Py4JException: Method __getstate__([]) does not exist
I operate in AWS, using s3 as hdfs, AWS Glue for reading_from_catalog and AWS Catalog as a MetaStore. Any tips or help on how to fix this issue? Thanks a lot!