Adding a custom function in SPARK NLP Pipeline - pyspark

Question

I am trying to add a custom function (to remove names from a document) and I want this udf to be incorporated into the SPARK NLP pipeline. Can you please tell me where I am doing it wrong?
Here is the udf I created to remove names
def remove_names(str):
#All_names = list(map(lambda x: x.lower(), All_names))
All_names = ['josh','alice']
split_str = str.split()
found_names = [name for name in split_str if name in All_names]
for name in found_names:
str = str.replace(name,'')
str = ' '.join(str.split())
return str
udf_txt_clean = udf(lambda x: remove_names(x),StringType())
My pipeline looks like this :
documentAssembler = DocumentAssembler()\
.setInputCol("combined_text")\
.setOutputCol("document")
remove_names = udf_txt_clean('document') \
.setInputCols(["document"]) \
.setOutputCol("noname")
tokenizer = Tokenizer() \
.setInputCols(["noname"]) \
.setOutputCol("token")
spellModel = ContextSpellCheckerModel\
.pretrained()\
.setInputCols("token")\
.setOutputCol("checked")
normalizer = Normalizer() \
.setInputCols(["checked"]) \
.setOutputCol("normalized")\
.setLowercase(True)\
stopwords_cleaner = StopWordsCleaner()\
.setInputCols("normalized")\
.setOutputCol("cleanTokens")\
.setCaseSensitive(False)\
tokenassembler = TokenAssembler()\
.setInputCols(["document", "cleanTokens"]) \
.setOutputCol("clean_text")
nlpPipeline = Pipeline(stages=[
documentAssembler,
remove_names,
tokenizer,
spellModel,
normalizer,
stopwords_cleaner,
tokenassembler
])
empty_df = spark.createDataFrame([['']]).toDF("text")
pipelineModel = nlpPipeline.fit(empty_df)
result = pipelineModel.transform(sdf)
result.select('text', explode('clean_text.result').alias('clean_text')).display()
Sample Data just to recreate the issue
text = 'josh and sam are buddies. They (both) like <b>running</b>. They got better at it over the weekend'
test = pd.DataFrame({"test": [text]})
sdf = spark.createDataFrame(test)