Compare two data frame row by row in pyspark - pyspark

Question

I am trying to compare two data frame row by row such that if any mismatch found it prints in below formatted way.
Example:
data = [("James", "M", 60000), ("Michael", "M", 70000),
("Robert", None, 400000), ("Maria", "F", 500000),
("Jen", "", None),(None,None,None)]
columns = ["name", "gender", "salary"]
source_df = spark.createDataFrame(data=data, schema=columns)
source_df.show()
+-------+------+------+
| name|gender|salary|
+-------+------+------+
| James| M| 60000|
|Michael| M| 70000|
| Robert| null|400000|
| Maria| F|500000|
| Jen| | null|
| null| null| null|
+-------+------+------+
data1 = [("Anurag", "M", 70000), ("Michael", "M", 70000),
("Sunil", None, 900000), ("Maria", "F", 500000),
("Jen", "", None),(None,None,None)]
columns = ["name_1", "gender_1", "salary_1"]
target_df = spark.createDataFrame(data=data1, schema=columns)
target_df.show()
| name_1|gender_1|salary_1|
+-------+--------+--------+
| Anurag| M| 70000|
|Michael| M| 70000|
| Sunil| null| 900000|
| Maria| F| 500000|
| Jen| | null|
| null| null| null|
+-------+--------+--------+
So, here I have to iterate over row of 1st dataframe James|M|60000 and compare with row of 2nd dataframe Anurag|M|70000 and so on.. and print the output in formatted way if any mismatch found
eg :
1st df: 'James|M|60000'
2nd df: 'Anurag|M|70000'
output: Mismatch: name-->James,name_1--> Anurag
salary-->60000, salary_1--> 70000..so on..
Please let me know if any other info required.
Very new to pyspark, so need your help. Thanks in Advance for your help.
The below code working fine for me.
from pyspark.sql.functions import concat, monotonically_increasing_id, udf, col
from pyspark.sql import SparkSession
from os import truncate
import findspark
findspark.init()
findspark.find()
findspark.find()
def print_mismatch(row):
output = ""
for i in range(len(source_cols)):
output += f"Row Index--> {row['id']}, "
if row[source_cols[i]] != row[target_cols[i]]:
output += f"{source_cols[i]}--> {row[source_cols[i]]}, {target_cols[i]}--> {row[target_cols[i]]} "
return output
spark = SparkSession \
.builder \
.appName("SparkExample") \
.getOrCreate()
data = [("James", "M", 60000), ("Michael", "M", 70000),
("Robert", None, 400000), ("Maria", "F", 500000),
("Jen", "", None), (None, None, None)]
columns = ["name", "gender", "salary"]
source_df = spark.createDataFrame(data=data, schema=columns)
rdd_df = source_df.rdd.zipWithIndex()
source_df = rdd_df.toDF().select(col("_1.*"), col("_2").alias("id"))
data1 = [("Anurag", "M", 70000), ("Michael", "M", 70000),
("Sunil", None, 900000), ("Maria", "F", 500000),
("Jen", "", None), (None, None, None)]
columns = ["name_1", "gender_1", "salary_1"]
target_df = spark.createDataFrame(data=data1, schema=columns)
rdd_df = target_df.rdd.zipWithIndex()
target_df = rdd_df.toDF().select(col("_1.*"), col("_2").alias("id1"))
final_df = source_df.join(target_df, source_df.id == target_df.id1)
source_cols = source_df.columns
target_cols = target_df.columns
final_coulmn = source_cols + target_cols
df = spark.createDataFrame([], schema=final_df.schema)
for i in range(len(source_cols)):
final = final_df.filter(
final_df[f'{source_cols[i]}'] != final_df[f'{target_cols[i]}'])
df = df.union(final)
# final.collect()
final_df.show()
#df.show()
df_rdd = df.rdd
df_rdd.map(print_mismatch).collect() ```

Answer 1

You can achieve the same as follows:
from pyspark.sql import SparkSession
from pyspark.sql.functions import concat, monotonically_increasing_id, udf, col
def print_mismatch(row):
output = ""
for i in range(len(source_cols)):
if row[source_cols[i]] != row[target_cols[i]]:
output += f"{source_cols[i]}--> {row[source_cols[i]]}, {target_cols[i]}--> {row[target_cols[i]]}"
print(output)
spark = SparkSession \
.builder \
.appName("SparkExample") \
.getOrCreate()
data = [("James", "M", 60000), ("Michael", "M", 70000),
("Robert", None, 400000), ("Maria", "F", 500000),
("Jen", "", None), (None, None, None)]
columns = ["name", "gender", "salary"]
source_df = spark.createDataFrame(data=data, schema=columns)
rdd_df = source_df.rdd.zipWithIndex()
source_df = rdd_df.toDF().select(col("_1.*"), col("_2").alias("id"))
# source_df = source_df.withColumn("id", monotonically_increasing_id())
source_df.show()
data1 = [("Anurag", "M", 70000), ("Michael", "M", 70000),
("Sunil", None, 900000), ("Maria", "F", 500000),
("Jen", "", None), (None, None, None)]
columns = ["name_1", "gender_1", "salary_1"]
target_df = spark.createDataFrame(data=data1, schema=columns)
rdd_df = target_df.rdd.zipWithIndex()
target_df = rdd_df.toDF().select(col("_1.*"), col("_2").alias("id"))
# target_df = target_df.withColumn("id", monotonically_increasing_id())
target_df.show()
final_df = source_df.join(target_df, source_df.id == target_df.id)
final_df.show()
source_cols = source_df.columns
target_cols = target_df.columns
final_df.foreach(lambda row: print_mismatch(row))
name--> James, name_1--> Anurag salary--> 60000, salary_1--> 70000
name--> Robert, name_1--> Sunil salary--> 400000, salary_1--> 900000
spark-submit --master local spark_combining_df.py
enter image description here