diff --git a/python/pyspark/pandas/tests/data_type_ops/testing_utils.py b/python/pyspark/pandas/tests/data_type_ops/testing_utils.py index 04d03a05e02dc..3c23fd8727f0d 100644 --- a/python/pyspark/pandas/tests/data_type_ops/testing_utils.py +++ b/python/pyspark/pandas/tests/data_type_ops/testing_utils.py @@ -219,3 +219,17 @@ def check_extension(self, left, right): pandas versions. Please refer to https://github.com/pandas-dev/pandas/issues/39410. """ self.assert_eq(left, right) + + def assert_eq(self, left, right, **kwargs): + """ + SPARK-55321: + Ignore null values when comparing dataframe or series. pyspark.pandas generates + a different null value than pandas and pandas 3 has a more strict testing + utility. + """ + try: + if left.isnull().to_numpy().any() or right.isnull().to_numpy().any(): + kwargs["ignore_null"] = True + except AttributeError: + pass + return super().assert_eq(left, right, **kwargs)