From 40639b08136e118ea857f878d6733e4613b5389e Mon Sep 17 00:00:00 2001 From: MithunR Date: Mon, 28 Oct 2024 22:20:58 +0000 Subject: [PATCH] Fix `misc_expr_test` for [databricks] 14.3 Fixes #11537. This commit addresses the failure of the `test_raise_error` test in `misc_expr_test.py` for Databricks 14.3. This is an extension of #11129, where this test was skipped for Apache Spark 4.0. The failure on Databricks 14.3 shares the same cause as in Spark 4.0, i.e. a backward-incompatible Spark change in the signature of RaiseError, as introduced in https://issues.apache.org/jira/browse/SPARK-44838. The work to support this change in a Spark-RAPIDS shim will be tracked in #10969. This test will be skipped until that work is completed. Signed-off-by: MithunR --- integration_tests/src/main/python/misc_expr_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/main/python/misc_expr_test.py b/integration_tests/src/main/python/misc_expr_test.py index 3251df08d27..0895d451b9d 100644 --- a/integration_tests/src/main/python/misc_expr_test.py +++ b/integration_tests/src/main/python/misc_expr_test.py @@ -19,7 +19,7 @@ from marks import incompat, approximate_float from pyspark.sql.types import * import pyspark.sql.functions as f -from spark_session import is_before_spark_400 +from spark_session import is_databricks_version_or_later, is_spark_400_or_later def test_mono_id(): assert_gpu_and_cpu_are_equal_collect( @@ -34,8 +34,8 @@ def test_part_id(): f.spark_partition_id())) -@pytest.mark.skipif(condition=not is_before_spark_400(), - reason="raise_error() not currently implemented for Spark 4.0. " +@pytest.mark.skipif(condition=is_spark_400_or_later() or is_databricks_version_or_later(14, 3), + reason="raise_error() not currently implemented for Spark 4.0, or Databricks 14.3. " "See https://github.com/NVIDIA/spark-rapids/issues/10107.") def test_raise_error(): data_gen = ShortGen(nullable=False, min_val=0, max_val=20, special_cases=[])