diff --git a/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala b/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala index 67aad858..7a5f5481 100644 --- a/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala +++ b/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala @@ -23,7 +23,9 @@ object SparkShimProvider { val SPARK330_DESCRIPTOR = SparkShimDescriptor(3, 3, 0) val SPARK331_DESCRIPTOR = SparkShimDescriptor(3, 3, 1) val SPARK332_DESCRIPTOR = SparkShimDescriptor(3, 3, 2) - val DESCRIPTOR_STRINGS = Seq(s"$SPARK330_DESCRIPTOR", s"$SPARK331_DESCRIPTOR", s"$SPARK332_DESCRIPTOR") + val SPARK333_DESCRIPTOR = SparkShimDescriptor(3, 3, 3) + val DESCRIPTOR_STRINGS = Seq(s"$SPARK330_DESCRIPTOR", s"$SPARK331_DESCRIPTOR", + s"$SPARK332_DESCRIPTOR", s"$SPARK333_DESCRIPTOR") val DESCRIPTOR = SPARK332_DESCRIPTOR } diff --git a/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala b/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala index ff757be7..a73dcacf 100644 --- a/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala +++ b/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala @@ -21,8 +21,9 @@ import com.intel.raydp.shims.{SparkShims, SparkShimDescriptor} object SparkShimProvider { val SPARK340_DESCRIPTOR = SparkShimDescriptor(3, 4, 0) - val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR") - val DESCRIPTOR = SPARK340_DESCRIPTOR + val SPARK341_DESCRIPTOR = SparkShimDescriptor(3, 4, 1) + val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR", s"$SPARK341_DESCRIPTOR") + val DESCRIPTOR = SPARK341_DESCRIPTOR } class SparkShimProvider extends com.intel.raydp.shims.SparkShimProvider { diff --git a/python/setup.py b/python/setup.py index 75b6d992..4af2ed44 100644 --- a/python/setup.py +++ b/python/setup.py @@ -100,7 +100,7 @@ def run(self): "psutil", "pyarrow >= 4.0.1", "ray >= 2.1.0", - "pyspark >= 3.1.1, <= 3.3.2", + "pyspark >= 3.1.1, <= 3.4.1", "netifaces", "protobuf > 3.19.5, <= 3.20.3" ]