Support Spark 3.4.1 and 3.3.3 (#382)

oap-project · Oct 8, 2023 · f3b136a · f3b136a
1 parent 7da5e92
commit f3b136a
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala b/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala
@@ -23,7 +23,9 @@ object SparkShimProvider {
   val SPARK330_DESCRIPTOR = SparkShimDescriptor(3, 3, 0)
   val SPARK331_DESCRIPTOR = SparkShimDescriptor(3, 3, 1)
   val SPARK332_DESCRIPTOR = SparkShimDescriptor(3, 3, 2)
-  val DESCRIPTOR_STRINGS = Seq(s"$SPARK330_DESCRIPTOR", s"$SPARK331_DESCRIPTOR", s"$SPARK332_DESCRIPTOR")
+  val SPARK333_DESCRIPTOR = SparkShimDescriptor(3, 3, 3)
+  val DESCRIPTOR_STRINGS = Seq(s"$SPARK330_DESCRIPTOR", s"$SPARK331_DESCRIPTOR",
+    s"$SPARK332_DESCRIPTOR", s"$SPARK333_DESCRIPTOR")
   val DESCRIPTOR = SPARK332_DESCRIPTOR
 }
 

diff --git a/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala b/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala
@@ -21,8 +21,9 @@ import com.intel.raydp.shims.{SparkShims, SparkShimDescriptor}
 
 object SparkShimProvider {
   val SPARK340_DESCRIPTOR = SparkShimDescriptor(3, 4, 0)
-  val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR")
-  val DESCRIPTOR = SPARK340_DESCRIPTOR
+  val SPARK341_DESCRIPTOR = SparkShimDescriptor(3, 4, 1)
+  val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR", s"$SPARK341_DESCRIPTOR")
+  val DESCRIPTOR = SPARK341_DESCRIPTOR
 }
 
 class SparkShimProvider extends com.intel.raydp.shims.SparkShimProvider {

diff --git a/python/setup.py b/python/setup.py
@@ -100,7 +100,7 @@ def run(self):
         "psutil",
         "pyarrow >= 4.0.1",
         "ray >= 2.1.0",
-        "pyspark >= 3.1.1, <= 3.3.2",
+        "pyspark >= 3.1.1, <= 3.4.1",
         "netifaces",
         "protobuf > 3.19.5, <= 3.20.3"
     ]