DATAFU-175 Support Spark 3.2.0

.github/workflows/tests.yml

@@ -20,18 +20,18 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Use Java 8
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v4
       with:
          java-version: '8'
          distribution: 'adopt'
          cache: 'gradle'
 
 
     - name: Set up Python      
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
          python-version: '3.7'
 

datafu-spark/README.md

@@ -11,7 +11,7 @@ This matrix represents versions of Spark that DataFu has been compiled and teste
 | 1.7.0 | 2.2.0 to 2.2.2, 2.3.0 to 2.3.2 and 2.4.0 to 2.4.3|
 | 1.8.0 | 2.2.3, 2.3.3, and 2.4.4 to 2.4.5|
 | 2.0.0 | 3.0.x - 3.1.x |
-| 2.1.0 (not released yet) | 3.2.x and up |
+| 2.1.0 (not released yet) | 3.0.x - 3.2.x |
 
 # Examples
 

datafu-spark/build.gradle

@@ -70,26 +70,15 @@ dependencies {
     	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":3.0.2_" + sparkTestingBaseVersion
     } else if (sparkVersion == "3.1.3") {
     	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":3.1.2_" + sparkTestingBaseVersion
-    } else if (sparkVersion == "3.2.4") {
+    } else if (sparkVersion > "3.2" && sparkVersion < "3.3") {
     	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":3.2.1_" + sparkTestingBaseVersion
+    } else if (sparkVersion > "3.3" && sparkVersion < "3.4") {
+    	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":3.3.0_" + sparkTestingBaseVersion
     } else {
 	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion + "_" + sparkTestingBaseVersion
     }
 }
 
-// we need to set up the build for hadoop 3
-if (hadoopVersion.startsWith("2.")) {
-  dependencies {
-    testRuntime "org.apache.hadoop:hadoop-common:$hadoopVersion"
-    testRuntime "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
-    testRuntime "org.apache.hadoop:hadoop-mapreduce-client-jobclient:$hadoopVersion"
-  }
-} else {
-  dependencies {
-    testRuntime "org.apache.hadoop:hadoop-core:$hadoopVersion"
-  }
-}
-
 project.ext.sparkFile = file("build/spark-zips/spark-" + sparkVersion + ".zip")
 project.ext.sparkUnzipped = "build/spark-unzipped/spark-" + sparkVersion
 

datafu-spark/build_and_test_spark.sh

@@ -17,8 +17,9 @@
 
 #!/bin/bash
 
-export SPARK_VERSIONS_FOR_SCALA_212="3.0.0 3.0.1 3.0.2 3.0.3 3.1.1 3.1.2 3.1.3"
-export LATEST_SPARK_VERSIONS_FOR_SCALA_212="3.0.3 3.1.3"
+export SPARK_VERSIONS_FOR_SCALA_212="3.0.1 3.0.2 3.0.3 3.1.1 3.1.2 3.1.3 3.2.0 3.2.1 3.2.2 3.2.3 3.2.4"
+export LATEST_SPARK_VERSIONS_FOR_SCALA_212="3.0.3 3.1.3 3.2.4"
+
 
 STARTTIME=$(date +%s)
 

datafu-spark/gradle/dependency-versions-scala-2.12.gradle

@@ -17,7 +17,7 @@
  * under the License.
  */
 ext {
-  scalaLibraryVersion = "2.12.10"
+  scalaLibraryVersion = "2.12.15"
   // Extra options for the compiler:
   // -feature: Give detailed warnings about language feature use (rather than just 'there were 4 warnings')
   scalaOptions = "-feature"

datafu-spark/src/main/resources/pyspark_utils/bridge_utils.py

@@ -48,8 +48,11 @@ def __init__(self):
         java_import(gateway.jvm, "org.apache.spark.SparkConf")
         java_import(gateway.jvm, "org.apache.spark.api.java.*")
         java_import(gateway.jvm, "org.apache.spark.api.python.*")
+        java_import(gateway.jvm, "org.apache.spark.ml.python.*")
         java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
+        java_import(gateway.jvm, "org.apache.spark.resource.*")
         java_import(gateway.jvm, "org.apache.spark.sql.*")
+        java_import(gateway.jvm, "org.apache.spark.sql.api.python.*")
         java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
 
         intp = gateway.entry_point