Issue 120: Support Spark 3.1.2 (#121)

Signed-off-by: David Maddison <david.maddison@dell.com>
pravega · Aug 9, 2021 · 909a137 · 909a137
1 parent 85c5797
commit 909a137
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -39,8 +39,9 @@ The [master](https://github.com/pravega/spark-connectors) branch will always hav
 
 | Spark Version | Pravega Version | Java Version To Build Connector | Java Version To Run Connector | Git Branch                                                                        |
 |---------------|-----------------|---------------------------------|-------------------------------|-----------------------------------------------------------------------------------|
-| 3.0           | 1.0             | Java 11                         | Java 8 or 11                  | [master](https://github.com/pravega/spark-connectors)                             |
-| 2.4           | 1.0             | Java 8                          | Java 8                        | [r0.10-spark2.4](https://github.com/pravega/spark-connectors/tree/r0.10-spark2.4) |
+| 3.1           | 1.10            | Java 11                         | Java 8 or 11                  | [master](https://github.com/pravega/spark-connectors)
+| 3.0           | 1.10            | Java 11                         | Java 8 or 11                  | [r0.10-spark3.0](https://github.com/pravega/spark-connectors/tree/r0.10-spark3.0)                             |
+| 2.4           | 0.10            | Java 8                          | Java 8                        | [r0.10-spark2.4](https://github.com/pravega/spark-connectors/tree/r0.10-spark2.4) |
 | 3.0           | 0.9             | Java 11                         | Java 8 or 11                  | [r0.9](https://github.com/pravega/spark-connectors/tree/r0.9)                     |
 | 2.4           | 0.9             | Java 8                          | Java 8                        | [r0.9-spark2.4](https://github.com/pravega/spark-connectors/tree/r0.9-spark2.4)   |
 | 3.0           | 0.8             | Java 8                          | Java 8                        | [r0.8-spark3.0](https://github.com/pravega/spark-connectors/tree/r0.8-spark3.0)   |

diff --git a/gradle.properties b/gradle.properties
@@ -17,12 +17,12 @@ jacksonVersion=2.11.0
 junitVersion=4.12
 logbackVersion=1.1.7
 scalaJava8CompatVersion=0.9.1
-scalaTestVersion=3.0.8
+scalaTestVersion=3.1.0
 scalaArmVersion=2.0
 scalaVersion=2.12.13
 shadowGradlePlugin=4.0.2
 slf4jApiVersion=1.7.25
-sparkVersion=3.0.1
+sparkVersion=3.1.2
 
 # Version and base tags can be overridden at build time.
 connectorVersion=0.10.0-SNAPSHOT

diff --git a/src/main/scala/io/pravega/connectors/spark/PravegaTable.scala b/src/main/scala/io/pravega/connectors/spark/PravegaTable.scala
@@ -8,7 +8,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Tabl
 import org.apache.spark.sql.connector.read.ScanBuilder
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
 import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, SupportsTruncate, WriteBuilder}
-import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdateAsAppend
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -30,7 +30,7 @@ class PravegaTable extends Table with SupportsWrite with SupportsRead with Loggi
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdateAsAppend {
       private val inputSchema: StructType = info.schema()
       private val options = info.options();
       private val parameters = options.asScala.toMap
@@ -114,8 +114,6 @@ class PravegaTable extends Table with SupportsWrite with SupportsRead with Loggi
       }
 
       override def truncate(): WriteBuilder = this
-      override def update(): WriteBuilder = this
-
     }
   }
 }