diff --git a/README.md b/README.md index b281dfa..967c8e7 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ The [master](https://github.com/pravega/spark-connectors) branch will always hav | Spark Version | Pravega Version | Java Version To Build Connector | Java Version To Run Connector | Git Branch | |---------------|-----------------|---------------------------------|-------------------------------|-----------------------------------------------------------------------------------| -| 3.0 | 1.0 | Java 11 | Java 8 or 11 | [master](https://github.com/pravega/spark-connectors) | -| 2.4 | 1.0 | Java 8 | Java 8 | [r0.10-spark2.4](https://github.com/pravega/spark-connectors/tree/r0.10-spark2.4) | +| 3.1 | 1.10 | Java 11 | Java 8 or 11 | [master](https://github.com/pravega/spark-connectors) +| 3.0 | 1.10 | Java 11 | Java 8 or 11 | [r0.10-spark3.0](https://github.com/pravega/spark-connectors/tree/r0.10-spark3.0) | +| 2.4 | 0.10 | Java 8 | Java 8 | [r0.10-spark2.4](https://github.com/pravega/spark-connectors/tree/r0.10-spark2.4) | | 3.0 | 0.9 | Java 11 | Java 8 or 11 | [r0.9](https://github.com/pravega/spark-connectors/tree/r0.9) | | 2.4 | 0.9 | Java 8 | Java 8 | [r0.9-spark2.4](https://github.com/pravega/spark-connectors/tree/r0.9-spark2.4) | | 3.0 | 0.8 | Java 8 | Java 8 | [r0.8-spark3.0](https://github.com/pravega/spark-connectors/tree/r0.8-spark3.0) | diff --git a/gradle.properties b/gradle.properties index 0a063f6..4359b24 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,12 +17,12 @@ jacksonVersion=2.11.0 junitVersion=4.12 logbackVersion=1.1.7 scalaJava8CompatVersion=0.9.1 -scalaTestVersion=3.0.8 +scalaTestVersion=3.1.0 scalaArmVersion=2.0 scalaVersion=2.12.13 shadowGradlePlugin=4.0.2 slf4jApiVersion=1.7.25 -sparkVersion=3.0.1 +sparkVersion=3.1.2 # Version and base tags can be overridden at build time. connectorVersion=0.10.0-SNAPSHOT diff --git a/src/main/scala/io/pravega/connectors/spark/PravegaTable.scala b/src/main/scala/io/pravega/connectors/spark/PravegaTable.scala index 0eb7c0b..2e55514 100644 --- a/src/main/scala/io/pravega/connectors/spark/PravegaTable.scala +++ b/src/main/scala/io/pravega/connectors/spark/PravegaTable.scala @@ -8,7 +8,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Tabl import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.connector.write.streaming.StreamingWrite import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, SupportsTruncate, WriteBuilder} -import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate +import org.apache.spark.sql.internal.connector.SupportsStreamingUpdateAsAppend import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -30,7 +30,7 @@ class PravegaTable extends Table with SupportsWrite with SupportsRead with Loggi } override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { - new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate { + new WriteBuilder with SupportsTruncate with SupportsStreamingUpdateAsAppend { private val inputSchema: StructType = info.schema() private val options = info.options(); private val parameters = options.asScala.toMap @@ -114,8 +114,6 @@ class PravegaTable extends Table with SupportsWrite with SupportsRead with Loggi } override def truncate(): WriteBuilder = this - override def update(): WriteBuilder = this - } } }