Skip to content

Commit

Permalink
Upgrade iceberg support to 1.6.1
Browse files Browse the repository at this point in the history 
Signed-off-by: liurenjie1024 <liurenjie2008@gmail.com>

Signed-off-by: liurenjie1024 <liurenjie2008@gmail.com>
  • Loading branch information
@liurenjie1024
liurenjie1024 committed Feb 24, 2025
1 parent 83b4626 commit c51c5f5
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 112 deletions.
111 changes: 70 additions & 41 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@
<spark.test.version>${spark320.version}</spark.test.version>
<parquet.hadoop.version>1.12.1</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-20x</rapids.delta.artifactId1>
<iceberg.artifact.suffix>${spark32x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark32x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-20x</module>
Expand All @@ -118,6 +120,8 @@
<spark.test.version>${spark321.version}</spark.test.version>
<parquet.hadoop.version>1.12.2</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-20x</rapids.delta.artifactId1>
<iceberg.artifact.suffix>${spark32x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark32x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-20x</module>
Expand Down Expand Up @@ -145,6 +149,8 @@
check https://github.com/NVIDIA/spark-rapids/issues/11112 -->
<!-- #if scala-2.12 -->
<scala.plugin.version>4.3.0</scala.plugin.version>
<iceberg.artifact.suffix>${spark32x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark32x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>shim-deps/cloudera</module>
Expand All @@ -167,6 +173,8 @@
<spark.test.version>${spark322.version}</spark.test.version>
<parquet.hadoop.version>1.12.2</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-20x</rapids.delta.artifactId1>
<iceberg.artifact.suffix>${spark32x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark32x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-20x</module>
Expand All @@ -188,6 +196,8 @@
<spark.test.version>${spark323.version}</spark.test.version>
<parquet.hadoop.version>1.12.2</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-20x</rapids.delta.artifactId1>
<iceberg.artifact.suffix>${spark32x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark32x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-20x</module>
Expand All @@ -209,6 +219,8 @@
<spark.test.version>${spark324.version}</spark.test.version>
<parquet.hadoop.version>1.12.2</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-20x</rapids.delta.artifactId1>
<iceberg.artifact.suffix>${spark32x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark32x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-20x</module>
Expand All @@ -234,7 +246,8 @@
<rapids.delta.artifactId1>rapids-4-spark-delta-21x</rapids.delta.artifactId1>
<rapids.delta.artifactId2>rapids-4-spark-delta-22x</rapids.delta.artifactId2>
<rapids.delta.artifactId3>rapids-4-spark-delta-23x</rapids.delta.artifactId3>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-21x</module>
Expand All @@ -258,7 +271,8 @@
<rapids.delta.artifactId1>rapids-4-spark-delta-21x</rapids.delta.artifactId1>
<rapids.delta.artifactId2>rapids-4-spark-delta-22x</rapids.delta.artifactId2>
<rapids.delta.artifactId3>rapids-4-spark-delta-23x</rapids.delta.artifactId3>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-21x</module>
Expand All @@ -282,7 +296,8 @@
<rapids.delta.artifactId1>rapids-4-spark-delta-21x</rapids.delta.artifactId1>
<rapids.delta.artifactId2>rapids-4-spark-delta-22x</rapids.delta.artifactId2>
<rapids.delta.artifactId3>rapids-4-spark-delta-23x</rapids.delta.artifactId3>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-21x</module>
Expand All @@ -306,7 +321,8 @@
<rapids.delta.artifactId1>rapids-4-spark-delta-21x</rapids.delta.artifactId1>
<rapids.delta.artifactId2>rapids-4-spark-delta-22x</rapids.delta.artifactId2>
<rapids.delta.artifactId3>rapids-4-spark-delta-23x</rapids.delta.artifactId3>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-21x</module>
Expand All @@ -330,7 +346,8 @@
<rapids.delta.artifactId1>rapids-4-spark-delta-21x</rapids.delta.artifactId1>
<rapids.delta.artifactId2>rapids-4-spark-delta-22x</rapids.delta.artifactId2>
<rapids.delta.artifactId3>rapids-4-spark-delta-23x</rapids.delta.artifactId3>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>delta-lake/delta-21x</module>
Expand All @@ -352,7 +369,8 @@
<spark.test.version>${spark340.version}</spark.test.version>
<parquet.hadoop.version>1.12.3</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-24x</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark34x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark34x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.6</slf4j.version>
</properties>
<modules>
Expand All @@ -373,7 +391,8 @@
<spark.test.version>${spark341.version}</spark.test.version>
<parquet.hadoop.version>1.12.3</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-24x</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark34x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark34x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.6</slf4j.version>
</properties>
<modules>
Expand All @@ -394,7 +413,8 @@
<spark.test.version>${spark342.version}</spark.test.version>
<parquet.hadoop.version>1.12.3</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-24x</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark34x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark34x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.6</slf4j.version>
</properties>
<modules>
Expand All @@ -415,7 +435,8 @@
<spark.test.version>${spark343.version}</spark.test.version>
<parquet.hadoop.version>1.12.3</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-24x</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark34x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark34x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.6</slf4j.version>
</properties>
<modules>
Expand All @@ -436,7 +457,8 @@
<spark.test.version>${spark344.version}</spark.test.version>
<parquet.hadoop.version>1.12.3</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-24x</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark34x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark34x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.6</slf4j.version>
</properties>
<modules>
Expand All @@ -459,7 +481,8 @@
<rapids.delta.artifactId1>rapids-4-spark-delta-21x</rapids.delta.artifactId1>
<rapids.delta.artifactId2>rapids-4-spark-delta-22x</rapids.delta.artifactId2>
<rapids.delta.artifactId3>rapids-4-spark-delta-23x</rapids.delta.artifactId3>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
<cloudera.repo.enabled>true</cloudera.repo.enabled>
<!-- Keeping the scala plugin version 4.3.0 for details
check https://github.com/NVIDIA/spark-rapids/issues/11112 -->
Expand Down Expand Up @@ -488,7 +511,8 @@
<rapids.delta.artifactId1>rapids-4-spark-delta-21x</rapids.delta.artifactId1>
<rapids.delta.artifactId2>rapids-4-spark-delta-22x</rapids.delta.artifactId2>
<rapids.delta.artifactId3>rapids-4-spark-delta-23x</rapids.delta.artifactId3>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
<cloudera.repo.enabled>true</cloudera.repo.enabled>
<!-- Keeping the scala plugin version 4.3.0 for details
check https://github.com/NVIDIA/spark-rapids/issues/11112 -->
Expand Down Expand Up @@ -519,7 +543,8 @@
<rat.consoleOutput>true</rat.consoleOutput>
<parquet.hadoop.version>1.12.0</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-${spark.version.classifier}</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>shim-deps/databricks</module>
Expand All @@ -544,7 +569,8 @@
<rat.consoleOutput>true</rat.consoleOutput>
<parquet.hadoop.version>1.12.0</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-${spark.version.classifier}</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark33x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark33x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>shim-deps/databricks</module>
Expand All @@ -568,7 +594,8 @@
<rat.consoleOutput>true</rat.consoleOutput>
<parquet.hadoop.version>1.12.0</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-${spark.version.classifier}</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark34x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark34x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>shim-deps/databricks</module>
Expand All @@ -589,7 +616,8 @@
<spark.test.version>${spark350.version}</spark.test.version>
<parquet.hadoop.version>1.13.1</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-stub</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark35x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark35x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.7</slf4j.version>
</properties>
<modules>
Expand All @@ -615,7 +643,8 @@
<rat.consoleOutput>true</rat.consoleOutput>
<parquet.hadoop.version>1.12.0</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-${spark.version.classifier}</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark35x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark35x.iceberg.version}</iceberg.version>
</properties>
<modules>
<module>shim-deps/databricks</module>
Expand All @@ -636,7 +665,8 @@
<spark.test.version>${spark351.version}</spark.test.version>
<parquet.hadoop.version>1.13.1</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-stub</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark35x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark35x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.7</slf4j.version>
</properties>
<modules>
Expand All @@ -657,7 +687,8 @@
<spark.test.version>${spark352.version}</spark.test.version>
<parquet.hadoop.version>1.13.1</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-stub</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark35x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark35x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.7</slf4j.version>
</properties>
<modules>
Expand All @@ -678,7 +709,8 @@
<spark.test.version>${spark353.version}</spark.test.version>
<parquet.hadoop.version>1.13.1</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-stub</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark35x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark35x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.7</slf4j.version>
</properties>
<modules>
Expand All @@ -699,7 +731,8 @@
<spark.test.version>${spark354.version}</spark.test.version>
<parquet.hadoop.version>1.13.1</parquet.hadoop.version>
<rapids.delta.artifactId1>rapids-4-spark-delta-stub</rapids.delta.artifactId1>
<iceberg.version>${spark330.iceberg.version}</iceberg.version>
<iceberg.artifact.suffix>${spark35x.iceberg.artifact.suffix}</iceberg.artifact.suffix>
<iceberg.version>${spark35x.iceberg.version}</iceberg.version>
<slf4j.version>2.0.7</slf4j.version>
</properties>
<modules>
Expand Down Expand Up @@ -927,14 +960,22 @@
<slf4j.version>1.7.30</slf4j.version>
<flatbuffers.java.version>1.11.0</flatbuffers.java.version>
<hadoop.client.version>3.3.1</hadoop.client.version>
<iceberg.version>0.13.2</iceberg.version>
<scala.local-lib.path>org/scala-lang/scala-library/${scala.version}/scala-library-${scala.version}.jar</scala.local-lib.path>
<target.classifier>${spark.version.classifier}</target.classifier>
<maven.clean.plugin.version>3.1.0</maven.clean.plugin.version>
<maven.scaladoc.skip>false</maven.scaladoc.skip>
<maven.scalastyle.skip>false</maven.scalastyle.skip>
<dist.jar.compress>true</dist.jar.compress>
<spark330.iceberg.version>0.14.1</spark330.iceberg.version>
<spark32x.iceberg.artifact.suffix>3.2</spark32x.iceberg.artifact.suffix>
<spark32x.iceberg.version>1.4.3</spark32x.iceberg.version>
<spark33x.iceberg.artifact.suffix>3.3</spark33x.iceberg.artifact.suffix>
<spark33x.iceberg.version>1.6.1</spark33x.iceberg.version>
<spark34x.iceberg.artifact.suffix>3.4</spark34x.iceberg.artifact.suffix>
<spark34x.iceberg.version>1.6.1</spark34x.iceberg.version>
<spark35x.iceberg.artifact.suffix>3.5</spark35x.iceberg.artifact.suffix>
<spark35x.iceberg.version>1.6.1</spark35x.iceberg.version>
<iceberg.artifact.suffix>3.3</iceberg.artifact.suffix>
<iceberg.version>1.6.1</iceberg.version>
<!--
If true, disables verification that all Shims be built as of one and the same git
commit hash. Do not use for CI!
Expand Down Expand Up @@ -1038,24 +1079,6 @@
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-api</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-bundled-guava</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-annotation_${scala.binary.version}</artifactId>
Expand Down Expand Up @@ -1100,6 +1123,12 @@
<version>${roaringbitmap.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-spark-runtime-${iceberg.artifact.suffix}_${scala.binary.version}</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
Expand Down
Loading

0 comments on commit c51c5f5

Please sign in to comment.