diff --git a/pom.xml b/pom.xml
index 82ccd62b3d7c4..6fbc26af4980b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2821,6 +2821,22 @@
${iceberg.version}
runtime
+
+
+
+ org.apache.iceberg
+ iceberg-spark-3.4_${scala.binary.version}
+ ${iceberg.version}
+ shadow-tests
+ test
+
+
+ org.apache.iceberg
+ iceberg-hive-metastore
+ ${iceberg.version}
+ shadow-tests
+ test
+
@@ -3001,6 +3017,10 @@
**/*TestCase.java
**/*Suite.java
+
+ org.apache.iceberg:iceberg-spark-3.4_${scala.binary.version}:jar:shadow-tests
+ org.apache.iceberg:iceberg-hive-metastore:jar:shadow-tests
+
${project.build.directory}/surefire-reports
-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true
@@ -3622,6 +3642,14 @@
+
+
+ iceberg
+
+ sql/iceberg
+
+
+
hadoop-cloud
diff --git a/rio.yml b/rio.yml
index 62caa1dc36d3f..e1ee809179b6e 100644
--- a/rio.yml
+++ b/rio.yml
@@ -501,6 +501,65 @@ builds:
- BOSON=false build/sbt -Dscala.version=2.13.8 -Pscala-2.13 -Phadoop-3 -Phive "hive/testOnly *.HiveCompatibilitySuite"
- BOSON=true build/sbt -Dscala.version=2.13.8 -Pscala-2.13 -Phadoop-3 -Phive "hive/testOnly *.HiveCompatibilitySuite"
+# Uncomment once shadow-test jar removes Parquet shading: https://github.pie.apple.com/IPR/apache-incubator-iceberg/pull/1025
+# - name: apple-spark-2.12_3.4.0-snapshot-iceberg-spark-test-jars
+# branchName: branch-3.4.0-apple
+# build:
+# template: freestyle:v4:build
+# steps:
+# - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/*" -DwildcardSuites=none test
+
+# - name: apple-spark-2.12_3.4.0-snapshot-iceberg-spark-test-jars-pr
+# branchName: branch-3.4.0-apple
+# build:
+# template: freestyle:v4:prb
+# steps:
+# - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/*" -DwildcardSuites=none test
+
+ - name: apple-spark-2.12_3.4.0-snapshot-iceberg
+ branchName: branch-3.4.0-apple
+ build:
+ template: freestyle:v4:build
+ steps:
+ - export NEW_VERSION="$(sed -n '29s/\(.*\)<\/version>/\1/p' pom.xml | xargs)-iceberg"
+ - build/mvn versions:set -DnewVersion="$NEW_VERSION" -DgenerateBackupPoms=false
+ - build/mvn install -Phive -Piceberg -Phadoop-3 -DskipTests
+ - git clone git@github.pie.apple.com:IPR/apache-incubator-iceberg.git
+ - cd apache-incubator-iceberg
+ - git checkout -b apple-1.3.x origin/apple-1.3.x
+ - sed -i "s/sparkVersion = \(.*\)/sparkVersion = '${NEW_VERSION}'/g" spark/v3.4/build.gradle
+ - sed -i "s/org\.apache\.hive:\* = \(.*\)/org\.apache\.hive:\* = $(sed -n 's/\(.*\)<\/hive\.version>/\1/p' ../pom.xml | xargs)/g" versions.props
+ - ./gradlew test
+
+ - name: apple-spark-2.12_3.4.0-snapshot-iceberg-pr
+ branchName: branch-3.4.0-apple
+ build:
+ template: freestyle:v4:prb
+ steps:
+ - export NEW_VERSION="$(sed -n '29s/\(.*\)<\/version>/\1/p' pom.xml | xargs)-iceberg"
+ - build/mvn versions:set -DnewVersion="$NEW_VERSION" -DgenerateBackupPoms=false
+ - build/mvn install -Phive -Piceberg -Phadoop-3 -DskipTests
+ - git clone git@github.pie.apple.com:IPR/apache-incubator-iceberg.git
+ - cd apache-incubator-iceberg
+ - git checkout -b apple-1.3.x origin/apple-1.3.x
+ - sed -i "s/sparkVersion = \(.*\)/sparkVersion = '${NEW_VERSION}'/g" spark/v3.4/build.gradle
+ - sed -i "s/org\.apache\.hive:\* = \(.*\)/org\.apache\.hive:\* = $(sed -n 's/\(.*\)<\/hive\.version>/\1/p' ../pom.xml | xargs)/g" versions.props
+ - ./gradlew test
+
+ - name: apple-spark-2.12_3.4.0-snapshot-iceberg-hive-test-jars
+ branchName: branch-3.4.0-apple
+ build:
+ template: freestyle:v4:build
+ steps:
+ - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/hive/*" -DwildcardSuites=none test
+
+ - name: apple-spark-2.12_3.4.0-snapshot-iceberg-hive-test-jars-pr
+ branchName: branch-3.4.0-apple
+ build:
+ template: freestyle:v4:prb
+ steps:
+ - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/hive/*" -DwildcardSuites=none test
+
- name: apple-spark-2.12_3.4.0-snapshot-launcher-repl-tools
branchName: branch-3.4.0-apple
build:
diff --git a/sql/iceberg/pom.xml b/sql/iceberg/pom.xml
new file mode 100644
index 0000000000000..d19a0706451d7
--- /dev/null
+++ b/sql/iceberg/pom.xml
@@ -0,0 +1,207 @@
+
+
+
+
+ 4.0.0
+
+ org.apache.spark
+ spark-parent_2.12
+ 3.4.0.25-apple-SNAPSHOT
+ ../../pom.xml
+
+
+ spark-iceberg_2.12
+ jar
+ Spark Project Iceberg Test
+ http://spark.apache.org/
+
+ iceberg
+
+
+
+
+ org.apache.spark
+ spark-core_${scala.binary.version}
+ ${project.version}
+
+
+ org.apache.spark
+ spark-core_${scala.binary.version}
+ ${project.version}
+ test-jar
+ test
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${project.version}
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${project.version}
+ test-jar
+ test
+
+
+ org.apache.spark
+ spark-catalyst_${scala.binary.version}
+ test-jar
+ ${project.version}
+ test
+
+
+ org.apache.spark
+ spark-hive_${scala.binary.version}
+ ${project.version}
+
+
+ org.apache.spark
+ spark-tags_${scala.binary.version}
+ test-jar
+ test
+
+
+ org.apache.parquet
+ parquet-hadoop
+ ${parquet.version}
+ test-jar
+ test
+
+
+
+ ${hive.group}
+ hive-common
+ ${hive.common.scope}
+
+
+ ${hive.group}
+ hive-exec
+ ${hive.classifier}
+
+
+ ${hive.group}
+ hive-metastore
+
+
+ ${hive.group}
+ hive-serde
+ ${hive.serde.scope}
+
+
+ ${hive.group}
+ hive-shims
+ ${hive.shims.scope}
+
+
+ org.apache.hive
+ hive-llap-common
+ ${hive.llap.scope}
+
+
+ org.apache.hive
+ hive-llap-client
+ ${hive.llap.scope}
+
+
+
+ org.apache.avro
+ avro
+
+
+ org.apache.avro
+ avro-mapred
+
+
+ org.apache.httpcomponents
+ httpclient
+
+
+ org.codehaus.jackson
+ jackson-mapper-asl
+
+
+
+ commons-codec
+ commons-codec
+
+
+ joda-time
+ joda-time
+
+
+ org.jodd
+ jodd-core
+
+
+ com.google.code.findbugs
+ jsr305
+
+
+ org.datanucleus
+ datanucleus-core
+
+
+ org.apache.hadoop
+ ${hadoop-client-runtime.artifact}
+ ${hadoop.deps.scope}
+
+
+ org.apache.thrift
+ libthrift
+
+
+ org.apache.thrift
+ libfb303
+
+
+ org.apache.derby
+ derby
+
+
+ org.scala-lang
+ scala-compiler
+ test
+
+
+ org.scalacheck
+ scalacheck_${scala.binary.version}
+ test
+
+
+
+ org.apache.iceberg
+ iceberg-spark-runtime-3.4_${scala.binary.version}
+
+
+
+
+ org.apache.iceberg
+ iceberg-spark-3.4_${scala.binary.version}
+ shadow-tests
+ test
+
+
+ org.apache.iceberg
+ iceberg-hive-metastore
+ shadow-tests
+ test
+
+
+