diff --git a/pom.xml b/pom.xml index 82ccd62b3d7c4..6fbc26af4980b 100644 --- a/pom.xml +++ b/pom.xml @@ -2821,6 +2821,22 @@ ${iceberg.version} runtime + + + + org.apache.iceberg + iceberg-spark-3.4_${scala.binary.version} + ${iceberg.version} + shadow-tests + test + + + org.apache.iceberg + iceberg-hive-metastore + ${iceberg.version} + shadow-tests + test + @@ -3001,6 +3017,10 @@ **/*TestCase.java **/*Suite.java + + org.apache.iceberg:iceberg-spark-3.4_${scala.binary.version}:jar:shadow-tests + org.apache.iceberg:iceberg-hive-metastore:jar:shadow-tests + ${project.build.directory}/surefire-reports -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true @@ -3622,6 +3642,14 @@ + + + iceberg + + sql/iceberg + + + hadoop-cloud diff --git a/rio.yml b/rio.yml index 62caa1dc36d3f..e1ee809179b6e 100644 --- a/rio.yml +++ b/rio.yml @@ -501,6 +501,65 @@ builds: - BOSON=false build/sbt -Dscala.version=2.13.8 -Pscala-2.13 -Phadoop-3 -Phive "hive/testOnly *.HiveCompatibilitySuite" - BOSON=true build/sbt -Dscala.version=2.13.8 -Pscala-2.13 -Phadoop-3 -Phive "hive/testOnly *.HiveCompatibilitySuite" +# Uncomment once shadow-test jar removes Parquet shading: https://github.pie.apple.com/IPR/apache-incubator-iceberg/pull/1025 +# - name: apple-spark-2.12_3.4.0-snapshot-iceberg-spark-test-jars +# branchName: branch-3.4.0-apple +# build: +# template: freestyle:v4:build +# steps: +# - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/*" -DwildcardSuites=none test + +# - name: apple-spark-2.12_3.4.0-snapshot-iceberg-spark-test-jars-pr +# branchName: branch-3.4.0-apple +# build: +# template: freestyle:v4:prb +# steps: +# - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/*" -DwildcardSuites=none test + + - name: apple-spark-2.12_3.4.0-snapshot-iceberg + branchName: branch-3.4.0-apple + build: + template: freestyle:v4:build + steps: + - export NEW_VERSION="$(sed -n '29s/\(.*\)<\/version>/\1/p' pom.xml | xargs)-iceberg" + - build/mvn versions:set -DnewVersion="$NEW_VERSION" -DgenerateBackupPoms=false + - build/mvn install -Phive -Piceberg -Phadoop-3 -DskipTests + - git clone git@github.pie.apple.com:IPR/apache-incubator-iceberg.git + - cd apache-incubator-iceberg + - git checkout -b apple-1.3.x origin/apple-1.3.x + - sed -i "s/sparkVersion = \(.*\)/sparkVersion = '${NEW_VERSION}'/g" spark/v3.4/build.gradle + - sed -i "s/org\.apache\.hive:\* = \(.*\)/org\.apache\.hive:\* = $(sed -n 's/\(.*\)<\/hive\.version>/\1/p' ../pom.xml | xargs)/g" versions.props + - ./gradlew test + + - name: apple-spark-2.12_3.4.0-snapshot-iceberg-pr + branchName: branch-3.4.0-apple + build: + template: freestyle:v4:prb + steps: + - export NEW_VERSION="$(sed -n '29s/\(.*\)<\/version>/\1/p' pom.xml | xargs)-iceberg" + - build/mvn versions:set -DnewVersion="$NEW_VERSION" -DgenerateBackupPoms=false + - build/mvn install -Phive -Piceberg -Phadoop-3 -DskipTests + - git clone git@github.pie.apple.com:IPR/apache-incubator-iceberg.git + - cd apache-incubator-iceberg + - git checkout -b apple-1.3.x origin/apple-1.3.x + - sed -i "s/sparkVersion = \(.*\)/sparkVersion = '${NEW_VERSION}'/g" spark/v3.4/build.gradle + - sed -i "s/org\.apache\.hive:\* = \(.*\)/org\.apache\.hive:\* = $(sed -n 's/\(.*\)<\/hive\.version>/\1/p' ../pom.xml | xargs)/g" versions.props + - ./gradlew test + + - name: apple-spark-2.12_3.4.0-snapshot-iceberg-hive-test-jars + branchName: branch-3.4.0-apple + build: + template: freestyle:v4:build + steps: + - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/hive/*" -DwildcardSuites=none test + + - name: apple-spark-2.12_3.4.0-snapshot-iceberg-hive-test-jars-pr + branchName: branch-3.4.0-apple + build: + template: freestyle:v4:prb + steps: + - build/mvn -Phadoop-3 -Phive -Piceberg -Dtest="org/apache/iceberg/hive/*" -DwildcardSuites=none test + - name: apple-spark-2.12_3.4.0-snapshot-launcher-repl-tools branchName: branch-3.4.0-apple build: diff --git a/sql/iceberg/pom.xml b/sql/iceberg/pom.xml new file mode 100644 index 0000000000000..d19a0706451d7 --- /dev/null +++ b/sql/iceberg/pom.xml @@ -0,0 +1,207 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.12 + 3.4.0.25-apple-SNAPSHOT + ../../pom.xml + + + spark-iceberg_2.12 + jar + Spark Project Iceberg Test + http://spark.apache.org/ + + iceberg + + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + test-jar + test + + + org.apache.spark + spark-sql_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${project.version} + test-jar + test + + + org.apache.spark + spark-catalyst_${scala.binary.version} + test-jar + ${project.version} + test + + + org.apache.spark + spark-hive_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-tags_${scala.binary.version} + test-jar + test + + + org.apache.parquet + parquet-hadoop + ${parquet.version} + test-jar + test + + + + ${hive.group} + hive-common + ${hive.common.scope} + + + ${hive.group} + hive-exec + ${hive.classifier} + + + ${hive.group} + hive-metastore + + + ${hive.group} + hive-serde + ${hive.serde.scope} + + + ${hive.group} + hive-shims + ${hive.shims.scope} + + + org.apache.hive + hive-llap-common + ${hive.llap.scope} + + + org.apache.hive + hive-llap-client + ${hive.llap.scope} + + + + org.apache.avro + avro + + + org.apache.avro + avro-mapred + + + org.apache.httpcomponents + httpclient + + + org.codehaus.jackson + jackson-mapper-asl + + + + commons-codec + commons-codec + + + joda-time + joda-time + + + org.jodd + jodd-core + + + com.google.code.findbugs + jsr305 + + + org.datanucleus + datanucleus-core + + + org.apache.hadoop + ${hadoop-client-runtime.artifact} + ${hadoop.deps.scope} + + + org.apache.thrift + libthrift + + + org.apache.thrift + libfb303 + + + org.apache.derby + derby + + + org.scala-lang + scala-compiler + test + + + org.scalacheck + scalacheck_${scala.binary.version} + test + + + + org.apache.iceberg + iceberg-spark-runtime-3.4_${scala.binary.version} + + + + + org.apache.iceberg + iceberg-spark-3.4_${scala.binary.version} + shadow-tests + test + + + org.apache.iceberg + iceberg-hive-metastore + shadow-tests + test + + +