diff --git a/.github/actions/setup-spark-builder/action.yaml b/.github/actions/setup-spark-builder/action.yaml index 1bd37d6b17..dc333e5d1b 100644 --- a/.github/actions/setup-spark-builder/action.yaml +++ b/.github/actions/setup-spark-builder/action.yaml @@ -26,10 +26,6 @@ inputs: description: 'The Apache Spark version (e.g., 3.4.3) to build' required: true default: '3.4.3' - comet-version: - description: 'The Comet version to use for Spark' - required: true - default: '0.5.0-SNAPSHOT' runs: using: "composite" steps: @@ -46,7 +42,6 @@ runs: run: | cd apache-spark git apply ../dev/diffs/${{inputs.spark-version}}.diff - ../mvnw -nsu -q versions:set-property -Dproperty=comet.version -DnewVersion=${{inputs.comet-version}} -DgenerateBackupPoms=false - name: Cache Maven dependencies uses: actions/cache@v4 diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml index 238fbb2715..b325a51930 100644 --- a/.github/workflows/spark_sql_test.yml +++ b/.github/workflows/spark_sql_test.yml @@ -71,7 +71,6 @@ jobs: with: spark-version: ${{ matrix.spark-version.full }} spark-short-version: ${{ matrix.spark-version.short }} - comet-version: '0.6.0-SNAPSHOT' # TODO: get this from pom.xml - name: Run Spark tests run: | cd apache-spark diff --git a/.github/workflows/spark_sql_test_ansi.yml b/.github/workflows/spark_sql_test_ansi.yml index 14ec6366f4..170c10644b 100644 --- a/.github/workflows/spark_sql_test_ansi.yml +++ b/.github/workflows/spark_sql_test_ansi.yml @@ -69,7 +69,6 @@ jobs: with: spark-version: ${{ matrix.spark-version.full }} spark-short-version: ${{ matrix.spark-version.short }} - comet-version: '0.6.0-SNAPSHOT' # TODO: get this from pom.xml - name: Run Spark tests run: | cd apache-spark diff --git a/Makefile b/Makefile index 80f334a34c..9382d1aeee 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,9 @@ define spark_jvm_17_extra_args $(shell ./mvnw help:evaluate -Dexpression=extraJavaTestArgs | grep -v '\[') endef +# Build optional Comet native features (like hdfs e.g) +FEATURES_ARG := $(shell ! [ -z $(COMET_FEATURES) ] && echo '--features=$(COMET_FEATURES)') + all: core jvm core: @@ -95,7 +98,7 @@ release-linux: clean cd native && RUSTFLAGS="-Ctarget-cpu=native -Ctarget-feature=-prefer-256-bit" cargo build --release ./mvnw install -Prelease -DskipTests $(PROFILES) release: - cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release + cd native && RUSTFLAGS="$(RUSTFLAGS) -Ctarget-cpu=native" cargo build --release $(FEATURES_ARG) ./mvnw install -Prelease -DskipTests $(PROFILES) release-nogit: cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release diff --git a/common/pom.xml b/common/pom.xml index b6cd75a32d..060da5a30c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -211,6 +211,7 @@ under the License. libcomet.dylib libcomet.so + comet.dll org/apache/comet/${platform}/${arch} diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index ed1dfa21cf..7d38c70932 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -7,7 +7,7 @@ index d3544881af1..26ab186c65d 100644 2.5.1 2.0.8 + 3.4 -+ 0.5.0-SNAPSHOT ++ 0.7.0-SNAPSHOT