diff --git a/.github/actions/setup-spark-builder/action.yaml b/.github/actions/setup-spark-builder/action.yaml
index 1bd37d6b17..dc333e5d1b 100644
--- a/.github/actions/setup-spark-builder/action.yaml
+++ b/.github/actions/setup-spark-builder/action.yaml
@@ -26,10 +26,6 @@ inputs:
description: 'The Apache Spark version (e.g., 3.4.3) to build'
required: true
default: '3.4.3'
- comet-version:
- description: 'The Comet version to use for Spark'
- required: true
- default: '0.5.0-SNAPSHOT'
runs:
using: "composite"
steps:
@@ -46,7 +42,6 @@ runs:
run: |
cd apache-spark
git apply ../dev/diffs/${{inputs.spark-version}}.diff
- ../mvnw -nsu -q versions:set-property -Dproperty=comet.version -DnewVersion=${{inputs.comet-version}} -DgenerateBackupPoms=false
- name: Cache Maven dependencies
uses: actions/cache@v4
diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml
index 238fbb2715..b325a51930 100644
--- a/.github/workflows/spark_sql_test.yml
+++ b/.github/workflows/spark_sql_test.yml
@@ -71,7 +71,6 @@ jobs:
with:
spark-version: ${{ matrix.spark-version.full }}
spark-short-version: ${{ matrix.spark-version.short }}
- comet-version: '0.6.0-SNAPSHOT' # TODO: get this from pom.xml
- name: Run Spark tests
run: |
cd apache-spark
diff --git a/.github/workflows/spark_sql_test_ansi.yml b/.github/workflows/spark_sql_test_ansi.yml
index 14ec6366f4..170c10644b 100644
--- a/.github/workflows/spark_sql_test_ansi.yml
+++ b/.github/workflows/spark_sql_test_ansi.yml
@@ -69,7 +69,6 @@ jobs:
with:
spark-version: ${{ matrix.spark-version.full }}
spark-short-version: ${{ matrix.spark-version.short }}
- comet-version: '0.6.0-SNAPSHOT' # TODO: get this from pom.xml
- name: Run Spark tests
run: |
cd apache-spark
diff --git a/Makefile b/Makefile
index 80f334a34c..9382d1aeee 100644
--- a/Makefile
+++ b/Makefile
@@ -21,6 +21,9 @@ define spark_jvm_17_extra_args
$(shell ./mvnw help:evaluate -Dexpression=extraJavaTestArgs | grep -v '\[')
endef
+# Build optional Comet native features (like hdfs e.g)
+FEATURES_ARG := $(shell ! [ -z $(COMET_FEATURES) ] && echo '--features=$(COMET_FEATURES)')
+
all: core jvm
core:
@@ -95,7 +98,7 @@ release-linux: clean
cd native && RUSTFLAGS="-Ctarget-cpu=native -Ctarget-feature=-prefer-256-bit" cargo build --release
./mvnw install -Prelease -DskipTests $(PROFILES)
release:
- cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release
+ cd native && RUSTFLAGS="$(RUSTFLAGS) -Ctarget-cpu=native" cargo build --release $(FEATURES_ARG)
./mvnw install -Prelease -DskipTests $(PROFILES)
release-nogit:
cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release
diff --git a/common/pom.xml b/common/pom.xml
index b6cd75a32d..060da5a30c 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -26,7 +26,7 @@ under the License.
org.apache.datafusion
comet-parent-spark${spark.version.short}_${scala.binary.version}
- 0.6.0-SNAPSHOT
+ 0.7.0-SNAPSHOT
../pom.xml
@@ -211,6 +211,7 @@ under the License.
libcomet.dylib
libcomet.so
+ comet.dll
org/apache/comet/${platform}/${arch}
diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff
index ed1dfa21cf..7d38c70932 100644
--- a/dev/diffs/3.4.3.diff
+++ b/dev/diffs/3.4.3.diff
@@ -7,7 +7,7 @@ index d3544881af1..26ab186c65d 100644
2.5.1
2.0.8
+ 3.4
-+ 0.5.0-SNAPSHOT
++ 0.7.0-SNAPSHOT