Skip to content

Commit

Permalink
apacheGH-527: [Release] Add support for .jar for Windows
Browse files Browse the repository at this point in the history
Fixes apacheGH-527.

It seems that arrow-testing with ARROW_BUILD_STATIC=ON and
GTest_SOURCE=BUNDLED is broken. We'll enable JNI test after we fix it
in apache/arrow.
  • Loading branch information
kou committed Jan 17, 2025
1 parent 37988c5 commit 5c5eea4
Show file tree
Hide file tree
Showing 8 changed files with 321 additions and 48 deletions.
93 changes: 82 additions & 11 deletions .github/workflows/rc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ jobs:
name: release-source
path: |
apache-arrow-java-*
jni-ubuntu:
jni-linux:
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
runs-on: ${{ matrix.platform.runs_on }}
needs:
Expand All @@ -100,7 +100,7 @@ jobs:
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: release-source
- name: Extract Download the latest Apache Arrow C++
- name: Extract source archive
run: |
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
- name: Download the latest Apache Arrow C++
Expand All @@ -127,7 +127,7 @@ jobs:
path: .docker
key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
restore-keys: jni-linux-${{ matrix.platform.arch }}-
- name: Build C++ libraries
- name: Build
run: |
docker compose run vcpkg-jni
- name: Push Docker image
Expand Down Expand Up @@ -159,7 +159,7 @@ jobs:
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: release-source
- name: Extract Download the latest Apache Arrow C++
- name: Extract source archive
run: |
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
- name: Download the latest Apache Arrow C++
Expand All @@ -176,7 +176,7 @@ jobs:
repository: apache/parquet-testing
path: arrow/cpp/submodules/parquet-testing
- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
cache: 'pip'
python-version: 3.12
Expand Down Expand Up @@ -246,7 +246,7 @@ jobs:
path: ccache
key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
restore-keys: jni-macos-${{ matrix.platform.arch }}-
- name: Build C++ libraries
- name: Build
run: |
set -e
# make brew Java available to CMake
Expand All @@ -259,12 +259,83 @@ jobs:
with:
name: jni-macos-${{ matrix.platform.arch }}
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
jni-windows:
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
runs-on: ${{ matrix.platform.runs_on }}
needs:
- source
strategy:
fail-fast: false
matrix:
platform:
- runs_on: windows-2019
arch: "x86_64"
steps:
- name: Download source archive
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: release-source
- name: Extract source archive
shell: bash
run: |
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
- name: Download the latest Apache Arrow C++
shell: bash
run: |
ci/scripts/download_cpp.sh
- name: Set up Java
uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0
with:
java-version: '11'
distribution: 'temurin'
- name: Download Timezone Database
shell: bash
run: |
arrow/ci/scripts/download_tz_database.sh
- name: Install ccache
shell: bash
run: |
env | sort
version=4.10.2
base_name="ccache-${version}-windows-x86_64"
url="https://github.com/ccache/ccache/releases/download/v${version}/${base_name}.zip"
curl --fail --location --remote-name "${url}"
unzip "${base_name}.zip"
chmod +x "${base_name}/ccache.exe"
mv "${base_name}/ccache.exe" /usr/bin/
rm -rf "${base_name}"{,.zip}
- name: Prepare ccache
shell: bash
run: |
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
- name: Cache ccache
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
with:
path: ccache
key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
restore-keys: jni-windows-${{ matrix.platform.arch }}-
- name: Build
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
REM For ORC
set TZDIR=/c/msys64/usr/share/zoneinfo
bash -c "ci/scripts/jni_windows_build.sh . arrow build jni"
- name: Compress into single artifact to keep directory structure
shell: bash
run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/
- name: Upload artifacts
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
with:
name: jni-windows-${{ matrix.platform.arch }}
path: jni-windows-${{ matrix.platform.arch }}.tar.gz
binaries:
name: Binaries
runs-on: ubuntu-latest
needs:
- jni-ubuntu
- jni-linux
- jni-macos
- jni-windows
steps:
- name: Download artifacts
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
Expand All @@ -278,7 +349,7 @@ jobs:
# tar -xvzf jni-linux-aarch_64.tar.gz
tar -xvzf jni-macos-x86_64.tar.gz
tar -xvzf jni-macos-aarch_64.tar.gz
# tar -xvzf jni-windows.tar.gz
tar -xvzf jni-windows-x86_64.tar.gz
- name: Test that shared libraries exist
run: |
set -x
Expand All @@ -303,9 +374,9 @@ jobs:
test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib
# test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
# test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
# test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
- name: Checkout apache/arrow-testing
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
Expand Down
2 changes: 2 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ repos:
rev: v0.10.0
hooks:
- id: shellcheck
args:
- "--external-sources"
- repo: https://github.com/scop/pre-commit-shfmt
rev: v3.9.0-1
hooks:
Expand Down
2 changes: 1 addition & 1 deletion ci/scripts/download_cpp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,6 @@ fi
if [ -z "${url}" ]; then
url="https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${version}/apache-arrow-${version}.tar.gz"
fi
wget --output-document "apache-arrow-${version}.tar.gz" "${url}"
curl --location --output "apache-arrow-${version}.tar.gz" "${url}"
tar xf "apache-arrow-${version}.tar.gz"
mv "apache-arrow-${version}" arrow
17 changes: 13 additions & 4 deletions ci/scripts/jni_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,25 @@
# specific language governing permissions and limitations
# under the License.

set -euxo pipefail
set -euo pipefail

# shellcheck source=ci/scripts/util_log.sh
. "$(dirname "${0}")/util_log.sh"

github_actions_group_begin "Prepare arguments"
source_dir=${1}
arrow_install_dir=${2}
build_dir=${3}/java_jni
# The directory where the final binaries will be stored when scripts finish
dist_dir=${4}
prefix_dir="${build_dir}/java-jni"
github_actions_group_end

echo "=== Clear output directories and leftovers ==="
# Clear output directories and leftovers
github_actions_group_begin "Clear output directories and leftovers"
rm -rf "${build_dir}"
github_actions_group_end

echo "=== Building Arrow Java C Data Interface native library ==="
github_actions_group_begin "Building Arrow Java C Data Interface native library"

case "$(uname)" in
Linux)
Expand Down Expand Up @@ -71,10 +76,14 @@ if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then
fi
cmake --build "${build_dir}" --target install

github_actions_group_end

github_actions_group_begin "Copying artifacts"
mkdir -p "${dist_dir}"
# For Windows. *.dll are installed into bin/ on Windows.
if [ -d "${prefix_dir}/bin" ]; then
mv "${prefix_dir}"/bin/* "${dist_dir}"/
else
mv "${prefix_dir}"/lib/* "${dist_dir}"/
fi
github_actions_group_end
36 changes: 24 additions & 12 deletions ci/scripts/jni_macos_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@
# This script is like java_jni_build.sh, but is meant for release artifacts
# and hardcodes assumptions about the environment it is being run in.

set -ex
set -euo pipefail

# shellcheck source=ci/scripts/util_log.sh
. "$(dirname "${0}")/util_log.sh"

github_actions_group_begin "Prepare arguments"
source_dir="$(cd "${1}" && pwd)"
arrow_dir="$(cd "${2}" && pwd)"
build_dir="${3}"
Expand All @@ -36,20 +40,29 @@ i386)
esac
# The directory where the final binaries will be stored when scripts finish
dist_dir="${4}"
github_actions_group_end

echo "=== Clear output directories and leftovers ==="
# Clear output directories and leftovers
github_actions_group_begin "Clear output directories and leftovers"
rm -rf "${build_dir}"
rm -rf "${dist_dir}"

mkdir -p "${build_dir}"
build_dir="$(cd "${build_dir}" && pwd)"
github_actions_group_end

: "${ARROW_USE_CCACHE:=ON}"
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
github_actions_group_begin "ccache statistics before build"
ccache -sv 2>/dev/null || ccache -s
github_actions_group_end
fi

echo "=== Building Arrow C++ libraries ==="
github_actions_group_begin "Building Arrow C++ libraries"
install_dir="${build_dir}/cpp-install"
: "${ARROW_ACERO:=ON}"
export ARROW_ACERO
: "${ARROW_BUILD_TESTS:=ON}"
export ARROW_BUILD_TESTS
: "${ARROW_DATASET:=ON}"
export ARROW_DATASET
: "${ARROW_GANDIVA:=ON}"
Expand All @@ -58,15 +71,9 @@ export ARROW_GANDIVA
export ARROW_ORC
: "${ARROW_PARQUET:=ON}"
: "${ARROW_S3:=ON}"
: "${ARROW_USE_CCACHE:=ON}"
: "${CMAKE_BUILD_TYPE:=Release}"
: "${CMAKE_UNITY_BUILD:=ON}"

if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
echo "=== ccache statistics before build ==="
ccache -sv 2>/dev/null || ccache -s
fi

export ARROW_TEST_DATA="${arrow_dir}/testing/data"
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
export AWS_EC2_METADATA_DISABLED=TRUE
Expand Down Expand Up @@ -99,8 +106,10 @@ cmake \
-Dre2_SOURCE=BUNDLED \
-GNinja
cmake --build "${build_dir}/cpp" --target install
github_actions_group_end

if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
github_actions_group_begin "Running Arrow C++ libraries tests"
# MinIO is required
exclude_tests="arrow-s3fs-test"
# unstable
Expand All @@ -113,6 +122,7 @@ if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
--parallel "$(sysctl -n hw.ncpu)" \
--test-dir "${build_dir}/cpp" \
--timeout 300
github_actions_group_end
fi

export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
Expand All @@ -123,11 +133,12 @@ export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install
"${dist_dir}"

if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
echo "=== ccache statistics after build ==="
github_actions_group_begin "ccache statistics after build"
ccache -sv 2>/dev/null || ccache -s
github_actions_group_end
fi

echo "=== Checking shared dependencies for libraries ==="
github_actions_group_begin "Checking shared dependencies for libraries"
pushd "${dist_dir}"
archery linking check-dependencies \
--allow CoreFoundation \
Expand All @@ -147,3 +158,4 @@ archery linking check-dependencies \
"arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \
"gandiva_jni/${normalized_arch}/libgandiva_jni.dylib"
popd
github_actions_group_end
Loading

0 comments on commit 5c5eea4

Please sign in to comment.