Skip to content

Commit

Permalink
GH-14: Add JNI CI test (#441)
Browse files Browse the repository at this point in the history
Fixes #14.
  • Loading branch information
lidavidm authored Dec 4, 2024
1 parent 8f8c62f commit d650aa0
Show file tree
Hide file tree
Showing 5 changed files with 155 additions and 10 deletions.
2 changes: 1 addition & 1 deletion .env
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,4 @@ ULIMIT_CORE=-1

# Default versions for various dependencies
JDK=11
MAVEN=3.9.6
MAVEN=3.9.9
11 changes: 8 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,21 @@ env:

jobs:
ubuntu:
name: AMD64 Ubuntu 22.04 JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
jdk: [11, 17, 21, 22]
maven: [3.9.6]
image: [java]
maven: [3.9.9]
image: [ubuntu, conda-jni-cdata]
include:
- image: ubuntu
name: "Ubuntu"
- image: conda-jni-cdata
name: "Conda JNI"
env:
JDK: ${{ matrix.jdk }}
MAVEN: ${{ matrix.maven }}
Expand Down
30 changes: 30 additions & 0 deletions ci/docker/conda-jni.dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

FROM ghcr.io/mamba-org/micromamba:ubuntu24.04

ARG jdk=11
ARG maven=3.9.9

RUN micromamba install -y \
-c conda-forge \
cmake \
compilers \
maven=${maven} \
ninja \
openjdk=${jdk} && \
micromamba clean --all
80 changes: 80 additions & 0 deletions ci/scripts/java_jni_build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -eo pipefail

arrow_dir=${1}
arrow_install_dir=${2}
build_dir=${3}/java_jni
# The directory where the final binaries will be stored when scripts finish
dist_dir=${4}
prefix_dir="${build_dir}/java-jni"

echo "=== Clear output directories and leftovers ==="
# Clear output directories and leftovers
rm -rf ${build_dir}

echo "=== Building Arrow Java C Data Interface native library ==="
mkdir -p "${build_dir}"
pushd "${build_dir}"

case "$(uname)" in
Linux)
n_jobs=$(nproc)
;;
Darwin)
n_jobs=$(sysctl -n hw.logicalcpu)
;;
*)
n_jobs=${NPROC:-1}
;;
esac

: ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}}
: ${CMAKE_BUILD_TYPE:=release}
cmake \
-DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \
-DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \
-DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \
-DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
-DCMAKE_PREFIX_PATH=${arrow_install_dir} \
-DCMAKE_INSTALL_PREFIX=${prefix_dir} \
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
-DProtobuf_USE_STATIC_LIBS=ON \
-GNinja \
${JAVA_JNI_CMAKE_ARGS:-} \
${arrow_dir}
export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs}
cmake --build . --config ${CMAKE_BUILD_TYPE}
if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then
ctest \
--output-on-failure \
--parallel ${n_jobs} \
--timeout 300
fi
cmake --build . --config ${CMAKE_BUILD_TYPE} --target install
popd

mkdir -p ${dist_dir}
# For Windows. *.dll are installed into bin/ on Windows.
if [ -d "${prefix_dir}/bin" ]; then
mv ${prefix_dir}/bin/* ${dist_dir}/
else
mv ${prefix_dir}/lib/* ${dist_dir}/
fi
42 changes: 36 additions & 6 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,48 @@ volumes:
name: maven-cache

services:
java:
ubuntu:
# Usage:
# docker compose build java
# docker compose run java
# docker compose build ubuntu
# docker compose run ubuntu
# Parameters:
# MAVEN: 3.9.6
# MAVEN: 3.9.9
# JDK: 11, 17, 21
image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK}
volumes: &java-volumes
volumes:
- .:/arrow-java:delegated
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
command: &java-command >
command:
/bin/bash -c "
/arrow-java/ci/scripts/java_build.sh /arrow-java /build &&
/arrow-java/ci/scripts/java_test.sh /arrow-java /build"

conda-jni-cdata:
# Usage:
# docker compose build conda-jni-cdata
# docker compose run conda-jni-cdata
# Parameters:
# MAVEN: 3.9.9
# JDK: 11, 17, 21
image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration
build:
context: .
dockerfile: ci/docker/conda-jni.dockerfile
cache_from:
- ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration
args:
jdk: ${JDK}
maven: ${MAVEN}
# required to use micromamba with rootless docker
# https://github.com/mamba-org/micromamba-docker/issues/407#issuecomment-2088523507
user: root
volumes:
- .:/arrow-java:delegated
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
environment:
ARROW_JAVA_CDATA: "ON"
command:
/bin/bash -c "
/arrow-java/ci/scripts/java_jni_build.sh /arrow-java /build/jni /build /jni &&
/arrow-java/ci/scripts/java_build.sh /arrow-java /build /jni &&
/arrow-java/ci/scripts/java_test.sh /arrow-java /build /jni"

0 comments on commit d650aa0

Please sign in to comment.