From 17ac4c333fd0697bdcc941e0e09467724880e2ff Mon Sep 17 00:00:00 2001 From: Christian Zentgraf Date: Fri, 13 Dec 2024 16:31:32 -0500 Subject: [PATCH] [native] Convert CircleCI jobs for PrestoC++ to Github actions This pull request converts the CircleCI workflows to GitHub actions workflows. The workflows use the standard runners with Ubuntu 22.04. Some modifications were made to allow a build with the limited resources available: |name|build type|comments|expected runtime (no cache)|unit tests|E2E tests| |----|----------|--------|----------------|----------|---------| | prestocpp-linux-build | debug | All adapters enabled | 45m | N | N | | prestocpp-linux-build-and-unit-test | release | JWT, remote function, Parquet enabled | 90m | Y | Y | | prestocpp-macos-build | debug | Setup script plus build | 160m | N | N | | prestocpp-linux-adapters-build | release | Adapters and release build | 120m | N | N | Caching is currently enabled for the prestocpp-linux-build-and-unit-test. If the cache can be used, the time to run tests drops to 50m. Additional workflows are prestocpp-format-check and prestocpp-header-check That run in a few seconds. For the original conversion see: https://github.com/prestodb/presto/pull/21545 Co-authored-by: Rob Anderson --- .../prestocpp-format-and-header-check.yml | 33 ++ .../prestocpp-linux-adapters-build.yml | 91 +++++ .../prestocpp-linux-build-and-unit-test.yml | 325 ++++++++++++++++++ .github/workflows/prestocpp-linux-build.yml | 75 ++++ .github/workflows/prestocpp-macos-build.yml | 73 ++++ 5 files changed, 597 insertions(+) create mode 100644 .github/workflows/prestocpp-format-and-header-check.yml create mode 100644 .github/workflows/prestocpp-linux-adapters-build.yml create mode 100644 .github/workflows/prestocpp-linux-build-and-unit-test.yml create mode 100644 .github/workflows/prestocpp-linux-build.yml create mode 100644 .github/workflows/prestocpp-macos-build.yml diff --git a/.github/workflows/prestocpp-format-and-header-check.yml b/.github/workflows/prestocpp-format-and-header-check.yml new file mode 100644 index 0000000000000..42845f670dc6f --- /dev/null +++ b/.github/workflows/prestocpp-format-and-header-check.yml @@ -0,0 +1,33 @@ +name: prestocpp-format-and-header-check + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + - '.github/workflows/prestocpp-format-and-header-check.yml' + +jobs: + prestocpp-format-and-header-check: + runs-on: ubuntu-latest + container: + image: public.ecr.aws/oss-presto/velox-dev:check + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Check formatting + run: | + git fetch origin master + cd presto-native-execution + make format-check + + - name: Check license headers + run: | + git fetch origin master + cd presto-native-execution + make header-check diff --git a/.github/workflows/prestocpp-linux-adapters-build.yml b/.github/workflows/prestocpp-linux-adapters-build.yml new file mode 100644 index 0000000000000..34f6e6af96b38 --- /dev/null +++ b/.github/workflows/prestocpp-linux-adapters-build.yml @@ -0,0 +1,91 @@ +name: prestocpp-linux-adapters-build + +on: + workflow_dispatch: + # Disable the automatic execution on PR because it currently will run out of disk space and + # we will address this subsequently. + # - use smaller image - in the works + # - remove the adapters downloaded files after install - JWT needs fixing because of the cmake files end up + #pull_request: + # paths: + # - 'presto-native-execution/scripts/**' + # - '.github/workflows/prestocpp-linux-adapters-build.yml' + +jobs: + prestocpp-linux-adapters-build: + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + CCACHE_DIR: "${{ github.workspace }}/ccache" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Update submodules + run: | + cd presto-native-execution + make submodules + + - name: Build all adapter dependencies + run: | + mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install + mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/download + source /opt/rh/gcc-toolset-12/enable + set -xu + cd presto-native-execution + export DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps/download + export INSTALL_PREFIX=${GITHUB_WORKSPACE}/adapter-deps/install + PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh + PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh + + - name: Install Github CLI for using apache/infrastructure-actions/stash + run: | + curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm + rpm -iv gh_2.63.2_linux_amd64.rpm + + - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-adapters-build + + - name: Zero ccache statistics + run: ccache -sz + + - name: Build engine + run: | + source /opt/rh/gcc-toolset-12/enable + cd presto-native-execution + cmake \ + -B _build/release \ + -GNinja \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_S3=ON \ + -DPRESTO_ENABLE_GCS=ON \ + -DPRESTO_ENABLE_ABFS=OFF \ + -DPRESTO_ENABLE_HDFS=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ + -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ + -DPRESTO_ENABLE_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DThrift_ROOT=/usr/local \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DMAX_LINK_JOBS=4 + ninja -C _build/release -j 4 + + - name: Ccache after + run: ccache -s + + - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-adapters-build diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml new file mode 100644 index 0000000000000..a3770fb1a8df5 --- /dev/null +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -0,0 +1,325 @@ +name: prestocpp-linux-build-and-unit-test + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + - '.github/workflows/prestocpp-linux-build-and-unit-test.yml' + +jobs: + prestocpp-linux-build-for-test: + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + CCACHE_DIR: "${{ github.workspace }}/ccache" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Update velox + run: | + cd presto-native-execution + make velox-submodule + + - name: Install Github CLI for using apache/infrastructure-actions/stash + run: | + curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm + rpm -iv gh_2.63.2_linux_amd64.rpm + + - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-build-for-test + + - name: Zero ccache statistics + run: ccache -sz + + - name: Build engine + run: | + source /opt/rh/gcc-toolset-12/enable + cd presto-native-execution + cmake \ + -B _build/release \ + -GNinja \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ + -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DThrift_ROOT=/usr/local \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DMAX_LINK_JOBS=4 + ninja -C _build/release -j 4 + + - name: Ccache after + run: ccache -s + + - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-build-for-test + + - name: Run Unit Tests + run: | + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + cd presto-native-execution/_build/release + ctest -j 4 -VV --output-on-failure --exclude-regex velox.* + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: presto-native-build + path: | + presto-native-execution/_build/release/presto_cpp/main/presto_server + presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + + prestocpp-linux-presto-e2e-tests: + needs: prestocpp-linux-build-for-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 + - name: Restore execute permissions and library path + run: | + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + + - name: Install OpenJDK8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8' + + - name: Cache local Maven repository + id: cache-maven + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-2- + + - name: Populate maven cache + if: steps.cache-maven.outputs.cache-hit != 'true' + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + + - name: Maven install + env: + # Use different Maven options to install. + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run presto-native e2e tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + # TODO: neeed to enable remote function tests with + # "-Ppresto-native-execution-remote-functions" once + # > https://github.com/facebookincubator/velox/discussions/6163 + # is fixed. + + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-execution' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C + + prestocpp-linux-spark-e2e-tests: + needs: prestocpp-linux-build-for-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 + - name: Restore execute permissions and library path + run: | + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + + - name: Check Java + run: java --version + + - name: Cache local Maven repository + id: cache-maven + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-2- + + - name: Populate maven cache + if: steps.cache-maven.outputs.cache-hit != 'true' + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + + - name: Maven install + env: + # Use different Maven options to install. + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run spark e2e tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-execution' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C + + prestocpp-linux-presto-sidecar-tests: + needs: prestocpp-linux-build-for-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 + - name: Restore execute permissions and library path + run: | + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + + - name: Install OpenJDK8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8' + + - name: Cache local Maven repository + id: cache-maven + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-2- + + - name: Populate maven cache + if: steps.cache-maven.outputs.cache-hit != 'true' + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + + - name: Maven install + env: + # Use different Maven options to install. + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run presto-native sidecar tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-sidecar-plugin/src/test -type f -name 'Test*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-sidecar-plugin' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C diff --git a/.github/workflows/prestocpp-linux-build.yml b/.github/workflows/prestocpp-linux-build.yml new file mode 100644 index 0000000000000..372be3b56fdfd --- /dev/null +++ b/.github/workflows/prestocpp-linux-build.yml @@ -0,0 +1,75 @@ +name: prestocpp-linux-build + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + - '.github/workflows/prestocpp-linux-build.yml' + +jobs: + prestocpp-linux-build-engine: + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + CCACHE_DIR: "${{ github.workspace }}/ccache" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Update velox + run: | + cd presto-native-execution + make velox-submodule + + - name: Install Github CLI for using apache/infrastructure-actions/stash + run: | + curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm + rpm -iv gh_2.63.2_linux_amd64.rpm + + - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-build-engine + + - name: Zero ccache statistics + run: ccache -sz + + - name: Build engine + run: | + source /opt/rh/gcc-toolset-12/enable + cd presto-native-execution + cmake \ + -B _build/debug \ + -GNinja \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_S3=ON \ + -DPRESTO_ENABLE_GCS=ON \ + -DPRESTO_ENABLE_ABFS=OFF \ + -DPRESTO_ENABLE_HDFS=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ + -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ + -DPRESTO_ENABLE_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DThrift_ROOT=/usr/local \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DMAX_LINK_JOBS=4 + ninja -C _build/debug -j 4 + + - name: Ccache after + run: ccache -s + + - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-build-engine diff --git a/.github/workflows/prestocpp-macos-build.yml b/.github/workflows/prestocpp-macos-build.yml new file mode 100644 index 0000000000000..05a656e87247f --- /dev/null +++ b/.github/workflows/prestocpp-macos-build.yml @@ -0,0 +1,73 @@ +name: prestocpp-macos-build + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + - '.github/workflows/prestocpp-macos-build.yml' + +jobs: + prestocpp-macos-build-engine: + runs-on: macos-13 + env: + CCACHE_DIR: "${{ github.workspace }}/ccache" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Update submodules + run: | + cd presto-native-execution + make submodules + + # The current Github runners for newer versions of macOS (14, 15) only provide 7GB memory while macOS 13 runners provide 14GB. + # Thus, we are installing XCode 15 on macOS 13 to build with Clang 15. + # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 15 + + - name: "Setup MacOS" + run: | + set -xu + mkdir ~/deps ~/deps-src + git clone --depth 1 https://github.com/Homebrew/brew ~/deps + PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh + # Calculate the prefix path before we delete brew's repos and taps. + rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%. + rm -rf ~/deps-src + + - name: Install Github CLI for using apache/infrastructure-actions/stash + run: | + PATH=~/deps/bin:${PATH} + brew install gh + + - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-macos-build-engine + + - name: Zero ccache statistics + run: ccache -sz + + - name: "Build presto_cpp on MacOS" + run: | + clang --version + export INSTALL_PREFIX=~/deps + export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH} + cd presto-native-execution + cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + ninja -C _build/debug + + - name: Ccache after + run: ccache -s + + - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-macos-prestocpp