[native] Convert CircleCI jobs for PrestoC++ to Github actions

This pull request converts the CircleCI workflows to GitHub actions workflows. The workflows use the standard runners with Ubuntu 22.04. Some modifications were made to allow a build with the limited resources available: |name|build type|comments|expected runtime (no cache)|unit tests|E2E tests| |----|----------|--------|----------------|----------|---------| | prestocpp-linux-build | debug | All adapters enabled | 45m | N | N | | prestocpp-linux-build-and-unit-test | release | JWT, remote function, Parquet enabled | 90m | Y | Y | | prestocpp-macos-build | debug | Setup script plus build | 160m | N | N | | prestocpp-linux-adapters-build | release | Adapters and release build | 120m | N | N | Caching is currently enabled for the prestocpp-linux-build-and-unit-test. If the cache can be used, the time to run tests drops to 50m. Additional workflows are prestocpp-format-check and prestocpp-header-check That run in a few seconds. For the original conversion see: #21545 Co-authored-by: Rob Anderson <robandpdx@github.com>
prestodb · Dec 14, 2024 · 17ac4c3 · 17ac4c3
1 parent 901da6d
commit 17ac4c3
Show file tree

Hide file tree

Showing 5 changed files with 597 additions and 0 deletions.
diff --git a/.github/workflows/prestocpp-format-and-header-check.yml b/.github/workflows/prestocpp-format-and-header-check.yml
@@ -0,0 +1,33 @@
+name: prestocpp-format-and-header-check
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'presto-native-execution/**'
+      - '.github/workflows/prestocpp-format-and-header-check.yml'
+
+jobs:
+  prestocpp-format-and-header-check:
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/oss-presto/velox-dev:check
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Check formatting
+        run: |
+          git fetch origin master
+          cd presto-native-execution
+          make format-check
+
+      - name: Check license headers
+        run: |
+          git fetch origin master
+          cd presto-native-execution
+          make header-check
diff --git a/.github/workflows/prestocpp-linux-adapters-build.yml b/.github/workflows/prestocpp-linux-adapters-build.yml
@@ -0,0 +1,91 @@
+name: prestocpp-linux-adapters-build
+
+on:
+  workflow_dispatch:
+  # Disable the automatic execution on PR because it currently will run out of disk space and
+  # we will address this subsequently.
+  # - use smaller image - in the works
+  # - remove the adapters downloaded files after install - JWT needs fixing because of the cmake files end up
+  #pull_request:
+  #  paths:
+  #    - 'presto-native-execution/scripts/**'
+  #    - '.github/workflows/prestocpp-linux-adapters-build.yml'
+
+jobs:
+  prestocpp-linux-adapters-build:
+    runs-on: ubuntu-22.04
+    container:
+      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/ccache"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Update submodules
+        run: |
+          cd presto-native-execution
+          make submodules
+
+      - name: Build all adapter dependencies
+        run: |
+          mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install
+          mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/download
+          source /opt/rh/gcc-toolset-12/enable
+          set -xu
+          cd presto-native-execution
+          export DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps/download
+          export INSTALL_PREFIX=${GITHUB_WORKSPACE}/adapter-deps/install
+          PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
+          PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
+
+      - name: Install Github CLI for using apache/infrastructure-actions/stash
+        run: |
+          curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm
+          rpm -iv gh_2.63.2_linux_amd64.rpm
+
+      - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-prestocpp-linux-adapters-build
+
+      - name: Zero ccache statistics
+        run: ccache -sz
+
+      - name: Build engine
+        run: |
+          source /opt/rh/gcc-toolset-12/enable
+          cd presto-native-execution
+          cmake \
+            -B _build/release \
+            -GNinja \
+            -DTREAT_WARNINGS_AS_ERRORS=1 \
+            -DENABLE_ALL_WARNINGS=1 \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DPRESTO_ENABLE_PARQUET=ON \
+            -DPRESTO_ENABLE_S3=ON \
+            -DPRESTO_ENABLE_GCS=ON \
+            -DPRESTO_ENABLE_ABFS=OFF \
+            -DPRESTO_ENABLE_HDFS=ON \
+            -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
+            -DPRESTO_ENABLE_JWT=ON \
+            -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
+            -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
+            -DPRESTO_ENABLE_TESTING=OFF \
+            -DCMAKE_PREFIX_PATH=/usr/local \
+            -DThrift_ROOT=/usr/local \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DMAX_LINK_JOBS=4
+          ninja -C _build/release -j 4
+
+      - name: Ccache after
+        run: ccache -s
+
+      - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-prestocpp-linux-adapters-build