diff --git a/.github/actions/build_clio/action.yml b/.github/actions/build_clio/action.yml index 6e00dac6e..2287ad22e 100644 --- a/.github/actions/build_clio/action.yml +++ b/.github/actions/build_clio/action.yml @@ -15,7 +15,7 @@ runs: uses: ./.github/actions/get_number_of_threads id: number_of_threads with: - substract_threads: inputs.substract_threads + substract_threads: ${{ inputs.substract_threads }} - name: Build Clio shell: bash diff --git a/.github/actions/generate/action.yml b/.github/actions/generate/action.yml index 4e96a5eb1..1926cf135 100644 --- a/.github/actions/generate/action.yml +++ b/.github/actions/generate/action.yml @@ -57,4 +57,4 @@ runs: '' }} run: | cd build - cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} ${SANITIZER_OPTION} .. -G Ninja + cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" ${SANITIZER_OPTION} .. -G Ninja diff --git a/.github/actions/get_number_of_threads/action.yml b/.github/actions/get_number_of_threads/action.yml index b32735838..088371b9e 100644 --- a/.github/actions/get_number_of_threads/action.yml +++ b/.github/actions/get_number_of_threads/action.yml @@ -27,9 +27,8 @@ runs: - name: Shift and export number of threads id: number_of_threads_export shell: bash - env: - NUM_OF_THREADS: steps.mac_threads.outputs.num || steps.linux_threads.outputs.num - SHIFT_BY: inputs.substract_threads run: | - shifted=$((NUM_OF_THREADS - SHIFT_BY)) + num_of_threads=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }} + shift_by=${{ inputs.substract_threads }} + shifted=$((num_of_threads - shift_by)) echo "num=$(( shifted > 1 ? shifted : 1 ))" >> $GITHUB_OUTPUT diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69b754a38..b4b5a5b89 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,192 +1,129 @@ -name: Reusable build +name: Build on: - workflow_call: - inputs: - runs_on: - description: Runner to run the job on - required: true - type: string - default: heavy + push: + branches: [master, release/*, develop] + pull_request: + branches: [master, release/*, develop] + workflow_dispatch: - container: - description: "The container object as a JSON string (leave empty to run natively)" - required: true - type: string - default: "" - - conan_profile: - description: Conan profile to use - required: true - type: string - - build_type: - description: Build type - required: true - type: string - - disable_cache: - description: Whether ccache and conan cache should be disabled - required: false - type: boolean - default: false - - code_coverage: - description: Whether to enable code coverage - required: true - type: boolean - default: false - - static: - description: Whether to build static binaries - required: true - type: boolean - default: true - - unit_tests: - description: Whether to run unit tests - required: true - type: boolean - default: false - - integration_tests: - description: Whether to run integration tests - required: true - type: boolean - default: false - - clio_server: - description: Whether to build clio_server - required: true - type: boolean - default: true - - target: - description: Build target name - required: false - type: string - default: all +jobs: + check_format: + name: Check format + runs-on: ubuntu-20.04 + container: + image: rippleci/clio_ci:latest + steps: + - name: Fix git permissions on Linux + shell: bash + run: git config --global --add safe.directory $PWD - sanitizer: - description: Sanitizer to use - required: false - type: string - default: 'false' + - uses: actions/checkout@v4 + - name: Run formatters + id: run_formatters + run: | + ./.githooks/check-format --diff + shell: bash + + check_docs: + name: Check documentation + runs-on: ubuntu-20.04 + container: + image: rippleci/clio_ci:latest + steps: + - uses: actions/checkout@v4 + - name: Run linter + id: run_linter + run: | + ./.githooks/check-docs + shell: bash -jobs: build: - name: Build ${{ inputs.container != '' && 'in container' || 'natively' }} - runs-on: ${{ inputs.runs_on }} - container: ${{ inputs.container != '' && fromJson(inputs.container) || null }} + name: Build + needs: + - check_format + - check_docs + strategy: + fail-fast: false + matrix: + include: + - os: heavy + conan_profile: gcc + build_type: Release + container: '{ "image": "rippleci/clio_ci:latest" }' + code_coverage: false + static: true + - os: heavy + conan_profile: gcc + build_type: Debug + container: '{ "image": "rippleci/clio_ci:latest" }' + code_coverage: true + static: true + - os: heavy + conan_profile: clang + build_type: Release + container: '{ "image": "rippleci/clio_ci:latest" }' + code_coverage: false + static: true + - os: heavy + conan_profile: clang + build_type: Debug + container: '{ "image": "rippleci/clio_ci:latest" }' + code_coverage: false + static: true + - os: macos15 + build_type: Release + code_coverage: false + static: false + uses: ./.github/workflows/build_impl.yml + with: + runs_on: ${{ matrix.os }} + container: ${{ matrix.container }} + conan_profile: ${{ matrix.conan_profile }} + build_type: ${{ matrix.build_type }} + code_coverage: ${{ matrix.code_coverage }} + static: ${{ matrix.static }} + unit_tests: true + integration_tests: true + clio_server: true + + test: + name: Run Tests + needs: build + strategy: + fail-fast: false + matrix: + include: + - os: heavy + conan_profile: gcc + build_type: Release + container: + image: rippleci/clio_ci:latest + - os: heavy + conan_profile: clang + build_type: Release + container: + image: rippleci/clio_ci:latest + - os: heavy + conan_profile: clang + build_type: Debug + container: + image: rippleci/clio_ci:latest + - os: macos15 + conan_profile: apple_clang_16 + build_type: Release + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} steps: - name: Clean workdir if: ${{ runner.os == 'macOS' }} uses: kuznetsss/workspace-cleanup@1.0 - - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: - fetch-depth: 0 + name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }} - - name: Prepare runner - uses: ./.github/actions/prepare_runner - with: - disable_ccache: ${{ inputs.disable_cache }} - - - name: Setup conan - uses: ./.github/actions/setup_conan - id: conan - with: - conan_profile: ${{ inputs.conan_profile }} - - - name: Restore cache - if: ${{ !inputs.disable_cache }} - uses: ./.github/actions/restore_cache - id: restore_cache - with: - conan_dir: ${{ env.CONAN_USER_HOME }}/.conan - conan_profile: ${{ steps.conan.outputs.conan_profile }} - ccache_dir: ${{ env.CCACHE_DIR }} - build_type: ${{ inputs.build_type }} - code_coverage: ${{ inputs.code_coverage }} - - - name: Run conan and cmake - uses: ./.github/actions/generate - with: - conan_profile: ${{ steps.conan.outputs.conan_profile }} - conan_cache_hit: ${{ !inputs.disable_cache && steps.restore_cache.outputs.conan_cache_hit }} - build_type: ${{ inputs.build_type }} - code_coverage: ${{ inputs.code_coverage }} - static: ${{ inputs.static }} - sanitizer: ${{ inputs.sanitizer }} - - - name: Build Clio - uses: ./.github/actions/build_clio - with: - target: ${{ inputs.target }} - - - name: Show ccache's statistics - if: ${{ !inputs.disable_cache }} - shell: bash - id: ccache_stats + - name: Run clio_tests run: | - ccache -s > /tmp/ccache.stats - miss_rate=$(cat /tmp/ccache.stats | grep 'Misses' | head -n1 | sed 's/.*(\(.*\)%).*/\1/') - echo "miss_rate=${miss_rate}" >> $GITHUB_OUTPUT - cat /tmp/ccache.stats - - - name: Strip unit_tests - if: ${{ inputs.unit_tests && !inputs.code_coverage && inputs.sanitizer == 'false' }} - run: strip build/clio_tests - - - name: Strip integration_tests - if: ${{ inputs.integration_tests && !inputs.code_coverage }} - run: strip build/clio_integration_tests - - - name: Upload clio_server - if: ${{ inputs.clio_server }} - uses: actions/upload-artifact@v4 - with: - name: clio_server_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} - path: build/clio_server - - - name: Upload clio_tests - if: ${{ inputs.unit_tests && !inputs.code_coverage }} - uses: actions/upload-artifact@v4 - with: - name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} - path: build/clio_tests - - - name: Upload clio_integration_tests - if: ${{ inputs.integration_tests && !inputs.code_coverage }} - uses: actions/upload-artifact@v4 - with: - name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} - path: build/clio_integration_tests - - - name: Save cache - if: ${{ !inputs.disable_cache }} - uses: ./.github/actions/save_cache - with: - conan_dir: ${{ env.CONAN_USER_HOME }}/.conan - conan_hash: ${{ steps.restore_cache.outputs.conan_hash }} - conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }} - ccache_dir: ${{ env.CCACHE_DIR }} - ccache_cache_hit: ${{ steps.restore_cache.outputs.ccache_cache_hit }} - ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }} - build_type: ${{ inputs.build_type }} - code_coverage: ${{ inputs.code_coverage }} - conan_profile: ${{ steps.conan.outputs.conan_profile }} - - # TODO: This is not a part of build process but it is the easiest way to do it here. - # It will be refactored in https://github.com/XRPLF/clio/issues/1075 - - name: Run code coverage - if: ${{ inputs.code_coverage }} - uses: ./.github/actions/code_coverage - - upload_coverage_report: - if: ${{ inputs.code_coverage }} - name: Codecov - needs: build - uses: ./.github/workflows/upload_coverage_report.yml - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + chmod +x ./clio_tests + ./clio_tests diff --git a/.github/workflows/build_impl.yml b/.github/workflows/build_impl.yml new file mode 100644 index 000000000..69b754a38 --- /dev/null +++ b/.github/workflows/build_impl.yml @@ -0,0 +1,192 @@ +name: Reusable build +on: + workflow_call: + inputs: + runs_on: + description: Runner to run the job on + required: true + type: string + default: heavy + + container: + description: "The container object as a JSON string (leave empty to run natively)" + required: true + type: string + default: "" + + conan_profile: + description: Conan profile to use + required: true + type: string + + build_type: + description: Build type + required: true + type: string + + disable_cache: + description: Whether ccache and conan cache should be disabled + required: false + type: boolean + default: false + + code_coverage: + description: Whether to enable code coverage + required: true + type: boolean + default: false + + static: + description: Whether to build static binaries + required: true + type: boolean + default: true + + unit_tests: + description: Whether to run unit tests + required: true + type: boolean + default: false + + integration_tests: + description: Whether to run integration tests + required: true + type: boolean + default: false + + clio_server: + description: Whether to build clio_server + required: true + type: boolean + default: true + + target: + description: Build target name + required: false + type: string + default: all + + sanitizer: + description: Sanitizer to use + required: false + type: string + default: 'false' + +jobs: + build: + name: Build ${{ inputs.container != '' && 'in container' || 'natively' }} + runs-on: ${{ inputs.runs_on }} + container: ${{ inputs.container != '' && fromJson(inputs.container) || null }} + + steps: + - name: Clean workdir + if: ${{ runner.os == 'macOS' }} + uses: kuznetsss/workspace-cleanup@1.0 + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Prepare runner + uses: ./.github/actions/prepare_runner + with: + disable_ccache: ${{ inputs.disable_cache }} + + - name: Setup conan + uses: ./.github/actions/setup_conan + id: conan + with: + conan_profile: ${{ inputs.conan_profile }} + + - name: Restore cache + if: ${{ !inputs.disable_cache }} + uses: ./.github/actions/restore_cache + id: restore_cache + with: + conan_dir: ${{ env.CONAN_USER_HOME }}/.conan + conan_profile: ${{ steps.conan.outputs.conan_profile }} + ccache_dir: ${{ env.CCACHE_DIR }} + build_type: ${{ inputs.build_type }} + code_coverage: ${{ inputs.code_coverage }} + + - name: Run conan and cmake + uses: ./.github/actions/generate + with: + conan_profile: ${{ steps.conan.outputs.conan_profile }} + conan_cache_hit: ${{ !inputs.disable_cache && steps.restore_cache.outputs.conan_cache_hit }} + build_type: ${{ inputs.build_type }} + code_coverage: ${{ inputs.code_coverage }} + static: ${{ inputs.static }} + sanitizer: ${{ inputs.sanitizer }} + + - name: Build Clio + uses: ./.github/actions/build_clio + with: + target: ${{ inputs.target }} + + - name: Show ccache's statistics + if: ${{ !inputs.disable_cache }} + shell: bash + id: ccache_stats + run: | + ccache -s > /tmp/ccache.stats + miss_rate=$(cat /tmp/ccache.stats | grep 'Misses' | head -n1 | sed 's/.*(\(.*\)%).*/\1/') + echo "miss_rate=${miss_rate}" >> $GITHUB_OUTPUT + cat /tmp/ccache.stats + + - name: Strip unit_tests + if: ${{ inputs.unit_tests && !inputs.code_coverage && inputs.sanitizer == 'false' }} + run: strip build/clio_tests + + - name: Strip integration_tests + if: ${{ inputs.integration_tests && !inputs.code_coverage }} + run: strip build/clio_integration_tests + + - name: Upload clio_server + if: ${{ inputs.clio_server }} + uses: actions/upload-artifact@v4 + with: + name: clio_server_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} + path: build/clio_server + + - name: Upload clio_tests + if: ${{ inputs.unit_tests && !inputs.code_coverage }} + uses: actions/upload-artifact@v4 + with: + name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} + path: build/clio_tests + + - name: Upload clio_integration_tests + if: ${{ inputs.integration_tests && !inputs.code_coverage }} + uses: actions/upload-artifact@v4 + with: + name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} + path: build/clio_integration_tests + + - name: Save cache + if: ${{ !inputs.disable_cache }} + uses: ./.github/actions/save_cache + with: + conan_dir: ${{ env.CONAN_USER_HOME }}/.conan + conan_hash: ${{ steps.restore_cache.outputs.conan_hash }} + conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }} + ccache_dir: ${{ env.CCACHE_DIR }} + ccache_cache_hit: ${{ steps.restore_cache.outputs.ccache_cache_hit }} + ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }} + build_type: ${{ inputs.build_type }} + code_coverage: ${{ inputs.code_coverage }} + conan_profile: ${{ steps.conan.outputs.conan_profile }} + + # TODO: This is not a part of build process but it is the easiest way to do it here. + # It will be refactored in https://github.com/XRPLF/clio/issues/1075 + - name: Run code coverage + if: ${{ inputs.code_coverage }} + uses: ./.github/actions/code_coverage + + upload_coverage_report: + if: ${{ inputs.code_coverage }} + name: Codecov + needs: build + uses: ./.github/workflows/upload_coverage_report.yml + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a5616d0f1..11b1e1d56 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -26,7 +26,7 @@ jobs: build_type: Debug static: true container: '{ "image": "rippleci/clio_ci:latest" }' - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build_impl.yml with: runs_on: ${{ matrix.os }} container: ${{ matrix.container }} diff --git a/.github/workflows/on_push_checks.yml b/.github/workflows/on_push_checks.yml deleted file mode 100644 index 9e1f4ebb4..000000000 --- a/.github/workflows/on_push_checks.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: Check codebase -on: - push: - branches: [master, release/*, develop] - pull_request: - branches: [master, release/*, develop] - workflow_dispatch: - -jobs: - check_format: - name: Check format - runs-on: ubuntu-20.04 - container: - image: rippleci/clio_ci:latest - steps: - - name: Fix git permissions on Linux - shell: bash - run: git config --global --add safe.directory $PWD - - - uses: actions/checkout@v4 - - name: Run formatters - id: run_formatters - run: | - ./.githooks/check-format --diff - shell: bash - - check_docs: - name: Check documentation - runs-on: ubuntu-20.04 - container: - image: rippleci/clio_ci:latest - steps: - - uses: actions/checkout@v4 - - name: Run linter - id: run_linter - run: | - ./.githooks/check-docs - shell: bash - - build: - name: Build - needs: - - check_format - - check_docs - strategy: - fail-fast: false - matrix: - include: - - os: heavy - conan_profile: gcc - build_type: Release - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: false - static: true - - os: heavy - conan_profile: gcc - build_type: Debug - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: true - static: true - - os: heavy - conan_profile: clang - build_type: Release - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: false - static: true - - os: heavy - conan_profile: clang - build_type: Debug - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: false - static: true - - os: macos15 - build_type: Release - code_coverage: false - static: false - uses: ./.github/workflows/build.yml - with: - runs_on: ${{ matrix.os }} - container: ${{ matrix.container }} - conan_profile: ${{ matrix.conan_profile }} - build_type: ${{ matrix.build_type }} - code_coverage: ${{ matrix.code_coverage }} - static: ${{ matrix.static }} - unit_tests: true - integration_tests: true - clio_server: true - - test: - name: Run Tests - needs: build - strategy: - fail-fast: false - matrix: - include: - - os: heavy - conan_profile: gcc - build_type: Release - container: - image: rippleci/clio_ci:latest - - os: heavy - conan_profile: clang - build_type: Release - container: - image: rippleci/clio_ci:latest - - os: heavy - conan_profile: clang - build_type: Debug - container: - image: rippleci/clio_ci:latest - - os: macos15 - conan_profile: apple_clang_16 - build_type: Release - runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} - - steps: - - name: Clean workdir - if: ${{ runner.os == 'macOS' }} - uses: kuznetsss/workspace-cleanup@1.0 - - - uses: actions/download-artifact@v4 - with: - name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }} - - - name: Run clio_tests - run: | - chmod +x ./clio_tests - ./clio_tests diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 0e573bc34..ce0f2bb33 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -20,7 +20,7 @@ jobs: compiler: gcc # - sanitizer: ubsan # todo: enable when heavy runners are available # compiler: gcc - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build_impl.yml with: runs_on: ubuntu-latest # todo: change to heavy container: '{ "image": "rippleci/clio_ci:latest" }'