From 8334a604e8f5c8e5402d1ff9551306adc98de021 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 11:42:31 +0100 Subject: [PATCH 01/10] Add workflow for networking benchmarks --- .github/workflows/networking-benchmarks.yml | 90 +++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/networking-benchmarks.yml diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml new file mode 100644 index 000000000000..6ba3ddc7743e --- /dev/null +++ b/.github/workflows/networking-benchmarks.yml @@ -0,0 +1,90 @@ +name: Networking Benchmarks + +on: + push: + branches: + - master + pull_request: + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + preflight: + uses: ./.github/workflows/reusable-preflight.yml + + build: + timeout-minutes: 80 + needs: [preflight] + runs-on: ${{ needs.preflight.outputs.RUNNER }} + container: + image: ${{ needs.preflight.outputs.IMAGE }} + strategy: + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Check Rust + run: | + rustup show + rustup +nightly show + + - name: Run Benchmarks + id: run-benchmarks + run: | + mkdir -p ./charts + forklift cargo bench -p sc-network --output-format bencher | tee ./charts/networking-bench.txt || echo "Benchmarks failed" + ls -lsa ./charts + + - name: Upload artifacts + uses: actions/upload-artifact@v4.3.6 + with: + name: + path: ./charts + + publish-benchmarks: + timeout-minutes: 60 + needs: [build] + if: github.ref == 'refs/heads/master' + environment: subsystem-benchmarks + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: gh-pages + fetch-depth: 0 + + - run: git checkout master -- + + - name: Download artifacts + uses: actions/download-artifact@v4.1.8 + with: + path: ./charts + + - name: Setup git + run: | + # Fixes "detected dubious ownership" error in the ci + git config --global --add safe.directory '*' + ls -lsR ./charts + + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.POLKADOTSDK_GHPAGES_APP_ID }} + private-key: ${{ secrets.POLKADOTSDK_GHPAGES_APP_KEY }} + + - name: Generate networking benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "cargo" + output-file-path: ./charts/networking-bench.txt + benchmark-data-dir-path: ./bench/networking + github-token: ${{ steps.app-token.outputs.token }} + auto-push: true From d1728ecf66b7347fc02fb95286958ae77e6d6731 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 11:44:57 +0100 Subject: [PATCH 02/10] Add debug --- .github/workflows/networking-benchmarks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index 6ba3ddc7743e..569f5e9a7f04 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -5,6 +5,7 @@ on: branches: - master pull_request: + types: [opened, synchronize, reopened, ready_for_review] # TODO: remove this merge_group: concurrency: From ab0acea86e008ebc40b8b2d68d2808a07a9b085d Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 11:57:19 +0100 Subject: [PATCH 03/10] Add missing name --- .github/workflows/networking-benchmarks.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index 569f5e9a7f04..23bcc5a81ff1 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -31,11 +31,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Check Rust - run: | - rustup show - rustup +nightly show - - name: Run Benchmarks id: run-benchmarks run: | @@ -46,7 +41,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4.3.6 with: - name: + name: networking-bench-${{ github.sha }} path: ./charts publish-benchmarks: @@ -67,6 +62,7 @@ jobs: - name: Download artifacts uses: actions/download-artifact@v4.1.8 with: + name: networking-bench-${{ github.sha }} path: ./charts - name: Setup git From 5b8ca9d51c2fae65725050727e820abfb38de875 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 12:07:34 +0100 Subject: [PATCH 04/10] Fix params --- .github/workflows/networking-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index 23bcc5a81ff1..865bd271c21d 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -35,7 +35,7 @@ jobs: id: run-benchmarks run: | mkdir -p ./charts - forklift cargo bench -p sc-network --output-format bencher | tee ./charts/networking-bench.txt || echo "Benchmarks failed" + forklift cargo bench -p sc-network -- --output-format bencher | tee ./charts/networking-bench.txt || echo "Benchmarks failed" ls -lsa ./charts - name: Upload artifacts From 69770873ebc35e806c7d5c57c719f23f0edd8d3f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 12:30:31 +0100 Subject: [PATCH 05/10] Split benches --- .github/workflows/networking-benchmarks.yml | 33 +++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index 865bd271c21d..d2a850fe8775 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -27,6 +27,16 @@ jobs: image: ${{ needs.preflight.outputs.IMAGE }} strategy: fail-fast: false + matrix: + features: + [ + { + bench: "notifications_protocol", + }, + { + bench: "request_response_protocol", + }, + ] steps: - name: Checkout uses: actions/checkout@v4 @@ -35,13 +45,13 @@ jobs: id: run-benchmarks run: | mkdir -p ./charts - forklift cargo bench -p sc-network -- --output-format bencher | tee ./charts/networking-bench.txt || echo "Benchmarks failed" + forklift cargo bench -p sc-network --bench ${{ matrix.features.bench }} -- --output-format bencher | tee ./charts/networking-bench.txt || echo "Benchmarks failed" ls -lsa ./charts - name: Upload artifacts uses: actions/upload-artifact@v4.3.6 with: - name: networking-bench-${{ github.sha }} + name: ${{ matrix.features.bench }}-${{ github.sha }} path: ./charts publish-benchmarks: @@ -77,11 +87,24 @@ jobs: app-id: ${{ secrets.POLKADOTSDK_GHPAGES_APP_ID }} private-key: ${{ secrets.POLKADOTSDK_GHPAGES_APP_KEY }} - - name: Generate networking benchmarks + - name: Generate ${{ env.BENCH }} + env: + BENCH: notifications_protocol + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "cargo" + output-file-path: ./charts/${{ env.BENCH }}.txt + benchmark-data-dir-path: ./bench/${{ env.BENCH }} + github-token: ${{ steps.app-token.outputs.token }} + auto-push: true + + - name: Generate ${{ env.BENCH }} + env: + BENCH: request_response_protocol uses: benchmark-action/github-action-benchmark@v1 with: tool: "cargo" - output-file-path: ./charts/networking-bench.txt - benchmark-data-dir-path: ./bench/networking + output-file-path: ./charts/${{ env.BENCH }}.txt + benchmark-data-dir-path: ./bench/${{ env.BENCH }} github-token: ${{ steps.app-token.outputs.token }} auto-push: true From d5f5bba40b17d73ca1dafc6c9f1ea835c867f17f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 12:50:21 +0100 Subject: [PATCH 06/10] Clean output file --- .github/workflows/networking-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index d2a850fe8775..5d9e661822ba 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -45,7 +45,7 @@ jobs: id: run-benchmarks run: | mkdir -p ./charts - forklift cargo bench -p sc-network --bench ${{ matrix.features.bench }} -- --output-format bencher | tee ./charts/networking-bench.txt || echo "Benchmarks failed" + forklift cargo bench -p sc-network --bench ${{ matrix.features.bench }} -- --output-format bencher | grep "${{ matrix.features.bench }}" | tee ./charts/networking-bench.txt || echo "Benchmarks failed" ls -lsa ./charts - name: Upload artifacts From 70f55a99d95e4b42781fc8a20731f63cecdc168a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 13:05:46 +0100 Subject: [PATCH 07/10] Fix grep --- .github/workflows/networking-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index 5d9e661822ba..3f11f5c1975c 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -45,7 +45,7 @@ jobs: id: run-benchmarks run: | mkdir -p ./charts - forklift cargo bench -p sc-network --bench ${{ matrix.features.bench }} -- --output-format bencher | grep "${{ matrix.features.bench }}" | tee ./charts/networking-bench.txt || echo "Benchmarks failed" + forklift cargo bench -p sc-network --bench ${{ matrix.features.bench }} -- --output-format bencher | grep "^test" | tee ./charts/networking-bench.txt || echo "Benchmarks failed" ls -lsa ./charts - name: Upload artifacts From e994c3e42ca6b13ceae3e3b123b2a39d846da4c1 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 13:19:55 +0100 Subject: [PATCH 08/10] Remove debug --- .github/workflows/networking-benchmarks.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index 3f11f5c1975c..8cc2fe0000e2 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -4,9 +4,6 @@ on: push: branches: - master - pull_request: - types: [opened, synchronize, reopened, ready_for_review] # TODO: remove this - merge_group: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From d8e2585912d48dc54dbf6a2098403bc97b173d84 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 13:54:04 +0100 Subject: [PATCH 09/10] Update .github/workflows/networking-benchmarks.yml --- .github/workflows/networking-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/networking-benchmarks.yml b/.github/workflows/networking-benchmarks.yml index 8cc2fe0000e2..e45ae601105d 100644 --- a/.github/workflows/networking-benchmarks.yml +++ b/.github/workflows/networking-benchmarks.yml @@ -19,7 +19,7 @@ jobs: build: timeout-minutes: 80 needs: [preflight] - runs-on: ${{ needs.preflight.outputs.RUNNER }} + runs-on: ${{ needs.preflight.outputs.RUNNER_BENCHMARK }} container: image: ${{ needs.preflight.outputs.IMAGE }} strategy: From cab2de79d0a85a7ddb68c960d21b17bd85640dd9 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 2 Jan 2025 16:45:54 +0100 Subject: [PATCH 10/10] Trigger CI