Skip to content

Commit

Permalink
chore: simplify reports (#7421)
Browse files Browse the repository at this point in the history
  • Loading branch information
@TomAFrench
TomAFrench authored Feb 19, 2025
1 parent 1d04f8b commit d0d04e8
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 110 deletions.
202 changes: 103 additions & 99 deletions .github/workflows/reports.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,9 +282,9 @@ jobs:
strategy:
fail-fast: false
matrix:
project: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}}
include: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}}

name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }}
name: External repo compilation and execution reports - ${{ matrix.repo }}/${{ matrix.path }}
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -302,32 +302,77 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
repository: ${{ matrix.project.repo }}
repository: ${{ matrix.repo }}
path: test-repo
ref: ${{ matrix.project.ref }}
ref: ${{ matrix.ref }}

- name: Fetch noir dependencies
working-directory: ./test-repo/${{ matrix.project.path }}
working-directory: ./test-repo/${{ matrix.path }}
run: |
# We run `nargo check` to pre-fetch any dependencies so we don't measure the time to download these
# when benchmarking.
nargo check
- name: Generate compilation report
working-directory: ./test-repo/${{ matrix.project.path }}
id: compilation_report
working-directory: ./test-repo/${{ matrix.path }}
run: |
mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
touch parse_time.sh
chmod +x parse_time.sh
cp /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
./compilation_report.sh 1 ${{ matrix.project.num_runs }}
./compilation_report.sh 1 ${{ matrix.num_runs }}
PACKAGE_NAME=${{ matrix.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
REPORT_NAME=compilation_report_$PACKAGE_NAME.json
REPORT_PATH=$(pwd)/$REPORT_NAME
mv ./compilation_report.json $REPORT_PATH
echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT
echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT
env:
FLAGS: ${{ matrix.project.flags }}
FLAGS: ${{ matrix.flags }}

- name: Upload compilation report
uses: actions/upload-artifact@v4
with:
name: ${{ steps.compilation_report.outputs.report_name }}
path: ${{ steps.compilation_report.outputs.report_path }}
retention-days: 3
overwrite: true

- name: Generate execution report
id: execution_report
working-directory: ./test-repo/${{ matrix.path }}
if: ${{ !matrix.cannot_execute }}
run: |
mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
./execution_report.sh 1 ${{ matrix.num_runs }}
PACKAGE_NAME=${{ matrix.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
REPORT_NAME=execution_report_$PACKAGE_NAME.json
REPORT_PATH=$(pwd)/$REPORT_NAME
mv ./execution_report.json $REPORT_PATH
echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT
echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT
- name: Upload execution report
if: ${{ !matrix.cannot_execute }}
uses: actions/upload-artifact@v4
with:
name: ${{ steps.execution_report.outputs.report_name }}
path: ${{ steps.execution_report.outputs.report_path }}
retention-days: 3
overwrite: true

- name: Check compilation time limit
run: |
TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_report.json)
TIME_LIMIT=${{ matrix.project.compilation-timeout }}
TIME=$(jq '.[0].value' ${{ steps.compilation_report.outputs.report_path }})
TIME_LIMIT=${{ matrix.compilation-timeout }}
if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then
# Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
echo "Failing due to compilation exceeding timeout..."
Expand All @@ -336,19 +381,11 @@ jobs:
exit 1
fi
- name: Generate execution report
working-directory: ./test-repo/${{ matrix.project.path }}
if: ${{ !matrix.project.cannot_execute }}
run: |
mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
./execution_report.sh 1 ${{ matrix.project.num_runs }}
- name: Check execution time limit
if: ${{ !matrix.project.cannot_execute }}
if: ${{ !matrix.cannot_execute }}
run: |
TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/execution_report.json)
TIME_LIMIT=${{ matrix.project.execution-timeout }}
TIME=$(jq '.[0].value' ${{ steps.execution_report.outputs.report_path }})
TIME_LIMIT=${{ matrix.execution-timeout }}
if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then
# Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
echo "Failing due to execution exceeding timeout..."
Expand All @@ -357,41 +394,6 @@ jobs:
exit 1
fi
- name: Move compilation report
id: compilation_report
shell: bash
run: |
PACKAGE_NAME=${{ matrix.project.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json
echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
- name: Move execution report
id: execution_report
shell: bash
if: ${{ !matrix.project.cannot_execute }}
run: |
PACKAGE_NAME=${{ matrix.project.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
mv ./test-repo/${{ matrix.project.path }}/execution_report.json ./execution_report_$PACKAGE_NAME.json
echo "execution_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
- name: Upload compilation report
uses: actions/upload-artifact@v4
with:
name: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}
path: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}.json
retention-days: 3
overwrite: true

- name: Upload execution report
uses: actions/upload-artifact@v4
with:
name: execution_report_${{ steps.execution_report.outputs.execution_report_name }}
path: execution_report_${{ steps.execution_report.outputs.execution_report_name }}.json
retention-days: 3
overwrite: true

external_repo_memory_report:
needs: [build-nargo, benchmark-projects-list]
runs-on: ubuntu-22.04
Expand Down Expand Up @@ -423,19 +425,61 @@ jobs:
ref: ${{ matrix.ref }}

- name: Generate compilation memory report
id: compilation_report
working-directory: ./test-repo/${{ matrix.path }}
run: |
mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
mv /home/runner/work/noir/noir/scripts/test_programs/parse_memory.sh ./parse_memory.sh
./memory_report.sh 1
# Rename the memory report as the execution report is about to write to the same file
cp memory_report.json compilation_memory_report.json
PACKAGE_NAME=${{ matrix.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
REPORT_NAME=compilation_mem_report_$PACKAGE_NAME.json
REPORT_PATH=$(pwd)/$REPORT_NAME
mv ./memory_report.json $REPORT_PATH
echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT
echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT
env:
FLAGS: ${{ matrix.flags }}

- name: Upload compilation memory report
uses: actions/upload-artifact@v4
with:
name: ${{ steps.compilation_report.outputs.report_name }}
path: ${{ steps.compilation_report.outputs.report_path }}
retention-days: 3
overwrite: true

- name: Generate execution memory report
id: execution_report
working-directory: ./test-repo/${{ matrix.path }}
if: ${{ !matrix.cannot_execute }}
run: |
./memory_report.sh 1 1
PACKAGE_NAME=${{ matrix.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
REPORT_NAME=execution_mem_report_$PACKAGE_NAME.json
REPORT_PATH=$(pwd)/$REPORT_NAME
mv ./memory_report.json $REPORT_PATH
echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT
echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT
- name: Upload execution memory report
if: ${{ !matrix.cannot_execute }}
uses: actions/upload-artifact@v4
with:
name: ${{ steps.execution_report.outputs.report_name }}
path: ${{ steps.execution_report.outputs.report_path }}
retention-days: 3
overwrite: true

- name: Check compilation memory limit
run: |
MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/compilation_memory_report.json)
MEMORY=$(jq '.[0].value' ${{ steps.compilation_report.outputs.report_path }})
MEMORY_LIMIT=${{ matrix.compilation-memory-limit }}
if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then
# Don't bump this limit without understanding why this has happened and confirming that you're not the cause.
Expand All @@ -445,16 +489,10 @@ jobs:
exit 1
fi
- name: Generate execution memory report
working-directory: ./test-repo/${{ matrix.path }}
if: ${{ !matrix.cannot_execute }}
run: |
./memory_report.sh 1 1
- name: Check execution memory limit
if: ${{ !matrix.cannot_execute }}
run: |
MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/memory_report.json)
MEMORY=$(jq '.[0].value' ${{ steps.execution_report.outputs.report_path }})
MEMORY_LIMIT=${{ matrix.execution-memory-limit }}
if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then
# Don't bump this limit without understanding why this has happened and confirming that you're not the cause.
Expand All @@ -464,40 +502,6 @@ jobs:
exit 1
fi
- name: Move compilation report
id: compilation_mem_report
shell: bash
run: |
PACKAGE_NAME=${{ matrix.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
mv ./test-repo/${{ matrix.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json
echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
- name: Upload compilation memory report
uses: actions/upload-artifact@v4
with:
name: compilation_mem_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}
path: memory_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}.json
retention-days: 3
overwrite: true

- name: Move execution report
id: execution_mem_report
if: ${{ !matrix.cannot_execute }}
run: |
PACKAGE_NAME=${{ matrix.path }}
PACKAGE_NAME=$(basename $PACKAGE_NAME)
mv ./test-repo/${{ matrix.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
- name: Upload execution memory report
uses: actions/upload-artifact@v4
with:
name: execution_mem_report_${{ steps.execution_mem_report.outputs.memory_report_name }}
path: memory_report_${{ steps.execution_mem_report.outputs.memory_report_name }}.json
retention-days: 3
overwrite: true

upload_compilation_report:
name: Upload compilation report
needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
Expand Down
22 changes: 11 additions & 11 deletions .github/workflows/test-js-packages.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,17 +541,6 @@ jobs:
fi
env:
NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true

- name: Check test time limit
run: |
TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json)
if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then
# Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
echo "Failing due to test suite exceeding timeout..."
echo "Timeout: ${{ matrix.timeout }}"
echo "Test suite took: $TIME".
exit 1
fi

- name: Compare test results
working-directory: ./noir-repo
Expand All @@ -566,6 +555,17 @@ jobs:
retention-days: 3
overwrite: true

- name: Check test time limit
run: |
TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json)
if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then
# Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
echo "Failing due to test suite exceeding timeout..."
echo "Timeout: ${{ matrix.timeout }}"
echo "Test suite took: $TIME".
exit 1
fi
compile-noir-contracts:
needs: [build-nargo]
runs-on: ubuntu-22.04
Expand Down

1 comment on commit d0d04e8

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for benchmark 'Test Suite Duration'.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold 1.20.

Benchmark suite Current: d0d04e8 Previous: 1d04f8b Ratio
AztecProtocol_aztec-packages_noir-projects_noir-protocol-circuits_crates_blob 65 s 54 s 1.20

This comment was automatically generated by workflow using github-action-benchmark.

CC: @TomAFrench

Please sign in to comment.