chore(ci): do PGO optimization in CI and benchmark it (#48) #291
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Test code base and upload coverage | |
| name: Test code | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| env: | |
| QT_QPA_PLATFORM: offscreen | |
| DISPLAY: :99 | |
| jobs: | |
| pytest: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, ubuntu-latest, windows-latest] | |
| pyversion: ['3.9', '3.10', '3.11', '3.12'] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install Linux dependencies | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install freeglut3-dev xvfb | |
| sudo apt-get install x11-utils | |
| nohup Xvfb $DISPLAY -screen 0 1400x900x24 -dpi 96 +extension RANDR +render & | |
| - name: Install Mesa (OpenGL on Windows) | |
| if: matrix.os == 'windows-latest' | |
| uses: ssciwr/setup-mesa-dist-win@v2 | |
| - name: Install PDM | |
| uses: pdm-project/setup-pdm@v4 | |
| with: | |
| python-version: ${{ matrix.pyversion }} | |
| cache: true | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install dependencies | |
| run: | | |
| pdm install -G test,github-action | |
| - name: Run Pytest | |
| if: matrix.os != 'ubuntu-latest' || matrix.pyversion != '3.12' | |
| run: | | |
| pdm run pytest -n auto | |
| - name: Run pytest and coverage | |
| if: matrix.os == 'ubuntu-latest' && matrix.pyversion == '3.12' | |
| run: pdm run pytest --cov-report xml --cov=differt tests/ | |
| - name: Upload to codecov.io | |
| if: matrix.os == 'ubuntu-latest' && matrix.pyversion == '3.12' | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: true | |
| cargo-test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, ubuntu-latest, windows-latest] | |
| pyversion: ['3.9', '3.10', '3.11', '3.12'] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.pyversion }} | |
| - name: Install NumPy | |
| run: pip install numpy | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Run Cargo test | |
| run: | | |
| cargo test | |
| python-benchmark: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install PDM | |
| uses: pdm-project/setup-pdm@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: true | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: llvm-tools | |
| - name: Install dependencies | |
| run: | | |
| pdm install -G test,github-action | |
| - name: Run Benchmarks on changes | |
| run: pdm run pytest --benchmark-only --benchmark-save=changes tests/benchmarks | |
| - name: Build wheel with profiling profile | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| args: > | |
| --release | |
| --out pgo-wheel | |
| env: | |
| RUSTFLAGS: -Cprofile-generate=${{ github.workspace }}/profdata | |
| - name: Detect Rust host | |
| run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: Generate profile data | |
| run: | | |
| pdm run python -m ensurepip | |
| pdm run python -m pip install differt --no-index --no-deps --find-links pgo-wheel --force-reinstall | |
| pdm run pytest --benchmark-only tests/benchmarks | |
| rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"' | |
| - name: Merge GO data | |
| run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata | |
| - name: Build PGO-optimized wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| args: > | |
| --release | |
| --out opt-wheel | |
| env: | |
| RUSTFLAGS: -Cprofile-use=${{ github.workspace }}/merged.profdata | |
| - name: Run Benchmarks on changes with PGO-optimized wheel | |
| run: | | |
| pdm run python -m pip install differt --no-index --no-deps --find-links opt-wheel --force-reinstall | |
| pdm run pytest --benchmark-only --benchmark-save=pgo tests/benchmarks | |
| - name: Checkout base branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| clean: false | |
| - name: Install dependencies (before changes) | |
| run: | | |
| pdm install -G test,github-action | |
| - name: Build wheel before changes | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| args: > | |
| --release | |
| --out dist | |
| - name: Run Benchmarks before changes | |
| run: | | |
| pdm run python -m pip install differt --no-index --no-deps --find-links dist --force-reinstall | |
| pdm run pytest --benchmark-only --benchmark-save=before tests/benchmarks | |
| - name: Compare benchmarks | |
| run: | | |
| echo 'results<<EOF' >> $GITHUB_OUTPUT | |
| pdm run pytest-benchmark compare --group-by=group --columns="mean,stddev,ops" >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| id: compare | |
| - name: Comment PR with benchmarks | |
| uses: thollander/actions-comment-pull-request@v2 | |
| continue-on-error: true | |
| with: | |
| message: | | |
| Python benchmark results: | |
| ``` | |
| ${{ steps.compare.outputs.results }} | |
| ``` | |
| comment_tag: python-benchmarks | |
| id: comment | |
| - name: If PR comment failed, write to PR summary | |
| if: steps.comment.outcome != 'success' | |
| run: | | |
| echo '### Python benchmark results' >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo '${{ steps.compare.outputs.results }}' >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| rust-benchmark: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install NumPy | |
| run: pip install numpy | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: critcmp@0.1.8 | |
| - name: Run Benchmarks on changes | |
| run: cargo bench --bench bench_main -- --save-baseline changes | |
| - name: Checkout base branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| clean: false | |
| - name: Run Benchmarks before changes | |
| run: cargo bench --bench bench_main -- --save-baseline before | |
| - name: Compare benchmarks | |
| run: | | |
| echo 'results<<EOF' >> $GITHUB_OUTPUT | |
| critcmp before changes >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| id: compare | |
| - name: Comment PR with benchmarks | |
| uses: thollander/actions-comment-pull-request@v2 | |
| continue-on-error: true | |
| with: | |
| message: | | |
| Rust benchmark results: | |
| ``` | |
| ${{ steps.compare.outputs.results }} | |
| ``` | |
| comment_tag: rust-benchmarks | |
| id: comment | |
| - name: If PR comment failed, write to PR summary | |
| if: steps.comment.outcome != 'success' | |
| run: | | |
| echo '### Rust benchmark results' >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo '${{ steps.compare.outputs.results }}' >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY |