From 476d4ae74d20209bd629dac3f44aede846756ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 19 Apr 2024 21:50:08 +0200 Subject: [PATCH] Use a dynamic environment for PR jobs --- .github/workflows/ci.yml | 16 ++----- library/backtrace | 2 +- library/stdarch | 2 +- src/ci/github-actions/calculate-job-matrix.py | 21 ++++++++- src/ci/github-actions/ci.yml | 10 ++-- src/ci/github-actions/jobs.yml | 47 +++++++++++++++++-- src/doc/book | 2 +- src/doc/edition-guide | 2 +- src/doc/embedded-book | 2 +- src/doc/nomicon | 2 +- src/doc/reference | 2 +- src/doc/rust-by-example | 2 +- src/doc/rustc-dev-guide | 2 +- src/llvm-project | 2 +- src/tools/cargo | 2 +- 15 files changed, 82 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90269f56c52a4..ce1cc5130c5ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,21 +45,15 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix - run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + env: + GITHUB_CTX: "${{ toJSON(github) }}" + run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs pr: - name: "PR - ${{ matrix.name }}" + name: "${{ matrix.name }}" needs: - calculate_matrix - env: - PR_CI_JOB: 1 - CI_JOB_NAME: "${{ matrix.name }}" - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" - DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - SCCACHE_BUCKET: rust-lang-ci-sccache2 - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - CACHE_DOMAIN: ci-caches.rust-lang.org + env: "${{ matrix.env }}" if: "github.event_name == 'pull_request'" continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}" strategy: diff --git a/library/backtrace b/library/backtrace index e15130618237e..6fa4b85b9962c 160000 --- a/library/backtrace +++ b/library/backtrace @@ -1 +1 @@ -Subproject commit e15130618237eb3e2d4b622549f9647b4c1d9ca3 +Subproject commit 6fa4b85b9962c3e1be8c2e5cc605cd078134152b diff --git a/library/stdarch b/library/stdarch index 7df81ba8c3e2d..967e7afd87cbe 160000 --- a/library/stdarch +++ b/library/stdarch @@ -1 +1 @@ -Subproject commit 7df81ba8c3e2d02c2ace0c5a6f4f32d800c09e56 +Subproject commit 967e7afd87cbea3232581a4a55031134ab88f595 diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index ba40f4da84752..6ed09754ccac7 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -11,6 +11,7 @@ """ import json +import os from pathlib import Path import yaml @@ -19,7 +20,23 @@ if __name__ == "__main__": + github_ctx = json.loads(os.environ["GITHUB_CTX"]) + + print(github_ctx) + with open(JOBS_YAML_PATH) as f: jobs = yaml.safe_load(f) - job_output = jobs["pr"] - print(f"jobs={json.dumps(job_output)}") + selected_jobs = jobs["pr"] + + for job in selected_jobs: + name = job.get("name") + if name is None: + raise Exception(f"Job {job} has no `name` attribute") + env = job.get("env", {}) + env["CI_JOB_NAME"] = name + # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. + env["HEAD_SHA"] = "" # ${{ github.event.pull_request.head.sha || github.sha }} + # env["DOCKER_TOKEN"] = secrets["github_token"] + job["env"] = env + + print(f"jobs={json.dumps(selected_jobs)}") diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index de71b9f874f8d..ad4787124d64f 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -351,15 +351,15 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix - run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + env: + GITHUB_CTX: ${{ toJSON(github) }} + run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs pr: <<: *base-ci-job - name: PR - ${{ matrix.name }} + name: ${{ matrix.name }} needs: [ calculate_matrix ] - env: - <<: [*shared-ci-variables, *public-variables] - PR_CI_JOB: 1 + env: ${{ matrix.env }} if: github.event_name == 'pull_request' continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }} strategy: diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 7e89eef2670e4..88c865118c6b8 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -2,7 +2,32 @@ # dynamically in CI from ci.yml. # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you # modify this file. -shared_defs: +variables: + - &shared-ci-variables + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + + - &public-variables + SCCACHE_BUCKET: rust-lang-ci-sccache2 + TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate + CACHE_DOMAIN: ci-caches.rust-lang.org + + - &prod-variables + SCCACHE_BUCKET: rust-lang-ci-sccache2 + DEPLOY_BUCKET: rust-lang-ci2 + TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate + TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues + TOOLSTATE_PUBLISH: 1 + # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named + # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to + # rotate them in a single branch while keeping the old key in another + # branch, which wouldn't be possible if the key was named with the kind + # (caches, artifacts...). + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 + AWS_REGION: us-west-1 + CACHE_DOMAIN: ci-caches.rust-lang.org + +runners: - &base-job env: { } @@ -37,14 +62,26 @@ shared_defs: - &job-aarch64-linux os: [ self-hosted, ARM64, linux ] +envs: + - &pr-env + <<: [ *shared-ci-variables, *public-variables ] + PR_CI_JOB: 1 + pr: - - name: mingw-check + - name: PR - mingw-check + env: + <<: *pr-env <<: *job-linux-4c - - name: mingw-check-tidy + - name: PR - mingw-check-tidy + env: + <<: *pr-env <<: *job-linux-4c - - name: x86_64-gnu-llvm-17 + - name: PR - x86_64-gnu-llvm-17 env: ENABLE_GCC_CODEGEN: "1" + <<: *pr-env <<: *job-linux-16c - - name: x86_64-gnu-tools + - name: PR - x86_64-gnu-tools + env: + <<: *pr-env <<: *job-linux-16c diff --git a/src/doc/book b/src/doc/book index 3131aa4642c62..19c40bfd2d576 160000 --- a/src/doc/book +++ b/src/doc/book @@ -1 +1 @@ -Subproject commit 3131aa4642c627a24f523c82566b94a7d920f68c +Subproject commit 19c40bfd2d57641d962f3119a1c343355f1b3c5e diff --git a/src/doc/edition-guide b/src/doc/edition-guide index eb3eb80e106d0..98b33e9a44145 160000 --- a/src/doc/edition-guide +++ b/src/doc/edition-guide @@ -1 +1 @@ -Subproject commit eb3eb80e106d03250c1fb7c5666b1c8c59672862 +Subproject commit 98b33e9a441457b0a491fe1be90e7de64eafc3e5 diff --git a/src/doc/embedded-book b/src/doc/embedded-book index aa7d4b0b4653d..2e95fc2fd31d6 160000 --- a/src/doc/embedded-book +++ b/src/doc/embedded-book @@ -1 +1 @@ -Subproject commit aa7d4b0b4653ddb47cb1de2036d090ec2ba9dab1 +Subproject commit 2e95fc2fd31d669947e993aa07ef10dc9828bee7 diff --git a/src/doc/nomicon b/src/doc/nomicon index 0d5f88475fe28..6bc2415218d4d 160000 --- a/src/doc/nomicon +++ b/src/doc/nomicon @@ -1 +1 @@ -Subproject commit 0d5f88475fe285affa6dbbc806e9e44d730797c0 +Subproject commit 6bc2415218d4dd0cb01433d8320f5ccf79c343a1 diff --git a/src/doc/reference b/src/doc/reference index 55694913b1301..984b36eca4b92 160000 --- a/src/doc/reference +++ b/src/doc/reference @@ -1 +1 @@ -Subproject commit 55694913b1301cc809f9bf4a1ad1b3d6920efbd9 +Subproject commit 984b36eca4b9293df04d5ba4eb5c4f77db0f51dc diff --git a/src/doc/rust-by-example b/src/doc/rust-by-example index 60d34b5fd33db..7601e0c5ad29d 160000 --- a/src/doc/rust-by-example +++ b/src/doc/rust-by-example @@ -1 +1 @@ -Subproject commit 60d34b5fd33db1346f9aabfc0c9d0bda6c8e42be +Subproject commit 7601e0c5ad29d5bd3b518700ea63fddfff5915a7 diff --git a/src/doc/rustc-dev-guide b/src/doc/rustc-dev-guide index b77a34bd46399..ffa246b7fd95a 160000 --- a/src/doc/rustc-dev-guide +++ b/src/doc/rustc-dev-guide @@ -1 +1 @@ -Subproject commit b77a34bd46399687b4ce6a17198e9f316c988794 +Subproject commit ffa246b7fd95a96e1cd54883e613aed42c32547d diff --git a/src/llvm-project b/src/llvm-project index 5399a24c66cb6..0af6c732ec6ca 160000 --- a/src/llvm-project +++ b/src/llvm-project @@ -1 +1 @@ -Subproject commit 5399a24c66cb6164cf32280e7d300488c90d5765 +Subproject commit 0af6c732ec6ca189cd7725e4a7d4290793046e83 diff --git a/src/tools/cargo b/src/tools/cargo index 6f06fe908a5ee..0637083df5bbd 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit 6f06fe908a5ee0f415c187f868ea627e82efe07d +Subproject commit 0637083df5bbdcc951845f0d2eff6999cdb6d30a