Use a dynamic environment for PR jobs

rust-lang · Apr 19, 2024 · 476d4ae · 476d4ae
1 parent 05f3c0a
commit 476d4ae
Show file tree

Hide file tree

Showing 15 changed files with 82 additions and 34 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -45,21 +45,15 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
-        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        env:
+          GITHUB_CTX: "${{ toJSON(github) }}"
+        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
   pr:
-    name: "PR - ${{ matrix.name }}"
+    name: "${{ matrix.name }}"
     needs:
       - calculate_matrix
-    env:
-      PR_CI_JOB: 1
-      CI_JOB_NAME: "${{ matrix.name }}"
-      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
-      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-      SCCACHE_BUCKET: rust-lang-ci-sccache2
-      TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
-      CACHE_DOMAIN: ci-caches.rust-lang.org
+    env: "${{ matrix.env }}"
     if: "github.event_name == 'pull_request'"
     continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
     strategy:

diff --git a/library/backtrace b/library/backtrace
diff --git a/library/stdarch b/library/stdarch
diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
@@ -11,6 +11,7 @@
 """
 
 import json
+import os
 from pathlib import Path
 
 import yaml
@@ -19,7 +20,23 @@
 
 
 if __name__ == "__main__":
+    github_ctx = json.loads(os.environ["GITHUB_CTX"])
+
+    print(github_ctx)
+
     with open(JOBS_YAML_PATH) as f:
         jobs = yaml.safe_load(f)
-    job_output = jobs["pr"]
-    print(f"jobs={json.dumps(job_output)}")
+    selected_jobs = jobs["pr"]
+
+    for job in selected_jobs:
+        name = job.get("name")
+        if name is None:
+            raise Exception(f"Job {job} has no `name` attribute")
+        env = job.get("env", {})
+        env["CI_JOB_NAME"] = name
+        # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+        env["HEAD_SHA"] = ""  # ${{ github.event.pull_request.head.sha || github.sha }}
+        # env["DOCKER_TOKEN"] = secrets["github_token"]
+        job["env"] = env
+
+    print(f"jobs={json.dumps(selected_jobs)}")
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
@@ -351,15 +351,15 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
-        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        env:
+          GITHUB_CTX: ${{ toJSON(github) }}
+        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
   pr:
     <<: *base-ci-job
-    name: PR - ${{ matrix.name }}
+    name: ${{ matrix.name }}
     needs: [ calculate_matrix ]
-    env:
-      <<: [*shared-ci-variables, *public-variables]
-      PR_CI_JOB: 1
+    env: ${{ matrix.env }}
     if: github.event_name == 'pull_request'
     continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
     strategy:

diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
@@ -2,7 +2,32 @@
 # dynamically in CI from ci.yml.
 # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
 # modify this file.
-shared_defs:
+variables:
+  - &shared-ci-variables
+    CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+
+  - &public-variables
+    SCCACHE_BUCKET: rust-lang-ci-sccache2
+    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+    CACHE_DOMAIN: ci-caches.rust-lang.org
+
+  - &prod-variables
+    SCCACHE_BUCKET: rust-lang-ci-sccache2
+    DEPLOY_BUCKET: rust-lang-ci2
+    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
+    TOOLSTATE_PUBLISH: 1
+    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
+    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
+    # rotate them in a single branch while keeping the old key in another
+    # branch, which wouldn't be possible if the key was named with the kind
+    # (caches, artifacts...).
+    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
+    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
+    AWS_REGION: us-west-1
+    CACHE_DOMAIN: ci-caches.rust-lang.org
+
+runners:
   - &base-job
     env: { }
 
@@ -37,14 +62,26 @@ shared_defs:
   - &job-aarch64-linux
     os: [ self-hosted, ARM64, linux ]
 
+envs:
+  - &pr-env
+    <<: [ *shared-ci-variables, *public-variables ]
+    PR_CI_JOB: 1
+
 pr:
-  - name: mingw-check
+  - name: PR - mingw-check
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: mingw-check-tidy
+  - name: PR - mingw-check-tidy
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: x86_64-gnu-llvm-17
+  - name: PR - x86_64-gnu-llvm-17
     env:
       ENABLE_GCC_CODEGEN: "1"
+      <<: *pr-env
     <<: *job-linux-16c
-  - name: x86_64-gnu-tools
+  - name: PR - x86_64-gnu-tools
+    env:
+      <<: *pr-env
     <<: *job-linux-16c
diff --git a/src/doc/book b/src/doc/book
diff --git a/src/doc/edition-guide b/src/doc/edition-guide
diff --git a/src/doc/embedded-book b/src/doc/embedded-book
diff --git a/src/doc/nomicon b/src/doc/nomicon
diff --git a/src/doc/reference b/src/doc/reference
diff --git a/src/doc/rust-by-example b/src/doc/rust-by-example
diff --git a/src/doc/rustc-dev-guide b/src/doc/rustc-dev-guide
diff --git a/src/llvm-project b/src/llvm-project
diff --git a/src/tools/cargo b/src/tools/cargo
+2 −3		.github/workflows/check-binary-size.yml
+5 −14		.github/workflows/main.yml
+8 −1		Cargo.lock
+8 −8		Cargo.toml
+20 −33		src/backtrace/dbghelp64.rs
+49 −0		src/capture.rs
+2 −0		src/lib.rs
+1 −8		src/symbolize/dbghelp.rs
+5 −2		src/symbolize/gimli.rs
+12 −0		tests/smoke.rs
+2 −0		crates/core_arch/src/arm_shared/hints.rs
+1 −2		crates/core_arch/src/lib.rs
+10 −8		crates/core_arch/src/x86/avx512f.rs
+0 −1		crates/intrinsic-test/src/json_parser.rs
+90 −92		crates/stdarch-gen-arm/src/main.rs
+2 −2		README.md
+1 −1		listings/ch09-error-handling/listing-09-13/src/main.rs
+1 −1		listings/ch11-writing-automated-tests/listing-11-08/src/lib.rs
+4 −2		listings/ch11-writing-automated-tests/listing-11-09/src/lib.rs
+1 −1		listings/ch11-writing-automated-tests/no-listing-08-guess-with-bug/src/lib.rs
+4 −2		listings/ch11-writing-automated-tests/no-listing-09-guess-with-panic-msg-bug/src/lib.rs
+1 −1		listings/ch12-an-io-project/listing-12-12/output.txt
+1 −1		listings/ch13-functional-features/listing-13-08/output.txt
+1 −1		listings/ch13-functional-features/listing-13-08/src/main.rs
+1 −1		listings/ch16-fearless-concurrency/listing-16-15/src/main.rs
+26 −20		listings/ch20-web-server/listing-20-25/src/main.rs
+26 −20		listings/ch20-web-server/no-listing-07-final-code/src/main.rs
+1 −1		src/appendix-02-operators.md
+1 −1		src/appendix-03-derivable-traits.md
+1 −1		src/appendix-06-translation.md
+0 −6		src/appendix-07-nightly-rust.md
+1 −1		src/ch03-05-control-flow.md
+1 −1		src/ch04-03-slices.md
+1 −1		src/ch05-01-defining-structs.md
+2 −2		src/ch07-03-paths-for-referring-to-an-item-in-the-module-tree.md
+1 −1		src/ch08-01-vectors.md
+1 −1		src/ch11-01-writing-tests.md
+1 −1		src/ch12-05-working-with-environment-variables.md
+2 −2		src/ch13-01-closures.md
+1 −1		src/ch15-02-deref.md
+1 −1		src/ch18-02-refutability.md
+1 −1		src/ch19-01-unsafe-rust.md
+1 −1		src/ch20-02-multithreaded.md
+0 −2		book.toml
+1 −1		src/editions/advanced-migrations.md
+61 −35		src/editions/index.md
+1 −4		src/attributes.md
+0 −70		src/attributes/diagnostics.md
+2 −2		src/procedural-macros.md
+2 −2		src/type-layout.md
+6 −0		.github/workflows/ci.yml
+19 −0		README.md
+43 −0		ci/lengthcheck.sh
+1 −7		src/SUMMARY.md
+1 −1		src/appendix/glossary.md
+1 −1		src/backend/debugging.md
+1 −1		src/backend/libs-and-metadata.md
+0 −264		src/borrow_check/opaque-types-region-inference-restrictions.md
+4 −4		src/building/bootstrapping/what-bootstrapping-does.md
+0 −5		src/building/how-to-build-and-run.md
+0 −64		src/building/quickstart.md
+1 −1		src/building/suggested.md
+1 −1		src/compiler-src.md
+3 −3		src/contributing.md
+1 −1		src/git.md
+1 −1		src/overview.md
+69 −0		src/param_env.md
+0 −44		src/param_env/param_env_acquisition.md
+0 −83		src/param_env/param_env_construction_internals.md
+0 −18		src/param_env/param_env_summary.md
+0 −60		src/param_env/param_env_what_is_it.md
+36 −148		src/rustdoc-internals/search.md
+1 −1		src/solve/invariants.md
+1 −9		src/tests/compiletest.md
+0 −1		src/tests/headers.md
+1 −1		src/what-does-early-late-bound-mean.md
+1 −1		.github/workflows/release-lit.yml
+0 −7		clang/docs/ReleaseNotes.rst
+2 −2		clang/include/clang/Sema/ScopeInfo.h
+16 −27		clang/lib/CodeGen/CGDecl.cpp
+10 −10		clang/lib/Format/TokenAnnotator.cpp
+0 −4		clang/lib/Headers/__stddef_unreachable.h
+26 −47		clang/lib/Sema/SemaDecl.cpp
+1 −21		clang/lib/Sema/SemaTemplate.cpp
+2 −2		clang/test/CodeGen/aapcs-align.cpp
+4 −4		clang/test/CodeGen/aapcs64-align.cpp
+18 −6		clang/test/CodeGen/attr-counted-by.c
+12 −15		clang/test/CodeGenCXX/auto-var-init.cpp
+8 −1		clang/test/CodeGenOpenCL/amdgpu-printf.cl
+3 −1		clang/test/OpenMP/bug54082.c
+3 −89		clang/test/SemaCXX/warn-shadow-in-lambdas.cpp
+0 −26		clang/test/SemaTemplate/concepts-friends.cpp
+1 −1		clang/test/SemaTemplate/ctad.cpp
+0 −14		clang/unittests/Format/TokenAnnotatorTest.cpp
+0 −42		compiler-rt/lib/builtins/riscv/restore.S
+0 −42		compiler-rt/lib/builtins/riscv/save.S
+1 −1		compiler-rt/lib/msan/msan.cpp
+11 −24		compiler-rt/lib/msan/msan.h
+0 −3		compiler-rt/lib/msan/msan_allocator.cpp
+13 −54		compiler-rt/lib/msan/msan_linux.cpp
+25 −18		compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+1 −1		libcxx/include/__format/formatter_floating_point.h
+19 −10		libcxx/include/stddef.h
+2 −13		libcxx/modules/CMakeLists.txt
+0 −7		lld/COFF/Chunks.cpp
+3 −4		lld/ELF/Arch/X86_64.cpp
+0 −41		lld/test/COFF/autoimport-gc.s
+1 −11		lld/test/ELF/x86-64-gotpc-relax-too-far.s
+1 −1		llvm/CMakeLists.txt
+0 −2		llvm/docs/LangRef.rst
+7 −44		llvm/include/llvm/Support/FormattedStream.h
+1 −2		llvm/lib/Analysis/InstructionSimplify.cpp
+1 −6		llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+2 −2		llvm/lib/IR/ConstantRange.cpp
+0 −3		llvm/lib/Support/FormattedStream.cpp
+34 −8		llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+0 −3		llvm/lib/Target/AArch64/AArch64ISelLowering.h
+4 −7		llvm/lib/Target/ARM/ARMFrameLowering.cpp
+0 −4		llvm/lib/Target/ARM/ARMFrameLowering.h
+13 −10		llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+0 −5		llvm/lib/Target/LoongArch/LoongArchTargetMachine.h
+6 −0		llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+0 −44		llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
+0 −34		llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+1 −3		llvm/lib/Target/X86/X86MCInstLower.cpp
+27 −53		llvm/lib/Transforms/Scalar/SROA.cpp
+1 −20		llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+0 −56		llvm/test/CodeGen/ARM/ldst-opt-lr-restored.ll
+0 −47		llvm/test/CodeGen/LoongArch/addrspacecast.ll
+45 −284		llvm/test/CodeGen/PowerPC/crsave.ll
+0 −9		llvm/test/CodeGen/SPARC/inlineasm-bad.ll
+0 −9		llvm/test/CodeGen/SPARC/inlineasm.ll
+0 −22		llvm/test/CodeGen/SystemZ/int-usub-12.ll
+0 −2		llvm/test/CodeGen/SystemZ/int-usub-13.ll
+0 −23		llvm/test/CodeGen/X86/addcarry.ll
+0 −17		llvm/test/CodeGen/X86/patchable-prologue.ll
+0 −20		llvm/test/Transforms/Float2Int/pr79158.ll
+0 −42		llvm/test/Transforms/InstSimplify/pr87042.ll
+0 −82		llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll
+0 −62		llvm/test/Transforms/SROA/vector-promotion.ll
+5 −0		llvm/tools/llvm-mc/llvm-mc.cpp
+7 −0		llvm/tools/llvm-objdump/llvm-objdump.cpp
+0 −18		llvm/unittests/IR/ConstantRangeTest.cpp
+1 −1		llvm/utils/lit/lit/__init__.py
+1 −1		llvm/utils/release/github-upload-release.py
+0 −1		mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+0 −1		mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir
+1 −1		mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+1 −3		openmp/runtime/src/kmp.h
+8 −121		openmp/runtime/src/kmp_affinity.cpp
+2 −71		openmp/runtime/src/kmp_affinity.h
+1 −2		openmp/runtime/src/kmp_os.h
+6 −31		openmp/runtime/src/z_Linux_util.cpp
+1 −1		openmp/runtime/test/lit.cfg
+1 −1		polly/lib/Exchange/JSONExporter.cpp
+1 −1		polly/lib/Transform/DeLICM.cpp
+1 −1		polly/lib/Transform/FlattenSchedule.cpp
+1 −1		polly/lib/Transform/ForwardOpTree.cpp