diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 09b8858d09f712..d1e9fff37d3c2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: with: fetch-depth: 1 - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Check Autoconf and aclocal versions run: | grep "Generated by GNU Autoconf 2.71" configure @@ -98,7 +98,7 @@ jobs: with: python-version: '3.x' - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -108,7 +108,7 @@ jobs: - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -247,7 +247,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -259,9 +259,9 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -270,16 +270,16 @@ jobs: key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl=$OPENSSL_DIR + run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython run: make -j4 - name: Display build info @@ -312,9 +312,9 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -323,24 +323,24 @@ jobs: key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -353,7 +353,7 @@ jobs: --config-cache \ --with-pydebug \ --enable-slower-safety \ - --with-openssl=$OPENSSL_DIR + --with-openssl="$OPENSSL_DIR" - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -362,18 +362,18 @@ jobs: run: make pythoninfo - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: "Create hypothesis venv" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | VENV_LOC=$(realpath -m .)/hypovenv VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV - echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt + echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" + echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" + ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database uses: actions/cache@v4 @@ -411,10 +411,13 @@ jobs: build_asan: name: 'Address sanitizer' - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + matrix: + os: [ubuntu-22.04] env: OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 @@ -422,7 +425,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -438,9 +441,9 @@ jobs: version: 10 - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -449,10 +452,10 @@ jobs: key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index ff5cbdf3eda749..bbedd22cc6d189 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -9,15 +9,53 @@ permissions: pull-requests: write jobs: - label: - name: DO-NOT-MERGE / unresolved review + label-dnm: + name: DO-NOT-MERGE if: github.repository_owner == 'python' runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: mheap/github-action-required-labels@v5 + - name: Check there's no DO-NOT-MERGE + uses: mheap/github-action-required-labels@v5 with: mode: exactly count: 0 - labels: "DO-NOT-MERGE, awaiting changes, awaiting change review" + labels: | + DO-NOT-MERGE + + label-reviews: + name: Unresolved review + if: github.repository_owner == 'python' + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + # Check that the PR is not awaiting changes from the author due to previous review. + - name: Check there's no required changes + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 0 + labels: | + awaiting changes + awaiting change review + - id: is-feature + name: Check whether this PR is a feature (contains a "type-feature" label) + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + type-feature + exit_type: success # don't fail the check if the PR is not a feature, just record the result + # In case of a feature PR, check for a complete review (contains an "awaiting merge" label). + - id: awaiting-merge + if: steps.is-feature.outputs.status == 'success' + name: Check for complete review + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + awaiting merge diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 5cd6fb39f1e12f..1a6fd33186840c 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -65,9 +65,9 @@ jobs: id: check run: | if [ -z "$GITHUB_BASE_REF" ]; then - echo "run-tests=true" >> $GITHUB_OUTPUT + echo "run-tests=true" >> "$GITHUB_OUTPUT" else - git fetch origin $GITHUB_BASE_REF --depth=1 + git fetch origin "$GITHUB_BASE_REF" --depth=1 # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), # but it requires to download more commits (this job uses @@ -81,18 +81,18 @@ jobs: # into the PR branch anyway. # # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run-tests=true" >> $GITHUB_OUTPUT || true + git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run-tests=true" >> "$GITHUB_OUTPUT" || true fi # Check if we should run hypothesis tests GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - echo $GIT_BRANCH + echo "$GIT_BRANCH" if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then echo "Branch too old for hypothesis tests" - echo "run-hypothesis=false" >> $GITHUB_OUTPUT + echo "run-hypothesis=false" >> "$GITHUB_OUTPUT" else echo "Run hypothesis tests" - echo "run-hypothesis=true" >> $GITHUB_OUTPUT + echo "run-hypothesis=true" >> "$GITHUB_OUTPUT" fi # oss-fuzz maintains a configuration for fuzzing the main branch of @@ -100,19 +100,19 @@ jobs: # merged into the main branch; compatibility with older branches may # be broken. FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' - if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then + if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then # The tests are pretty slow so they are executed only for PRs # changing relevant files. echo "Run CIFuzz tests" - echo "run-cifuzz=true" >> $GITHUB_OUTPUT + echo "run-cifuzz=true" >> "$GITHUB_OUTPUT" else echo "Branch too old for CIFuzz tests; or no C files were changed" - echo "run-cifuzz=false" >> $GITHUB_OUTPUT + echo "run-cifuzz=false" >> "$GITHUB_OUTPUT" fi - name: Compute hash for config cache key id: config-hash run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT + echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" - name: Get a list of the changed documentation-related files if: github.event_name == 'pull_request' id: changed-docs-files diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index b3a160fbbf8053..bfa5f69defb87c 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index f4c976ca996410..65072efa8e9023 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -26,7 +26,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -47,12 +47,12 @@ jobs: sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> $GITHUB_ENV - echo "CC=clang" >> $GITHUB_ENV - echo "CXX=clang++" >> $GITHUB_ENV + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> "$GITHUB_ENV" + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -68,7 +68,7 @@ jobs: run: ./python -m test --tsan -j4 - name: Display TSAN logs if: always() - run: find ${GITHUB_WORKSPACE} -name 'tsan_log.*' | xargs head -n 1000 + run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 - name: Archive TSAN logs if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 0cf40ba8a9b03b..18e2e471e60e8d 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -34,9 +34,9 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -45,10 +45,10 @@ jobs: key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -56,14 +56,14 @@ jobs: max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -77,7 +77,7 @@ jobs: --with-pydebug --enable-slower-safety --enable-safety - --with-openssl=$OPENSSL_DIR + --with-openssl="$OPENSSL_DIR" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree if: ${{ inputs.free-threading }} @@ -95,14 +95,14 @@ jobs: run: >- python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt - --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + --warning-ignore-file-path "${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu" --compiler-output-type=gcc --fail-on-regression --fail-on-improvement --path-prefix="../cpython-ro-srcdir/" - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} run: xvfb-run make test diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 4c8137c958a312..abc617a317cc0f 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -43,7 +43,7 @@ jobs: save: ${{ github.event_name == 'push' }} max-size: "200M" - name: "Add ccache to PATH" - run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" uses: actions/setup-python@v5 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 891934bc70a64f..ec769d7ff70314 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 + rev: v0.7.1 hooks: - id: ruff name: Run Ruff (lint) on Doc/ @@ -24,7 +24,7 @@ repos: files: ^Doc/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black name: Run Black on Tools/build/check_warnings.py @@ -37,7 +37,7 @@ repos: language_version: python3.12 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -50,6 +50,21 @@ repos: - id: trailing-whitespace types_or: [c, inc, python, rst] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.29.4 + hooks: + - id: check-dependabot + - id: check-github-workflows + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.3 + hooks: + - id: actionlint + args: [ + -ignore=1st argument of function call is not assignable, + -ignore=SC2(015|038|086|091|097|098|129|155), + ] + - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 hooks: @@ -57,6 +72,22 @@ repos: args: [--enable=default-role] files: ^Doc/|^Misc/NEWS.d/ + - repo: local + hooks: + - id: blurb-no-space-c-api + name: Check C API news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/C_API/ + files: Misc/NEWS.d/next/C API/20.*.rst + + - repo: local + hooks: + - id: blurb-no-space-core-and-builtins + name: Check Core and Builtins news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/Core_and_Builtins/ + files: Misc/NEWS.d/next/Core and Builtins/20.*.rst + - repo: meta hooks: - id: check-hooks-apply diff --git a/Doc/library/time.rst b/Doc/library/time.rst index a0bf13fc0a3577..8e29e57d00f9b2 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -483,6 +483,9 @@ Functions | | | | | | | | +-----------+------------------------------------------------+-------+ + | ``%u`` | Day of the week (Monday is 1; Sunday is 7) | | + | | as a decimal number [1, 7]. | | + +-----------+------------------------------------------------+-------+ | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | | | | | +-----------+------------------------------------------------+-------+ @@ -515,6 +518,16 @@ Functions | ``%Z`` | Time zone name (no characters if no time zone | | | | exists). Deprecated. [1]_ | | +-----------+------------------------------------------------+-------+ + | ``%G`` | ISO 8601 year (similar to ``%Y`` but follows | | + | | the rules for the ISO 8601 calendar year). | | + | | The year starts with the week that contains | | + | | the first Thursday of the calendar year. | | + +-----------+------------------------------------------------+-------+ + | ``%V`` | ISO 8601 week number (as a decimal number | | + | | [01,53]). The first week of the year is the | | + | | one that contains the first Thursday of the | | + | | year. Weeks start on Monday. | | + +-----------+------------------------------------------------+-------+ | ``%%`` | A literal ``'%'`` character. | | +-----------+------------------------------------------------+-------+ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-10-18-10-11-43.gh-issue-125593.Q97m3A.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-10-11-43.gh-issue-125593.Q97m3A.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2024-10-18-10-11-43.gh-issue-125593.Q97m3A.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-10-11-43.gh-issue-125593.Q97m3A.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d71b55cbe1c68d..f40ad1e5744fd1 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -331,6 +331,24 @@ dummy_func(void) { } } + op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { + _Py_UopsSymbol *res; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } + // _STORE_FAST: + GETLOCAL(this_instr->operand) = res; + } + op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { (void)container; (void)sub; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 6ec9e69d1dbc44..243b3efa41b2d0 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -484,6 +484,25 @@ } case _BINARY_OP_INPLACE_ADD_UNICODE: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + _Py_UopsSymbol *res; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } + // _STORE_FAST: + GETLOCAL(this_instr->operand) = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py new file mode 100644 index 00000000000000..767aeae9349070 --- /dev/null +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -0,0 +1,324 @@ +# This script runs a set of small benchmarks to help identify scaling +# bottlenecks in the free-threaded interpreter. The benchmarks consist +# of patterns that ought to scale well, but haven't in the past. This is +# typically due to reference count contention or lock contention. +# +# This is not intended to be a general multithreading benchmark suite, nor +# are the benchmarks intended to be representative of real-world workloads. +# +# On Linux, to avoid confounding hardware effects, the script attempts to: +# * Use a single CPU socket (to avoid NUMA effects) +# * Use distinct physical cores (to avoid hyperthreading/SMT effects) +# * Use "performance" cores (Intel, ARM) on CPUs that have performance and +# efficiency cores +# +# It also helps to disable dynamic frequency scaling (i.e., "Turbo Boost") +# +# Intel: +# > echo "1" | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo +# +# AMD: +# > echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost +# + +import math +import os +import queue +import sys +import threading +import time + +# The iterations in individual benchmarks are scaled by this factor. +WORK_SCALE = 100 + +ALL_BENCHMARKS = {} + +threads = [] +in_queues = [] +out_queues = [] + + +def register_benchmark(func): + ALL_BENCHMARKS[func.__name__] = func + return func + +@register_benchmark +def object_cfunction(): + accu = 0 + tab = [1] * 100 + for i in range(1000 * WORK_SCALE): + tab.pop(0) + tab.append(i) + accu += tab[50] + return accu + +@register_benchmark +def cmodule_function(): + for i in range(1000 * WORK_SCALE): + math.floor(i * i) + +@register_benchmark +def mult_constant(): + x = 1.0 + for i in range(3000 * WORK_SCALE): + x *= 1.01 + +def simple_gen(): + for i in range(10): + yield i + +@register_benchmark +def generator(): + accu = 0 + for i in range(100 * WORK_SCALE): + for v in simple_gen(): + accu += v + return accu + +class Counter: + def __init__(self): + self.i = 0 + + def next_number(self): + self.i += 1 + return self.i + +@register_benchmark +def pymethod(): + c = Counter() + for i in range(1000 * WORK_SCALE): + c.next_number() + return c.i + +def next_number(i): + return i + 1 + +@register_benchmark +def pyfunction(): + accu = 0 + for i in range(1000 * WORK_SCALE): + accu = next_number(i) + return accu + +def double(x): + return x + x + +module = sys.modules[__name__] + +@register_benchmark +def module_function(): + total = 0 + for i in range(1000 * WORK_SCALE): + total += module.double(i) + return total + +class MyObject: + pass + +@register_benchmark +def load_string_const(): + accu = 0 + for i in range(1000 * WORK_SCALE): + if i == 'a string': + accu += 7 + else: + accu += 1 + return accu + +@register_benchmark +def load_tuple_const(): + accu = 0 + for i in range(1000 * WORK_SCALE): + if i == (1, 2): + accu += 7 + else: + accu += 1 + return accu + +@register_benchmark +def create_pyobject(): + for i in range(1000 * WORK_SCALE): + o = MyObject() + +@register_benchmark +def create_closure(): + for i in range(1000 * WORK_SCALE): + def foo(x): + return x + foo(i) + +@register_benchmark +def create_dict(): + for i in range(1000 * WORK_SCALE): + d = { + "key": "value", + } + +thread_local = threading.local() + +@register_benchmark +def thread_local_read(): + tmp = thread_local + tmp.x = 10 + for i in range(500 * WORK_SCALE): + _ = tmp.x + _ = tmp.x + _ = tmp.x + _ = tmp.x + _ = tmp.x + + +def bench_one_thread(func): + t0 = time.perf_counter_ns() + func() + t1 = time.perf_counter_ns() + return t1 - t0 + + +def bench_parallel(func): + t0 = time.perf_counter_ns() + for inq in in_queues: + inq.put(func) + for outq in out_queues: + outq.get() + t1 = time.perf_counter_ns() + return t1 - t0 + + +def benchmark(func): + delta_one_thread = bench_one_thread(func) + delta_many_threads = bench_parallel(func) + + speedup = delta_one_thread * len(threads) / delta_many_threads + if speedup >= 1: + factor = speedup + direction = "faster" + else: + factor = 1 / speedup + direction = "slower" + + use_color = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + color = reset_color = "" + if use_color: + if speedup <= 1.1: + color = "\x1b[31m" # red + elif speedup < len(threads)/2: + color = "\x1b[33m" # yellow + reset_color = "\x1b[0m" + + print(f"{color}{func.__name__:<18} {round(factor, 1):>4}x {direction}{reset_color}") + +def determine_num_threads_and_affinity(): + if sys.platform != "linux": + return [None] * os.cpu_count() + + # Try to use `lscpu -p` on Linux + import subprocess + try: + output = subprocess.check_output(["lscpu", "-p=cpu,node,core,MAXMHZ"], + text=True, env={"LC_NUMERIC": "C"}) + except (FileNotFoundError, subprocess.CalledProcessError): + return [None] * os.cpu_count() + + table = [] + for line in output.splitlines(): + if line.startswith("#"): + continue + cpu, node, core, maxhz = line.split(",") + if maxhz == "": + maxhz = "0" + table.append((int(cpu), int(node), int(core), float(maxhz))) + + cpus = [] + cores = set() + max_mhz_all = max(row[3] for row in table) + for cpu, node, core, maxmhz in table: + # Choose only CPUs on the same node, unique cores, and try to avoid + # "efficiency" cores. + if node == 0 and core not in cores and maxmhz == max_mhz_all: + cpus.append(cpu) + cores.add(core) + return cpus + + +def thread_run(cpu, in_queue, out_queue): + if cpu is not None and hasattr(os, "sched_setaffinity"): + # Set the affinity for the current thread + os.sched_setaffinity(0, (cpu,)) + + while True: + func = in_queue.get() + if func is None: + break + func() + out_queue.put(None) + + +def initialize_threads(opts): + if opts.threads == -1: + cpus = determine_num_threads_and_affinity() + else: + cpus = [None] * opts.threads # don't set affinity + + print(f"Running benchmarks with {len(cpus)} threads") + for cpu in cpus: + inq = queue.Queue() + outq = queue.Queue() + in_queues.append(inq) + out_queues.append(outq) + t = threading.Thread(target=thread_run, args=(cpu, inq, outq), daemon=True) + threads.append(t) + t.start() + + +def main(opts): + global WORK_SCALE + if not hasattr(sys, "_is_gil_enabled") or sys._is_gil_enabled(): + sys.stderr.write("expected to be run with the GIL disabled\n") + + benchmark_names = opts.benchmarks + if benchmark_names: + for name in benchmark_names: + if name not in ALL_BENCHMARKS: + sys.stderr.write(f"Unknown benchmark: {name}\n") + sys.exit(1) + else: + benchmark_names = ALL_BENCHMARKS.keys() + + WORK_SCALE = opts.scale + + if not opts.baseline_only: + initialize_threads(opts) + + do_bench = not opts.baseline_only and not opts.parallel_only + for name in benchmark_names: + func = ALL_BENCHMARKS[name] + if do_bench: + benchmark(func) + continue + + if opts.parallel_only: + delta_ns = bench_parallel(func) + else: + delta_ns = bench_one_thread(func) + + time_ms = delta_ns / 1_000_000 + print(f"{func.__name__:<18} {time_ms:.1f} ms") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--threads", type=int, default=-1, + help="number of threads to use") + parser.add_argument("--scale", type=int, default=100, + help="work scale factor for the benchmark (default=100)") + parser.add_argument("--baseline-only", default=False, action="store_true", + help="only run the baseline benchmarks (single thread)") + parser.add_argument("--parallel-only", default=False, action="store_true", + help="only run the parallel benchmark (many threads)") + parser.add_argument("benchmarks", nargs="*", + help="benchmarks to run") + options = parser.parse_args() + main(options)