Merge remote-tracking branch 'upstream/main' into tail-call

Fidget-Spinner · Jan 20, 2025 · 8f1f0b9 · 8f1f0b9
2 parents aa3d408 + c463270
commit 8f1f0b9
Show file tree

Hide file tree

Showing 92 changed files with 1,993 additions and 745 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -231,10 +231,14 @@ jobs:
     name: >-
       Ubuntu
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
+      ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }}
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     strategy:
       matrix:
+        bolt:
+        - false
+        - true
         free-threading:
         - false
         - true
@@ -246,9 +250,16 @@ jobs:
         exclude:
         - os: ubuntu-24.04-aarch64
           is-fork: true
+        # Do not test BOLT with free-threading, to conserve resources
+        - bolt: true
+          free-threading: true
+        # BOLT currently crashes during instrumentation on aarch64
+        - os: ubuntu-24.04-aarch64
+          bolt: true
     uses: ./.github/workflows/reusable-ubuntu.yml
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
+      bolt-optimizations: ${{ matrix.bolt }}
       free-threading: ${{ matrix.free-threading }}
       os: ${{ matrix.os }}
 

diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
@@ -65,8 +65,8 @@ jobs:
       continue-on-error: true
       run: |
         set -Eeuo pipefail
-        # Build docs with the '-n' (nit-picky) option; write warnings to file
-        make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html
+        # Build docs with the nit-picky option; write warnings to file
+        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --keep-going --warning-file sphinx-warnings.txt" html
     - name: 'Check warnings'
       if: github.event_name == 'pull_request'
       run: |
@@ -76,26 +76,6 @@ jobs:
           --fail-if-improved \
           --fail-if-new-news-nit
 
-  # This build doesn't use problem matchers or check annotations
-  build_doc_oldest_supported_sphinx:
-    name: 'Docs (Oldest Sphinx)'
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        persist-credentials: false
-    - name: 'Set up Python'
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.13'  # known to work with Sphinx 7.2.6
-        cache: 'pip'
-        cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
-    - name: 'Install build dependencies'
-      run: make -C Doc/ venv REQUIREMENTS="requirements-oldest-sphinx.txt"
-    - name: 'Build HTML documentation'
-      run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html
-
   # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release
   doctest:
     name: 'Doctest'
@@ -121,4 +101,4 @@ jobs:
       run: make -C Doc/ PYTHON=../python venv
     # Use "xvfb-run" since some doctest tests open GUI windows
     - name: 'Run documentation doctest'
-      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest
+      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning --keep-going" doctest
diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml
@@ -6,6 +6,11 @@ on:
       config_hash:
         required: true
         type: string
+      bolt-optimizations:
+        description: Whether to enable BOLT optimizations
+        required: false
+        type: boolean
+        default: false
       free-threading:
         description: Whether to use free-threaded mode
         required: false
@@ -34,6 +39,12 @@ jobs:
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Install Clang and BOLT
+      if: ${{ fromJSON(inputs.bolt-optimizations) }}
+      run: |
+        sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19
+        sudo apt-get install bolt-19
+        echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV
     - name: Configure OpenSSL env vars
       run: |
         echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
@@ -73,14 +84,18 @@ jobs:
         key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      # `test_unpickle_module_race` writes to the source directory, which is
+      # read-only during builds — so we exclude it from profiling with BOLT.
       run: >-
+        PROFILE_TASK='-m test --pgo --ignore test_unpickle_module_race'
         ../cpython-ro-srcdir/configure
         --config-cache
         --with-pydebug
         --enable-slower-safety
         --enable-safety
         --with-openssl="$OPENSSL_DIR"
         ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
+        ${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }}
     - name: Build CPython out-of-tree
       if: ${{ inputs.free-threading }}
       working-directory: ${{ env.CPYTHON_BUILDDIR }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.2
+    rev: v0.9.1
     hooks:
       - id: ruff
         name: Run Ruff (lint) on Doc/
@@ -29,12 +29,10 @@ repos:
       - id: black
         name: Run Black on Tools/build/check_warnings.py
         files: ^Tools/build/check_warnings.py
-        language_version: python3.12
         args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
         files: ^Tools/jit/
-        language_version: python3.12
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
@@ -51,19 +49,19 @@ repos:
         types_or: [c, inc, python, rst]
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.30.0
+    rev: 0.31.0
     hooks:
       - id: check-dependabot
       - id: check-github-workflows
       - id: check-readthedocs
 
   - repo: https://github.com/rhysd/actionlint
-    rev: v1.7.4
+    rev: v1.7.6
     hooks:
       - id: actionlint
 
   - repo: https://github.com/woodruffw/zizmor-pre-commit
-    rev: v0.8.0
+    rev: v1.1.1
     hooks:
       - id: zizmor
 

diff --git a/Doc/Makefile b/Doc/Makefile
@@ -14,15 +14,15 @@ PAPER        =
 SOURCES      =
 DISTVERSION  = $(shell $(PYTHON) tools/extensions/patchlevel.py)
 REQUIREMENTS = requirements.txt
-SPHINXERRORHANDLING = -W
+SPHINXERRORHANDLING = --fail-on-warning
 
 # Internal variables.
-PAPEROPT_a4     = -D latex_elements.papersize=a4paper
-PAPEROPT_letter = -D latex_elements.papersize=letterpaper
+PAPEROPT_a4     = --define latex_elements.papersize=a4paper
+PAPEROPT_letter = --define latex_elements.papersize=letterpaper
 
-ALLSPHINXOPTS = -b $(BUILDER) \
-                -d build/doctrees \
-                -j $(JOBS) \
+ALLSPHINXOPTS = --builder $(BUILDER) \
+                --doctree-dir build/doctrees \
+                --jobs $(JOBS) \
                 $(PAPEROPT_$(PAPER)) \
                 $(SPHINXOPTS) $(SPHINXERRORHANDLING) \
                 . build/$(BUILDER) $(SOURCES)
@@ -144,7 +144,7 @@ pydoc-topics: build
 
 .PHONY: gettext
 gettext: BUILDER = gettext
-gettext: override SPHINXOPTS := -d build/doctrees-gettext $(SPHINXOPTS)
+gettext: override SPHINXOPTS := --doctree-dir build/doctrees-gettext $(SPHINXOPTS)
 gettext: build
 
 .PHONY: htmlview
@@ -300,20 +300,20 @@ serve:
 # By default, Sphinx only rebuilds pages where the page content has changed.
 # This means it doesn't always pick up changes to preferred link targets, etc
 # To ensure such changes are picked up, we build the published docs with
-# `-E` (to ignore the cached environment) and `-a` (to ignore already existing
-# output files)
+# ``--fresh-env`` (to ignore the cached environment) and ``--write-all``
+# (to ignore already existing output files)
 
 # for development releases: always build
 .PHONY: autobuild-dev
 autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
 autobuild-dev:
-	$(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
+	$(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION)
 
 # for HTML-only rebuilds
 .PHONY: autobuild-dev-html
 autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
 autobuild-dev-html:
-	$(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
+	$(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION)
 
 # for stable releases: only build if not in pre-release stage (alpha, beta)
 # release candidate downloads are okay, since the stable tree can be in that stage

diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
@@ -109,7 +109,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    Set by the :option:`-b` option.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_DebugFlag
 
@@ -123,7 +123,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment
    variable.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_DontWriteBytecodeFlag
 
@@ -137,7 +137,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE`
    environment variable.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_FrozenFlag
 
@@ -150,7 +150,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    Private flag used by ``_freeze_module`` and ``frozenmain`` programs.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_HashRandomizationFlag
 
@@ -165,7 +165,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment
    variable to initialize the secret hash seed.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_IgnoreEnvironmentFlag
 
@@ -178,7 +178,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    Set by the :option:`-E` and :option:`-I` options.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_InspectFlag
 
@@ -193,7 +193,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment
    variable.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_InteractiveFlag
 
@@ -218,7 +218,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    .. versionadded:: 3.4
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_LegacyWindowsFSEncodingFlag
 
@@ -237,7 +237,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    .. availability:: Windows.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_LegacyWindowsStdioFlag
 
@@ -255,7 +255,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    .. availability:: Windows.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_NoSiteFlag
 
@@ -270,7 +270,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    Set by the :option:`-S` option.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_NoUserSiteDirectory
 
@@ -284,7 +284,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Set by the :option:`-s` and :option:`-I` options, and the
    :envvar:`PYTHONNOUSERSITE` environment variable.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_OptimizeFlag
 
@@ -295,7 +295,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment
    variable.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_QuietFlag
 
@@ -309,7 +309,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
    .. versionadded:: 3.2
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_UnbufferedStdioFlag
 
@@ -322,7 +322,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED`
    environment variable.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 .. c:var:: int Py_VerboseFlag
 
@@ -338,7 +338,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment
    variable.
 
-   .. deprecated-removed:: 3.12 3.14
+   .. deprecated-removed:: 3.12 3.15
 
 
 Initializing and finalizing the interpreter
@@ -606,7 +606,7 @@ Process-wide parameters
    Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
    :c:expr:`wchar_*` string.
 
-   .. deprecated:: 3.11
+   .. deprecated-removed:: 3.11 3.15
 
 
 .. c:function:: wchar_t* Py_GetProgramName()
@@ -868,7 +868,7 @@ Process-wide parameters
    .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params;
       check w/ Guido.
 
-   .. deprecated:: 3.11
+   .. deprecated-removed:: 3.11 3.15
 
 
 .. c:function:: void PySys_SetArgv(int argc, wchar_t **argv)
@@ -889,7 +889,7 @@ Process-wide parameters
 
    .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`.
 
-   .. deprecated:: 3.11
+   .. deprecated-removed:: 3.11 3.15
 
 
 .. c:function:: void Py_SetPythonHome(const wchar_t *home)
@@ -910,7 +910,7 @@ Process-wide parameters
    Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
    :c:expr:`wchar_*` string.
 
-   .. deprecated:: 3.11
+   .. deprecated-removed:: 3.11 3.15
 
 
 .. c:function:: wchar_t* Py_GetPythonHome()