Merge branch 'main' into decimal-format-N

.github/workflows/jit.yml

@@ -0,0 +1,112 @@
+name: JIT
+on:
+  pull_request:
+    paths: '**jit**'
+  push:
+    paths: '**jit**'
+  workflow_dispatch:
+jobs:
+  jit:
+    name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - i686-pc-windows-msvc/msvc
+          - x86_64-pc-windows-msvc/msvc
+          - x86_64-apple-darwin/clang
+          - x86_64-unknown-linux-gnu/gcc
+          - x86_64-unknown-linux-gnu/clang
+          - aarch64-unknown-linux-gnu/gcc
+          - aarch64-unknown-linux-gnu/clang
+        debug:
+          - true
+          - false
+        llvm:
+          - 16
+        include:
+          - target: i686-pc-windows-msvc/msvc
+            architecture: Win32
+            runner: windows-latest
+            compiler: msvc
+          - target: x86_64-pc-windows-msvc/msvc
+            architecture: x64
+            runner: windows-latest
+            compiler: msvc
+          - target: x86_64-apple-darwin/clang
+            architecture: x86_64
+            runner: macos-latest
+            compiler: clang
+            exclude: test_embed
+          - target: x86_64-unknown-linux-gnu/gcc
+            architecture: x86_64
+            runner: ubuntu-latest
+            compiler: gcc
+          - target: x86_64-unknown-linux-gnu/clang
+            architecture: x86_64
+            runner: ubuntu-latest
+            compiler: clang
+          - target: aarch64-unknown-linux-gnu/gcc
+            architecture: aarch64
+            runner: ubuntu-latest
+            compiler: gcc
+            # These fail because of emulation, not because of the JIT:
+            exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
+          - target: aarch64-unknown-linux-gnu/clang
+            architecture: aarch64
+            runner: ubuntu-latest
+            compiler: clang
+            # These fail because of emulation, not because of the JIT:
+            exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
+    env:
+      CC: ${{ matrix.compiler }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Windows
+        if: runner.os == 'Windows'
+        run: |
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
+          ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+
+      - name: macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew install llvm@${{ matrix.llvm }}
+          export SDKROOT="$(xcrun --show-sdk-path)"
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          make all --jobs 3
+          ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+
+      - name: Native Linux
+        if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          make all --jobs 4
+          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+      - name: Emulated Linux
+        if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          ./configure --prefix="$(pwd)/../build"
+          make install --jobs 4
+          make clean --jobs 4
+          export HOST=${{ matrix.architecture }}-linux-gnu
+          sudo apt install --yes "gcc-$HOST" qemu-user
+          ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }}
+          ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }}
+          export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}"
+          export CPP="$CC --preprocess"
+          export HOSTRUNNER=qemu-${{ matrix.architecture }}
+          export QEMU_LD_PREFIX="/usr/$HOST"
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+          make all --jobs 4
+          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3

.github/workflows/mypy.yml

@@ -12,6 +12,7 @@ on:
       - "Tools/build/generate_sbom.py"
       - "Tools/cases_generator/**"
       - "Tools/clinic/**"
+      - "Tools/jit/**"
       - "Tools/peg_generator/**"
       - "Tools/requirements-dev.txt"
       - "Tools/wasm/**"
@@ -38,6 +39,7 @@ jobs:
           "Tools/build/",
           "Tools/cases_generator",
           "Tools/clinic",
+          "Tools/jit",
           "Tools/peg_generator",
           "Tools/wasm",
         ]

.gitignore

@@ -126,6 +126,7 @@ Tools/unicode/data/
 # hendrikmuhs/ccache-action@v1
 /.ccache
 /cross-build/
+/jit_stencils.h
 /platform
 /profile-clean-stamp
 /profile-run-stamp

Doc/c-api/memoryview.rst

@@ -20,6 +20,17 @@ any other object.
    read/write, otherwise it may be either read-only or read/write at the
    discretion of the exporter.
 
+
+.. c:macro:: PyBUF_READ
+
+   Flag to request a readonly buffer.
+
+
+.. c:macro:: PyBUF_WRITE
+
+   Flag to request a writable buffer.
+
+
 .. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags)
 
    Create a memoryview object using *mem* as the underlying buffer.
@@ -41,6 +52,8 @@ any other object.
    original memory. Otherwise, a copy is made and the memoryview points to a
    new bytes object.
 
+   *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`.
+
 
 .. c:function:: int PyMemoryView_Check(PyObject *obj)
 

Doc/library/__future__.rst

@@ -8,20 +8,68 @@
 
 --------------
 
-:mod:`__future__` is a real module, and serves three purposes:
+Imports of the form ``from __future__ import feature`` are called
+:ref:`future statements <future>`. These are special-cased by the Python compiler
+to allow the use of new Python features in modules containing the future statement
+before the release in which the feature becomes standard.
+
+While these future statements are given additional special meaning by the
+Python compiler, they are still executed like any other import statement and
+the :mod:`__future__` exists and is handled by the import system the same way
+any other Python module would be. This design serves three purposes:
 
 * To avoid confusing existing tools that analyze import statements and expect to
   find the modules they're importing.
 
-* To ensure that :ref:`future statements <future>` run under releases prior to
-  2.1 at least yield runtime exceptions (the import of :mod:`__future__` will
-  fail, because there was no module of that name prior to 2.1).
-
 * To document when incompatible changes were introduced, and when they will be
   --- or were --- made mandatory.  This is a form of executable documentation, and
   can be inspected programmatically via importing :mod:`__future__` and examining
   its contents.
 
+* To ensure that :ref:`future statements <future>` run under releases prior to
+  Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__`
+  will fail, because there was no module of that name prior to 2.1).
+
+Module Contents
+---------------
+
+No feature description will ever be deleted from :mod:`__future__`. Since its
+introduction in Python 2.1 the following features have found their way into the
+language using this mechanism:
+
++------------------+-------------+--------------+---------------------------------------------+
+| feature          | optional in | mandatory in | effect                                      |
++==================+=============+==============+=============================================+
+| nested_scopes    | 2.1.0b1     | 2.2          | :pep:`227`:                                 |
+|                  |             |              | *Statically Nested Scopes*                  |
++------------------+-------------+--------------+---------------------------------------------+
+| generators       | 2.2.0a1     | 2.3          | :pep:`255`:                                 |
+|                  |             |              | *Simple Generators*                         |
++------------------+-------------+--------------+---------------------------------------------+
+| division         | 2.2.0a2     | 3.0          | :pep:`238`:                                 |
+|                  |             |              | *Changing the Division Operator*            |
++------------------+-------------+--------------+---------------------------------------------+
+| absolute_import  | 2.5.0a1     | 3.0          | :pep:`328`:                                 |
+|                  |             |              | *Imports: Multi-Line and Absolute/Relative* |
++------------------+-------------+--------------+---------------------------------------------+
+| with_statement   | 2.5.0a1     | 2.6          | :pep:`343`:                                 |
+|                  |             |              | *The "with" Statement*                      |
++------------------+-------------+--------------+---------------------------------------------+
+| print_function   | 2.6.0a2     | 3.0          | :pep:`3105`:                                |
+|                  |             |              | *Make print a function*                     |
++------------------+-------------+--------------+---------------------------------------------+
+| unicode_literals | 2.6.0a2     | 3.0          | :pep:`3112`:                                |
+|                  |             |              | *Bytes literals in Python 3000*             |
++------------------+-------------+--------------+---------------------------------------------+
+| generator_stop   | 3.5.0b1     | 3.7          | :pep:`479`:                                 |
+|                  |             |              | *StopIteration handling inside generators*  |
++------------------+-------------+--------------+---------------------------------------------+
+| annotations      | 3.7.0b1     | TBD [1]_     | :pep:`563`:                                 |
+|                  |             |              | *Postponed evaluation of annotations*       |
++------------------+-------------+--------------+---------------------------------------------+
+
+.. XXX Adding a new entry?  Remember to update simple_stmts.rst, too.
+
 .. _future-classes:
 
 .. class:: _Feature
@@ -65,43 +113,6 @@
    dynamically compiled code.  This flag is stored in the :attr:`_Feature.compiler_flag`
    attribute on :class:`_Feature` instances.
 
-No feature description will ever be deleted from :mod:`__future__`. Since its
-introduction in Python 2.1 the following features have found their way into the
-language using this mechanism:
-
-+------------------+-------------+--------------+---------------------------------------------+
-| feature          | optional in | mandatory in | effect                                      |
-+==================+=============+==============+=============================================+
-| nested_scopes    | 2.1.0b1     | 2.2          | :pep:`227`:                                 |
-|                  |             |              | *Statically Nested Scopes*                  |
-+------------------+-------------+--------------+---------------------------------------------+
-| generators       | 2.2.0a1     | 2.3          | :pep:`255`:                                 |
-|                  |             |              | *Simple Generators*                         |
-+------------------+-------------+--------------+---------------------------------------------+
-| division         | 2.2.0a2     | 3.0          | :pep:`238`:                                 |
-|                  |             |              | *Changing the Division Operator*            |
-+------------------+-------------+--------------+---------------------------------------------+
-| absolute_import  | 2.5.0a1     | 3.0          | :pep:`328`:                                 |
-|                  |             |              | *Imports: Multi-Line and Absolute/Relative* |
-+------------------+-------------+--------------+---------------------------------------------+
-| with_statement   | 2.5.0a1     | 2.6          | :pep:`343`:                                 |
-|                  |             |              | *The "with" Statement*                      |
-+------------------+-------------+--------------+---------------------------------------------+
-| print_function   | 2.6.0a2     | 3.0          | :pep:`3105`:                                |
-|                  |             |              | *Make print a function*                     |
-+------------------+-------------+--------------+---------------------------------------------+
-| unicode_literals | 2.6.0a2     | 3.0          | :pep:`3112`:                                |
-|                  |             |              | *Bytes literals in Python 3000*             |
-+------------------+-------------+--------------+---------------------------------------------+
-| generator_stop   | 3.5.0b1     | 3.7          | :pep:`479`:                                 |
-|                  |             |              | *StopIteration handling inside generators*  |
-+------------------+-------------+--------------+---------------------------------------------+
-| annotations      | 3.7.0b1     | TBD [1]_     | :pep:`563`:                                 |
-|                  |             |              | *Postponed evaluation of annotations*       |
-+------------------+-------------+--------------+---------------------------------------------+
-
-.. XXX Adding a new entry?  Remember to update simple_stmts.rst, too.
-
 .. [1]
    ``from __future__ import annotations`` was previously scheduled to
    become mandatory in Python 3.10, but the Python Steering Council
@@ -115,3 +126,6 @@ language using this mechanism:
 
    :ref:`future`
       How the compiler treats future imports.
+
+   :pep:`236` - Back to the __future__
+      The original proposal for the __future__ mechanism.

Doc/library/dataclasses.rst

@@ -538,8 +538,8 @@ that has to be called, it is common to call this method in a
 
     class Rectangle:
         def __init__(self, height, width):
-          self.height = height
-          self.width = width
+            self.height = height
+            self.width = width
 
     @dataclass
     class Square(Rectangle):

Doc/library/tarfile.rst

@@ -673,6 +673,7 @@ be finalized; only the internally used file object will be closed. See the
 
 
 .. attribute:: TarFile.pax_headers
+   :type: dict
 
    A dictionary containing key-value pairs of pax global headers.
 
@@ -838,26 +839,31 @@ A ``TarInfo`` object has the following public data attributes:
       attribute.
 
 .. attribute:: TarInfo.chksum
+   :type: int
 
    Header checksum.
 
 
 .. attribute:: TarInfo.devmajor
+   :type: int
 
    Device major number.
 
 
 .. attribute:: TarInfo.devminor
+   :type: int
 
    Device minor number.
 
 
 .. attribute:: TarInfo.offset
+   :type: int
 
    The tar header starts here.
 
 
 .. attribute:: TarInfo.offset_data
+   :type: int
 
    The file's data starts here.
 

Doc/tools/.nitignore

@@ -9,7 +9,6 @@ Doc/c-api/gcsupport.rst
 Doc/c-api/init.rst
 Doc/c-api/init_config.rst
 Doc/c-api/intro.rst
-Doc/c-api/memoryview.rst
 Doc/c-api/module.rst
 Doc/c-api/stable.rst
 Doc/c-api/sys.rst

Doc/whatsnew/3.11.rst

@@ -2701,4 +2701,30 @@ Removed
   (Contributed by Inada Naoki in :issue:`44029`.)
 
 
+Notable changes in 3.11.4
+=========================
+
+tarfile
+-------
+
+* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`,
+  have a new a *filter* argument that allows limiting tar features than may be
+  surprising or dangerous, such as creating files outside the destination
+  directory.
+  See :ref:`tarfile-extraction-filter` for details.
+  In Python 3.12, use without the *filter* argument will show a
+  :exc:`DeprecationWarning`.
+  In Python 3.14, the default will switch to ``'data'``.
+  (Contributed by Petr Viktorin in :pep:`706`.)
+
+
+Notable changes in 3.11.5
+=========================
+
+OpenSSL
+-------
+
+* Windows builds and macOS installers from python.org now use OpenSSL 3.0.
+
+
 .. _libb2: https://www.blake2.net/

Include/cpython/optimizer.h

@@ -39,6 +39,8 @@ typedef struct {
 typedef struct _PyExecutorObject {
     PyObject_VAR_HEAD
     _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */
+    void *jit_code;
+    size_t jit_size;
     _PyUOpInstruction trace[1];
 } _PyExecutorObject;