Merge branch 'main' into pythongh-110109-pathbase-resolve-speedups

.github/workflows/build.yml

@@ -120,7 +120,9 @@ jobs:
 
   check_generated_files:
     name: 'Check if generated files are up to date'
-    runs-on: ubuntu-latest
+    # Don't use ubuntu-latest but a specific version to make the job
+    # reproducible: to get the same tools versions (autoconf, aclocal, ...)
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
@@ -143,15 +145,16 @@ jobs:
       - name: Check Autoconf and aclocal versions
         run: |
           grep "Generated by GNU Autoconf 2.71" configure
-          grep "aclocal 1.16.4" aclocal.m4
+          grep "aclocal 1.16.5" aclocal.m4
           grep -q "runstatedir" configure
           grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
       - name: Configure CPython
         run: |
           # Build Python with the libpython dynamic library
           ./configure --config-cache --with-pydebug --enable-shared
-      - name: Regenerate autoconf files with container image
-        run: make regen-configure
+      - name: Regenerate autoconf files
+        # Same command used by Tools/build/regen-configure.sh ($AUTORECONF)
+        run: autoreconf -ivf -Werror
       - name: Build CPython
         run: |
           make -j4 regen-all

.github/workflows/posix-deps-apt.sh

@@ -1,9 +1,11 @@
 #!/bin/sh
 apt-get update
 
+# autoconf-archive is needed by autoreconf (check_generated_files job)
 apt-get -yq install \
     build-essential \
     pkg-config \
+    autoconf-archive \
     ccache \
     gdb \
     lcov \

Doc/c-api/bool.rst

@@ -26,19 +26,19 @@ are available, however.
 .. c:var:: PyObject* Py_False
 
    The Python ``False`` object.  This object has no methods and is
-   `immortal <https://peps.python.org/pep-0683/>`_.
+   :term:`immortal`.
 
-.. versionchanged:: 3.12
-   :c:data:`Py_False` is immortal.
+   .. versionchanged:: 3.12
+      :c:data:`Py_False` is :term:`immortal`.
 
 
 .. c:var:: PyObject* Py_True
 
    The Python ``True`` object.  This object has no methods and is
-   `immortal <https://peps.python.org/pep-0683/>`_.
+   :term:`immortal`.
 
-.. versionchanged:: 3.12
-   :c:data:`Py_True` is immortal.
+   .. versionchanged:: 3.12
+      :c:data:`Py_True` is :term:`immortal`.
 
 
 .. c:macro:: Py_RETURN_FALSE

Doc/c-api/dict.rst

@@ -173,6 +173,33 @@ Dictionary Objects
 
    .. versionadded:: 3.4
 
+
+.. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result)
+
+   Remove *key* from dictionary *p* and optionally return the removed value.
+   Do not raise :exc:`KeyError` if the key missing.
+
+   - If the key is present, set *\*result* to a new reference to the removed
+     value if *result* is not ``NULL``, and return ``1``.
+   - If the key is missing, set *\*result* to ``NULL`` if *result* is not
+     ``NULL``, and return ``0``.
+   - On error, raise an exception and return ``-1``.
+
+   This is similar to :meth:`dict.pop`, but without the default value and
+   not raising :exc:`KeyError` if the key missing.
+
+   .. versionadded:: 3.13
+
+
+.. c:function:: int PyDict_PopString(PyObject *p, const char *key, PyObject **result)
+
+   Similar to :c:func:`PyDict_Pop`, but *key* is specified as a
+   :c:expr:`const char*` UTF-8 encoded bytes string, rather than a
+   :c:expr:`PyObject*`.
+
+   .. versionadded:: 3.13
+
+
 .. c:function:: PyObject* PyDict_Items(PyObject *p)
 
    Return a :c:type:`PyListObject` containing all the items from the dictionary.

Doc/c-api/hash.rst

@@ -0,0 +1,48 @@
+.. highlight:: c
+
+PyHash API
+----------
+
+See also the :c:member:`PyTypeObject.tp_hash` member.
+
+.. c:type:: Py_hash_t
+
+   Hash value type: signed integer.
+
+   .. versionadded:: 3.2
+
+.. c:type:: Py_uhash_t
+
+   Hash value type: unsigned integer.
+
+   .. versionadded:: 3.2
+
+
+.. c:type:: PyHash_FuncDef
+
+   Hash function definition used by :c:func:`PyHash_GetFuncDef`.
+
+   .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t)
+
+      Hash function.
+
+   .. c:member:: const char *name
+
+      Hash function name (UTF-8 encoded string).
+
+   .. c:member:: const int hash_bits
+
+      Internal size of the hash value in bits.
+
+   .. c:member:: const int seed_bits
+
+      Size of seed input in bits.
+
+   .. versionadded:: 3.4
+
+
+.. c:function:: PyHash_FuncDef* PyHash_GetFuncDef(void)
+
+   Get the hash function definition.
+
+   .. versionadded:: 3.4

Doc/c-api/init.rst

@@ -1485,7 +1485,7 @@ otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared
 because of the refcount.  One simple but less-efficient approach around
 this is to use a global lock around all use of some state (or object).
 Alternately, effectively immutable objects (like integers or strings)
-can be made safe in spite of their refcounts by making them "immortal".
+can be made safe in spite of their refcounts by making them :term:`immortal`.
 In fact, this has been done for the builtin singletons, small integers,
 and a number of other builtin objects.
 

Doc/c-api/init_config.rst

@@ -1170,7 +1170,7 @@ PyConfig
 
    .. c:member:: int show_ref_count
 
-      Show total reference count at exit (excluding immortal objects)?
+      Show total reference count at exit (excluding :term:`immortal` objects)?
 
       Set to ``1`` by :option:`-X showrefcount <-X>` command line option.
 

Doc/c-api/none.rst

@@ -16,10 +16,10 @@ same reason.
 .. c:var:: PyObject* Py_None
 
    The Python ``None`` object, denoting lack of value.  This object has no methods
-   and is `immortal <https://peps.python.org/pep-0683/>`_.
+   and is :term:`immortal`.
 
-.. versionchanged:: 3.12
-   :c:data:`Py_None` is immortal.
+   .. versionchanged:: 3.12
+      :c:data:`Py_None` is :term:`immortal`.
 
 .. c:macro:: Py_RETURN_NONE
 

Doc/c-api/refcounting.rst

@@ -17,7 +17,7 @@ of Python objects.
 
    Note that the returned value may not actually reflect how many
    references to the object are actually held.  For example, some
-   objects are "immortal" and have a very high refcount that does not
+   objects are :term:`immortal` and have a very high refcount that does not
    reflect the actual number of references.  Consequently, do not rely
    on the returned value to be accurate, other than a value of 0 or 1.
 
@@ -34,9 +34,7 @@ of Python objects.
 
    Set the object *o* reference counter to *refcnt*.
 
-   Note that this function has no effect on
-   `immortal <https://peps.python.org/pep-0683/>`_
-   objects.
+   This function has no effect on :term:`immortal` objects.
 
    .. versionadded:: 3.9
 
@@ -49,6 +47,8 @@ of Python objects.
    Indicate taking a new :term:`strong reference` to object *o*,
    indicating it is in use and should not be destroyed.
 
+   This function has no effect on :term:`immortal` objects.
+
    This function is usually used to convert a :term:`borrowed reference` to a
    :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be
    used to create a new :term:`strong reference`.
@@ -113,6 +113,8 @@ of Python objects.
    Release a :term:`strong reference` to object *o*, indicating the
    reference is no longer used.
 
+   This function has no effect on :term:`immortal` objects.
+
    Once the last :term:`strong reference` is released
    (i.e. the object's reference count reaches 0),
    the object's type's deallocation

Doc/c-api/slice.rst

@@ -119,8 +119,7 @@ Ellipsis Object
 .. c:var:: PyObject *Py_Ellipsis
 
    The Python ``Ellipsis`` object.  This object has no methods.  Like
-   :c:data:`Py_None`, it is an `immortal <https://peps.python.org/pep-0683/>`_.
-   singleton object.
+   :c:data:`Py_None`, it is an :term:`immortal` singleton object.
 
    .. versionchanged:: 3.12
       :c:data:`Py_Ellipsis` is immortal.

Doc/c-api/utilities.rst

@@ -17,6 +17,7 @@ and parsing function arguments and constructing Python values from C values.
    marshal.rst
    arg.rst
    conversion.rst
+   hash.rst
    reflection.rst
    codec.rst
    perfmaps.rst

Doc/conf.py

@@ -288,9 +288,8 @@
     "pr_id": os.getenv("READTHEDOCS_VERSION")
 }
 
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-html_last_updated_fmt = '%b %d, %Y'
+# This 'Last updated on:' timestamp is inserted at the bottom of every page.
+html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime())
 
 # Path to find HTML templates.
 templates_path = ['tools/templates']

Doc/glossary.rst

@@ -579,6 +579,16 @@ Glossary
       :ref:`idle` is a basic editor and interpreter environment
       which ships with the standard distribution of Python.
 
+   immortal
+      If an object is immortal, its reference count is never modified, and
+      therefore it is never deallocated.
+
+      Built-in strings and singletons are immortal objects. For example,
+      :const:`True` and :const:`None` singletons are immmortal.
+
+      See `PEP 683 – Immortal Objects, Using a Fixed Refcount
+      <https://peps.python.org/pep-0683/>`_ for more information.
+
    immutable
       An object with a fixed value.  Immutable objects include numbers, strings and
       tuples.  Such an object cannot be altered.  A new object has to
@@ -1056,7 +1066,7 @@ Glossary
    reference count
       The number of references to an object.  When the reference count of an
       object drops to zero, it is deallocated.  Some objects are
-      "immortal" and have reference counts that are never modified, and
+      :term:`immortal` and have reference counts that are never modified, and
       therefore the objects are never deallocated.  Reference counting is
       generally not visible to Python code, but it is a key element of the
       :term:`CPython` implementation.  Programmers can call the

Doc/howto/isolating-extensions.rst

@@ -339,12 +339,44 @@ That is, heap types should:
 - Define a traverse function using ``Py_tp_traverse``, which
   visits the type (e.g. using :c:expr:`Py_VISIT(Py_TYPE(self))`).
 
-Please refer to the :ref:`the documentation <type-structs>` of
+Please refer to the the documentation of
 :c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse`
 for additional considerations.
 
-If your traverse function delegates to the ``tp_traverse`` of its base class
-(or another type), ensure that ``Py_TYPE(self)`` is visited only once.
+The API for defining heap types grew organically, leaving it
+somewhat awkward to use in its current state.
+The following sections will guide you through common issues.
+
+
+``tp_traverse`` in Python 3.8 and lower
+.......................................
+
+The requirement to visit the type from ``tp_traverse`` was added in Python 3.9.
+If you support Python 3.8 and lower, the traverse function must *not*
+visit the type, so it must be more complicated::
+
+   static int my_traverse(PyObject *self, visitproc visit, void *arg)
+   {
+       if (Py_Version >= 0x03090000) {
+           Py_VISIT(Py_TYPE(self));
+       }
+       return 0;
+   }
+
+Unfortunately, :c:data:`Py_Version` was only added in Python 3.11.
+As a replacement, use:
+
+* :c:macro:`PY_VERSION_HEX`, if not using the stable ABI, or
+* :py:data:`sys.version_info` (via :c:func:`PySys_GetObject` and
+  :c:func:`PyArg_ParseTuple`).
+
+
+Delegating ``tp_traverse``
+..........................
+
+If your traverse function delegates to the :c:member:`~PyTypeObject.tp_traverse`
+of its base class (or another type), ensure that ``Py_TYPE(self)`` is visited
+only once.
 Note that only heap type are expected to visit the type in ``tp_traverse``.
 
 For example, if your traverse function includes::
@@ -356,11 +388,70 @@ For example, if your traverse function includes::
     if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) {
         // a heap type's tp_traverse already visited Py_TYPE(self)
     } else {
-        Py_VISIT(Py_TYPE(self));
+        if (Py_Version >= 0x03090000) {
+            Py_VISIT(Py_TYPE(self));
+        }
     }
 
-It is not necessary to handle the type's reference count in ``tp_new``
-and ``tp_clear``.
+It is not necessary to handle the type's reference count in
+:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_clear`.
+
+
+Defining ``tp_dealloc``
+.......................
+
+If your type has a custom :c:member:`~PyTypeObject.tp_dealloc` function,
+it needs to:
+
+- call :c:func:`PyObject_GC_UnTrack` before any fields are invalidated, and
+- decrement the reference count of the type.
+
+To keep the type valid while ``tp_free`` is called, the type's refcount needs
+to be decremented *after* the instance is deallocated. For example::
+
+   static void my_dealloc(PyObject *self)
+   {
+       PyObject_GC_UnTrack(self);
+       ...
+       PyTypeObject *type = Py_TYPE(self);
+       type->tp_free(self);
+       Py_DECREF(type);
+   }
+
+The default ``tp_dealloc`` function does this, so
+if your type does *not* override
+``tp_dealloc`` you don't need to add it.
+
+
+Not overriding ``tp_free``
+..........................
+
+The :c:member:`~PyTypeObject.tp_free` slot of a heap type must be set to
+:c:func:`PyObject_GC_Del`.
+This is the default; do not override it.
+
+
+Avoiding ``PyObject_New``
+.........................
+
+GC-tracked objects need to be allocated using GC-aware functions.
+
+If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`:
+
+- Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible.
+  That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with::
+
+      TYPE *o = typeobj->tp_alloc(typeobj, 0);
+
+  Replace ``o = PyObject_NewVar(TYPE, typeobj, size)`` with the same,
+  but use size instead of the 0.
+
+- If the above is not possible (e.g. inside a custom ``tp_alloc``),
+  call :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`::
+
+      TYPE *o = PyObject_GC_New(TYPE, typeobj);
+
+      TYPE *o = PyObject_GC_NewVar(TYPE, typeobj, size);
 
 
 Module State Access from Classes