diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0a18913bf6a09..ec3acf77563f7e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,9 @@ jobs: - name: Check limited ABI symbols run: make check-limited-abi - name: Check Autoconf version 2.69 - run: grep "Generated by GNU Autoconf 2.69" configure + run: | + grep "Generated by GNU Autoconf 2.69" configure + grep "PKG_PROG_PKG_CONFIG" aclocal.m4 build_win32: name: 'Windows (x86)' @@ -126,8 +128,12 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v2 + - name: Prepare homebrew environment variables + run: | + echo "LDFLAGS=-L$(brew --prefix tcl-tk)/lib" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=$(brew --prefix openssl@1.1)/lib/pkgconfig:$(brew --prefix tcl-tk)/lib/pkgconfig" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-dev + run: ./configure --with-pydebug --prefix=/opt/python-dev - name: Build CPython run: make -j4 - name: Display build info diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 31dc9c8031fdb6..739b5e97d15150 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -185,7 +185,7 @@ Object Calling API Various functions are available for calling a Python object. Each converts its arguments to a convention supported by the called object – either *tp_call* or vectorcall. -In order to do as litle conversion as possible, pick one that best fits +In order to do as little conversion as possible, pick one that best fits the format of data you have available. The following table summarizes the available functions; diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 77036726b0a923..09dfc68fee57dc 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1173,6 +1173,26 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.9 +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) Get the current interpreter. @@ -1623,6 +1643,8 @@ Python-level trace functions in previous versions. profile function is called for all monitored events except :const:`PyTrace_LINE` :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`. + See also the :func:`sys.setprofile` function. + The caller must hold the :term:`GIL`. @@ -1635,6 +1657,8 @@ Python-level trace functions in previous versions. will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or :const:`PyTrace_C_RETURN` as a value for the *what* parameter. + See also the :func:`sys.settrace` function. + The caller must hold the :term:`GIL`. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 2e52679ebc5b5d..989660caeda06e 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -22,7 +22,7 @@ There are two kinds of configuration: * The :ref:`Isolated Configuration ` can be used to embed Python into an application. It isolates Python from the system. For example, environments variables are ignored, the LC_CTYPE locale is left unchanged and - no signal handler is registred. + no signal handler is registered. The :c:func:`Py_RunMain` function can be used to write a customized Python program. @@ -706,7 +706,7 @@ PyConfig * Otherwise, use the :term:`locale encoding`: ``nl_langinfo(CODESET)`` result. - At Python statup, the encoding name is normalized to the Python codec + At Python startup, the encoding name is normalized to the Python codec name. For example, ``"ANSI_X3.4-1968"`` is replaced with ``"ascii"``. See also the :c:member:`~PyConfig.filesystem_errors` member. diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index ced7ee5bd09f7c..39ee3215ccec3d 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -570,6 +570,7 @@ function,PyStructSequence_GetItem,3.2, function,PyStructSequence_New,3.2, function,PyStructSequence_NewType,3.2, function,PyStructSequence_SetItem,3.2, +var,PyStructSequence_UnnamedField,3.11, var,PySuper_Type,3.2, function,PySys_AddWarnOption,3.2, function,PySys_AddWarnOptionUnicode,3.2, diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 66ba1e9fb9b347..136cf4e77b1543 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -31,7 +31,7 @@ installing other Python projects, refer to the Key terms ========= -* the `Python Packaging Index `__ is a public +* the `Python Package Index `__ is a public repository of open source licensed packages made available for use by other Python users * the `Python Packaging Authority @@ -127,14 +127,14 @@ involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ -* `Uploading the project to the Python Packaging Index`_ +* `Uploading the project to the Python Package Index`_ * `The .pypirc file`_ .. _Project structure: \ https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects .. _Building and packaging the project: \ https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Packaging Index: \ +.. _Uploading the project to the Python Package Index: \ https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives .. _The .pypirc file: \ https://packaging.python.org/specifications/pypirc/ @@ -150,7 +150,7 @@ These are quick answers or links for some common tasks. This isn't an easy topic, but here are a few tips: -* check the Python Packaging Index to see if the name is already in use +* check the Python Package Index to see if the name is already in use * check popular hosting sites like GitHub, Bitbucket, etc to see if there is already a project with that name * check what comes up in a web search for the name you're considering diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 720b1e496eb848..0437b59d55da6e 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -714,7 +714,7 @@ Why don't generators support the with statement? For technical reasons, a generator used directly as a context manager would not work correctly. When, as is most common, a generator is used as an iterator run to completion, no closing is needed. When it is, wrap -it as "contextlib.closing(generator)" in the 'with' statment. +it as "contextlib.closing(generator)" in the 'with' statement. Why are colons required for the if/while/def/class statements? diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 3379e41d9de072..fd32b097335e53 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -290,9 +290,6 @@ complete example using the GNU readline library (you may want to ignore #define PY_SSIZE_T_CLEAN #include - #include - #include - #include int main (int argc, char* argv[]) { diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ef80808a1a4d5e..12b70dbbe73029 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -836,6 +836,27 @@ ago? ``-190 % 12 == 2`` is useful; ``-190 % 12 == -10`` is a bug waiting to bite. +How do I get int literal attribute instead of SyntaxError? +---------------------------------------------------------- + +Trying to lookup an ``int`` literal attribute in the normal manner gives +a syntax error because the period is seen as a decimal point:: + + >>> 1.__class__ + File "", line 1 + 1.__class__ + ^ + SyntaxError: invalid decimal literal + +The solution is to separate the literal from the period +with either a space or parentheses. + + >>> 1 .__class__ + + >>> (1).__class__ + + + How do I convert a string to a number? -------------------------------------- diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index 0153a4f316ee82..6b95819c8ee855 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -212,7 +212,7 @@ Embedding the Python interpreter in a Windows app can be summarized as follows: .. code-block:: c - #include "python.h" + #include ... Py_Initialize(); // Initialize Python. initmyAppc(); // Initialize (import) the helper class. diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 3a3653a5ee3a3e..a3c4330296bedb 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -567,9 +567,6 @@ expression. Currently the following are explicitly supported: * Simple symbolic constants like ``sys.maxsize``, which must start with the name of the module -In case you're curious, this is implemented in ``from_builtin()`` -in ``Lib/inspect.py``. - (In the future, this may need to get even more elaborate, to allow full expressions like ``CONSTANT - 1``.) diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 4621b66270fcc4..d9cfad972cfa9d 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -936,7 +936,8 @@ and raise an error if the two do not match:: _Private__names """"""""""""""" -Private names are not converted to enum members, but remain normal attributes. +:ref:`Private names ` are not converted to enum members, +but remain normal attributes. .. versionchanged:: 3.10 diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 74e861480d2ff8..c7f8bc8f17f43b 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -65,11 +65,10 @@ output must only depend on its input. Some languages are very strict about purity and don't even have assignment statements such as ``a=3`` or ``c = a + b``, but it's difficult to avoid all -side effects. Printing to the screen or writing to a disk file are side -effects, for example. For example, in Python a call to the :func:`print` or -:func:`time.sleep` function both return no useful value; they're only called for -their side effects of sending some text to the screen or pausing execution for a -second. +side effects, such as printing to the screen or writing to a disk file. Another +example is a call to the :func:`print` or :func:`time.sleep` function, neither +of which returns a useful value. Both are called only for their side effects +of sending some text to the screen or pausing execution for a second. Python programs written in functional style usually won't go to the extreme of avoiding all I/O or all assignments; instead, they'll provide a diff --git a/Doc/includes/sqlite3/text_factory.py b/Doc/includes/sqlite3/text_factory.py index a857a155cdd4ff..c0d87cd559118c 100644 --- a/Doc/includes/sqlite3/text_factory.py +++ b/Doc/includes/sqlite3/text_factory.py @@ -3,9 +3,9 @@ con = sqlite3.connect(":memory:") cur = con.cursor() -AUSTRIA = "\xd6sterreich" +AUSTRIA = "Österreich" -# by default, rows are returned as Unicode +# by default, rows are returned as str cur.execute("select ?", (AUSTRIA,)) row = cur.fetchone() assert row[0] == AUSTRIA diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 5e7e03045b2acc..4bacc7ba0c2cf2 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -44,7 +44,7 @@ Key terms ``venv``. It allows virtual environments to be used on versions of Python prior to 3.4, which either don't provide ``venv`` at all, or aren't able to automatically install ``pip`` into created environments. -* The `Python Packaging Index `__ is a public +* The `Python Package Index `__ is a public repository of open source licensed packages made available for use by other Python users. * the `Python Packaging Authority @@ -78,7 +78,7 @@ The standard packaging tools are all designed to be used from the command line. The following command will install the latest version of a module and its -dependencies from the Python Packaging Index:: +dependencies from the Python Package Index:: python -m pip install SomePackage @@ -226,7 +226,7 @@ the installation process. With the introduction of support for the binary ``wheel`` format, and the ability to publish wheels for at least Windows and macOS through the -Python Packaging Index, this problem is expected to diminish over time, +Python Package Index, this problem is expected to diminish over time, as users are more regularly able to install pre-built extensions rather than needing to build them themselves. diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index 2a13776e29336c..9a1644a97c0aea 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -11,6 +11,11 @@ contains a rich set of fixers that will handle almost all code. 2to3 supporting library :mod:`lib2to3` is, however, a flexible and generic library, so it is possible to write your own fixers for 2to3. +.. deprecated-removed:: 3.11 3.13 + The ``lib2to3`` module was marked pending for deprecation in Python 3.9 + (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated + in Python 3.11 (raising :exc:`DeprecationWarning`). The ``2to3`` tool is + part of that. It will be removed in Python 3.13. .. _2to3-using: diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index ac8a2fd6195f40..2edf84e5f0444c 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1109,7 +1109,7 @@ Anything with more interesting error-handling or resource management should be done downstream after the arguments are parsed. For example, JSON or YAML conversions have complex error cases that require -better reporting than can be given by the ``type`` keyword. An +better reporting than can be given by the ``type`` keyword. A :exc:`~json.JSONDecodeError` would not be well formatted and a :exc:`FileNotFound` exception would not be handled at all. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index e21151bd4ef792..e29b5e88d71d41 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1266,7 +1266,7 @@ Pattern matching the pattern matches the subject. ``body`` contains a list of nodes to execute if the pattern matches and - the result of evaluating the guard expression is truthy. + the result of evaluating the guard expression is true. .. doctest:: @@ -1920,7 +1920,7 @@ and classes for traversing abstract syntax trees: If source contains a null character ('\0'), :exc:`ValueError` is raised. .. warning:: - Note that succesfully parsing souce code into an AST object doesn't + Note that successfully parsing source code into an AST object doesn't guarantee that the source code provided is valid Python code that can be executed as the compilation step can raise further :exc:`SyntaxError` exceptions. For instance, the source ``return 42`` generates a valid diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 02a00033152aba..77f1128de50c95 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -57,7 +57,7 @@ When the debug mode is enabled: * The execution time of the I/O selector is logged if it takes too long to perform an I/O operation. -* Callbacks taking longer than 100ms are logged. The +* Callbacks taking longer than 100 milliseconds are logged. The :attr:`loop.slow_callback_duration` attribute can be used to set the minimum execution duration in seconds that is considered "slow". diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 390ee1969d096f..50ad8a2ab70324 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -63,7 +63,7 @@ All event loops on Windows do not support the following methods: methods are not supported. The resolution of the monotonic clock on Windows is usually around 15.6 -msec. The best resolution is 0.5 msec. The resolution depends on the +milliseconds. The best resolution is 0.5 milliseconds. The resolution depends on the hardware (availability of `HPET `_) and on the Windows configuration. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index b3e229c24f07d1..95a8e4649beede 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -395,8 +395,8 @@ TCP echo server using the :func:`asyncio.start_server` function:: server = await asyncio.start_server( handle_echo, '127.0.0.1', 8888) - addr = server.sockets[0].getsockname() - print(f'Serving on {addr}') + addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets) + print(f'Serving on {addrs}') async with server: await server.serve_forever() diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 29e52ad48e1159..4ff038c8d29f1a 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -154,7 +154,7 @@ The modern interface provides: This version does not allow the digit 0 (zero) to the letter O (oh) and digit 1 (one) to either the letter I (eye) or letter L (el) mappings, all these characters are included in the Extended Hex Alphabet and are not - interchangable. + interchangeable. .. versionadded:: 3.10 diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 07df1873bba588..132b0ce7192ac1 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -104,6 +104,9 @@ example, knowing that a class supplies ``__getitem__``, ``__len__``, and ``__iter__`` is insufficient for distinguishing a :class:`Sequence` from a :class:`Mapping`. +.. versionadded:: 3.9 + These abstract classes now support ``[]``. See :ref:`types-genericalias` + and :pep:`585`. .. _collections-abstract-base-classes: diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index bc38a63a52d97c..a800a0b8dee4b8 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -353,6 +353,23 @@ Functions and classes provided: .. versionadded:: 3.5 +.. function:: chdir(path) + + Non parallel-safe context manager to change the current working directory. + As this changes a global state, the working directory, it is not suitable + for use in most threaded or async contexts. It is also not suitable for most + non-linear code execution, like generators, where the program execution is + temporarily relinquished -- unless explicitely desired, you should not yield + when this context manager is active. + + This is a simple wrapper around :func:`~os.chdir`, it changes the current + working directory upon entering and restores the old one on exit. + + This context manager is :ref:`reentrant `. + + .. versionadded:: 3.11 + + .. class:: ContextDecorator() A base class that enables a context manager to also be used as a decorator. @@ -900,8 +917,8 @@ but may also be used *inside* a :keyword:`!with` statement that is already using the same context manager. :class:`threading.RLock` is an example of a reentrant context manager, as are -:func:`suppress` and :func:`redirect_stdout`. Here's a very simple example of -reentrant use:: +:func:`suppress`, :func:`redirect_stdout`, and :func:`chdir`. Here's a very +simple example of reentrant use:: >>> from contextlib import redirect_stdout >>> from io import StringIO diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 899ce0225ce7f3..3a7817cfdfad87 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -383,6 +383,8 @@ Dialects support the following attributes: :const:`False`. On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. + .. versionchanged:: 3.11 + An empty *escapechar* is not allowed. .. attribute:: Dialect.lineterminator @@ -402,6 +404,8 @@ Dialects support the following attributes: as the *delimiter* or *quotechar*, or which contain new-line characters. It defaults to ``'"'``. + .. versionchanged:: 3.11 + An empty *quotechar* is not allowed. .. attribute:: Dialect.quoting diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index efbece437af2dd..37e822c0e2b207 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -368,7 +368,7 @@ The module :mod:`curses` defines the following functions: Set the maximum time in milliseconds that can elapse between press and release events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 msec, or one fifth of a second. + interval value. The default value is 200 milliseconds, or one fifth of a second. .. function:: mousemask(mousemask) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 196aa84473f135..0f8b70cdedb456 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -27,6 +27,9 @@ on efficient attribute extraction for output formatting and manipulation. Module :mod:`time` Time access and conversions. + Module :mod:`zoneinfo` + Concrete time zones representing the IANA time zone database. + Package `dateutil `_ Third-party library with expanded time zone and parsing support. @@ -2174,14 +2177,13 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: - `dateutil.tz `_ + :mod:`zoneinfo` The :mod:`datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` attribute (a UTC timezone instance). - *dateutil.tz* library brings the *IANA timezone database* - (also known as the Olson database) to Python, and its usage is - recommended. + ``zoneinfo`` brings the *IANA timezone database* (also known as the Olson + database) to Python, and its usage is recommended. `IANA timezone database `_ The Time Zone Database (often called tz, tzdata or zoneinfo) contains code diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 9b683083f6b622..9958dea5587f17 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -725,7 +725,7 @@ iterations of the loop. of the stack and sets the ``f_lasti`` attribute of the frame with that value. Then pops the next exception from the stack uses it to restore the current exception. Finally it re-raises the originally popped exception. - Used in excpetion handler cleanup. + Used in exception handler cleanup. .. versionadded:: 3.11 diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index a77322f83acbde..0bbb640bea26b1 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -485,25 +485,24 @@ Some details you should read once, but won't need to remember: .. index:: single: ^ (caret); marker -* For some :exc:`SyntaxError`\ s, Python displays the character position of the - syntax error, using a ``^`` marker:: +* For some exceptions, Python displays the position of the error using ``^`` + markers and tildes:: - >>> 1 1 + >>> 1 + None File "", line 1 - 1 1 - ^ - SyntaxError: invalid syntax + 1 + None + ~~^~~~~~ + TypeError: unsupported operand type(s) for +: 'int' and 'NoneType' Since the lines showing the position of the error come before the exception type and detail, they are not checked by doctest. For example, the following test would pass, even though it puts the ``^`` marker in the wrong location:: - >>> 1 1 - Traceback (most recent call last): + >>> 1 + None File "", line 1 - 1 1 - ^ - SyntaxError: invalid syntax + 1 + None + ^~~~~~~~ + TypeError: unsupported operand type(s) for +: 'int' and 'NoneType' .. _option-flags-and-directives: diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index f28505a6d2ccf8..86bf705af77e71 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -651,8 +651,8 @@ Data Types --------------- -Utilites and Decorators ------------------------ +Utilities and Decorators +------------------------ .. class:: auto diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index b665c6079ab0a3..8fa82a98a199d6 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -34,6 +34,10 @@ class or one of its subclasses, and not from :exc:`BaseException`. More information on defining exceptions is available in the Python Tutorial under :ref:`tut-userexceptions`. + +Exception context +----------------- + When raising (or re-raising) an exception in an :keyword:`except` or :keyword:`finally` clause :attr:`__context__` is automatically set to the last exception caught; if the @@ -67,6 +71,25 @@ exceptions so that the final line of the traceback always shows the last exception that was raised. +Inheriting from built-in exceptions +----------------------------------- + +User code can create subclasses that inherit from an exception type. +It's recommended to only subclass one exception type at a time to avoid +any possible conflicts between how the bases handle the ``args`` +attribute, as well as due to possible memory layout incompatibilities. + +.. impl-detail:: + + Most built-in exceptions are implemented in C for efficiency, see: + :source:`Objects/exceptions.c`. Some have custom memory layouts + which makes it impossible to create a subclass that inherits from + multiple exception types. The memory layout of a type is an implementation + detail and might change between Python versions, leading to new + conflicts in the future. Therefore, it's recommended to avoid + subclassing multiple exception types altogether. + + Base classes ------------ diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index ae31341a9e63ee..2a2c1b3c2b921c 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -50,7 +50,7 @@ You can control how files are opened by providing an opening hook via the *openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The hook must be a function that takes two arguments, *filename* and *mode*, and returns an accordingly opened file-like object. If *encoding* and/or *errors* -are specified, they will be passed to the hook as aditional keyword arguments. +are specified, they will be passed to the hook as additional keyword arguments. This module provides a :func:`hook_compressed` to support compressed files. The following function is the primary interface of this module: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 19d67e9d754c2c..a651d8829cd29c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1345,8 +1345,8 @@ are always available. They are listed here in alphabetical order. coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are - converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. + converted to float and a float result is delivered. For example, ``pow(10, 2)`` + returns ``100``, but ``pow(10, -2)`` returns ``0.01``. For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must also be of integer type and *mod* must be nonzero. If *mod* is present and diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 37addee6cda809..0c3bd7b5ac2c99 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -376,10 +376,10 @@ Constructor functions also accept the following tree hashing parameters: * *depth*: maximal depth of tree (1 to 255, 255 if unlimited, 1 in sequential mode). -* *leaf_size*: maximal byte length of leaf (0 to 2**32-1, 0 if unlimited or in +* *leaf_size*: maximal byte length of leaf (0 to ``2**32-1``, 0 if unlimited or in sequential mode). -* *node_offset*: node offset (0 to 2**64-1 for BLAKE2b, 0 to 2**48-1 for +* *node_offset*: node offset (0 to ``2**64-1`` for BLAKE2b, 0 to ``2**48-1`` for BLAKE2s, 0 for the first, leftmost, leaf, or in sequential mode). * *node_depth*: node depth (0 to 255, 0 for leaves, or in sequential mode). @@ -500,7 +500,7 @@ Keyed hashing Keyed hashing can be used for authentication as a faster and simpler replacement for `Hash-based message authentication code -`_ (HMAC). +`_ (HMAC). BLAKE2 can be securely used in prefix-MAC mode thanks to the indifferentiability property inherited from BLAKE. diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 067e1b1e5adb3e..7d836fe7380245 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -44,4 +44,4 @@ This module defines four dictionaries, :data:`html5`, .. rubric:: Footnotes -.. [#] See https://www.w3.org/TR/html5/syntax.html#named-character-references +.. [#] See https://html.spec.whatwg.org/multipage/syntax.html#named-character-references diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 6234e65629ea08..3d59665be4f254 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -122,7 +122,7 @@ The following classes are provided: :mod:`http.cookiejar` and :mod:`http.cookies` modules do not depend on each other. - https://curl.haxx.se/rfc/cookie_spec.html + https://curl.se/rfc/cookie_spec.html The specification of the original Netscape cookie protocol. Though this is still the dominant protocol, the 'Netscape cookie protocol' implemented by all the major browsers (and :mod:`http.cookiejar`) only bears a passing resemblance to diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index c43457a3850678..58c582d7124137 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -255,7 +255,7 @@ function:: Package distributions --------------------- -A convience method to resolve the distribution or +A convenience method to resolve the distribution or distributions (in the case of a namespace package) for top-level Python packages or modules:: diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index b5ee7a6b9659af..a25f5145cac835 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -815,14 +815,55 @@ ABC hierarchy:: .. versionadded:: 3.9 + .. abstractmethod:: name() + + The base name of this object without any parent references. + + .. abstractmethod:: iterdir() + + Yield Traversable objects in self. + + .. abstractmethod:: is_dir() + + Return True if self is a directory. + + .. abstractmethod:: is_file() + + Return True if self is a file. + + .. abstractmethod:: joinpath(child) + + Return Traversable child in self. + + .. abstractmethod:: __truediv__(child) + + Return Traversable child in self. + + .. abstractmethod:: open(mode='r', *args, **kwargs) + + *mode* may be 'r' or 'rb' to open as text or binary. Return a handle + suitable for reading (same as :attr:`pathlib.Path.open`). + + When opening as text, accepts encoding parameters such as those + accepted by :attr:`io.TextIOWrapper`. + + .. method:: read_bytes() + + Read contents of self as bytes. + + .. method:: read_text(encoding=None) + + Read contents of self as text. + .. class:: TraversableResources An abstract base class for resource readers capable of serving - the ``files`` interface. Subclasses ResourceReader and provides - concrete implementations of the ResourceReader's abstract - methods. Therefore, any loader supplying TraversableReader - also supplies ResourceReader. + the :meth:`importlib.resources.files` interface. Subclasses + :class:`importlib.abc.ResourceReader` and provides + concrete implementations of the :class:`importlib.abc.ResourceReader`'s + abstract methods. Therefore, any loader supplying + :class:`importlib.abc.TraversableReader` also supplies ResourceReader. Loaders that wish to support resource reading are expected to implement this interface. @@ -1353,6 +1394,24 @@ find and load modules. .. versionadded:: 3.4 +.. class:: NamespaceLoader(name, path, path_finder): + + A concrete implementation of :class:`importlib.abc.InspectLoader` for + namespace packages. This is an alias for a private class and is only made + public for introspecting the ``__loader__`` attribute on namespace + packages:: + + >>> from importlib.machinery import NamespaceLoader + >>> import my_namespace + >>> isinstance(my_namespace.__loader__, NamespaceLoader) + True + >>> import importlib.abc + >>> isinstance(my_namespace.__loader__, importlib.abc.Loader) + True + + .. versionadded:: 3.11 + + .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ed95c72c9a626d..1074f977184bb5 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -935,26 +935,6 @@ Classes and functions times. -.. function:: getargspec(func) - - Get the names and default values of a Python function's parameters. A - :term:`named tuple` ``ArgSpec(args, varargs, keywords, defaults)`` is - returned. *args* is a list of the parameter names. *varargs* and *keywords* - are the names of the ``*`` and ``**`` parameters or ``None``. *defaults* is a - tuple of default argument values or ``None`` if there are no default - arguments; if this tuple has *n* elements, they correspond to the last - *n* elements listed in *args*. - - .. deprecated:: 3.0 - Use :func:`getfullargspec` for an updated API that is usually a drop-in - replacement, but also correctly handles function annotations and - keyword-only parameters. - - Alternatively, use :func:`signature` and - :ref:`Signature Object `, which provide a - more structured introspection API for callables. - - .. function:: getfullargspec(func) Get the names and default values of a Python function's parameters. A @@ -1015,33 +995,6 @@ Classes and functions This function was inadvertently marked as deprecated in Python 3.5. -.. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) - - Format a pretty argument spec from the values returned by - :func:`getfullargspec`. - - The first seven arguments are (``args``, ``varargs``, ``varkw``, - ``defaults``, ``kwonlyargs``, ``kwonlydefaults``, ``annotations``). - - The other six arguments are functions that are called to turn argument names, - ``*`` argument name, ``**`` argument name, default values, return annotation - and individual annotations into strings, respectively. - - For example: - - >>> from inspect import formatargspec, getfullargspec - >>> def f(a: int, b: float): - ... pass - ... - >>> formatargspec(*getfullargspec(f)) - '(a: int, b: float)' - - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. - - .. function:: formatargvalues(args[, varargs, varkw, locals, formatarg, formatvarargs, formatvarkw, formatvalue]) Format a pretty argument spec from the four values returned by diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 2ab4dd83ad4dee..9c2dff55703273 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -41,7 +41,7 @@ IP addresses, networks and interfaces: Return an :class:`IPv4Address` or :class:`IPv6Address` object depending on the IP address passed as argument. Either IPv4 or IPv6 addresses may be - supplied; integers less than 2**32 will be considered to be IPv4 by default. + supplied; integers less than ``2**32`` will be considered to be IPv4 by default. A :exc:`ValueError` is raised if *address* does not represent a valid IPv4 or IPv6 address. @@ -56,7 +56,7 @@ IP addresses, networks and interfaces: Return an :class:`IPv4Network` or :class:`IPv6Network` object depending on the IP address passed as argument. *address* is a string or integer representing the IP network. Either IPv4 or IPv6 networks may be supplied; - integers less than 2**32 will be considered to be IPv4 by default. *strict* + integers less than ``2**32`` will be considered to be IPv4 by default. *strict* is passed to :class:`IPv4Network` or :class:`IPv6Network` constructor. A :exc:`ValueError` is raised if *address* does not represent a valid IPv4 or IPv6 address, or if the network has host bits set. @@ -70,7 +70,7 @@ IP addresses, networks and interfaces: Return an :class:`IPv4Interface` or :class:`IPv6Interface` object depending on the IP address passed as argument. *address* is a string or integer representing the IP address. Either IPv4 or IPv6 addresses may be supplied; - integers less than 2**32 will be considered to be IPv4 by default. A + integers less than ``2**32`` will be considered to be IPv4 by default. A :exc:`ValueError` is raised if *address* does not represent a valid IPv4 or IPv6 address. @@ -682,7 +682,7 @@ dictionaries. Note that currently expanded netmasks are not supported. That means ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` - not. + is not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and diff --git a/Doc/library/json.rst b/Doc/library/json.rst index c8184da80fe43e..6fa89f578a2cfd 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -11,9 +11,9 @@ -------------- -`JSON (JavaScript Object Notation) `_, specified by +`JSON (JavaScript Object Notation) `_, specified by :rfc:`7159` (which obsoletes :rfc:`4627`) and by -`ECMA-404 `_, +`ECMA-404 `_, is a lightweight data interchange format inspired by `JavaScript `_ object literal syntax (although it is not a strict subset of JavaScript [#rfc-errata]_ ). @@ -544,7 +544,7 @@ Standard Compliance and Interoperability ---------------------------------------- The JSON format is specified by :rfc:`7159` and by -`ECMA-404 `_. +`ECMA-404 `_. This section details this module's level of compliance with the RFC. For simplicity, :class:`JSONEncoder` and :class:`JSONDecoder` subclasses, and parameters other than those explicitly mentioned, are not considered. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index c5d07dc5954842..f3d725b822529e 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -412,7 +412,7 @@ For example:: multiple_results = [pool.apply_async(os.getpid, ()) for i in range(4)] print([res.get(timeout=1) for res in multiple_results]) - # make a single worker sleep for 10 secs + # make a single worker sleep for 10 seconds res = pool.apply_async(time.sleep, (10,)) try: print(res.get(timeout=1)) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index cba576a29e2e2b..2ba42b7e579a77 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -342,3 +342,30 @@ behind it: >>> c.shm.close() >>> c.shm.unlink() +The following examples demonstrates that ``ShareableList`` +(and underlying ``SharedMemory``) objects +can be pickled and unpickled if needed. +Note, that it will still be the same shared object. +This happens, because the deserialized object has +the same unique name and is just attached to an existing +object with the same name (if the object is still alive): + + >>> import pickle + >>> from multiprocessing import shared_memory + >>> sl = shared_memory.ShareableList(range(10)) + >>> list(sl) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> deserialized_sl = pickle.loads(pickle.dumps(sl)) + >>> list(deserialized_sl) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> sl[0] = -1 + >>> deserialized_sl[1] = -2 + >>> list(sl) + [-1, -2, 2, 3, 4, 5, 6, 7, 8, 9] + >>> list(deserialized_sl) + [-1, -2, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> sl.shm.close() + >>> sl.shm.unlink() diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index ce6d4a85bf5e9d..5ded9661f08014 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -133,7 +133,7 @@ The following classes are available: encoded data, which contains UID (see PList manual). It has one attribute, :attr:`data`, which can be used to retrieve the int value - of the UID. :attr:`data` must be in the range `0 <= data < 2**64`. + of the UID. :attr:`data` must be in the range ``0 <= data < 2**64``. .. versionadded:: 3.8 diff --git a/Doc/library/random.rst b/Doc/library/random.rst index e444f9573246a6..36f232dc319e40 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -135,15 +135,10 @@ Functions for integers values. Formerly it used a style like ``int(random()*n)`` which could produce slightly uneven distributions. - .. deprecated:: 3.10 - The automatic conversion of non-integer types to equivalent integers is - deprecated. Currently ``randrange(10.0)`` is losslessly converted to - ``randrange(10)``. In the future, this will raise a :exc:`TypeError`. - - .. deprecated:: 3.10 - The exception raised for non-integral values such as ``randrange(10.5)`` - or ``randrange('10')`` will be changed from :exc:`ValueError` to - :exc:`TypeError`. + .. versionchanged:: 3.11 + Automatic conversion of non-integer types is no longer supported. + Calls such as ``randrange(10.0)`` and ``randrange(Fraction(10, 1))`` + now raise a :exc:`TypeError`. .. function:: randint(a, b) @@ -508,7 +503,7 @@ between the effects of a drug versus a placebo:: Simulation of arrival times and service deliveries for a multiserver queue:: - from heapq import heappush, heappop + from heapq import heapify, heapreplace from random import expovariate, gauss from statistics import mean, quantiles @@ -520,14 +515,15 @@ Simulation of arrival times and service deliveries for a multiserver queue:: waits = [] arrival_time = 0.0 servers = [0.0] * num_servers # time when each server becomes available - for i in range(100_000): + heapify(servers) + for i in range(1_000_000): arrival_time += expovariate(1.0 / average_arrival_interval) - next_server_available = heappop(servers) + next_server_available = servers[0] wait = max(0.0, next_server_available - arrival_time) waits.append(wait) - service_duration = gauss(average_service_time, stdev_service_time) + service_duration = max(0.0, gauss(average_service_time, stdev_service_time)) service_completed = arrival_time + wait + service_duration - heappush(servers, service_completed) + heapreplace(servers, service_completed) print(f'Mean wait: {mean(waits):.1f} Max wait: {max(waits):.1f}') print('Quartiles:', [round(q, 1) for q in quantiles(waits)]) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index ff7687cc936ec9..b12ce4b9744f94 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1572,7 +1572,7 @@ find all of the adverbs in some text, they might use :func:`findall` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." - >>> re.findall(r"\w+ly", text) + >>> re.findall(r"\w+ly\b", text) ['carefully', 'quickly'] @@ -1586,7 +1586,7 @@ a writer wanted to find all of the adverbs *and their positions* in some text, they would use :func:`finditer` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." - >>> for m in re.finditer(r"\w+ly", text): + >>> for m in re.finditer(r"\w+ly\b", text): ... print('%02d-%02d: %s' % (m.start(), m.end(), m.group(0))) 07-16: carefully 40-47: quickly diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 11c67074921672..22d6dba9e1a9c6 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -804,6 +804,10 @@ Querying the size of the output terminal .. versionadded:: 3.3 + .. versionchanged:: 3.11 + The ``fallback`` values are also used if :func:`os.get_terminal_size` + returns zeroes. + .. _`fcopyfile`: http://www.manpagez.com/man/3/copyfile/ diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 84a569d03eb293..63821866a012bb 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -68,10 +68,34 @@ Module contents signal (SIG*), handler (:const:`SIG_DFL`, :const:`SIG_IGN`) and sigmask (:const:`SIG_BLOCK`, :const:`SIG_UNBLOCK`, :const:`SIG_SETMASK`) related constants listed below were turned into - :class:`enums `. + :class:`enums ` (:class:`Signals`, :class:`Handlers` and :class:`Sigmasks` respectively). :func:`getsignal`, :func:`pthread_sigmask`, :func:`sigpending` and :func:`sigwait` functions return human-readable - :class:`enums `. + :class:`enums ` as :class:`Signals` objects. + + +The signal module defines three enums: + +.. class:: Signals + + :class:`enum.IntEnum` collection of SIG* constants and the CTRL_* constants. + + .. versionadded:: 3.5 + +.. class:: Handlers + + :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. + + .. versionadded:: 3.5 + +.. class:: Sigmasks + + :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. + + Availability: Unix. See the man page :manpage:`sigprocmask(3)` and + :manpage:`pthread_sigmask(3)` for further information. + + .. versionadded:: 3.5 The variables defined in the :mod:`signal` module are: @@ -618,8 +642,8 @@ The :mod:`signal` module defines the following functions: .. _signal-example: -Example -------- +Examples +-------- Here is a minimal example program. It uses the :func:`alarm` function to limit the time spent waiting to open a file; this is useful if the file is for a @@ -631,7 +655,8 @@ be sent, and the handler raises an exception. :: import signal, os def handler(signum, frame): - print('Signal handler called with signal', signum) + signame = signal.Signals(signum).name + print(f'Signal handler called with signal {signame} ({signum})') raise OSError("Couldn't open device!") # Set the signal handler and a 5-second alarm diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 397ba54733ca8c..27ad5c7fd1d91f 100755 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -827,8 +827,8 @@ The :mod:`socket` module also offers various network-related services: .. function:: gethostbyname_ex(hostname) Translate a host name to IPv4 address format, extended interface. Return a - triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the primary - host name responding to the given *ip_address*, *aliaslist* is a (possibly + triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the host's + primary host name, *aliaslist* is a (possibly empty) list of alternative host names for the same address, and *ipaddrlist* is a list of IPv4 addresses for the same interface on the same host (often but not always a single address). :func:`gethostbyname_ex` does not support IPv6 name diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e6b8b95d2aa52e..eaea7ae390b972 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -537,8 +537,8 @@ Connection Objects Using this attribute you can control what objects are returned for the ``TEXT`` data type. By default, this attribute is set to :class:`str` and the - :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to - return bytestrings instead, you can set it to :class:`bytes`. + :mod:`sqlite3` module will return :class:`str` objects for ``TEXT``. + If you want to return :class:`bytes` instead, you can set it to :class:`bytes`. You can also set it to any other callable that accepts a single bytestring parameter and returns the resulting object. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ec12e02fb37d45..ee07ba1042f623 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1073,7 +1073,11 @@ always available. This is a dictionary that maps module names to modules which have already been loaded. This can be manipulated to force reloading of modules and other tricks. However, replacing the dictionary will not necessarily work as expected and - deleting essential items from the dictionary may cause Python to fail. + deleting essential items from the dictionary may cause Python to fail. If + you want to iterate over this global dictionary always use + ``sys.modules.copy()`` or ``tuple(sys.modules)`` to avoid exceptions as its + size may change during iteration as a side effect of code or activity in + other threads. .. data:: orig_argv @@ -1725,13 +1729,13 @@ always available. .. code-block:: shell-session - $ ./python -Xa=b -Xc + $ ./python -Xpycache_prefix=some_path -Xdev Python 3.2a3+ (py3k, Oct 16 2010, 20:14:50) [GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import sys >>> sys._xoptions - {'a': 'b', 'c': True} + {'pycache_prefix': 'some_path', 'dev': True} .. impl-detail:: diff --git a/Doc/library/test.rst b/Doc/library/test.rst index a6cc2be4d55222..699db14596f250 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -607,6 +607,15 @@ The :mod:`test.support` module defines the following functions: target of the "as" clause, if there is one. +.. function:: flush_std_streams() + + Call the ``flush()`` method on :data:`sys.stdout` and then on + :data:`sys.stderr`. It can be used to make sure that the logs order is + consistent before writing into stderr. + + .. versionadded:: 3.11 + + .. function:: print_warning(msg) Print a warning into :data:`sys.__stderr__`. Format the message as: @@ -1249,7 +1258,7 @@ The :mod:`test.support.threading_helper` module provides support for threading t Context manager catching :class:`threading.Thread` exception using :func:`threading.excepthook`. - Attributes set when an exception is catched: + Attributes set when an exception is caught: * ``exc_type`` * ``exc_value`` @@ -1458,7 +1467,7 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. function:: unlink(filename) Call :func:`os.unlink` on *filename*. On Windows platforms, this is - wrapped with a wait loop that checks for the existence fo the file. + wrapped with a wait loop that checks for the existence of the file. :mod:`test.support.import_helper` --- Utilities for import tests diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 69e5274f1cee74..0f880910c6ee68 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -366,11 +366,11 @@ Functions Unix implementation: - * Use ``clock_nanosleep()`` if available (resolution: 1 ns); - * Or use ``nanosleep()`` if available (resolution: 1 ns); - * Or use ``select()`` (resolution: 1 us). + * Use ``clock_nanosleep()`` if available (resolution: 1 nanosecond); + * Or use ``nanosleep()`` if available (resolution: 1 nanosecond); + * Or use ``select()`` (resolution: 1 microsecond). - On Windows, a waitable timer is used (resolution: 100 ns). If *secs* is + On Windows, a waitable timer is used (resolution: 100 nanosecond). If *secs* is zero, ``Sleep(0)`` is used. .. versionchanged:: 3.11 diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index d4e8b749db4808..7f1c41d46399ed 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -233,7 +233,7 @@ Where the following options are understood: .. cmdoption:: -u, --unit=U - specify a time unit for timer output; can select nsec, usec, msec, or sec + specify a time unit for timer output; can select ``nsec``, ``usec``, ``msec``, or ``sec`` .. versionadded:: 3.5 diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 88fe47a8d4a7fb..2314b02c7449c0 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -243,7 +243,7 @@ Standard names are defined for the following types: .. note:: A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferrably read from the + To guard against this potential change, preferably read from the :attr:`__spec__` attribute instead or use ``getattr(module, "__loader__", None)`` if you explicitly need to use this attribute. @@ -268,7 +268,7 @@ Standard names are defined for the following types: .. note:: A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferrably read from the + To guard against this potential change, preferably read from the :attr:`__spec__` attribute instead or use ``getattr(module, "__package__", None)`` if you explicitly need to use this attribute. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 13760c19214ee7..e5e7941833bebb 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -19,7 +19,7 @@ This module provides runtime support for type hints as specified by :pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, :pep:`591`, -:pep:`612` and :pep:`613`. +:pep:`593`, :pep:`612`, :pep:`613` and :pep:`647`. The most fundamental support consists of the types :data:`Any`, :data:`Union`, :data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For full specification please see :pep:`484`. For @@ -975,16 +975,16 @@ These can be used as types in annotations using ``[]``, each having a unique syn For example:: - def is_str_list(val: List[object]) -> TypeGuard[List[str]]: + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: '''Determines whether all objects in the list are strings''' return all(isinstance(x, str) for x in val) - def func1(val: List[object]): + def func1(val: list[object]): if is_str_list(val): - # Type of ``val`` is narrowed to ``List[str]``. + # Type of ``val`` is narrowed to ``list[str]``. print(" ".join(val)) else: - # Type of ``val`` remains as ``List[object]``. + # Type of ``val`` remains as ``list[object]``. print("Not a list of strings!") If ``is_str_list`` is a class or instance method, then the type in @@ -999,8 +999,8 @@ These can be used as types in annotations using ``[]``, each having a unique syn ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a wider form. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter - is not a subtype of the former, since ``List`` is invariant. + narrowing ``list[object]`` to ``list[str]`` even though the latter + is not a subtype of the former, since ``list`` is invariant. The responsibility of writing type-safe type guards is left to the user. ``TypeGuard`` also works with type variables. For more information, see @@ -2065,8 +2065,8 @@ Introspection helpers .. class:: ForwardRef A class used for internal typing representation of string forward references. - For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by + For example, ``list["SomeClass"]`` is implicitly transformed into + ``list[ForwardRef("SomeClass")]``. This class should not be instantiated by a user, but may be used by introspection tools. .. note:: diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 659a3632ac9be8..88e93ba6b002eb 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -876,7 +876,17 @@ HTTPRedirectHandler Objects .. method:: HTTPRedirectHandler.http_error_307(req, fp, code, msg, hdrs) The same as :meth:`http_error_301`, but called for the 'temporary redirect' - response. + response. It does not allow changing the request method from ``POST`` + to ``GET``. + + +.. method:: HTTPRedirectHandler.http_error_308(req, fp, code, msg, hdrs) + + The same as :meth:`http_error_301`, but called for the 'permanent redirect' + response. It does not allow changing the request method from ``POST`` + to ``GET``. + + .. versionadded:: 3.11 .. _http-cookie-processor: diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 5a8df4c27dc377..1102c634edaf32 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -88,6 +88,10 @@ support weak references but can add support through subclassing:: Extension types can easily be made to support weak references; see :ref:`weakref-support`. +When ``__slots__`` are defined for a given type, weak reference support is +disabled unless a ``'__weakref__'`` string is also present in the sequence of +strings in the ``__slots__`` declaration. +See :ref:`__slots__ documentation ` for details. .. class:: ref(object[, callback]) diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index d3a5f872204a35..20984b98b1778c 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -156,6 +156,9 @@ module documentation. This section lists the differences between the API and The :meth:`writexml` method now preserves the attribute order specified by the user. + .. versionchanged:: 3.9 + The *standalone* parameter was added. + .. method:: Node.toxml(encoding=None, standalone=None) Return a string or byte string containing the XML represented by @@ -174,6 +177,9 @@ module documentation. This section lists the differences between the API and The :meth:`toxml` method now preserves the attribute order specified by the user. + .. versionchanged:: 3.9 + The *standalone* parameter was added. + .. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, \ standalone=None) @@ -190,6 +196,8 @@ module documentation. This section lists the differences between the API and The :meth:`toprettyxml` method now preserves the attribute order specified by the user. + .. versionchanged:: 3.9 + The *standalone* parameter was added. .. _dom-example: diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index f24222f5fd9e94..3011a168ede42c 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -585,8 +585,8 @@ Here's an overview of the logical flow of a match statement: #. If the pattern succeeds, the corresponding guard (if present) is evaluated. In this case all name bindings are guaranteed to have happened. - * If the guard evaluates as truthy or missing, the ``block`` inside ``case_block`` is - executed. + * If the guard evaluates as true or is missing, the ``block`` inside + ``case_block`` is executed. * Otherwise, the next ``case_block`` is attempted as described above. @@ -637,10 +637,10 @@ The logical flow of a ``case`` block with a ``guard`` follows: #. If the pattern succeeded, evaluate the ``guard``. - * If the ``guard`` condition evaluates to "truthy", the case block is + * If the ``guard`` condition evaluates as true, the case block is selected. - * If the ``guard`` condition evaluates to "falsy", the case block is not + * If the ``guard`` condition evaluates as false, the case block is not selected. * If the ``guard`` raises an exception during evaluation, the exception @@ -1017,7 +1017,7 @@ A class pattern represents a class and its positional and keyword arguments The same keyword should not be repeated in class patterns. -The following is the logical flow for matching a mapping pattern against a +The following is the logical flow for matching a class pattern against a subject value: #. If ``name_or_attr`` is not an instance of the builtin :class:`type` , raise diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 181e445bb7bc6e..f1334f047d4b73 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1558,7 +1558,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully-chosen inputs that exploit the worst case performance of a - dict insertion, O(n^2) complexity. See + dict insertion, O(n\ :sup:`2`) complexity. See http://www.ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 55ac01b6a844dc..5c85dd7052d673 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -119,14 +119,14 @@ is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. -If the :keyword:`global` statement occurs within a block, all uses of the name -specified in the statement refer to the binding of that name in the top-level +If the :keyword:`global` statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the top-level namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The -global namespace is searched first. If the name is not found there, the +global namespace is searched first. If the names are not found there, the builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the name. +all uses of the listed names. The :keyword:`global` statement has the same scope as a name binding operation in the same block. If the nearest enclosing scope for a free variable contains diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 39fcba015b6947..69e2a947274497 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -605,7 +605,7 @@ the module. ``__file__`` is optional (if set, value must be a string). It indicates the pathname of the file from which the module was loaded (if - loaded from a file), or the pathname of the shared libray file + loaded from a file), or the pathname of the shared library file for extension modules loaded dynamically from a shared library. It might be missing for certain types of modules, such as C modules that are statically linked into the interpreter, and the diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 91ca8554adabfb..4f7f00192d7499 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -385,10 +385,20 @@ classes are identified by the patterns of leading and trailing underscore characters: ``_*`` - Not imported by ``from module import *``. The special identifier ``_`` is used - in the interactive interpreter to store the result of the last evaluation; it is - stored in the :mod:`builtins` module. When not in interactive mode, ``_`` - has no special meaning and is not defined. See section :ref:`import`. + Not imported by ``from module import *``. + +``_`` + In a ``case`` pattern within a :keyword:`match` statement, ``_`` is a + :ref:`soft keyword ` that denotes a + :ref:`wildcard `. + + Separately, the interactive interpreter makes the result of the last evaluation + available in the variable ``_``. + (It is stored in the :mod:`builtins` module, alongside built-in + functions like ``print``.) + + Elsewhere, ``_`` is a regular identifier. It is often used to name + "special" items, but it is not special to Python itself. .. note:: @@ -396,6 +406,8 @@ characters: refer to the documentation for the :mod:`gettext` module for more information on this convention. + It is also commonly used for unused variables. + ``__*__`` System-defined names, informally known as "dunder" names. These names are defined by the interpreter and its implementation (including the standard library). diff --git a/Doc/tools/rstlint.py b/Doc/tools/rstlint.py index 91aed80b1e9a6c..33cbaadfce944e 100755 --- a/Doc/tools/rstlint.py +++ b/Doc/tools/rstlint.py @@ -43,10 +43,10 @@ ] roles = [ - ":class:", - ":func:", - ":meth:", - ":mod:", + "(?>>`` + in the upper-right corner of an example box. If you hide the prompts + and output for an example, then you can easily copy and paste the input + lines into your interpreter. + .. index:: single: # (hash); comment Many of the examples in this manual, even those entered at the interactive diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 221c11ca74b4ef..58ad31972f603b 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -88,6 +88,11 @@ For example: '~/envs/tutorial-env/lib/python3.5/site-packages'] >>> +To deactivate a virtual environment, type:: + + deactivate + +into the terminal. Managing Packages with pip ========================== diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index a9c0931363bfca..d341ea8bb43c88 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -322,7 +322,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully-chosen inputs that exploit the worst - case performance of a dict construction, O(n^2) complexity. See + case performance of a dict construction, O(n\ :sup:`2`) complexity. See http://www.ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash @@ -477,13 +477,14 @@ Miscellaneous options * ``-X no_debug_ranges`` disables the inclusion of the tables mapping extra location information (end line, start column offset and end column offset) to every instruction in code objects. This is useful when smaller code - objects and pyc files are desired as well as supressing the extra visual + objects and pyc files are desired as well as suppressing the extra visual location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. * ``-X frozen_modules`` determines whether or not frozen modules are ignored by the import machinery. A value of "on" means they get imported and "off" means they are ignored. The default is "on" - for non-debug builds (the normal case) and "off" for debug builds. + if this is an installed Python (the normal case). If it's under + development (running from the source tree) then the default is "off". Note that the "importlib_bootstrap" and "importlib_bootstrap_external" frozen modules are always used, even if this flag is set to "off". @@ -959,7 +960,7 @@ conflict. If this variable is set, it disables the inclusion of the tables mapping extra location information (end line, start column offset and end column offset) to every instruction in code objects. This is useful when smaller - code objects and pyc files are desired as well as supressing the extra visual + code objects and pyc files are desired as well as suppressing the extra visual location indicators when the interpreter displays tracebacks. .. versionadded:: 3.11 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index a545d5a9372ac0..75f572c61877fe 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -416,15 +416,19 @@ Libraries options Security Options ---------------- -.. cmdoption:: --with-hash-algorithm=[fnv|siphash24] +.. cmdoption:: --with-hash-algorithm=[fnv|siphash13|siphash24] Select hash algorithm for use in ``Python/pyhash.c``: - * ``siphash24`` (default). - * ``fnv``; + * ``siphash13`` (default); + * ``siphash24``; + * ``fnv``. .. versionadded:: 3.4 + .. versionadded:: 3.11 + ``siphash13`` is added and it is the new default. + .. cmdoption:: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 Built-in hash modules: @@ -553,7 +557,7 @@ Built-in modules have no ``__file__`` attribute:: File "", line 1, in AttributeError: module 'sys' has no attribute '__file__' -Other C extensions are built as dynamic libraires, like the ``_asyncio`` module. +Other C extensions are built as dynamic libraries, like the ``_asyncio`` module. They are built with the ``Py_BUILD_CORE_MODULE`` macro defined. Example on Linux x86-64:: diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 1d1fa8bd85d7ed..0a1834453a0ee8 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -162,6 +162,7 @@ Custom OpenSSL $ pushd openssl-VERSION $ ./config \ --prefix=/usr/local/custom-openssl \ + --libdir=lib \ --openssldir=/etc/ssl $ make -j1 depend $ make -j8 diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index c0a6692049788b..0e1cf1fd0ce46c 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -791,7 +791,7 @@ Previously the Python virtual machine used 16-bit numbers in its bytecode, limiting the size of source files. In particular, this affected the maximum size of literal lists and dictionaries in Python source; occasionally people who are generating Python code would run into this limit. A patch by Charles G. -Waldman raises the limit from ``2^16`` to ``2^{32}``. +Waldman raises the limit from ``2**16`` to ``2**32``. Three new convenience functions intended for adding constants to a module's dictionary at module initialization time were added: :func:`PyModule_AddObject`, diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index d19c8e01ad8a00..abb65222ddd3d0 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -953,12 +953,12 @@ Several performance enhancements have been added: considered and traversed by the collector. (Contributed by Antoine Pitrou; :issue:`4688`.) -* Long integers are now stored internally either in base 2**15 or in base - 2**30, the base being determined at build time. Previously, they - were always stored in base 2**15. Using base 2**30 gives +* Long integers are now stored internally either in base ``2**15`` or in base + ``2**30``, the base being determined at build time. Previously, they + were always stored in base ``2**15``. Using base ``2**30`` gives significant performance improvements on 64-bit machines, but benchmark results on 32-bit machines have been mixed. Therefore, - the default is to use base 2**30 on 64-bit machines and base 2**15 + the default is to use base ``2**30`` on 64-bit machines and base ``2**15`` on 32-bit machines; on Unix, there's a new configure option :option:`!--enable-big-digits` that can be used to override this default. diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 919fbeeb2ad857..f1e6d0c4f3dd68 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -474,12 +474,12 @@ Build and C API Changes Changes to Python's build process and to the C API include: -* Integers are now stored internally either in base 2**15 or in base - 2**30, the base being determined at build time. Previously, they - were always stored in base 2**15. Using base 2**30 gives +* Integers are now stored internally either in base ``2**15`` or in base + ``2**30``, the base being determined at build time. Previously, they + were always stored in base ``2**15``. Using base ``2**30`` gives significant performance improvements on 64-bit machines, but benchmark results on 32-bit machines have been mixed. Therefore, - the default is to use base 2**30 on 64-bit machines and base 2**15 + the default is to use base ``2**30`` on 64-bit machines and base ``2**15`` on 32-bit machines; on Unix, there's a new configure option ``--enable-big-digits`` that can be used to override this default. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index de25d158bd54af..b1040d4888ed82 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -50,13 +50,6 @@ This article explains the new features in Python 3.10, compared to 3.9. For full details, see the :ref:`changelog `. -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.10 moves towards release, - so it's worth checking back even after reading earlier versions. - - Summary -- Release highlights ============================= @@ -1003,7 +996,7 @@ Added ``slots`` parameter in :func:`dataclasses.dataclass` decorator. Keyword-only fields ~~~~~~~~~~~~~~~~~~~ -dataclassses now supports fields that are keyword-only in the +dataclasses now supports fields that are keyword-only in the generated __init__ method. There are a number of ways of specifying keyword-only fields. @@ -1583,7 +1576,7 @@ Deprecated * Currently Python accepts numeric literals immediately followed by keywords, for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing - and ambigious expressions like ``[0x1for x in y]`` (which can be + and ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as ``[0x1 for x in y]`` or ``[0x1f or x in y]``). Starting in this release, a deprecation warning is raised if the numeric literal is immediately followed by one of keywords :keyword:`and`, :keyword:`else`, @@ -1920,7 +1913,7 @@ Changes in the Python API if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` - in Python is not affected, globals cannot be overriden with this syntax: it + in Python is not affected, globals cannot be overridden with this syntax: it also inherits the current builtins. (Contributed by Victor Stinner in :issue:`42990`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 0c3327f6f281bd..1fa8c57138feee 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -175,6 +175,11 @@ Other CPython Implementation Changes support :class:`typing.SupportsComplex` and :class:`typing.SupportsBytes` protocols. (Contributed by Mark Dickinson and Dong-hee Na in :issue:`24234`.) +* ``siphash13`` is added as a new internal hashing algorithms. It's has similar security + properties as ``siphash24`` but it is slightly faster for long inputs. ``str``, ``bytes``, + and some other types now use it as default algorithm for ``hash()``. :pep:`552` + hash-based pyc files now use ``siphash13``, too. + (Contributed by Inada Naoki in :issue:`29410`.) New Modules =========== @@ -239,19 +244,29 @@ sqlite3 (Contributed by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland in :issue:`16379`.) +threading +--------- + +* On Unix, if the ``sem_clockwait()`` function is available in the C library + (glibc 2.30 and newer), the :meth:`threading.Lock.acquire` method now uses + the monotonic clock (:data:`time.CLOCK_MONOTONIC`) for the timeout, rather + than using the system clock (:data:`time.CLOCK_REALTIME`), to not be affected + by system clock changes. + (Contributed by Victor Stinner in :issue:`41710`.) + time ---- * On Unix, :func:`time.sleep` now uses the ``clock_nanosleep()`` or - ``nanosleep()`` function, if available, which has a resolution of 1 ns (10^-6 - sec), rather than using ``select()`` which has a resolution of 1 us (10^-9 - sec). - (Contributed by Livius and Victor Stinner in :issue:`21302`.) + ``nanosleep()`` function, if available, which has a resolution of 1 nanosecond + (10\ :sup:`-9` seconds), rather than using ``select()`` which has a resolution + of 1 microsecond (10\ :sup:`-6` seconds). + (Contributed by Benjamin Szőke and Victor Stinner in :issue:`21302`.) * On Windows, :func:`time.sleep` now uses a waitable timer which has a - resolution of 100 ns (10^-7 sec). Previously, it had a solution of 1 ms - (10^-3 sec). - (Contributed by Livius and Victor Stinner in :issue:`21302`.) + resolution of 100 nanoseconds (10\ :sup:`-7` seconds). Previously, it had + a resolution of 1 millisecond (10\ :sup:`-3` seconds). + (Contributed by Benjamin Szőke and Victor Stinner in :issue:`21302`.) unicodedata ----------- @@ -293,9 +308,9 @@ CPython bytecode changes Deprecated ========== -* The :mod:`lib2to3` package is now deprecated and may not be able to parse - Python 3.10 or newer. See the :pep:`617` (New PEG parser for CPython). - (Contributed by Victor Stinner in :issue:`40360`.) +* The :mod:`lib2to3` package and ``2to3`` tool are now deprecated and may not + be able to parse Python 3.10 or newer. See the :pep:`617` (New PEG parser for + CPython). (Contributed by Victor Stinner in :issue:`40360`.) * :class:`webbrowser.MacOSX` is deprecated and will be removed in Python 3.13. It is untested and undocumented and also not used by webbrowser itself. @@ -348,7 +363,7 @@ Removed ``SO_REUSEADDR`` in UDP. (Contributed by Hugo van Kemenade in :issue:`45129`.) -* Remove :meth:`__getitem__` methods of +* Removed :meth:`__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput`, deprecated since Python 3.9. (Contributed by Hugo van Kemenade in :issue:`45132`.) @@ -387,7 +402,7 @@ Removed the ``l*gettext()`` functions. (Contributed by Dong-hee Na and Serhiy Storchaka in :issue:`44235`.) -* Remove from the :mod:`configparser` module: +* Removed from the :mod:`configparser` module: the :class:`SafeConfigParser` class, the :attr:`filename` property of the :class:`ParsingError` class, the :meth:`readfp` method of the :class:`ConfigParser` class, @@ -404,9 +419,25 @@ Removed generator-based coroutine objects in the debug mode. (Contributed by Illia Volochii in :issue:`43216`.) -* Remove the deprecated ``split()`` method of :class:`_tkinter.TkappType`. +* Removed the deprecated ``split()`` method of :class:`_tkinter.TkappType`. (Contributed by Erlend E. Aasland in :issue:`38371`.) +* Removed from the :mod:`inspect` module: + + * the ``getargspec`` function, deprecated since Python 3.0; + use :func:`inspect.signature` or :func:`inspect.getfullargspec` instead. + + * the ``formatargspec`` function, deprecated since Python 3.5; + use the :func:`inspect.signature` function and :class:`Signature` object + directly. + + * the undocumented ``Signature.from_callable`` and ``Signature.from_function`` + functions, deprecated since Python 3.5; use the + :meth:`Signature.from_callable() ` method + instead. + + (Contributed by Hugo van Kemenade in :issue:`45320`.) + Porting to Python 3.11 ====================== @@ -442,6 +473,18 @@ Build Changes * CPython can now be built with the ThinLTO option via ``--with-lto=thin``. (Contributed by Dong-hee Na and Brett Holman in :issue:`44340`.) +* libpython is no longer linked against libcrypt. + (Contributed by Mike Gilbert in :issue:`45433`.) + +* Building Python now requires a C99 ```` header file providing + the following functions: ``copysign()``, ``hypot()``, ``isfinite()``, + ``isinf()``, ``isnan()``, ``round()``. + (Contributed by Victor Stinner in :issue:`45440`.) + +* Freelists for object structs can now be disabled. A new :program:`configure` + option :option:`!--without-freelists` can be used to disable all freelists + except empty tuple singleton. + (Contributed by Christian Heimes in :issue:`45522`) C API Changes ============= @@ -455,6 +498,11 @@ New Features * Add a new :c:func:`PyType_GetQualName` function to get type's qualified name. (Contributed by Hai Shi in :issue:`42035`.) +* Add new :c:func:`PyThreadState_EnterTracing` and + :c:func:`PyThreadState_LeaveTracing` functions to the limited C API to + suspend and resume tracing and profiling. + (Contributed by Victor Stinner in :issue:`43760`.) + * Add a new :c:func:`PyStructSequence_FromModuleAndDesc` function to create a new struct sequence type from the desc. (Contributed by Hai Shi in :issue:`45113`.) @@ -541,15 +589,22 @@ Porting to Python 3.11 (Contributed by Victor Stinner in :issue:`39573`.) -Deprecated ----------- +* ```` no longer includes the header files ````, + ````, ```` and ```` when the ``Py_LIMITED_API`` + macro is set to ``0x030b0000`` (Python 3.11) or higher. C extensions should + explicitly include the header files after ``#include ``. + (Contributed by Victor Stinner in :issue:`45434`.) -Removed -------- +* The non-limited API files ``cellobject.h``, ``classobject.h``, ``context.h``, + ``funcobject.h``, ``genobject.h`` and ``longintrepr.h`` have been moved to + the ``Include/cpython`` directory. Moreover, the ``eval.h`` header file was + removed. These files must not be included directly, as they are already + included in ``Python.h``: :ref:`Include Files `. If they have + been included directly, consider including ``Python.h`` instead. + (Contributed by Victor Stinner in :issue:`35134`.) -* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been - removed. - (Contributed by Mark Shannon in :issue:`40222`.) +Deprecated +---------- * Deprecate the following functions to configure the Python initialization: @@ -566,3 +621,51 @@ Removed Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). (Contributed by Victor Stinner in :issue:`44113`.) + +Removed +------- + +* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been + removed. + (Contributed by Mark Shannon in :issue:`40222`.) + +* Remove the following math macros using the ``errno`` variable: + + * ``Py_ADJUST_ERANGE1()`` + * ``Py_ADJUST_ERANGE2()`` + * ``Py_OVERFLOWED()`` + * ``Py_SET_ERANGE_IF_OVERFLOW()`` + * ``Py_SET_ERRNO_ON_MATH_ERROR()`` + + (Contributed by Victor Stinner in :issue:`45412`.) + +* Remove ``Py_UNICODE_COPY()`` and ``Py_UNICODE_FILL()`` macros, deprecated + since Python 3.3. Use ``PyUnicode_CopyCharacters()`` or ``memcpy()`` + (``wchar_t*`` string), and ``PyUnicode_Fill()`` functions instead. + (Contributed by Victor Stinner in :issue:`41123`.) + +* Remove the ``pystrhex.h`` header file. It only contains private functions. + C extensions should only include the main ```` header file. + (Contributed by Victor Stinner in :issue:`45434`.) + +* Remove the ``Py_FORCE_DOUBLE()`` macro. It was used by the + ``Py_IS_INFINITY()`` macro. + (Contributed by Victor Stinner in :issue:`45440`.) + +* The following items are no longer available when :c:macro:`Py_LIMITED_API` + is defined: + + * :c:func:`PyMarshal_WriteLongToFile` + * :c:func:`PyMarshal_WriteObjectToFile` + * :c:func:`PyMarshal_ReadObjectFromString` + * :c:func:`PyMarshal_WriteObjectToString` + * the ``Py_MARSHAL_VERSION`` macro + + These are not part of the :ref:`limited API `. + + (Contributed by Victor Stinner in :issue:`45474`.) + +* Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never + worked since the :c:type:`PyWeakReference` structure is opaque in the + limited C API. + (Contributed by Victor Stinner in :issue:`35134`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index f1725e7df02204..941fae5fb87813 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -671,7 +671,7 @@ time On AIX, :func:`~time.thread_time` is now implemented with ``thread_cputime()`` which has nanosecond resolution, rather than -``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` which has a resolution of 10 ms. +``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` which has a resolution of 10 milliseconds. (Contributed by Batuhan Taskaya in :issue:`40192`) sys diff --git a/Grammar/python.gram b/Grammar/python.gram index 97114b20f5dc64..4443a5fda7c245 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1057,7 +1057,7 @@ invalid_arguments: RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, comprehension_ty)->target, "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")} - | a=args for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a) } + | a=args b=for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a, b) } | args ',' a=expression b=for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") } | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } diff --git a/Include/Python.h b/Include/Python.h index a83befa3117361..6e3303ac9a3b0c 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -1,88 +1,54 @@ +// Entry point of the Python C API. +// C extensions should only #include , and not include directly +// the other Python header files included by . + #ifndef Py_PYTHON_H #define Py_PYTHON_H -/* Since this is a "meta-include" file, no #ifdef __cplusplus / extern "C" { */ -/* Include nearly all Python header files */ +// Since this is a "meta-include" file, no #ifdef __cplusplus / extern "C" { +// Include Python header files #include "patchlevel.h" #include "pyconfig.h" #include "pymacconfig.h" -#include - -#ifndef UCHAR_MAX -#error "Something's broken. UCHAR_MAX should be defined in limits.h." -#endif - -#if UCHAR_MAX != 255 -#error "Python's source code assumes C's unsigned char is an 8-bit type." -#endif - #if defined(__sgi) && !defined(_SGI_MP_SOURCE) -#define _SGI_MP_SOURCE +# define _SGI_MP_SOURCE #endif -#include -#ifndef NULL -# error "Python.h requires that stdio.h define NULL." +// stdlib.h, stdio.h, errno.h and string.h headers are not used by Python +// headers, but kept for backward compatibility. They are excluded from the +// limited C API of Python 3.11. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# include +# include // FILE* +# include // errno +# include // memcpy() #endif - -#include -#ifdef HAVE_ERRNO_H -#include -#endif -#include #ifndef MS_WINDOWS -#include +# include #endif - -/* For size_t? */ #ifdef HAVE_STDDEF_H -#include +# include // size_t #endif -/* CAUTION: Build setups should ensure that NDEBUG is defined on the - * compiler command line when building Python in release mode; else - * assert() calls won't be removed. - */ -#include +#include // assert() +#include // wchar_t #include "pyport.h" #include "pymacro.h" - -/* A convenient way for code to know if sanitizers are enabled. */ -#if defined(__has_feature) -# if __has_feature(memory_sanitizer) -# if !defined(_Py_MEMORY_SANITIZER) -# define _Py_MEMORY_SANITIZER -# endif -# endif -# if __has_feature(address_sanitizer) -# if !defined(_Py_ADDRESS_SANITIZER) -# define _Py_ADDRESS_SANITIZER -# endif -# endif -#elif defined(__GNUC__) -# if defined(__SANITIZE_ADDRESS__) -# define _Py_ADDRESS_SANITIZER -# endif -#endif - #include "pymath.h" #include "pymem.h" - #include "object.h" #include "objimpl.h" #include "typeslots.h" #include "pyhash.h" - #include "cpython/pydebug.h" - #include "bytearrayobject.h" #include "bytesobject.h" #include "unicodeobject.h" #include "longobject.h" -#include "longintrepr.h" +#include "cpython/longintrepr.h" #include "boolobject.h" #include "floatobject.h" #include "complexobject.h" @@ -96,34 +62,30 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" -#include "funcobject.h" -#include "classobject.h" +#include "cpython/funcobject.h" +#include "cpython/classobject.h" #include "fileobject.h" #include "pycapsule.h" #include "code.h" #include "pyframe.h" #include "traceback.h" #include "sliceobject.h" -#include "cellobject.h" +#include "cpython/cellobject.h" #include "iterobject.h" -#include "genobject.h" +#include "pystate.h" +#include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" -#include "namespaceobject.h" #include "cpython/picklebufobject.h" #include "cpython/pytime.h" - #include "codecs.h" #include "pyerrors.h" - #include "cpython/initconfig.h" #include "pythread.h" -#include "pystate.h" -#include "context.h" - +#include "cpython/context.h" #include "modsupport.h" #include "compile.h" #include "pythonrun.h" @@ -133,12 +95,8 @@ #include "osmodule.h" #include "intrcheck.h" #include "import.h" - #include "abstract.h" #include "bltinmodule.h" - -#include "eval.h" - #include "cpython/pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" diff --git a/Include/abstract.h b/Include/abstract.h index e8d3f9293b98d3..9e06fbbb749138 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -318,7 +318,7 @@ PyAPI_FUNC(int) PyObject_DelItem(PyObject *o, PyObject *key); /* Takes an arbitrary object which must support the (character, single segment) buffer interface and returns a pointer to a read-only memory location - useable as character based input for subsequent processing. + usable as character based input for subsequent processing. Return 0 on success. buffer and buffer_len are only set in case no error occurs. Otherwise, -1 is returned and an exception set. */ @@ -863,7 +863,7 @@ PyAPI_FUNC(int) PyObject_IsSubclass(PyObject *object, PyObject *typeorclass); #ifndef Py_LIMITED_API # define Py_CPYTHON_ABSTRACTOBJECT_H -# include "cpython/abstract.h" +# include "cpython/abstract.h" # undef Py_CPYTHON_ABSTRACTOBJECT_H #endif diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index 9e95433f0f26f0..ae2bde1c303565 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -6,8 +6,6 @@ extern "C" { #endif -#include - /* Type PyByteArrayObject represents a mutable array of bytes. * The Python API is that of a sequence; * the bytes are mapped to ints in [0, 256). @@ -36,7 +34,7 @@ PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); #ifndef Py_LIMITED_API # define Py_CPYTHON_BYTEARRAYOBJECT_H -# include "cpython/bytearrayobject.h" +# include "cpython/bytearrayobject.h" # undef Py_CPYTHON_BYTEARRAYOBJECT_H #endif diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 39c241a2dcf5f7..4c4dc40d705d71 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -7,7 +7,7 @@ extern "C" { #endif -#include +#include // va_list /* Type PyBytesObject represents a byte string. An extra zero byte is @@ -59,7 +59,7 @@ PyAPI_FUNC(int) PyBytes_AsStringAndSize( #ifndef Py_LIMITED_API # define Py_CPYTHON_BYTESOBJECT_H -# include "cpython/bytesobject.h" +# include "cpython/bytesobject.h" # undef Py_CPYTHON_BYTESOBJECT_H #endif diff --git a/Include/ceval.h b/Include/ceval.h index 0f687666e2bccf..1b57f6ea20f6f0 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -1,3 +1,5 @@ +/* Interface to random parts in ceval.c */ + #ifndef Py_CEVAL_H #define Py_CEVAL_H #ifdef __cplusplus @@ -5,7 +7,15 @@ extern "C" { #endif -/* Interface to random parts in ceval.c */ +PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyObject *, PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co, + PyObject *globals, + PyObject *locals, + PyObject *const *args, int argc, + PyObject *const *kwds, int kwdc, + PyObject *const *defs, int defc, + PyObject *kwdefs, PyObject *closure); /* PyEval_CallObjectWithKeywords(), PyEval_CallObject(), PyEval_CallFunction * and PyEval_CallMethod are deprecated. Since they are officially part of the @@ -148,7 +158,7 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); #ifndef Py_LIMITED_API # define Py_CPYTHON_CEVAL_H -# include "cpython/ceval.h" +# include "cpython/ceval.h" # undef Py_CPYTHON_CEVAL_H #endif diff --git a/Include/code.h b/Include/code.h index b9e23eb816529b..2dea3c26105318 100644 --- a/Include/code.h +++ b/Include/code.h @@ -10,7 +10,7 @@ typedef struct PyCodeObject PyCodeObject; #ifndef Py_LIMITED_API # define Py_CPYTHON_CODE_H -# include "cpython/code.h" +# include "cpython/code.h" # undef Py_CPYTHON_CODE_H #endif diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index db850219645281..55a742c31fada0 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -58,71 +58,13 @@ PyVectorcall_NARGS(size_t n) return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET; } -static inline vectorcallfunc -PyVectorcall_Function(PyObject *callable) -{ - PyTypeObject *tp; - Py_ssize_t offset; - vectorcallfunc ptr; - - assert(callable != NULL); - tp = Py_TYPE(callable); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { - return NULL; - } - assert(PyCallable_Check(callable)); - offset = tp->tp_vectorcall_offset; - assert(offset > 0); - memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); - return ptr; -} - -/* Call the callable object 'callable' with the "vectorcall" calling - convention. - - args is a C array for positional arguments. - - nargsf is the number of positional arguments plus optionally the flag - PY_VECTORCALL_ARGUMENTS_OFFSET which means that the caller is allowed to - modify args[-1]. - - kwnames is a tuple of keyword names. The values of the keyword arguments - are stored in "args" after the positional arguments (note that the number - of keyword arguments does not change nargsf). kwnames can also be NULL if - there are no keyword arguments. - - keywords must only contain strings and all keys must be unique. +PyAPI_FUNC(vectorcallfunc) PyVectorcall_Function(PyObject *callable); - Return the result on success. Raise an exception and return NULL on - error. */ -static inline PyObject * -_PyObject_VectorcallTstate(PyThreadState *tstate, PyObject *callable, - PyObject *const *args, size_t nargsf, - PyObject *kwnames) -{ - vectorcallfunc func; - PyObject *res; - - assert(kwnames == NULL || PyTuple_Check(kwnames)); - assert(args != NULL || PyVectorcall_NARGS(nargsf) == 0); - - func = PyVectorcall_Function(callable); - if (func == NULL) { - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(tstate, callable, args, nargs, kwnames); - } - res = func(callable, args, nargsf, kwnames); - return _Py_CheckFunctionResult(tstate, callable, res, NULL); -} - -static inline PyObject * -PyObject_Vectorcall(PyObject *callable, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyThreadState *tstate = PyThreadState_Get(); - return _PyObject_VectorcallTstate(tstate, callable, - args, nargsf, kwnames); -} +PyAPI_FUNC(PyObject *) PyObject_Vectorcall( + PyObject *callable, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames); // Backwards compatibility aliases for API that was provisional in Python 3.8 #define _PyObject_Vectorcall PyObject_Vectorcall @@ -145,44 +87,13 @@ PyAPI_FUNC(PyObject *) PyObject_VectorcallDict( "tuple" and keyword arguments "dict". "dict" may also be NULL */ PyAPI_FUNC(PyObject *) PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict); -static inline PyObject * -_PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const *args, Py_ssize_t nargs) -{ - return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); -} - -/* Same as PyObject_Vectorcall except without keyword arguments */ -static inline PyObject * -_PyObject_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs) -{ - PyThreadState *tstate = PyThreadState_Get(); - return _PyObject_FastCallTstate(tstate, func, args, nargs); -} - -/* Call a callable without any arguments - Private static inline function variant of public function - PyObject_CallNoArgs(). */ -static inline PyObject * -_PyObject_CallNoArg(PyObject *func) { - PyThreadState *tstate = PyThreadState_Get(); - return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); -} - -static inline PyObject * -PyObject_CallOneArg(PyObject *func, PyObject *arg) -{ - PyObject *_args[2]; - PyObject **args; - PyThreadState *tstate; - size_t nargsf; +// Same as PyObject_Vectorcall(), except without keyword arguments +PyAPI_FUNC(PyObject *) _PyObject_FastCall( + PyObject *func, + PyObject *const *args, + Py_ssize_t nargs); - assert(arg != NULL); - args = _args + 1; // For PY_VECTORCALL_ARGUMENTS_OFFSET - args[0] = arg; - tstate = PyThreadState_Get(); - nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallTstate(tstate, func, args, nargsf, NULL); -} +PyAPI_FUNC(PyObject *) PyObject_CallOneArg(PyObject *func, PyObject *arg); PyAPI_FUNC(PyObject *) PyObject_VectorcallMethod( PyObject *name, PyObject *const *args, diff --git a/Include/cellobject.h b/Include/cpython/cellobject.h similarity index 89% rename from Include/cellobject.h rename to Include/cpython/cellobject.h index 81bc784d36f3e0..8dc7b8f4cf6f88 100644 --- a/Include/cellobject.h +++ b/Include/cpython/cellobject.h @@ -1,4 +1,5 @@ /* Cell object interface */ + #ifndef Py_LIMITED_API #ifndef Py_CELLOBJECT_H #define Py_CELLOBJECT_H @@ -8,7 +9,8 @@ extern "C" { typedef struct { PyObject_HEAD - PyObject *ob_ref; /* Content of the cell or NULL when empty */ + /* Content of the cell or NULL when empty */ + PyObject *ob_ref; } PyCellObject; PyAPI_DATA(PyTypeObject) PyCell_Type; diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 44b78f6d223120..caf64401307c07 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -2,6 +2,8 @@ # error "this header file must not be included directly" #endif +PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args); + PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); diff --git a/Include/classobject.h b/Include/cpython/classobject.h similarity index 96% rename from Include/classobject.h rename to Include/cpython/classobject.h index 1952f673b7d865..80df8842eb4f78 100644 --- a/Include/classobject.h +++ b/Include/cpython/classobject.h @@ -53,5 +53,5 @@ PyAPI_FUNC(PyObject *) PyInstanceMethod_Function(PyObject *); #ifdef __cplusplus } #endif -#endif /* !Py_CLASSOBJECT_H */ -#endif /* Py_LIMITED_API */ +#endif // !Py_CLASSOBJECT_H +#endif // !Py_LIMITED_API diff --git a/Include/context.h b/Include/cpython/context.h similarity index 99% rename from Include/context.h rename to Include/cpython/context.h index 4e5007089dd94b..4db079f7633f48 100644 --- a/Include/context.h +++ b/Include/cpython/context.h @@ -1,12 +1,10 @@ +#ifndef Py_LIMITED_API #ifndef Py_CONTEXT_H #define Py_CONTEXT_H #ifdef __cplusplus extern "C" { #endif -#ifndef Py_LIMITED_API - - PyAPI_DATA(PyTypeObject) PyContext_Type; typedef struct _pycontextobject PyContext; @@ -73,9 +71,8 @@ PyAPI_FUNC(int) PyContextVar_Reset(PyObject *var, PyObject *token); PyAPI_FUNC(PyObject *) _PyContext_NewHamtForTests(void); -#endif /* !Py_LIMITED_API */ - #ifdef __cplusplus } #endif #endif /* !Py_CONTEXT_H */ +#endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 7c63374c566c7c..e97969be4de0cc 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -3,6 +3,7 @@ #endif typedef struct _dictkeysobject PyDictKeysObject; +typedef struct _dictvalues PyDictValues; /* The ma_values pointer is NULL for a combined table * or points to an array of PyObject* for a split table @@ -22,9 +23,9 @@ typedef struct { /* If ma_values is NULL, the table is "combined": keys and values are stored in ma_keys. - If ma_values is not NULL, the table is splitted: + If ma_values is not NULL, the table is split: keys are stored in ma_keys and values are stored in ma_values */ - PyObject **ma_values; + PyDictValues *ma_values; } PyDictObject; PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, diff --git a/Include/cpython/floatobject.h b/Include/cpython/floatobject.h new file mode 100644 index 00000000000000..fffd468690274e --- /dev/null +++ b/Include/cpython/floatobject.h @@ -0,0 +1,12 @@ +#ifndef Py_CPYTHON_FLOATOBJECT_H +# error "this header file must not be included directly" +#endif + +typedef struct { + PyObject_HEAD + double ob_fval; +} PyFloatObject; + +// Macro version of PyFloat_AsDouble() trading safety for speed. +// It doesn't check if op is a double object. +#define PyFloat_AS_DOUBLE(op) (((PyFloatObject *)(op))->ob_fval) diff --git a/Include/funcobject.h b/Include/cpython/funcobject.h similarity index 99% rename from Include/funcobject.h rename to Include/cpython/funcobject.h index 6bc03f57d4cb39..60b702218a1f40 100644 --- a/Include/funcobject.h +++ b/Include/cpython/funcobject.h @@ -1,5 +1,5 @@ - /* Function object interface */ + #ifndef Py_LIMITED_API #ifndef Py_FUNCOBJECT_H #define Py_FUNCOBJECT_H @@ -76,7 +76,6 @@ PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *); PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); -#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( PyObject *func, PyObject *const *stack, @@ -84,7 +83,6 @@ PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( PyObject *kwnames); uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); -#endif /* Macros for direct access to these values. Type checks are *not* done, so use with care. */ diff --git a/Include/genobject.h b/Include/cpython/genobject.h similarity index 93% rename from Include/genobject.h rename to Include/cpython/genobject.h index 55a8b34afd60ed..8f87cf5fff7579 100644 --- a/Include/genobject.h +++ b/Include/cpython/genobject.h @@ -1,4 +1,3 @@ - /* Generator object interface */ #ifndef Py_LIMITED_API @@ -8,8 +7,7 @@ extern "C" { #endif -#include "pystate.h" /* _PyErr_StackItem */ -#include "abstract.h" /* PySendResult */ +/* --- Generators --------------------------------------------------------- */ /* _PyGenObject_HEAD defines the initial segment of generator and coroutine objects. */ @@ -45,7 +43,9 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyObject *_PyGen_yf(PyGenObject *); PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); -#ifndef Py_LIMITED_API + +/* --- PyCoroObject ------------------------------------------------------- */ + typedef struct { _PyGenObject_HEAD(cr) PyObject *cr_origin; @@ -59,7 +59,8 @@ PyObject *_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); -/* Asynchronous Generators */ + +/* --- Asynchronous Generators -------------------------------------------- */ typedef struct { _PyGenObject_HEAD(ag) @@ -89,7 +90,6 @@ PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *_PyAsyncGenValueWrapperNew(PyObject *); -#endif #undef _PyGenObject_HEAD diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 65d52c45783f18..05641001bcd749 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -184,6 +184,7 @@ typedef struct PyConfig { /* --- Path configuration outputs ----------- */ int module_search_paths_set; PyWideStringList module_search_paths; + wchar_t *stdlib_dir; wchar_t *executable; wchar_t *base_executable; wchar_t *prefix; diff --git a/Include/longintrepr.h b/Include/cpython/longintrepr.h similarity index 100% rename from Include/longintrepr.h rename to Include/cpython/longintrepr.h diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h new file mode 100644 index 00000000000000..1a73799d658fe0 --- /dev/null +++ b/Include/cpython/longobject.h @@ -0,0 +1,95 @@ +#ifndef Py_CPYTHON_LONGOBJECT_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); + +PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); + +/* _PyLong_Frexp returns a double x and an exponent e such that the + true value is approximately equal to x * 2**e. e is >= 0. x is + 0.0 if and only if the input is 0 (in which case, e and x are both + zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is + possible if the number of bits doesn't fit into a Py_ssize_t, sets + OverflowError and returns -1.0 for x, 0 for e. */ +PyAPI_FUNC(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); + +PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); +PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); + +/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. + v must not be NULL, and must be a normalized long. + There are no error cases. +*/ +PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); + +/* _PyLong_NumBits. Return the number of bits needed to represent the + absolute value of a long. For example, this returns 1 for 1 and -1, 2 + for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. + v must not be NULL, and must be a normalized long. + (size_t)-1 is returned and OverflowError set if the true result doesn't + fit in a size_t. +*/ +PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); + +/* _PyLong_DivmodNear. Given integers a and b, compute the nearest + integer q to the exact quotient a / b, rounding to the nearest even integer + in the case of a tie. Return (q, r), where r = a - q*b. The remainder r + will satisfy abs(r) <= abs(b)/2, with equality possible only if q is + even. +*/ +PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); + +/* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in + base 256, and return a Python int with the same numeric value. + If n is 0, the integer is 0. Else: + If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; + else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the + LSB. + If is_signed is 0/false, view the bytes as a non-negative integer. + If is_signed is 1/true, view the bytes as a 2's-complement integer, + non-negative if bit 0x80 of the MSB is clear, negative if set. + Error returns: + + Return NULL with the appropriate exception set if there's not + enough memory to create the Python int. +*/ +PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( + const unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long + v to a base-256 integer, stored in array bytes. Normally return 0, + return -1 on error. + If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at + bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and + the LSB at bytes[n-1]. + If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes + are filled and there's nothing special about bit 0x80 of the MSB. + If is_signed is 1/true, bytes is filled with the 2's-complement + representation of v's value. Bit 0x80 of the MSB is the sign bit. + Error returns (-1): + + is_signed is 0 and v < 0. TypeError is set in this case, and bytes + isn't altered. + + n isn't big enough to hold the full mathematical value of v. For + example, if is_signed is 0 and there are more digits in the v than + fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of + being large enough to hold a sign bit. OverflowError is set in this + case, but bytes holds the least-significant n bytes of the true value. +*/ +PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, + unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +/* _PyLong_Format: Convert the long to a string object with given base, + appending a base prefix of 0[box] if base is 2, 8 or 16. */ +PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); + +/* For use by the gcd function in mathmodule.c */ +PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); +PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 5ae6f367c6048a..3a8a256e3b9ae6 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -270,6 +270,7 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + Py_ssize_t tp_inline_values_offset; }; /* The *real* layout of a type object when allocated on the heap */ @@ -289,6 +290,7 @@ typedef struct _heaptypeobject { PyObject *ht_name, *ht_slots, *ht_qualname; struct _dictkeysobject *ht_cached_keys; PyObject *ht_module; + char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec /* here are optional user slots, followed by the members. */ } PyHeapTypeObject; diff --git a/Include/cpython/pydebug.h b/Include/cpython/pydebug.h index 78bcb118be4659..cab799f0b38e0c 100644 --- a/Include/cpython/pydebug.h +++ b/Include/cpython/pydebug.h @@ -29,7 +29,7 @@ PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; /* this is a wrapper around getenv() that pays attention to Py_IgnoreEnvironmentFlag. It should be used for getting variables like PYTHONPATH and PYTHONHOME from the environment */ -#define Py_GETENV(s) (Py_IgnoreEnvironmentFlag ? NULL : getenv(s)) +PyAPI_DATA(char*) Py_GETENV(const char *name); #ifdef __cplusplus } diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ab4bf8bf8483c7..528d2a2998c997 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -185,6 +185,13 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); PyAPI_FUNC(PyObject *) _PyThreadState_GetDict(PyThreadState *tstate); +// Disable tracing and profiling. +PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); + +// Reset tracing and profiling: enable them if a trace function or a profile +// function is set, otherwise disable them. +PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread @@ -263,7 +270,7 @@ PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( PyAPI_FUNC(int) _PyInterpreterState_SetConfig( const struct PyConfig *config); -// Get the configuration of the currrent interpreter. +// Get the configuration of the current interpreter. // The caller must hold the GIL. PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h index 8c2958501f7967..23d4f16a8fd847 100644 --- a/Include/cpython/pytime.h +++ b/Include/cpython/pytime.h @@ -1,3 +1,46 @@ +// The _PyTime_t API is written to use timestamp and timeout values stored in +// various formats and to read clocks. +// +// The _PyTime_t type is an integer to support directly common arithmetic +// operations like t1 + t2. +// +// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type +// is signed to support negative timestamps. The supported range is around +// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the +// supported date range is around [1677-09-21; 2262-04-11]. +// +// Formats: +// +// * seconds +// * seconds as a floating pointer number (C double) +// * milliseconds (10^-3 seconds) +// * microseconds (10^-6 seconds) +// * 100 nanoseconds (10^-7 seconds) +// * nanoseconds (10^-9 seconds) +// * timeval structure, 1 microsecond resolution (10^-6 seconds) +// * timespec structure, 1 nanosecond resolution (10^-9 seconds) +// +// Integer overflows are detected and raise OverflowError. Conversion to a +// resolution worse than 1 nanosecond is rounded correctly with the requested +// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling +// (towards +inf), half even and up (away from zero). +// +// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so +// the caller doesn't have to handle errors and doesn't need to hold the GIL. +// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on +// overflow. +// +// Clocks: +// +// * System clock +// * Monotonic clock +// * Performance counter +// +// Operations like (t * k / q) with integers are implemented in a way to reduce +// the risk of integer overflow. Such operation is used to convert a clock +// value expressed in ticks with a frequency to _PyTime_t, like +// QueryPerformanceCounter() with QueryPerformanceFrequency(). + #ifndef Py_LIMITED_API #ifndef Py_PYTIME_H #define Py_PYTIME_H @@ -14,8 +57,11 @@ extern "C" { store a duration, and so indirectly a date (related to another date, like UNIX epoch). */ typedef int64_t _PyTime_t; +// _PyTime_MIN nanoseconds is around -292.3 years #define _PyTime_MIN INT64_MIN +// _PyTime_MAX nanoseconds is around +292.3 years #define _PyTime_MAX INT64_MAX +#define _SIZEOF_PYTIME_T 8 typedef enum { /* Round towards minus infinity (-inf). @@ -88,13 +134,13 @@ PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, PyObject *obj); -/* Convert a number of seconds (Python float or int) to a timetamp. +/* Convert a number of seconds (Python float or int) to a timestamp. Raise an exception and return -1 on error, return 0 on success. */ PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round); -/* Convert a number of milliseconds (Python float or int, 10^-3) to a timetamp. +/* Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. Raise an exception and return -1 on error, return 0 on success. */ PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, @@ -124,9 +170,11 @@ PyAPI_FUNC(_PyTime_t) _PyTime_As100Nanoseconds(_PyTime_t t, object. */ PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); +#ifndef MS_WINDOWS /* Create a timestamp from a timeval structure. Raise an exception and return -1 on overflow, return 0 on success. */ PyAPI_FUNC(int) _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +#endif /* Convert a timestamp to a timeval structure (microsecond resolution). tv_usec is always positive. @@ -136,8 +184,9 @@ PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); -/* Similar to _PyTime_AsTimeval(), but don't raise an exception on error. */ -PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t, +/* Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. + On overflow, clamp tv_sec to _PyTime_t min/max. */ +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); @@ -162,9 +211,18 @@ PyAPI_FUNC(int) _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); tv_nsec is always positive. Raise an exception and return -1 on error, return 0 on success. */ PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); + +/* Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. + On overflow, clamp tv_sec to _PyTime_t min/max. */ +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); #endif + +// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +PyAPI_FUNC(_PyTime_t) _PyTime_Add(_PyTime_t t1, _PyTime_t t2); + /* Compute ticks * mul / div. + Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. The caller must ensure that ((div - 1) * mul) cannot overflow. */ PyAPI_FUNC(_PyTime_t) _PyTime_MulDiv(_PyTime_t ticks, _PyTime_t mul, @@ -181,8 +239,8 @@ typedef struct { /* Get the current time from the system clock. If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and truncated the clock to - _PyTime_MIN or _PyTime_MAX. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. Use _PyTime_GetSystemClockWithInfo() to check for failure. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); @@ -201,8 +259,8 @@ PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( results of consecutive calls is valid. If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and truncated the clock to - _PyTime_MIN or _PyTime_MAX. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); @@ -232,8 +290,8 @@ PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); measure a short duration. If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and truncated the clock to - _PyTime_MIN or _PyTime_MAX. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); @@ -248,6 +306,15 @@ PyAPI_FUNC(int) _PyTime_GetPerfCounterWithInfo( _PyTime_t *t, _Py_clock_info_t *info); + +// Create a deadline. +// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); + +// Get remaining time from a deadline. +// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); + #ifdef __cplusplus } #endif diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index b40e2ea1011f5c..ab4aebf5e70b93 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -50,19 +50,6 @@ Py_UNICODE_ISDIGIT(ch) || \ Py_UNICODE_ISNUMERIC(ch)) -Py_DEPRECATED(3.3) static inline void -Py_UNICODE_COPY(Py_UNICODE *target, const Py_UNICODE *source, Py_ssize_t length) { - memcpy(target, source, (size_t)(length) * sizeof(Py_UNICODE)); -} - -Py_DEPRECATED(3.3) static inline void -Py_UNICODE_FILL(Py_UNICODE *target, Py_UNICODE value, Py_ssize_t length) { - Py_ssize_t i; - for (i = 0; i < length; i++) { - target[i] = value; - } -} - /* macros to work with surrogates */ #define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF) #define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF) @@ -790,17 +777,35 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( /* --- Unicode-Escape Codecs ---------------------------------------------- */ +/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed /* bytes consumed */ +); /* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape chars. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape( +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( const char *string, /* Unicode-Escape encoded string */ Py_ssize_t length, /* size of string */ const char *errors, /* error handling */ + Py_ssize_t *consumed, /* bytes consumed */ const char **first_invalid_escape /* on return, points to first invalid escaped char in string. */ ); +/* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */ + +/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeRawUnicodeEscapeStateful( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed /* bytes consumed */ +); + /* --- Latin-1 Codecs ----------------------------------------------------- */ PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( diff --git a/Include/cpython/warnings.h b/Include/cpython/warnings.h new file mode 100644 index 00000000000000..2ef8e3ce9435f4 --- /dev/null +++ b/Include/cpython/warnings.h @@ -0,0 +1,20 @@ +#ifndef Py_CPYTHON_WARNINGS_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(int) PyErr_WarnExplicitObject( + PyObject *category, + PyObject *message, + PyObject *filename, + int lineno, + PyObject *module, + PyObject *registry); + +PyAPI_FUNC(int) PyErr_WarnExplicitFormat( + PyObject *category, + const char *filename, int lineno, + const char *module, PyObject *registry, + const char *format, ...); + +// DEPRECATED: Use PyErr_WarnEx() instead. +#define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1) diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h new file mode 100644 index 00000000000000..9efcc412df9bed --- /dev/null +++ b/Include/cpython/weakrefobject.h @@ -0,0 +1,47 @@ +#ifndef Py_CPYTHON_WEAKREFOBJECT_H +# error "this header file must not be included directly" +#endif + +/* PyWeakReference is the base struct for the Python ReferenceType, ProxyType, + * and CallableProxyType. + */ +struct _PyWeakReference { + PyObject_HEAD + + /* The object to which this is a weak reference, or Py_None if none. + * Note that this is a stealth reference: wr_object's refcount is + * not incremented to reflect this pointer. + */ + PyObject *wr_object; + + /* A callable to invoke when wr_object dies, or NULL if none. */ + PyObject *wr_callback; + + /* A cache for wr_object's hash code. As usual for hashes, this is -1 + * if the hash code isn't known yet. + */ + Py_hash_t hash; + + /* If wr_object is weakly referenced, wr_object has a doubly-linked NULL- + * terminated list of weak references to it. These are the list pointers. + * If wr_object goes away, wr_object is set to Py_None, and these pointers + * have no meaning then. + */ + PyWeakReference *wr_prev; + PyWeakReference *wr_next; +}; + +PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); + +PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); + +/* Explanation for the Py_REFCNT() check: when a weakref's target is part + of a long chain of deallocations which triggers the trashcan mechanism, + clearing the weakrefs can be delayed long after the target's refcount + has dropped to zero. In the meantime, code accessing the weakref will + be able to "see" the target object even though it is supposed to be + unreachable. See issue #16602. */ +#define PyWeakref_GET_OBJECT(ref) \ + (Py_REFCNT(((PyWeakReference *)(ref))->wr_object) > 0 \ + ? ((PyWeakReference *)(ref))->wr_object \ + : Py_None) diff --git a/Include/dictobject.h b/Include/dictobject.h index da5a36ba07f32e..a6233d8ae2512a 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -87,7 +87,7 @@ PyAPI_DATA(PyTypeObject) PyDictRevIterValue_Type; #ifndef Py_LIMITED_API # define Py_CPYTHON_DICTOBJECT_H -# include "cpython/dictobject.h" +# include "cpython/dictobject.h" # undef Py_CPYTHON_DICTOBJECT_H #endif diff --git a/Include/eval.h b/Include/eval.h deleted file mode 100644 index eda28df8f65281..00000000000000 --- a/Include/eval.h +++ /dev/null @@ -1,27 +0,0 @@ - -/* Interface to execute compiled code */ - -#ifndef Py_EVAL_H -#define Py_EVAL_H -#ifdef __cplusplus -extern "C" { -#endif - -PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyObject *, PyObject *, PyObject *); - -PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co, - PyObject *globals, - PyObject *locals, - PyObject *const *args, int argc, - PyObject *const *kwds, int kwdc, - PyObject *const *defs, int defc, - PyObject *kwdefs, PyObject *closure); - -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args); -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_EVAL_H */ diff --git a/Include/fileobject.h b/Include/fileobject.h index 6ec2994aa859b6..4c983e7b5daa8a 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -39,7 +39,7 @@ PyAPI_DATA(int) Py_UTF8Mode; #ifndef Py_LIMITED_API # define Py_CPYTHON_FILEOBJECT_H -# include "cpython/fileobject.h" +# include "cpython/fileobject.h" # undef Py_CPYTHON_FILEOBJECT_H #endif diff --git a/Include/fileutils.h b/Include/fileutils.h index 16f3b635deed89..ba5acc84fcb185 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -16,7 +16,7 @@ PyAPI_FUNC(char*) Py_EncodeLocale( #ifndef Py_LIMITED_API # define Py_CPYTHON_FILEUTILS_H -# include "cpython/fileutils.h" +# include "cpython/fileutils.h" # undef Py_CPYTHON_FILEUTILS_H #endif diff --git a/Include/floatobject.h b/Include/floatobject.h index e994aa8f29da48..3b6ca478eaef20 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -11,106 +11,44 @@ PyFloatObject represents a (double precision) floating point number. extern "C" { #endif -#ifndef Py_LIMITED_API -typedef struct { - PyObject_HEAD - double ob_fval; -} PyFloatObject; -#endif - PyAPI_DATA(PyTypeObject) PyFloat_Type; #define PyFloat_Check(op) PyObject_TypeCheck(op, &PyFloat_Type) #define PyFloat_CheckExact(op) Py_IS_TYPE(op, &PyFloat_Type) #ifdef Py_NAN -#define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) +# define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) #endif -#define Py_RETURN_INF(sign) do \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_HUGE_VAL); \ - } else { \ - return PyFloat_FromDouble(-Py_HUGE_VAL); \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(Py_HUGE_VAL); \ + } \ + else { \ + return PyFloat_FromDouble(-Py_HUGE_VAL); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); PyAPI_FUNC(double) PyFloat_GetMin(void); -PyAPI_FUNC(PyObject *) PyFloat_GetInfo(void); +PyAPI_FUNC(PyObject*) PyFloat_GetInfo(void); /* Return Python float from string PyObject. */ -PyAPI_FUNC(PyObject *) PyFloat_FromString(PyObject*); +PyAPI_FUNC(PyObject*) PyFloat_FromString(PyObject*); /* Return Python float from C double. */ -PyAPI_FUNC(PyObject *) PyFloat_FromDouble(double); +PyAPI_FUNC(PyObject*) PyFloat_FromDouble(double); /* Extract C double from Python float. The macro version trades safety for speed. */ -PyAPI_FUNC(double) PyFloat_AsDouble(PyObject *); -#ifndef Py_LIMITED_API -#define PyFloat_AS_DOUBLE(op) (((PyFloatObject *)(op))->ob_fval) -#endif +PyAPI_FUNC(double) PyFloat_AsDouble(PyObject*); #ifndef Py_LIMITED_API -/* _PyFloat_{Pack,Unpack}{4,8} - * - * The struct and pickle (at least) modules need an efficient platform- - * independent way to store floating-point values as byte strings. - * The Pack routines produce a string from a C double, and the Unpack - * routines produce a C double from such a string. The suffix (4 or 8) - * specifies the number of bytes in the string. - * - * On platforms that appear to use (see _PyFloat_Init()) IEEE-754 formats - * these functions work by copying bits. On other platforms, the formats the - * 4- byte format is identical to the IEEE-754 single precision format, and - * the 8-byte format to the IEEE-754 double precision format, although the - * packing of INFs and NaNs (if such things exist on the platform) isn't - * handled correctly, and attempting to unpack a string containing an IEEE - * INF or NaN will raise an exception. - * - * On non-IEEE platforms with more precision, or larger dynamic range, than - * 754 supports, not all values can be packed; on non-IEEE platforms with less - * precision, or smaller dynamic range, not all values can be unpacked. What - * happens in such cases is partly accidental (alas). - */ - -/* The pack routines write 2, 4 or 8 bytes, starting at p. le is a bool - * argument, true if you want the string in little-endian format (exponent - * last, at p+1, p+3 or p+7), false if you want big-endian format (exponent - * first, at p). - * Return value: 0 if all is OK, -1 if error (and an exception is - * set, most likely OverflowError). - * There are two problems on non-IEEE platforms: - * 1): What this does is undefined if x is a NaN or infinity. - * 2): -0.0 and +0.0 produce the same string. - */ -PyAPI_FUNC(int) _PyFloat_Pack2(double x, unsigned char *p, int le); -PyAPI_FUNC(int) _PyFloat_Pack4(double x, unsigned char *p, int le); -PyAPI_FUNC(int) _PyFloat_Pack8(double x, unsigned char *p, int le); - -/* The unpack routines read 2, 4 or 8 bytes, starting at p. le is a bool - * argument, true if the string is in little-endian format (exponent - * last, at p+1, p+3 or p+7), false if big-endian (exponent first, at p). - * Return value: The unpacked double. On error, this is -1.0 and - * PyErr_Occurred() is true (and an exception is set, most likely - * OverflowError). Note that on a non-IEEE platform this will refuse - * to unpack a string that represents a NaN or infinity. - */ -PyAPI_FUNC(double) _PyFloat_Unpack2(const unsigned char *p, int le); -PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); -PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); - -PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); - -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); -#endif /* Py_LIMITED_API */ +# define Py_CPYTHON_FLOATOBJECT_H +# include "cpython/floatobject.h" +# undef Py_CPYTHON_FLOATOBJECT_H +#endif #ifdef __cplusplus } diff --git a/Include/frameobject.h b/Include/frameobject.h index c118af1201a4c8..adb628f6314fcf 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -10,7 +10,7 @@ extern "C" { #ifndef Py_LIMITED_API # define Py_CPYTHON_FRAMEOBJECT_H -# include "cpython/frameobject.h" +# include "cpython/frameobject.h" # undef Py_CPYTHON_FRAMEOBJECT_H #endif diff --git a/Include/import.h b/Include/import.h index aeef3efd0bcee8..a87677bb10c7f4 100644 --- a/Include/import.h +++ b/Include/import.h @@ -88,7 +88,7 @@ PyAPI_FUNC(int) PyImport_AppendInittab( #ifndef Py_LIMITED_API # define Py_CPYTHON_IMPORT_H -# include "cpython/import.h" +# include "cpython/import.h" # undef Py_CPYTHON_IMPORT_H #endif diff --git a/Include/internal/pycore_asdl.h b/Include/internal/pycore_asdl.h index c0b07c31810b99..2929e030872d94 100644 --- a/Include/internal/pycore_asdl.h +++ b/Include/internal/pycore_asdl.h @@ -91,7 +91,7 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a (S)->typed_elements[_asdl_i] = (V); \ } while (0) #else -# define asdl_seq_SET(S, I, V) (S)->typed_elements[I] = (V) +# define asdl_seq_SET(S, I, V) ((void)((S)->typed_elements[I] = (V))) #endif #ifdef Py_DEBUG @@ -103,7 +103,7 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a (S)->elements[_asdl_i] = (V); \ } while (0) #else -# define asdl_seq_SET_UNTYPED(S, I, V) (S)->elements[I] = (V) +# define asdl_seq_SET_UNTYPED(S, I, V) ((void)((S)->elements[I] = (V))) #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index f7d856a54bd4a9..f2cfd2fd53ed9c 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_pystate.h" // _PyThreadState_GET() + PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend( PyThreadState *tstate, PyObject *callable, @@ -28,11 +30,88 @@ PyAPI_FUNC(PyObject *) _PyObject_Call( PyObject *args, PyObject *kwargs); + +// Static inline variant of public PyVectorcall_Function(). +static inline vectorcallfunc +_PyVectorcall_FunctionInline(PyObject *callable) +{ + assert(callable != NULL); + + PyTypeObject *tp = Py_TYPE(callable); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + + +/* Call the callable object 'callable' with the "vectorcall" calling + convention. + + args is a C array for positional arguments. + + nargsf is the number of positional arguments plus optionally the flag + PY_VECTORCALL_ARGUMENTS_OFFSET which means that the caller is allowed to + modify args[-1]. + + kwnames is a tuple of keyword names. The values of the keyword arguments + are stored in "args" after the positional arguments (note that the number + of keyword arguments does not change nargsf). kwnames can also be NULL if + there are no keyword arguments. + + keywords must only contain strings and all keys must be unique. + + Return the result on success. Raise an exception and return NULL on + error. */ static inline PyObject * -_PyObject_CallNoArgTstate(PyThreadState *tstate, PyObject *func) { +_PyObject_VectorcallTstate(PyThreadState *tstate, PyObject *callable, + PyObject *const *args, size_t nargsf, + PyObject *kwnames) +{ + vectorcallfunc func; + PyObject *res; + + assert(kwnames == NULL || PyTuple_Check(kwnames)); + assert(args != NULL || PyVectorcall_NARGS(nargsf) == 0); + + func = _PyVectorcall_FunctionInline(callable); + if (func == NULL) { + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + return _PyObject_MakeTpCall(tstate, callable, args, nargs, kwnames); + } + res = func(callable, args, nargsf, kwnames); + return _Py_CheckFunctionResult(tstate, callable, res, NULL); +} + + +static inline PyObject * +_PyObject_CallNoArgsTstate(PyThreadState *tstate, PyObject *func) { + return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); +} + + +// Private static inline function variant of public PyObject_CallNoArgs() +static inline PyObject * +_PyObject_CallNoArgs(PyObject *func) { + PyThreadState *tstate = _PyThreadState_GET(); return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); } + +static inline PyObject * +_PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const *args, Py_ssize_t nargs) +{ + return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); +} + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 66ddc991a9b11a..53580b99d33ac6 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -12,7 +12,8 @@ extern "C" { struct pyruntimestate; struct _ceval_runtime_state; -#include "pycore_interp.h" /* PyInterpreterState.eval_frame */ +#include "pycore_interp.h" // PyInterpreterState.eval_frame +#include "pycore_pystate.h" // _PyThreadState_GET() extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *); @@ -43,6 +44,9 @@ extern PyObject *_PyEval_BuiltinsFromGlobals( static inline PyObject* _PyEval_EvalFrame(PyThreadState *tstate, struct _interpreter_frame *frame, int throwflag) { + if (tstate->interp->eval_frame == NULL) { + return _PyEval_EvalFrameDefault(tstate, frame, throwflag); + } return tstate->interp->eval_frame(tstate, frame, throwflag); } @@ -90,7 +94,7 @@ static inline int _Py_EnterRecursiveCall(PyThreadState *tstate, } static inline int _Py_EnterRecursiveCall_inline(const char *where) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCall(tstate, where); } @@ -101,7 +105,7 @@ static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) { } static inline void _Py_LeaveRecursiveCall_inline(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); _Py_LeaveRecursiveCall(tstate); } diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0b127ed28993b7..622829fccdd0dd 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -35,6 +35,12 @@ typedef struct { PyObject *obj; } _PyObjectCache; +typedef struct { + uint32_t func_version; + uint16_t defaults_start; + uint16_t defaults_len; +} _PyCallCache; + /* Add specialized versions of entries to this union. * * Do not break the invariant: sizeof(SpecializedCacheEntry) == 8 @@ -51,6 +57,7 @@ typedef union { _PyAttrCache attr; _PyLoadGlobalCache load_global; _PyObjectCache obj; + _PyCallCache call; } SpecializedCacheEntry; #define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT)) @@ -246,53 +253,6 @@ PyAPI_FUNC(PyObject *) _PyCode_GetVarnames(PyCodeObject *); PyAPI_FUNC(PyObject *) _PyCode_GetCellvars(PyCodeObject *); PyAPI_FUNC(PyObject *) _PyCode_GetFreevars(PyCodeObject *); - -/* Cache hits and misses */ - -static inline uint8_t -saturating_increment(uint8_t c) -{ - return c<<1; -} - -static inline uint8_t -saturating_decrement(uint8_t c) -{ - return (c>>1) + 128; -} - -static inline uint8_t -saturating_zero(void) -{ - return 255; -} - -/* Starting value for saturating counter. - * Technically this should be 1, but that is likely to - * cause a bit of thrashing when we optimize then get an immediate miss. - * We want to give the counter a change to stabilize, so we start at 3. - */ -static inline uint8_t -saturating_start(void) -{ - return saturating_zero()<<3; -} - -static inline void -record_cache_hit(_PyAdaptiveEntry *entry) { - entry->counter = saturating_increment(entry->counter); -} - -static inline void -record_cache_miss(_PyAdaptiveEntry *entry) { - entry->counter = saturating_decrement(entry->counter); -} - -static inline int -too_many_cache_misses(_PyAdaptiveEntry *entry) { - return entry->counter == saturating_zero(); -} - #define ADAPTIVE_CACHE_BACKOFF 64 static inline void @@ -307,7 +267,9 @@ int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *nam int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr); -int _Py_Specialize_BinaryAdd(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr); +int _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr); +int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr); +int _Py_Specialize_CallFunction(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins); #define PRINT_SPECIALIZATION_STATS 0 #define PRINT_SPECIALIZATION_STATS_DETAILED 0 diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 2becc30beb4d8d..13cb7ccc7b8192 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -71,6 +71,14 @@ struct _dictkeysobject { see the DK_ENTRIES() macro */ }; +/* This must be no more than 16, for the order vector to fit in 64 bits */ +#define SHARED_KEYS_MAX_SIZE 16 + +struct _dictvalues { + uint64_t mv_order; + PyObject *values[1]; +}; + #define DK_LOG_SIZE(dk) ((dk)->dk_log2_size) #if SIZEOF_VOID_P > 4 #define DK_SIZE(dk) (((int64_t)1)< /* struct lconv */ +// This is used after getting NULL back from Py_DecodeLocale(). +#define DECODE_LOCALE_ERR(NAME, LEN) \ + ((LEN) == (size_t)-2) \ + ? _PyStatus_ERR("cannot decode " NAME) \ + : _PyStatus_NO_MEMORY() + PyAPI_DATA(int) _Py_HasFileSystemDefaultEncodeErrors; PyAPI_FUNC(int) _Py_DecodeUTF8Ex( @@ -33,6 +39,9 @@ PyAPI_FUNC(wchar_t*) _Py_DecodeUTF8_surrogateescape( Py_ssize_t arglen, size_t *wlen); +extern int +_Py_wstat(const wchar_t *, struct stat *); + PyAPI_FUNC(int) _Py_GetForceASCII(void); /* Reset "force ASCII" mode (if it was initialized). @@ -65,6 +74,35 @@ extern int _Py_EncodeNonUnicodeWchar_InPlace( Py_ssize_t size); #endif +extern wchar_t * _Py_join_relfile(const wchar_t *dirname, + const wchar_t *relfile); +extern int _Py_add_relfile(wchar_t *dirname, + const wchar_t *relfile, + size_t bufsize); +extern size_t _Py_find_basename(const wchar_t *filename); + +// Macros to protect CRT calls against instant termination when passed an +// invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. +// Usage: +// +// _Py_BEGIN_SUPPRESS_IPH +// ... +// _Py_END_SUPPRESS_IPH +#if defined _MSC_VER && _MSC_VER >= 1900 + +# include // _set_thread_local_invalid_parameter_handler() + + extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; +# define _Py_BEGIN_SUPPRESS_IPH \ + { _invalid_parameter_handler _Py_old_handler = \ + _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); +# define _Py_END_SUPPRESS_IPH \ + _set_thread_local_invalid_parameter_handler(_Py_old_handler); } +#else +# define _Py_BEGIN_SUPPRESS_IPH +# define _Py_END_SUPPRESS_IPH +#endif /* _MSC_VER >= 1900 */ + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h new file mode 100644 index 00000000000000..18227c9e36925d --- /dev/null +++ b/Include/internal/pycore_floatobject.h @@ -0,0 +1,75 @@ +#ifndef Py_INTERNAL_FLOATOBJECT_H +#define Py_INTERNAL_FLOATOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* _PyFloat_{Pack,Unpack}{4,8} + * + * The struct and pickle (at least) modules need an efficient platform- + * independent way to store floating-point values as byte strings. + * The Pack routines produce a string from a C double, and the Unpack + * routines produce a C double from such a string. The suffix (4 or 8) + * specifies the number of bytes in the string. + * + * On platforms that appear to use (see _PyFloat_Init()) IEEE-754 formats + * these functions work by copying bits. On other platforms, the formats the + * 4- byte format is identical to the IEEE-754 single precision format, and + * the 8-byte format to the IEEE-754 double precision format, although the + * packing of INFs and NaNs (if such things exist on the platform) isn't + * handled correctly, and attempting to unpack a string containing an IEEE + * INF or NaN will raise an exception. + * + * On non-IEEE platforms with more precision, or larger dynamic range, than + * 754 supports, not all values can be packed; on non-IEEE platforms with less + * precision, or smaller dynamic range, not all values can be unpacked. What + * happens in such cases is partly accidental (alas). + */ + +/* The pack routines write 2, 4 or 8 bytes, starting at p. le is a bool + * argument, true if you want the string in little-endian format (exponent + * last, at p+1, p+3 or p+7), false if you want big-endian format (exponent + * first, at p). + * Return value: 0 if all is OK, -1 if error (and an exception is + * set, most likely OverflowError). + * There are two problems on non-IEEE platforms: + * 1): What this does is undefined if x is a NaN or infinity. + * 2): -0.0 and +0.0 produce the same string. + */ +PyAPI_FUNC(int) _PyFloat_Pack2(double x, unsigned char *p, int le); +PyAPI_FUNC(int) _PyFloat_Pack4(double x, unsigned char *p, int le); +PyAPI_FUNC(int) _PyFloat_Pack8(double x, unsigned char *p, int le); + +/* The unpack routines read 2, 4 or 8 bytes, starting at p. le is a bool + * argument, true if the string is in little-endian format (exponent + * last, at p+1, p+3 or p+7), false if big-endian (exponent first, at p). + * Return value: The unpacked double. On error, this is -1.0 and + * PyErr_Occurred() is true (and an exception is set, most likely + * OverflowError). Note that on a non-IEEE platform this will refuse + * to unpack a string that represents a NaN or infinity. + */ +PyAPI_FUNC(double) _PyFloat_Unpack2(const unsigned char *p, int le); +PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); +PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); + + +PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); + + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FLOATOBJECT_H */ diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 6afb95c3ad62e0..7e63f584eb3b01 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -31,6 +31,7 @@ typedef struct _interpreter_frame { int f_lasti; /* Last instruction if called */ int stacktop; /* Offset of TOS from localsplus */ PyFrameState f_state; /* What state the frame is in */ + int depth; /* Depth of the frame in a ceval loop */ PyObject *localsplus[1]; } InterpreterFrame; @@ -85,6 +86,7 @@ _PyFrame_InitializeSpecials( frame->generator = NULL; frame->f_lasti = -1; frame->f_state = FRAME_CREATED; + frame->depth = 0; } /* Gets the pointer to the locals array @@ -128,7 +130,7 @@ _PyFrame_GetFrameObject(InterpreterFrame *frame) /* Clears all references in the frame. * If take is non-zero, then the InterpreterFrame frame - * may be transfered to the frame object it references + * may be transferred to the frame object it references * instead of being cleared. Either way * the caller no longer owns the references * in the frame. diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 9db4a4716fa588..45e85b54c2ec2d 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -43,7 +43,7 @@ typedef struct { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. #define _PyGCHead_NEXT(g) ((PyGC_Head*)(g)->_gc_next) -#define _PyGCHead_SET_NEXT(g, p) ((g)->_gc_next = (uintptr_t)(p)) +#define _PyGCHead_SET_NEXT(g, p) ((void)((g)->_gc_next = (uintptr_t)(p))) // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. #define _PyGCHead_PREV(g) ((PyGC_Head*)((g)->_gc_prev & _PyGC_PREV_MASK)) @@ -56,7 +56,7 @@ typedef struct { #define _PyGCHead_FINALIZED(g) \ (((g)->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0) #define _PyGCHead_SET_FINALIZED(g) \ - ((g)->_gc_prev |= _PyGC_PREV_MASK_FINALIZED) + ((void)((g)->_gc_prev |= _PyGC_PREV_MASK_FINALIZED)) #define _PyGC_FINALIZED(o) \ _PyGCHead_FINALIZED(_Py_AS_GC(o)) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index e21ed0a7a06a2a..6439b7369fb593 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -10,6 +10,13 @@ extern PyStatus _PyImport_ReInitLock(void); #endif extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate); +struct _module_alias { + const char *name; /* ASCII encoded string */ + const char *orig; /* ASCII encoded string */ +}; + +extern const struct _module_alias * _PyImport_FrozenAliases; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0e6edf4ec26702..64ac3abe00fa01 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -85,12 +85,31 @@ struct _Py_unicode_state { struct _Py_unicode_ids ids; }; +#ifndef WITH_FREELISTS +// without freelists +# define PyFloat_MAXFREELIST 0 +// for tuples only store empty tuple singleton +# define PyTuple_MAXSAVESIZE 1 +# define PyTuple_MAXFREELIST 1 +# define PyList_MAXFREELIST 0 +# define PyDict_MAXFREELIST 0 +# define PyFrame_MAXFREELIST 0 +# define _PyAsyncGen_MAXFREELIST 0 +# define PyContext_MAXFREELIST 0 +#endif + +#ifndef PyFloat_MAXFREELIST +# define PyFloat_MAXFREELIST 100 +#endif + struct _Py_float_state { +#if PyFloat_MAXFREELIST > 0 /* Special free list free_list is a singly-linked list of available PyFloatObjects, linked via abuse of their ob_type members. */ int numfree; PyFloatObject *free_list; +#endif }; /* Speed optimization to avoid frequent malloc/free of small tuples */ @@ -119,8 +138,10 @@ struct _Py_tuple_state { #endif struct _Py_list_state { +#if PyList_MAXFREELIST > 0 PyListObject *free_list[PyList_MAXFREELIST]; int numfree; +#endif }; #ifndef PyDict_MAXFREELIST @@ -128,17 +149,25 @@ struct _Py_list_state { #endif struct _Py_dict_state { +#if PyDict_MAXFREELIST > 0 /* Dictionary reuse scheme to save calls to malloc and free */ PyDictObject *free_list[PyDict_MAXFREELIST]; int numfree; PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; int keys_numfree; +#endif }; +#ifndef PyFrame_MAXFREELIST +# define PyFrame_MAXFREELIST 200 +#endif + struct _Py_frame_state { +#if PyFrame_MAXFREELIST > 0 PyFrameObject *free_list; /* number of frames currently in free_list */ int numfree; +#endif }; #ifndef _PyAsyncGen_MAXFREELIST @@ -146,6 +175,7 @@ struct _Py_frame_state { #endif struct _Py_async_gen_state { +#if _PyAsyncGen_MAXFREELIST > 0 /* Freelists boost performance 6-10%; they also reduce memory fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend are short-living objects that are instantiated for every @@ -155,12 +185,19 @@ struct _Py_async_gen_state { struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; int asend_numfree; +#endif }; +#ifndef PyContext_MAXFREELIST +# define PyContext_MAXFREELIST 255 +#endif + struct _Py_context_state { +#if PyContext_MAXFREELIST > 0 // List of free PyContext objects PyContext *freelist; int numfree; +#endif }; struct _Py_exc_state { diff --git a/Include/cpython/interpreteridobject.h b/Include/internal/pycore_interpreteridobject.h similarity index 51% rename from Include/cpython/interpreteridobject.h rename to Include/internal/pycore_interpreteridobject.h index 5076584209b90b..804831e76deaea 100644 --- a/Include/cpython/interpreteridobject.h +++ b/Include/internal/pycore_interpreteridobject.h @@ -1,11 +1,22 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" +/* Interpreter ID Object */ + +#ifndef Py_INTERNAL_INTERPRETERIDOBJECT_H +#define Py_INTERNAL_INTERPRETERIDOBJECT_H +#ifdef __cplusplus +extern "C" { #endif -/* Interpreter ID Object */ +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t); PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_INTERPRETERIDOBJECT_H diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 7336c317c3f2c8..8bdf8e5736d20b 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -35,6 +35,33 @@ static inline PyObject* _PyLong_GetOne(void) { return __PyLong_GetSmallInt_internal(1); } PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); +PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right); + +/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), + _PyBytes_DecodeEscape(), etc. */ +PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +PyAPI_FUNC(int) _PyLong_FormatWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + int base, + int alternate); + +PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( + _PyBytesWriter *writer, + char *str, + PyObject *obj, + int base, + int alternate); #ifdef __cplusplus } diff --git a/Include/internal/pycore_namespace.h b/Include/internal/pycore_namespace.h new file mode 100644 index 00000000000000..cb76f040693d10 --- /dev/null +++ b/Include/internal/pycore_namespace.h @@ -0,0 +1,20 @@ +// Simple namespace object interface + +#ifndef Py_INTERNAL_NAMESPACE_H +#define Py_INTERNAL_NAMESPACE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyAPI_DATA(PyTypeObject) _PyNamespace_Type; + +PyAPI_FUNC(PyObject *) _PyNamespace_New(PyObject *kwds); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_NAMESPACE_H diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 82dddf119a47a1..3c126aaef1187a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -181,6 +181,16 @@ extern int _Py_CheckSlotResult( extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); extern int _PyObject_InitializeDict(PyObject *obj); +extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, + PyObject *name, PyObject *value); +PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, + PyObject *name); +PyDictValues ** _PyObject_ValuesPointer(PyObject *); +PyObject ** _PyObject_DictPointer(PyObject *); +int _PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg); +void _PyObject_ClearInstanceAttributes(PyObject *self); +void _PyObject_FreeInstanceAttributes(PyObject *self); +int _PyObject_IsInstanceDictEmpty(PyObject *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 15447f54490fb4..a258aab2397660 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -13,6 +13,7 @@ typedef struct _PyPathConfig { wchar_t *program_full_path; wchar_t *prefix; wchar_t *exec_prefix; + wchar_t *stdlib_dir; /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */ wchar_t *module_search_path; /* Python program name */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 524be9d4cbb940..4f12fef8d65466 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -122,6 +122,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const PyConfig *config, const struct _PyArgv *args); +PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h new file mode 100644 index 00000000000000..395b71452e40ad --- /dev/null +++ b/Include/internal/pycore_pymath.h @@ -0,0 +1,184 @@ +#ifndef Py_INTERNAL_PYMATH_H +#define Py_INTERNAL_PYMATH_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Extra declarations +#if !defined(_MSC_VER) && !defined(__STDC__) +extern double fmod (double, double); +extern double frexp (double, int *); +extern double ldexp (double, int); +extern double modf (double, double *); +extern double pow(double, double); +#endif // !defined(_MSC_VER) && !defined(__STDC__) + + +/* _Py_ADJUST_ERANGE1(x) + * _Py_ADJUST_ERANGE2(x, y) + * Set errno to 0 before calling a libm function, and invoke one of these + * macros after, passing the function result(s) (_Py_ADJUST_ERANGE2 is useful + * for functions returning complex results). This makes two kinds of + * adjustments to errno: (A) If it looks like the platform libm set + * errno=ERANGE due to underflow, clear errno. (B) If it looks like the + * platform libm overflowed but didn't set errno, force errno to ERANGE. In + * effect, we're trying to force a useful implementation of C89 errno + * behavior. + * Caution: + * This isn't reliable. C99 no longer requires libm to set errno under + * any exceptional condition, but does require +- HUGE_VAL return + * values on overflow. A 754 box *probably* maps HUGE_VAL to a + * double infinity, and we're cool if that's so, unless the input + * was an infinity and an infinity is the expected result. A C89 + * system sets errno to ERANGE, so we check for that too. We're + * out of luck if a C99 754 box doesn't map HUGE_VAL to +Inf, or + * if the returned result is a NaN, or if a C89 box returns HUGE_VAL + * in non-overflow cases. + */ +static inline void _Py_ADJUST_ERANGE1(double x) +{ + if (errno == 0) { + if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL) { + errno = ERANGE; + } + } + else if (errno == ERANGE && x == 0.0) { + errno = 0; + } +} + +static inline void _Py_ADJUST_ERANGE2(double x, double y) +{ + if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL || + y == Py_HUGE_VAL || y == -Py_HUGE_VAL) + { + if (errno == 0) { + errno = ERANGE; + } + } + else if (errno == ERANGE) { + errno = 0; + } +} + +// Return whether integral type *type* is signed or not. +#define _Py_IntegralTypeSigned(type) \ + ((type)(-1) < 0) + +// Return the maximum value of integral type *type*. +#define _Py_IntegralTypeMax(type) \ + ((_Py_IntegralTypeSigned(type)) ? (((((type)1 << (sizeof(type)*CHAR_BIT - 2)) - 1) << 1) + 1) : ~(type)0) + +// Return the minimum value of integral type *type*. +#define _Py_IntegralTypeMin(type) \ + ((_Py_IntegralTypeSigned(type)) ? -_Py_IntegralTypeMax(type) - 1 : 0) + +// Check whether *v* is in the range of integral type *type*. This is most +// useful if *v* is floating-point, since demoting a floating-point *v* to an +// integral type that cannot represent *v*'s integral part is undefined +// behavior. +#define _Py_InIntegralTypeRange(type, v) \ + (_Py_IntegralTypeMin(type) <= v && v <= _Py_IntegralTypeMax(type)) + + +//--- Implementation of the HAVE_PY_SET_53BIT_PRECISION macro ------------- +//--- defined in pyport.h ------------------------------------------------- +// +// Give appropriate definitions for the following three macros: +// +// _Py_SET_53BIT_PRECISION_HEADER : any variable declarations needed to +// use the two macros below. +// _Py_SET_53BIT_PRECISION_START : store original FPU settings, and +// set FPU to 53-bit precision/round-half-to-even +// _Py_SET_53BIT_PRECISION_END : restore original FPU settings + +// Get and set x87 control word for gcc/x86 +#ifdef HAVE_GCC_ASM_FOR_X87 + +// Functions defined in Python/pymath.c +extern unsigned short _Py_get_387controlword(void); +extern void _Py_set_387controlword(unsigned short); + +#define _Py_SET_53BIT_PRECISION_HEADER \ + unsigned short old_387controlword, new_387controlword +#define _Py_SET_53BIT_PRECISION_START \ + do { \ + old_387controlword = _Py_get_387controlword(); \ + new_387controlword = (old_387controlword & ~0x0f00) | 0x0200; \ + if (new_387controlword != old_387controlword) { \ + _Py_set_387controlword(new_387controlword); \ + } \ + } while (0) +#define _Py_SET_53BIT_PRECISION_END \ + do { \ + if (new_387controlword != old_387controlword) { \ + _Py_set_387controlword(old_387controlword); \ + } \ + } while (0) +#endif + +// Get and set x87 control word for VisualStudio/x86. +// x87 is not supported in 64-bit or ARM. +#if defined(_MSC_VER) && !defined(_WIN64) && !defined(_M_ARM) + +#include // __control87_2() + +#define _Py_SET_53BIT_PRECISION_HEADER \ + unsigned int old_387controlword, new_387controlword, out_387controlword + // We use the __control87_2 function to set only the x87 control word. + // The SSE control word is unaffected. +#define _Py_SET_53BIT_PRECISION_START \ + do { \ + __control87_2(0, 0, &old_387controlword, NULL); \ + new_387controlword = \ + (old_387controlword & ~(_MCW_PC | _MCW_RC)) | (_PC_53 | _RC_NEAR); \ + if (new_387controlword != old_387controlword) { \ + __control87_2(new_387controlword, _MCW_PC | _MCW_RC, \ + &out_387controlword, NULL); \ + } \ + } while (0) +#define _Py_SET_53BIT_PRECISION_END \ + do { \ + if (new_387controlword != old_387controlword) { \ + __control87_2(old_387controlword, _MCW_PC | _MCW_RC, \ + &out_387controlword, NULL); \ + } \ + } while (0) +#endif + +#ifdef HAVE_GCC_ASM_FOR_MC68881 +#define _Py_SET_53BIT_PRECISION_HEADER \ + unsigned int old_fpcr, new_fpcr +#define _Py_SET_53BIT_PRECISION_START \ + do { \ + __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ + /* Set double precision / round to nearest. */ \ + new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ + if (new_fpcr != old_fpcr) { \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr));\ + } \ + } while (0) +#define _Py_SET_53BIT_PRECISION_END \ + do { \ + if (new_fpcr != old_fpcr) { \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ + } \ + } while (0) +#endif + +// Default definitions are empty +#ifndef _Py_SET_53BIT_PRECISION_HEADER +# define _Py_SET_53BIT_PRECISION_HEADER +# define _Py_SET_53BIT_PRECISION_START +# define _Py_SET_53BIT_PRECISION_END +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYMATH_H */ diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 30db3f2b6701ec..d70deee710e30d 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -103,4 +103,4 @@ void _PyObject_VirtualFree(void *, size_t size); #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_PYMEM_H */ +#endif // !Py_INTERNAL_PYMEM_H diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 4b894f3eff4967..9a570b08bc5839 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -30,6 +30,17 @@ _Py_IsMainInterpreter(PyInterpreterState *interp) } +static inline const PyConfig * +_Py_GetMainConfig(void) +{ + PyInterpreterState *interp = _PyRuntime.interpreters.main; + if (interp == NULL) { + return NULL; + } + return _PyInterpreterState_GetConfig(interp); +} + + /* Only handle signals on the main thread of the main interpreter. */ static inline int _Py_ThreadCanHandleSignals(PyInterpreterState *interp) @@ -71,7 +82,7 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) The caller must hold the GIL. - See also PyThreadState_Get() and PyThreadState_GET(). */ + See also PyThreadState_Get() and _PyThreadState_UncheckedGet(). */ static inline PyThreadState* _PyThreadState_GET(void) { @@ -82,10 +93,6 @@ _PyThreadState_GET(void) #endif } -/* Redefine PyThreadState_GET() as an alias to _PyThreadState_GET() */ -#undef PyThreadState_GET -#define PyThreadState_GET() _PyThreadState_GET() - PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalError_TstateNULL(const char *func); static inline void @@ -118,7 +125,7 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { } -/* Other */ +// PyThreadState functions PyAPI_FUNC(void) _PyThreadState_Init( PyThreadState *tstate); @@ -126,6 +133,23 @@ PyAPI_FUNC(void) _PyThreadState_DeleteExcept( _PyRuntimeState *runtime, PyThreadState *tstate); +static inline void +_PyThreadState_PauseTracing(PyThreadState *tstate) +{ + tstate->cframe->use_tracing = 0; +} + +static inline void +_PyThreadState_ResumeTracing(PyThreadState *tstate) +{ + int use_tracing = (tstate->c_tracefunc != NULL + || tstate->c_profilefunc != NULL); + tstate->cframe->use_tracing = (use_tracing ? 255 : 0); +} + + +/* Other */ + PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( struct _gilstate_runtime_state *gilstate, PyThreadState *newts); diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h new file mode 100644 index 00000000000000..1633671da0f4c2 --- /dev/null +++ b/Include/internal/pycore_strhex.h @@ -0,0 +1,36 @@ +#ifndef Py_INTERNAL_STRHEX_H +#define Py_INTERNAL_STRHEX_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Returns a str() containing the hex representation of argbuf. +PyAPI_FUNC(PyObject*) _Py_strhex(const + char* argbuf, + const Py_ssize_t arglen); + +// Returns a bytes() containing the ASCII hex representation of argbuf. +PyAPI_FUNC(PyObject*) _Py_strhex_bytes( + const char* argbuf, + const Py_ssize_t arglen); + +// These variants include support for a separator between every N bytes: +PyAPI_FUNC(PyObject*) _Py_strhex_with_sep( + const char* argbuf, + const Py_ssize_t arglen, + const PyObject* sep, + const int bytes_per_group); +PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( + const char* argbuf, + const Py_ssize_t arglen, + const PyObject* sep, + const int bytes_per_group); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STRHEX_H */ diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 4d282308769dc8..c01a47639d5e30 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -51,7 +51,7 @@ PyAPI_FUNC(void) _Py_DumpTraceback( _PyGILState_GetInterpreterStateUnsafe() in last resort. It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve these informations. + different options to retrieve this information. This function is signal safe. */ diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index d1d0d2a92e49fb..79c827fe8800a7 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -13,6 +13,7 @@ extern "C" { #define _PyTuple_ITEMS(op) (_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); +extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); #ifdef __cplusplus } diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index f728ec3077b3c5..efb4f1cd7eac80 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -19,6 +19,10 @@ struct _warnings_runtime_state { extern int _PyWarnings_InitState(PyInterpreterState *interp); +PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); + +extern void _PyErr_WarnUnawaitedCoroutine(PyObject *coro); + #ifdef __cplusplus } #endif diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h deleted file mode 100644 index e744fcdc9ff189..00000000000000 --- a/Include/interpreteridobject.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef Py_INTERPRETERIDOBJECT_H -#define Py_INTERPRETERIDOBJECT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERPRETERIDOBJECT_H -# include "cpython/interpreteridobject.h" -# undef Py_CPYTHON_INTERPRETERIDOBJECT_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Include/listobject.h b/Include/listobject.h index 2a8a25525d1d7b..eff42c188f1ff1 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -42,7 +42,7 @@ PyAPI_FUNC(PyObject *) PyList_AsTuple(PyObject *); #ifndef Py_LIMITED_API # define Py_CPYTHON_LISTOBJECT_H -# include "cpython/listobject.h" +# include "cpython/listobject.h" # undef Py_CPYTHON_LISTOBJECT_H #endif diff --git a/Include/longobject.h b/Include/longobject.h index e2301d7abfcccf..7fe8f58cb3e959 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -26,9 +26,6 @@ PyAPI_FUNC(Py_ssize_t) PyLong_AsSsize_t(PyObject *); PyAPI_FUNC(size_t) PyLong_AsSize_t(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLong(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLongMask(PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); -#endif PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); /* It may be useful in the future. I've added it in the PyInt -> PyLong @@ -65,30 +62,6 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); # error "void* different in size from int, long and long long" #endif /* SIZEOF_VOID_P */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); -#endif - -/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), - _PyBytes_DecodeEscape(), etc. */ -#ifndef Py_LIMITED_API -PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; -#endif - -/* _PyLong_Frexp returns a double x and an exponent e such that the - true value is approximately equal to x * 2**e. e is >= 0. x is - 0.0 if and only if the input is 0 (in which case, e and x are both - zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is - possible if the number of bits doesn't fit into a Py_ssize_t, sets - OverflowError and returns -1.0 for x, 0 for e. */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); -#endif - PyAPI_FUNC(double) PyLong_AsDouble(PyObject *); PyAPI_FUNC(PyObject *) PyLong_FromVoidPtr(void *); PyAPI_FUNC(void *) PyLong_AsVoidPtr(PyObject *); @@ -101,102 +74,6 @@ PyAPI_FUNC(unsigned long long) PyLong_AsUnsignedLongLongMask(PyObject *); PyAPI_FUNC(long long) PyLong_AsLongLongAndOverflow(PyObject *, int *); PyAPI_FUNC(PyObject *) PyLong_FromString(const char *, char **, int); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); -#endif - -#ifndef Py_LIMITED_API -/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. - v must not be NULL, and must be a normalized long. - There are no error cases. -*/ -PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); - - -/* _PyLong_NumBits. Return the number of bits needed to represent the - absolute value of a long. For example, this returns 1 for 1 and -1, 2 - for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. - v must not be NULL, and must be a normalized long. - (size_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. -*/ -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - -/* _PyLong_DivmodNear. Given integers a and b, compute the nearest - integer q to the exact quotient a / b, rounding to the nearest even integer - in the case of a tie. Return (q, r), where r = a - q*b. The remainder r - will satisfy abs(r) <= abs(b)/2, with equality possible only if q is - even. -*/ -PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); - -/* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in - base 256, and return a Python int with the same numeric value. - If n is 0, the integer is 0. Else: - If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; - else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the - LSB. - If is_signed is 0/false, view the bytes as a non-negative integer. - If is_signed is 1/true, view the bytes as a 2's-complement integer, - non-negative if bit 0x80 of the MSB is clear, negative if set. - Error returns: - + Return NULL with the appropriate exception set if there's not - enough memory to create the Python int. -*/ -PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( - const unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long - v to a base-256 integer, stored in array bytes. Normally return 0, - return -1 on error. - If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at - bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and - the LSB at bytes[n-1]. - If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes - are filled and there's nothing special about bit 0x80 of the MSB. - If is_signed is 1/true, bytes is filled with the 2's-complement - representation of v's value. Bit 0x80 of the MSB is the sign bit. - Error returns (-1): - + is_signed is 0 and v < 0. TypeError is set in this case, and bytes - isn't altered. - + n isn't big enough to hold the full mathematical value of v. For - example, if is_signed is 0 and there are more digits in the v than - fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of - being large enough to hold a sign bit. OverflowError is set in this - case, but bytes holds the least-significant n bytes of the true value. -*/ -PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, - unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -/* _PyLong_Format: Convert the long to a string object with given base, - appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); - -PyAPI_FUNC(int) _PyLong_FormatWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - int base, - int alternate); - -PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( - _PyBytesWriter *writer, - char *str, - PyObject *obj, - int base, - int alternate); - -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); -#endif /* Py_LIMITED_API */ /* These aren't really part of the int object, but they're handy. The functions are in Python/mystrtoul.c. @@ -205,13 +82,9 @@ PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int); PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int); #ifndef Py_LIMITED_API -/* For use by the gcd function in mathmodule.c */ -PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); -#endif /* !Py_LIMITED_API */ - -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); -PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); +# define Py_CPYTHON_LONGOBJECT_H +# include "cpython/longobject.h" +# undef Py_CPYTHON_LONGOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/marshal.h b/Include/marshal.h index 09d9337e57b0aa..f8b0de80cfc38d 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -3,26 +3,29 @@ #ifndef Py_MARSHAL_H #define Py_MARSHAL_H +#ifndef Py_LIMITED_API + #ifdef __cplusplus extern "C" { #endif -#define Py_MARSHAL_VERSION 4 - -PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); -PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, + Py_ssize_t); PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); -#ifndef Py_LIMITED_API +#define Py_MARSHAL_VERSION 4 + PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); -#endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); + +PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); +PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); #ifdef __cplusplus } #endif + +#endif /* Py_LIMITED_API */ #endif /* !Py_MARSHAL_H */ diff --git a/Include/methodobject.h b/Include/methodobject.h index 9ffe8e1a3ddfcb..1be5873a30569f 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -103,11 +103,9 @@ PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *, #ifndef Py_LIMITED_API - -#define Py_CPYTHON_METHODOBJECT_H -#include "cpython/methodobject.h" -#undef Py_CPYTHON_METHODOBJECT_H - +# define Py_CPYTHON_METHODOBJECT_H +# include "cpython/methodobject.h" +# undef Py_CPYTHON_METHODOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/modsupport.h b/Include/modsupport.h index 7d37b499422559..baf47f0038d08d 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -7,7 +7,7 @@ extern "C" { /* Module support interface */ -#include +#include // va_list /* If PY_SSIZE_T_CLEAN is defined, each functions treats #-specifier to mean Py_ssize_t */ diff --git a/Include/namespaceobject.h b/Include/namespaceobject.h deleted file mode 100644 index 0c8d95c0f137c6..00000000000000 --- a/Include/namespaceobject.h +++ /dev/null @@ -1,19 +0,0 @@ - -/* simple namespace object interface */ - -#ifndef NAMESPACEOBJECT_H -#define NAMESPACEOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -PyAPI_DATA(PyTypeObject) _PyNamespace_Type; - -PyAPI_FUNC(PyObject *) _PyNamespace_New(PyObject *kwds); -#endif /* !Py_LIMITED_API */ - -#ifdef __cplusplus -} -#endif -#endif /* !NAMESPACEOBJECT_H */ diff --git a/Include/object.h b/Include/object.h index 490cbffd5f1074..33df303a44eb7e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -333,6 +333,7 @@ given type object has a specified feature. */ #ifndef Py_LIMITED_API + /* Set if instances of the type object are treated as sequences for pattern matching */ #define Py_TPFLAGS_SEQUENCE (1 << 5) /* Set if instances of the type object are treated as mappings for pattern matching */ @@ -600,7 +601,7 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) } // Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. -// Names overriden with macros by static inline functions for best +// Names overridden with macros by static inline functions for best // performances. #define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) #define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) @@ -723,7 +724,7 @@ times. #ifndef Py_LIMITED_API # define Py_CPYTHON_OBJECT_H -# include "cpython/object.h" +# include "cpython/object.h" # undef Py_CPYTHON_OBJECT_H #endif diff --git a/Include/objimpl.h b/Include/objimpl.h index 450befad679e70..9b98c112ac2cc5 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -205,7 +205,7 @@ PyAPI_FUNC(int) PyObject_GC_IsFinalized(PyObject *); #ifndef Py_LIMITED_API # define Py_CPYTHON_OBJIMPL_H -# include "cpython/objimpl.h" +# include "cpython/objimpl.h" # undef Py_CPYTHON_OBJIMPL_H #endif diff --git a/Include/opcode.h b/Include/opcode.h index 27895255947837..f8c02b840e0521 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -141,32 +141,43 @@ extern "C" { #define BINARY_ADD_FLOAT 13 #define BINARY_ADD_UNICODE 14 #define BINARY_ADD_UNICODE_INPLACE_FAST 18 -#define BINARY_SUBSCR_ADAPTIVE 21 -#define BINARY_SUBSCR_LIST_INT 36 -#define BINARY_SUBSCR_TUPLE_INT 38 -#define BINARY_SUBSCR_DICT 39 -#define JUMP_ABSOLUTE_QUICK 40 -#define LOAD_ATTR_ADAPTIVE 41 -#define LOAD_ATTR_SPLIT_KEYS 42 -#define LOAD_ATTR_WITH_HINT 43 -#define LOAD_ATTR_SLOT 44 -#define LOAD_ATTR_MODULE 45 -#define LOAD_GLOBAL_ADAPTIVE 46 -#define LOAD_GLOBAL_MODULE 47 -#define LOAD_GLOBAL_BUILTIN 48 -#define LOAD_METHOD_ADAPTIVE 58 -#define LOAD_METHOD_CACHED 80 -#define LOAD_METHOD_CLASS 81 -#define LOAD_METHOD_MODULE 87 -#define STORE_ATTR_ADAPTIVE 88 -#define STORE_ATTR_SPLIT_KEYS 120 -#define STORE_ATTR_SLOT 122 -#define STORE_ATTR_WITH_HINT 123 -#define LOAD_FAST__LOAD_FAST 127 -#define STORE_FAST__LOAD_FAST 128 -#define LOAD_FAST__LOAD_CONST 134 -#define LOAD_CONST__LOAD_FAST 140 -#define STORE_FAST__STORE_FAST 143 +#define BINARY_MULTIPLY_ADAPTIVE 21 +#define BINARY_MULTIPLY_INT 36 +#define BINARY_MULTIPLY_FLOAT 38 +#define BINARY_SUBSCR_ADAPTIVE 39 +#define BINARY_SUBSCR_LIST_INT 40 +#define BINARY_SUBSCR_TUPLE_INT 41 +#define BINARY_SUBSCR_DICT 42 +#define CALL_FUNCTION_ADAPTIVE 43 +#define CALL_FUNCTION_BUILTIN_O 44 +#define CALL_FUNCTION_BUILTIN_FAST 45 +#define CALL_FUNCTION_LEN 46 +#define CALL_FUNCTION_ISINSTANCE 47 +#define CALL_FUNCTION_PY_SIMPLE 48 +#define JUMP_ABSOLUTE_QUICK 58 +#define LOAD_ATTR_ADAPTIVE 80 +#define LOAD_ATTR_INSTANCE_VALUE 81 +#define LOAD_ATTR_WITH_HINT 87 +#define LOAD_ATTR_SLOT 88 +#define LOAD_ATTR_MODULE 120 +#define LOAD_GLOBAL_ADAPTIVE 122 +#define LOAD_GLOBAL_MODULE 123 +#define LOAD_GLOBAL_BUILTIN 127 +#define LOAD_METHOD_ADAPTIVE 128 +#define LOAD_METHOD_CACHED 134 +#define LOAD_METHOD_CLASS 140 +#define LOAD_METHOD_MODULE 143 +#define LOAD_METHOD_NO_DICT 149 +#define STORE_ATTR_ADAPTIVE 150 +#define STORE_ATTR_INSTANCE_VALUE 151 +#define STORE_ATTR_SLOT 153 +#define STORE_ATTR_WITH_HINT 154 +#define LOAD_FAST__LOAD_FAST 158 +#define STORE_FAST__LOAD_FAST 159 +#define LOAD_FAST__LOAD_CONST 167 +#define LOAD_CONST__LOAD_FAST 168 +#define STORE_FAST__STORE_FAST 169 +#define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { 0U, diff --git a/Include/patchlevel.h b/Include/patchlevel.h index f37c4d48e37607..591b41736f338e 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 11 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 0 +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.11.0a0" +#define PY_VERSION "3.11.0a1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pyerrors.h b/Include/pyerrors.h index f5d1c711577186..c6c443a2d7d0f0 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -314,7 +314,7 @@ PyAPI_FUNC(int) PyOS_vsnprintf(char *str, size_t size, const char *format, va_l #ifndef Py_LIMITED_API # define Py_CPYTHON_ERRORS_H -# include "cpython/pyerrors.h" +# include "cpython/pyerrors.h" # undef Py_CPYTHON_ERRORS_H #endif diff --git a/Include/pyhash.h b/Include/pyhash.h index a314ea907b7fe2..182d223fab1cac 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -114,11 +114,10 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); /* hash algorithm selection * - * The values for Py_HASH_SIPHASH24 and Py_HASH_FNV are hard-coded in the + * The values for Py_HASH_* are hard-coded in the * configure script. * - * - FNV is available on all platforms and architectures. - * - SIPHASH24 only works on platforms that don't require aligned memory for integers. + * - FNV and SIPHASH* are available on all platforms and architectures. * - With EXTERNAL embedders can provide an alternative implementation with:: * * PyHash_FuncDef PyHash_Func = {...}; @@ -128,10 +127,11 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); #define Py_HASH_EXTERNAL 0 #define Py_HASH_SIPHASH24 1 #define Py_HASH_FNV 2 +#define Py_HASH_SIPHASH13 3 #ifndef Py_HASH_ALGORITHM # ifndef HAVE_ALIGNED_REQUIRED -# define Py_HASH_ALGORITHM Py_HASH_SIPHASH24 +# define Py_HASH_ALGORITHM Py_HASH_SIPHASH13 # else # define Py_HASH_ALGORITHM Py_HASH_FNV # endif /* uint64_t && uint32_t && aligned */ diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 9b2dd0868eb25b..4aecda235abf76 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -64,7 +64,7 @@ PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t); #ifndef Py_LIMITED_API # define Py_CPYTHON_PYLIFECYCLE_H -# include "cpython/pylifecycle.h" +# include "cpython/pylifecycle.h" # undef Py_CPYTHON_PYLIFECYCLE_H #endif diff --git a/Include/pymath.h b/Include/pymath.h index ebb3b05f1b53ca..57310fc097e732 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -1,42 +1,9 @@ +// Symbols and macros to supply platform-independent interfaces to mathematical +// functions and constants. + #ifndef Py_PYMATH_H #define Py_PYMATH_H -#include "pyconfig.h" /* include for defines */ - -/************************************************************************** -Symbols and macros to supply platform-independent interfaces to mathematical -functions and constants -**************************************************************************/ - -/* Python provides implementations for copysign, round and hypot in - * Python/pymath.c just in case your math library doesn't provide the - * functions. - * - *Note: PC/pyconfig.h defines copysign as _copysign - */ -#ifndef HAVE_COPYSIGN -extern double copysign(double, double); -#endif - -#ifndef HAVE_ROUND -extern double round(double); -#endif - -#ifndef HAVE_HYPOT -extern double hypot(double, double); -#endif - -/* extra declarations */ -#ifndef _MSC_VER -#ifndef __STDC__ -extern double fmod (double, double); -extern double frexp (double, int *); -extern double ldexp (double, int); -extern double modf (double, double *); -extern double pow(double, double); -#endif /* __STDC__ */ -#endif /* _MSC_VER */ - /* High precision definition of pi and e (Euler) * The values are taken from libc6's math.h. */ @@ -60,84 +27,17 @@ extern double pow(double, double); #define Py_MATH_TAU 6.2831853071795864769252867665590057683943L #endif - -/* On x86, Py_FORCE_DOUBLE forces a floating-point number out of an x87 FPU - register and into a 64-bit memory location, rounding from extended - precision to double precision in the process. On other platforms it does - nothing. */ - -/* we take double rounding as evidence of x87 usage */ -#ifndef Py_LIMITED_API -#ifndef Py_FORCE_DOUBLE -# ifdef X87_DOUBLE_ROUNDING -PyAPI_FUNC(double) _Py_force_double(double); -# define Py_FORCE_DOUBLE(X) (_Py_force_double(X)) -# else -# define Py_FORCE_DOUBLE(X) (X) -# endif -#endif -#endif - -#ifndef Py_LIMITED_API -#ifdef HAVE_GCC_ASM_FOR_X87 -PyAPI_FUNC(unsigned short) _Py_get_387controlword(void); -PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); -#endif -#endif - -/* Py_IS_NAN(X) - * Return 1 if float or double arg is a NaN, else 0. - * Caution: - * X is evaluated more than once. - * This may not work on all platforms. Each platform has *some* - * way to spell this, though -- override in pyconfig.h if you have - * a platform where it doesn't work. - * Note: PC/pyconfig.h defines Py_IS_NAN as _isnan - */ -#ifndef Py_IS_NAN -#if defined HAVE_DECL_ISNAN && HAVE_DECL_ISNAN == 1 +// Py_IS_NAN(X) +// Return 1 if float or double arg is a NaN, else 0. #define Py_IS_NAN(X) isnan(X) -#else -#define Py_IS_NAN(X) ((X) != (X)) -#endif -#endif -/* Py_IS_INFINITY(X) - * Return 1 if float or double arg is an infinity, else 0. - * Caution: - * X is evaluated more than once. - * This implementation may set the underflow flag if |X| is very small; - * it really can't be implemented correctly (& easily) before C99. - * Override in pyconfig.h if you have a better spelling on your platform. - * Py_FORCE_DOUBLE is used to avoid getting false negatives from a - * non-infinite value v sitting in an 80-bit x87 register such that - * v becomes infinite when spilled from the register to 64-bit memory. - * Note: PC/pyconfig.h defines Py_IS_INFINITY as _isinf - */ -#ifndef Py_IS_INFINITY -# if defined HAVE_DECL_ISINF && HAVE_DECL_ISINF == 1 -# define Py_IS_INFINITY(X) isinf(X) -# else -# define Py_IS_INFINITY(X) ((X) && \ - (Py_FORCE_DOUBLE(X)*0.5 == Py_FORCE_DOUBLE(X))) -# endif -#endif +// Py_IS_INFINITY(X) +// Return 1 if float or double arg is an infinity, else 0. +#define Py_IS_INFINITY(X) isinf(X) -/* Py_IS_FINITE(X) - * Return 1 if float or double arg is neither infinite nor NAN, else 0. - * Some compilers (e.g. VisualStudio) have intrinsics for this, so a special - * macro for this particular test is useful - * Note: PC/pyconfig.h defines Py_IS_FINITE as _finite - */ -#ifndef Py_IS_FINITE -#if defined HAVE_DECL_ISFINITE && HAVE_DECL_ISFINITE == 1 +// Py_IS_FINITE(X) +// Return 1 if float or double arg is neither infinite nor NAN, else 0. #define Py_IS_FINITE(X) isfinite(X) -#elif defined HAVE_FINITE -#define Py_IS_FINITE(X) finite(X) -#else -#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X)) -#endif -#endif /* HUGE_VAL is supposed to expand to a positive double infinity. Python * uses Py_HUGE_VAL instead because some platforms are broken in this @@ -147,7 +47,7 @@ PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); * config to #define Py_HUGE_VAL to something that works on your platform. */ #ifndef Py_HUGE_VAL -#define Py_HUGE_VAL HUGE_VAL +# define Py_HUGE_VAL HUGE_VAL #endif /* Py_NAN @@ -156,10 +56,10 @@ PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); * doesn't support NaNs. */ #if !defined(Py_NAN) && !defined(Py_NO_NAN) -#if !defined(__INTEL_COMPILER) - #define Py_NAN (Py_HUGE_VAL * 0.) -#else /* __INTEL_COMPILER */ - #if defined(ICC_NAN_STRICT) +# if !defined(__INTEL_COMPILER) +# define Py_NAN (Py_HUGE_VAL * 0.) +# else /* __INTEL_COMPILER */ +# if defined(ICC_NAN_STRICT) #pragma float_control(push) #pragma float_control(precise, on) #pragma float_control(except, on) @@ -168,58 +68,12 @@ PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); return sqrt(-1.0); } #pragma float_control (pop) - #define Py_NAN __icc_nan() - #else /* ICC_NAN_RELAXED as default for Intel Compiler */ +# define Py_NAN __icc_nan() +# else /* ICC_NAN_RELAXED as default for Intel Compiler */ static const union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; - #define Py_NAN (__nan_store.__icc_nan) - #endif /* ICC_NAN_STRICT */ -#endif /* __INTEL_COMPILER */ +# define Py_NAN (__nan_store.__icc_nan) +# endif /* ICC_NAN_STRICT */ +# endif /* __INTEL_COMPILER */ #endif -/* Py_OVERFLOWED(X) - * Return 1 iff a libm function overflowed. Set errno to 0 before calling - * a libm function, and invoke this macro after, passing the function - * result. - * Caution: - * This isn't reliable. C99 no longer requires libm to set errno under - * any exceptional condition, but does require +- HUGE_VAL return - * values on overflow. A 754 box *probably* maps HUGE_VAL to a - * double infinity, and we're cool if that's so, unless the input - * was an infinity and an infinity is the expected result. A C89 - * system sets errno to ERANGE, so we check for that too. We're - * out of luck if a C99 754 box doesn't map HUGE_VAL to +Inf, or - * if the returned result is a NaN, or if a C89 box returns HUGE_VAL - * in non-overflow cases. - * X is evaluated more than once. - * Some platforms have better way to spell this, so expect some #ifdef'ery. - * - * OpenBSD uses 'isinf()' because a compiler bug on that platform causes - * the longer macro version to be mis-compiled. This isn't optimal, and - * should be removed once a newer compiler is available on that platform. - * The system that had the failure was running OpenBSD 3.2 on Intel, with - * gcc 2.95.3. - * - * According to Tim's checkin, the FreeBSD systems use isinf() to work - * around a FPE bug on that platform. - */ -#if defined(__FreeBSD__) || defined(__OpenBSD__) -#define Py_OVERFLOWED(X) isinf(X) -#else -#define Py_OVERFLOWED(X) ((X) != 0.0 && (errno == ERANGE || \ - (X) == Py_HUGE_VAL || \ - (X) == -Py_HUGE_VAL)) -#endif - -/* Return whether integral type *type* is signed or not. */ -#define _Py_IntegralTypeSigned(type) ((type)(-1) < 0) -/* Return the maximum value of integral type *type*. */ -#define _Py_IntegralTypeMax(type) ((_Py_IntegralTypeSigned(type)) ? (((((type)1 << (sizeof(type)*CHAR_BIT - 2)) - 1) << 1) + 1) : ~(type)0) -/* Return the minimum value of integral type *type*. */ -#define _Py_IntegralTypeMin(type) ((_Py_IntegralTypeSigned(type)) ? -_Py_IntegralTypeMax(type) - 1 : 0) -/* Check whether *v* is in the range of integral type *type*. This is most - * useful if *v* is floating-point, since demoting a floating-point *v* to an - * integral type that cannot represent *v*'s integral part is undefined - * behavior. */ -#define _Py_InIntegralTypeRange(type, v) (_Py_IntegralTypeMin(type) <= v && v <= _Py_IntegralTypeMax(type)) - #endif /* Py_PYMATH_H */ diff --git a/Include/pymem.h b/Include/pymem.h index 66cdb0d2973cd0..c15ad10dfcf831 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -93,7 +93,7 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifndef Py_LIMITED_API # define Py_CPYTHON_PYMEM_H -# include "cpython/pymem.h" +# include "cpython/pymem.h" # undef Py_CPYTHON_PYMEM_H #endif diff --git a/Include/pyport.h b/Include/pyport.h index d27d0838f1b2cb..61ca3a97c186bf 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -5,6 +5,14 @@ #include +#include +#ifndef UCHAR_MAX +# error "limits.h must define UCHAR_MAX" +#endif +#if UCHAR_MAX != 255 +# error "Python's source code assumes C's unsigned char is an 8-bit type" +#endif + /* Defines to build Python and its standard library: * @@ -193,11 +201,10 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif -/* Py_MEMCPY is kept for backwards compatibility, - * see https://bugs.python.org/issue28126 */ -#define Py_MEMCPY memcpy - -#include +// bpo-28126: Py_MEMCPY is kept for backwards compatibility, +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_MEMCPY memcpy +#endif #ifdef HAVE_IEEEFP_H #include /* needed for 'finite' declaration on some platforms */ @@ -316,69 +323,6 @@ extern "C" { #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE) #endif -/* Py_SET_ERRNO_ON_MATH_ERROR(x) - * If a libm function did not set errno, but it looks like the result - * overflowed or not-a-number, set errno to ERANGE or EDOM. Set errno - * to 0 before calling a libm function, and invoke this macro after, - * passing the function result. - * Caution: - * This isn't reliable. See Py_OVERFLOWED comments. - * X is evaluated more than once. - */ -#if defined(__FreeBSD__) || defined(__OpenBSD__) || (defined(__hpux) && defined(__ia64)) -#define _Py_SET_EDOM_FOR_NAN(X) if (isnan(X)) errno = EDOM; -#else -#define _Py_SET_EDOM_FOR_NAN(X) ; -#endif -#define Py_SET_ERRNO_ON_MATH_ERROR(X) \ - do { \ - if (errno == 0) { \ - if ((X) == Py_HUGE_VAL || (X) == -Py_HUGE_VAL) \ - errno = ERANGE; \ - else _Py_SET_EDOM_FOR_NAN(X) \ - } \ - } while(0) - -/* Py_SET_ERANGE_IF_OVERFLOW(x) - * An alias of Py_SET_ERRNO_ON_MATH_ERROR for backward-compatibility. - */ -#define Py_SET_ERANGE_IF_OVERFLOW(X) Py_SET_ERRNO_ON_MATH_ERROR(X) - -/* Py_ADJUST_ERANGE1(x) - * Py_ADJUST_ERANGE2(x, y) - * Set errno to 0 before calling a libm function, and invoke one of these - * macros after, passing the function result(s) (Py_ADJUST_ERANGE2 is useful - * for functions returning complex results). This makes two kinds of - * adjustments to errno: (A) If it looks like the platform libm set - * errno=ERANGE due to underflow, clear errno. (B) If it looks like the - * platform libm overflowed but didn't set errno, force errno to ERANGE. In - * effect, we're trying to force a useful implementation of C89 errno - * behavior. - * Caution: - * This isn't reliable. See Py_OVERFLOWED comments. - * X and Y may be evaluated more than once. - */ -#define Py_ADJUST_ERANGE1(X) \ - do { \ - if (errno == 0) { \ - if ((X) == Py_HUGE_VAL || (X) == -Py_HUGE_VAL) \ - errno = ERANGE; \ - } \ - else if (errno == ERANGE && (X) == 0.0) \ - errno = 0; \ - } while(0) - -#define Py_ADJUST_ERANGE2(X, Y) \ - do { \ - if ((X) == Py_HUGE_VAL || (X) == -Py_HUGE_VAL || \ - (Y) == Py_HUGE_VAL || (Y) == -Py_HUGE_VAL) { \ - if (errno == 0) \ - errno = ERANGE; \ - } \ - else if (errno == ERANGE) \ - errno = 0; \ - } while(0) - /* The functions _Py_dg_strtod and _Py_dg_dtoa in Python/dtoa.c (which are * required to support the short float repr introduced in Python 3.1) require * that the floating-point unit that's being used for arithmetic operations @@ -390,86 +334,25 @@ extern "C" { * * #define HAVE_PY_SET_53BIT_PRECISION 1 * - * and also give appropriate definitions for the following three macros: - * - * _PY_SET_53BIT_PRECISION_START : store original FPU settings, and - * set FPU to 53-bit precision/round-half-to-even - * _PY_SET_53BIT_PRECISION_END : restore original FPU settings - * _PY_SET_53BIT_PRECISION_HEADER : any variable declarations needed to - * use the two macros above. - * * The macros are designed to be used within a single C function: see * Python/pystrtod.c for an example of their use. */ -/* get and set x87 control word for gcc/x86 */ +// HAVE_PY_SET_53BIT_PRECISION macro must be kept in sync with pycore_pymath.h #ifdef HAVE_GCC_ASM_FOR_X87 -#define HAVE_PY_SET_53BIT_PRECISION 1 -/* _Py_get/set_387controlword functions are defined in Python/pymath.c */ -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned short old_387controlword, new_387controlword -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - old_387controlword = _Py_get_387controlword(); \ - new_387controlword = (old_387controlword & ~0x0f00) | 0x0200; \ - if (new_387controlword != old_387controlword) \ - _Py_set_387controlword(new_387controlword); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - if (new_387controlword != old_387controlword) \ - _Py_set_387controlword(old_387controlword) -#endif - -/* get and set x87 control word for VisualStudio/x86 */ -#if defined(_MSC_VER) && !defined(_WIN64) && !defined(_M_ARM) /* x87 not supported in 64-bit or ARM */ -#define HAVE_PY_SET_53BIT_PRECISION 1 -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned int old_387controlword, new_387controlword, out_387controlword -/* We use the __control87_2 function to set only the x87 control word. - The SSE control word is unaffected. */ -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - __control87_2(0, 0, &old_387controlword, NULL); \ - new_387controlword = \ - (old_387controlword & ~(_MCW_PC | _MCW_RC)) | (_PC_53 | _RC_NEAR); \ - if (new_387controlword != old_387controlword) \ - __control87_2(new_387controlword, _MCW_PC | _MCW_RC, \ - &out_387controlword, NULL); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - do { \ - if (new_387controlword != old_387controlword) \ - __control87_2(old_387controlword, _MCW_PC | _MCW_RC, \ - &out_387controlword, NULL); \ - } while (0) + // Get and set x87 control word for gcc/x86 +# define HAVE_PY_SET_53BIT_PRECISION 1 +#endif +#if defined(_MSC_VER) && !defined(_WIN64) && !defined(_M_ARM) + // Get and set x87 control word for VisualStudio/x86. + // x87 not supported in 64-bit or ARM. +# define HAVE_PY_SET_53BIT_PRECISION 1 #endif - #ifdef HAVE_GCC_ASM_FOR_MC68881 -#define HAVE_PY_SET_53BIT_PRECISION 1 -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned int old_fpcr, new_fpcr -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ - /* Set double precision / round to nearest. */ \ - new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ - if (new_fpcr != old_fpcr) \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr)); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - do { \ - if (new_fpcr != old_fpcr) \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ - } while (0) -#endif - -/* default definitions are empty */ -#ifndef HAVE_PY_SET_53BIT_PRECISION -#define _Py_SET_53BIT_PRECISION_HEADER -#define _Py_SET_53BIT_PRECISION_START -#define _Py_SET_53BIT_PRECISION_END +# define HAVE_PY_SET_53BIT_PRECISION 1 #endif + /* If we can't guarantee 53-bit precision, don't use the code in Python/dtoa.c, but fall back to standard code. This means that repr of a float will be long (17 sig digits). @@ -485,14 +368,14 @@ extern "C" { #if !defined(DOUBLE_IS_LITTLE_ENDIAN_IEEE754) && \ !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) && \ !defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754) -#define PY_NO_SHORT_FLOAT_REPR +# define PY_NO_SHORT_FLOAT_REPR #endif /* double rounding is symptomatic of use of extended precision on x86. If we're seeing double rounding, and we don't have any mechanism available for changing the FPU rounding precision, then don't use Python/dtoa.c. */ #if defined(X87_DOUBLE_ROUNDING) && !defined(HAVE_PY_SET_53BIT_PRECISION) -#define PY_NO_SHORT_FLOAT_REPR +# define PY_NO_SHORT_FLOAT_REPR #endif @@ -828,26 +711,6 @@ extern char * _getpty(int *, int, mode_t, int); # define PY_LITTLE_ENDIAN 1 #endif -#ifdef Py_BUILD_CORE -/* - * Macros to protect CRT calls against instant termination when passed an - * invalid parameter (issue23524). - */ -#if defined _MSC_VER && _MSC_VER >= 1900 - -extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; -#define _Py_BEGIN_SUPPRESS_IPH { _invalid_parameter_handler _Py_old_handler = \ - _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); -#define _Py_END_SUPPRESS_IPH _set_thread_local_invalid_parameter_handler(_Py_old_handler); } - -#else - -#define _Py_BEGIN_SUPPRESS_IPH -#define _Py_END_SUPPRESS_IPH - -#endif /* _MSC_VER >= 1900 */ -#endif /* Py_BUILD_CORE */ - #ifdef __ANDROID__ /* The Android langinfo.h header is not used. */ # undef HAVE_LANGINFO_H @@ -914,4 +777,22 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif +/* A convenient way for code to know if sanitizers are enabled. */ +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# if !defined(_Py_MEMORY_SANITIZER) +# define _Py_MEMORY_SANITIZER +# endif +# endif +# if __has_feature(address_sanitizer) +# if !defined(_Py_ADDRESS_SANITIZER) +# define _Py_ADDRESS_SANITIZER +# endif +# endif +#elif defined(__GNUC__) +# if defined(__SANITIZE_ADDRESS__) +# define _Py_ADDRESS_SANITIZER +# endif +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/pystate.h b/Include/pystate.h index bae440778b261a..b6ee0ede81dea0 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -66,18 +66,10 @@ PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); The caller must hold the GIL. - See also PyThreadState_GET() and _PyThreadState_GET(). */ + See also _PyThreadState_UncheckedGet() and _PyThreadState_GET(). */ PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); -/* Get the current Python thread state. - - Macro using PyThreadState_Get() or _PyThreadState_GET() depending if - pycore_pystate.h is included or not (this header redefines the macro). - - If PyThreadState_Get() is used, issue a fatal error if the current thread - state is NULL. - - See also PyThreadState_Get() and _PyThreadState_GET(). */ +// Alias to PyThreadState_Get() #define PyThreadState_GET() PyThreadState_Get() PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *); @@ -140,7 +132,7 @@ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H -# include "cpython/pystate.h" +# include "cpython/pystate.h" # undef Py_CPYTHON_PYSTATE_H #endif diff --git a/Include/pystrhex.h b/Include/pystrhex.h deleted file mode 100644 index a4f36305bac69b..00000000000000 --- a/Include/pystrhex.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef Py_STRHEX_H -#define Py_STRHEX_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -/* Returns a str() containing the hex representation of argbuf. */ -PyAPI_FUNC(PyObject*) _Py_strhex(const char* argbuf, const Py_ssize_t arglen); -/* Returns a bytes() containing the ASCII hex representation of argbuf. */ -PyAPI_FUNC(PyObject*) _Py_strhex_bytes(const char* argbuf, const Py_ssize_t arglen); -/* These variants include support for a separator between every N bytes: */ -PyAPI_FUNC(PyObject*) _Py_strhex_with_sep(const char* argbuf, const Py_ssize_t arglen, const PyObject* sep, const int bytes_per_group); -PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep(const char* argbuf, const Py_ssize_t arglen, const PyObject* sep, const int bytes_per_group); -#endif /* !Py_LIMITED_API */ - -#ifdef __cplusplus -} -#endif - -#endif /* !Py_STRHEX_H */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index b0a2fc3002d37d..02715775581c63 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -34,7 +34,7 @@ PyAPI_FUNC(int) PyOS_CheckStack(void); #ifndef Py_LIMITED_API # define Py_CPYTHON_PYTHONRUN_H -# include "cpython/pythonrun.h" +# include "cpython/pythonrun.h" # undef Py_CPYTHON_PYTHONRUN_H #endif diff --git a/Include/pythread.h b/Include/pythread.h index bb9d86412218ad..1a6092c4ad0be2 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -61,9 +61,11 @@ PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock); convert microseconds to nanoseconds. */ # define PY_TIMEOUT_MAX (LLONG_MAX / 1000) #elif defined (NT_THREADS) - /* In the NT API, the timeout is a DWORD and is expressed in milliseconds */ -# if 0xFFFFFFFFLL * 1000 < LLONG_MAX -# define PY_TIMEOUT_MAX (0xFFFFFFFFLL * 1000) + // WaitForSingleObject() accepts timeout in milliseconds in the range + // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no + // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. +# if 0xFFFFFFFELL * 1000 < LLONG_MAX +# define PY_TIMEOUT_MAX (0xFFFFFFFELL * 1000) # else # define PY_TIMEOUT_MAX LLONG_MAX # endif diff --git a/Include/structseq.h b/Include/structseq.h index fc3fcd77f2d414..ccafe8d87f8f51 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -19,7 +19,7 @@ typedef struct PyStructSequence_Desc { int n_in_sequence; } PyStructSequence_Desc; -extern const char * const PyStructSequence_UnnamedField; +PyAPI_DATA(const char * const) PyStructSequence_UnnamedField; #ifndef Py_LIMITED_API PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 8c8f7c425942a5..3463c622309009 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -31,7 +31,7 @@ PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); #ifndef Py_LIMITED_API # define Py_CPYTHON_SYSMODULE_H -# include "cpython/sysmodule.h" +# include "cpython/sysmodule.h" # undef Py_CPYTHON_SYSMODULE_H #endif diff --git a/Include/traceback.h b/Include/traceback.h index 781e5a6eec4edd..2dfa2ada4f2c37 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -16,7 +16,7 @@ PyAPI_DATA(PyTypeObject) PyTraceBack_Type; #ifndef Py_LIMITED_API # define Py_CPYTHON_TRACEBACK_H -# include "cpython/traceback.h" +# include "cpython/traceback.h" # undef Py_CPYTHON_TRACEBACK_H #endif diff --git a/Include/tupleobject.h b/Include/tupleobject.h index e796a320192c20..dc68e3fc5c6d88 100644 --- a/Include/tupleobject.h +++ b/Include/tupleobject.h @@ -36,7 +36,7 @@ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); #ifndef Py_LIMITED_API # define Py_CPYTHON_TUPLEOBJECT_H -# include "cpython/tupleobject.h" +# include "cpython/tupleobject.h" # undef Py_CPYTHON_TUPLEOBJECT_H #endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index b0ac086a6be23d..abce967caff78e 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1,7 +1,7 @@ #ifndef Py_UNICODEOBJECT_H #define Py_UNICODEOBJECT_H -#include +#include // va_list /* @@ -1043,7 +1043,7 @@ PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s); #ifndef Py_LIMITED_API # define Py_CPYTHON_UNICODEOBJECT_H -# include "cpython/unicodeobject.h" +# include "cpython/unicodeobject.h" # undef Py_CPYTHON_UNICODEOBJECT_H #endif diff --git a/Include/warnings.h b/Include/warnings.h index a675bb5dfcb9f5..18ac1543a3ca9e 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -4,14 +4,11 @@ extern "C" { #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); -#endif - PyAPI_FUNC(int) PyErr_WarnEx( PyObject *category, const char *message, /* UTF-8 encoded string */ Py_ssize_t stack_level); + PyAPI_FUNC(int) PyErr_WarnFormat( PyObject *category, Py_ssize_t stack_level, @@ -26,15 +23,7 @@ PyAPI_FUNC(int) PyErr_ResourceWarning( const char *format, /* ASCII-encoded string */ ...); #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) PyErr_WarnExplicitObject( - PyObject *category, - PyObject *message, - PyObject *filename, - int lineno, - PyObject *module, - PyObject *registry); -#endif + PyAPI_FUNC(int) PyErr_WarnExplicit( PyObject *category, const char *message, /* UTF-8 encoded string */ @@ -44,20 +33,9 @@ PyAPI_FUNC(int) PyErr_WarnExplicit( PyObject *registry); #ifndef Py_LIMITED_API -PyAPI_FUNC(int) -PyErr_WarnExplicitFormat(PyObject *category, - const char *filename, int lineno, - const char *module, PyObject *registry, - const char *format, ...); -#endif - -/* DEPRECATED: Use PyErr_WarnEx() instead. */ -#ifndef Py_LIMITED_API -#define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1) -#endif - -#ifndef Py_LIMITED_API -void _PyErr_WarnUnawaitedCoroutine(PyObject *coro); +# define Py_CPYTHON_WARNINGS_H +# include "cpython/warnings.h" +# undef Py_CPYTHON_WARNINGS_H #endif #ifdef __cplusplus diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index ac4b4821c8a147..f071e9c759a641 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -6,40 +6,8 @@ extern "C" { #endif - typedef struct _PyWeakReference PyWeakReference; -/* PyWeakReference is the base struct for the Python ReferenceType, ProxyType, - * and CallableProxyType. - */ -#ifndef Py_LIMITED_API -struct _PyWeakReference { - PyObject_HEAD - - /* The object to which this is a weak reference, or Py_None if none. - * Note that this is a stealth reference: wr_object's refcount is - * not incremented to reflect this pointer. - */ - PyObject *wr_object; - - /* A callable to invoke when wr_object dies, or NULL if none. */ - PyObject *wr_callback; - - /* A cache for wr_object's hash code. As usual for hashes, this is -1 - * if the hash code isn't known yet. - */ - Py_hash_t hash; - - /* If wr_object is weakly referenced, wr_object has a doubly-linked NULL- - * terminated list of weak references to it. These are the list pointers. - * If wr_object goes away, wr_object is set to Py_None, and these pointers - * have no meaning then. - */ - PyWeakReference *wr_prev; - PyWeakReference *wr_next; -}; -#endif - PyAPI_DATA(PyTypeObject) _PyWeakref_RefType; PyAPI_DATA(PyTypeObject) _PyWeakref_ProxyType; PyAPI_DATA(PyTypeObject) _PyWeakref_CallableProxyType; @@ -56,30 +24,18 @@ PyAPI_DATA(PyTypeObject) _PyWeakref_CallableProxyType; PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, - PyObject *callback); + PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, - PyObject *callback); + PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); -#ifndef Py_LIMITED_API -PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); -PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); +#ifndef Py_LIMITED_API +# define Py_CPYTHON_WEAKREFOBJECT_H +# include "cpython/weakrefobject.h" +# undef Py_CPYTHON_WEAKREFOBJECT_H #endif -/* Explanation for the Py_REFCNT() check: when a weakref's target is part - of a long chain of deallocations which triggers the trashcan mechanism, - clearing the weakrefs can be delayed long after the target's refcount - has dropped to zero. In the meantime, code accessing the weakref will - be able to "see" the target object even though it is supposed to be - unreachable. See issue #16602. */ - -#define PyWeakref_GET_OBJECT(ref) \ - (Py_REFCNT(((PyWeakReference *)(ref))->wr_object) > 0 \ - ? ((PyWeakReference *)(ref))->wr_object \ - : Py_None) - - #ifdef __cplusplus } #endif diff --git a/Lib/__phello__.foo.py b/Lib/__phello__.foo.py deleted file mode 100644 index 8e8623ee1daacb..00000000000000 --- a/Lib/__phello__.foo.py +++ /dev/null @@ -1 +0,0 @@ -# This file exists as a helper for the test.test_frozen module. diff --git a/Lib/__phello__/__init__.py b/Lib/__phello__/__init__.py new file mode 100644 index 00000000000000..d37bd2766ac1c6 --- /dev/null +++ b/Lib/__phello__/__init__.py @@ -0,0 +1,7 @@ +initialized = True + +def main(): + print("Hello world!") + +if __name__ == '__main__': + main() diff --git a/Lib/__phello__/ham/__init__.py b/Lib/__phello__/ham/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/__phello__/ham/eggs.py b/Lib/__phello__/ham/eggs.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/__phello__/spam.py b/Lib/__phello__/spam.py new file mode 100644 index 00000000000000..d37bd2766ac1c6 --- /dev/null +++ b/Lib/__phello__/spam.py @@ -0,0 +1,7 @@ +initialized = True + +def main(): + print("Hello world!") + +if __name__ == '__main__': + main() diff --git a/Lib/argparse.py b/Lib/argparse.py index df98999f875d9e..4cc14f23722463 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -526,12 +526,13 @@ def _format_action(self, action): parts = [action_header] # if there was help for the action, add lines of help text - if action.help: + if action.help and action.help.strip(): help_text = self._expand_help(action) - help_lines = self._split_lines(help_text, help_width) - parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) - for line in help_lines[1:]: - parts.append('%*s%s\n' % (help_position, '', line)) + if help_text: + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) # or add a newline if the description doesn't end with one elif not action_header.endswith('\n'): diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index b966ad26bf467b..7abaaca2d2b284 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -479,7 +479,7 @@ async def connect_read_pipe(self, protocol_factory, pipe): # The reason to accept file-like object instead of just file descriptor # is: we need to own pipe and close it at transport finishing # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. + # close fd in pipe transport then close f and vice versa. raise NotImplementedError async def connect_write_pipe(self, protocol_factory, pipe): @@ -492,7 +492,7 @@ async def connect_write_pipe(self, protocol_factory, pipe): # The reason to accept file-like object instead of just file descriptor # is: we need to own pipe and close it at transport finishing # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. + # close fd in pipe transport then close f and vice versa. raise NotImplementedError async def subprocess_shell(self, protocol_factory, cmd, *, diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index a7453fb1c77287..4fef64e3921e17 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -230,8 +230,6 @@ def __init__(self, lock=None, *, loop=mixins._marker): super().__init__(loop=loop) if lock is None: lock = Lock() - elif lock._loop is not self._get_loop(): - raise ValueError("loop argument must agree with lock") self._lock = lock # Export the lock's locked(), acquire() and release() methods. diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index e4f445e95026b5..c88b818de62a6f 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1379,7 +1379,7 @@ def add_child_handler(self, pid, callback, *args): def remove_child_handler(self, pid): # asyncio never calls remove_child_handler() !!! # The method is no-op but is implemented because - # abstract base classe requires it + # abstract base classes require it. return True def attach_loop(self, loop): diff --git a/Lib/base64.py b/Lib/base64.py index b25156ddb35d3f..7e9c2a2ca477ff 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -567,15 +567,17 @@ def decodebytes(s): def main(): """Small main program""" import sys, getopt + usage = """usage: %s [-h|-d|-e|-u|-t] [file|-] + -h: print this help message and exit + -d, -u: decode + -e: encode (default) + -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0] try: - opts, args = getopt.getopt(sys.argv[1:], 'deut') + opts, args = getopt.getopt(sys.argv[1:], 'hdeut') except getopt.error as msg: sys.stdout = sys.stderr print(msg) - print("""usage: %s [-d|-e|-u|-t] [file|-] - -d, -u: decode - -e: encode (default) - -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0]) + print(usage) sys.exit(2) func = encode for o, a in opts: @@ -583,6 +585,7 @@ def main(): if o == '-d': func = decode if o == '-u': func = decode if o == '-t': test(); return + if o == '-h': print(usage); return if args and args[0] != '-': with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) diff --git a/Lib/bz2.py b/Lib/bz2.py index 7f1d20632ef139..fabe4f73c8d808 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -197,10 +197,6 @@ def readline(self, size=-1): self._check_can_read() return self._buffer.readline(size) - def __iter__(self): - self._check_can_read() - return self._buffer.__iter__() - def readlines(self, size=-1): """Read a list of lines of uncompressed bytes from the file. diff --git a/Lib/cgitb.py b/Lib/cgitb.py index 17ddda376884df..ec156843099d36 100644 --- a/Lib/cgitb.py +++ b/Lib/cgitb.py @@ -31,6 +31,7 @@ import time import tokenize import traceback +from html import escape as html_escape def reset(): """Return a string that resets the CGI and browser to a known state.""" @@ -105,10 +106,16 @@ def html(einfo, context=5): etype = etype.__name__ pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable date = time.ctime(time.time()) - head = '' + pydoc.html.heading( - '%s' % - strong(pydoc.html.escape(str(etype))), - '#ffffff', '#6622aa', pyver + '
' + date) + ''' + head = f''' + + + + + +
 
+ 
+{html_escape(str(etype))}		 +{pyver}
{date}

A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred.

''' diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 1384d8903d17bf..ee72258505714a 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,5 +1,6 @@ """Utilities for with-statement contexts. See PEP 343.""" import abc +import os import sys import _collections_abc from collections import deque @@ -9,7 +10,8 @@ __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", "AsyncExitStack", "ContextDecorator", "ExitStack", - "redirect_stdout", "redirect_stderr", "suppress", "aclosing"] + "redirect_stdout", "redirect_stderr", "suppress", "aclosing", + "chdir"] class AbstractContextManager(abc.ABC): @@ -553,10 +555,10 @@ def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: exc_context = new_exc.__context__ - if exc_context is old_exc: + if exc_context is None or exc_context is old_exc: # Context is already set correctly (see issue 20317) return - if exc_context is None or exc_context is frame_exc: + if exc_context is frame_exc: break new_exc = exc_context # Change the end of the chain to point to the exception @@ -693,10 +695,10 @@ def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: exc_context = new_exc.__context__ - if exc_context is old_exc: + if exc_context is None or exc_context is old_exc: # Context is already set correctly (see issue 20317) return - if exc_context is None or exc_context is frame_exc: + if exc_context is frame_exc: break new_exc = exc_context # Change the end of the chain to point to the exception @@ -762,3 +764,18 @@ async def __aenter__(self): async def __aexit__(self, *excinfo): pass + + +class chdir(AbstractContextManager): + """Non thread-safe context manager to change the current working directory.""" + + def __init__(self, path): + self.path = path + self._old_cwd = [] + + def __enter__(self): + self._old_cwd.append(os.getcwd()) + os.chdir(self.path) + + def __exit__(self, *excinfo): + os.chdir(self._old_cwd.pop()) diff --git a/Lib/ctypes/_aix.py b/Lib/ctypes/_aix.py index 26959d90a4dd6a..fc3e95cbcc88a5 100644 --- a/Lib/ctypes/_aix.py +++ b/Lib/ctypes/_aix.py @@ -163,7 +163,7 @@ def get_legacy(members): return member else: # 32-bit legacy names - both shr.o and shr4.o exist. - # shr.o is the preffered name so we look for shr.o first + # shr.o is the preferred name so we look for shr.o first # i.e., shr4.o is returned only when shr.o does not exist for name in ['shr.o', 'shr4.o']: member = get_one_match(re.escape(name), members) diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 245cd94c5cdd9e..97ad2b8ed8a50d 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -443,7 +443,7 @@ def __del__(self): s = Test(1, 2, 3) # Test the StructUnionType_paramfunc() code path which copies the - # structure: if the stucture is larger than sizeof(void*). + # structure: if the structure is larger than sizeof(void*). self.assertGreater(sizeof(s), sizeof(c_void_p)) dll = CDLL(_ctypes_test.__file__) @@ -451,7 +451,7 @@ def __del__(self): func.argtypes = (Test,) func.restype = None func(s) - # bpo-37140: Passing the structure by refrence must not call + # bpo-37140: Passing the structure by reference must not call # its finalizer! self.assertEqual(finalizer_calls, []) self.assertEqual(s.first, 1) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 95ff39287bed61..1e98bf9b9bb97b 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1387,7 +1387,7 @@ def exec_body_callback(ns): ns['__annotations__'] = annotations # We use `types.new_class()` instead of simply `type()` to allow dynamic creation - # of generic dataclassses. + # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) # Apply the normal decorator. diff --git a/Lib/difflib.py b/Lib/difflib.py index 480bad2224c8e1..afd8a0c7c5b61e 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -62,7 +62,7 @@ class SequenceMatcher: notion, pairing up elements that appear uniquely in each sequence. That, and the method here, appear to yield more intuitive difference reports than does diff. This method appears to be the least vulnerable - to synching up on blocks of "junk lines", though (like blank lines in + to syncing up on blocks of "junk lines", though (like blank lines in ordinary text files, or maybe "

" lines in HTML files). That may be because this is the only method of the 3 that has a *concept* of "junk" . diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py index b5ef143e72c564..4c47f2ed245d4f 100644 --- a/Lib/distutils/ccompiler.py +++ b/Lib/distutils/ccompiler.py @@ -392,7 +392,7 @@ def _fix_compile_args(self, output_dir, macros, include_dirs): return output_dir, macros, include_dirs def _prep_compile(self, sources, output_dir, depends=None): - """Decide which souce files must be recompiled. + """Decide which source files must be recompiled. Determine the list of object files corresponding to 'sources', and figure out which ones really need to be recompiled. diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 26696cfb9dcf9c..01d5331a63069b 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -31,7 +31,7 @@ # while making the sysconfig module the single point of truth. # This makes it easier for OS distributions where they need to # alter locations for packages installations in a single place. -# Note that this module is depracated (PEP 632); all consumers +# Note that this module is deprecated (PEP 632); all consumers # of this information should switch to using sysconfig directly. INSTALL_SCHEMES = {"unix_prefix": {}, "unix_home": {}, "nt": {}} @@ -43,7 +43,7 @@ sys_key = key sys_scheme = sysconfig._INSTALL_SCHEMES[sys_scheme_name] if key == "headers" and key not in sys_scheme: - # On POSIX-y platofrms, Python will: + # On POSIX-y platforms, Python will: # - Build from .h files in 'headers' (only there when # building CPython) # - Install .h files to 'include' diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py index f0792de74a1a48..d00c48981eb6d6 100644 --- a/Lib/distutils/unixccompiler.py +++ b/Lib/distutils/unixccompiler.py @@ -215,7 +215,8 @@ def library_dir_option(self, dir): return "-L" + dir def _is_gcc(self, compiler_name): - return "gcc" in compiler_name or "g++" in compiler_name + # clang uses same syntax for rpath as gcc + return any(name in compiler_name for name in ("gcc", "g++", "clang")) def runtime_library_dir_option(self, dir): # XXX Hackish, at the very least. See Python bug #445902: diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 977fedf67b1591..ba5ad5a36d06b7 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -128,6 +128,8 @@ def _parsedate_tz(data): tss = 0 elif len(tm) == 3: [thh, tmm, tss] = tm + else: + return None else: return None try: diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 1d258c34fc9d4a..3ad00565549968 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -110,4 +110,4 @@ class NonASCIILocalPartDefect(HeaderDefect): # parsing messages decoded from binary. class InvalidDateDefect(HeaderDefect): - """Header has unparseable or invalid date""" + """Header has unparsable or invalid date""" diff --git a/Lib/encodings/raw_unicode_escape.py b/Lib/encodings/raw_unicode_escape.py index 2b919b40d3788a..46c8e070dd192e 100644 --- a/Lib/encodings/raw_unicode_escape.py +++ b/Lib/encodings/raw_unicode_escape.py @@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): return codecs.raw_unicode_escape_encode(input, self.errors)[0] -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.raw_unicode_escape_decode(input, self.errors)[0] +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return codecs.raw_unicode_escape_decode(input, errors, final) class StreamWriter(Codec,codecs.StreamWriter): pass class StreamReader(Codec,codecs.StreamReader): - pass + def decode(self, input, errors='strict'): + return codecs.raw_unicode_escape_decode(input, errors, False) ### encodings module API diff --git a/Lib/encodings/unicode_escape.py b/Lib/encodings/unicode_escape.py index 817f93265a4634..9b1ce99b339ae0 100644 --- a/Lib/encodings/unicode_escape.py +++ b/Lib/encodings/unicode_escape.py @@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): return codecs.unicode_escape_encode(input, self.errors)[0] -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.unicode_escape_decode(input, self.errors)[0] +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return codecs.unicode_escape_decode(input, errors, final) class StreamWriter(Codec,codecs.StreamWriter): pass class StreamReader(Codec,codecs.StreamReader): - pass + def decode(self, input, errors='strict'): + return codecs.unicode_escape_decode(input, errors, False) ### encodings module API diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 651f876e24bc8b..94f1b6604cb7ab 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,8 +10,8 @@ __all__ = ["version", "bootstrap"] _PACKAGE_NAMES = ('setuptools', 'pip') -_SETUPTOOLS_VERSION = "57.4.0" -_PIP_VERSION = "21.2.3" +_SETUPTOOLS_VERSION = "58.1.0" +_PIP_VERSION = "21.2.4" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py3"), @@ -41,7 +41,7 @@ def _find_packages(path): # comparison since this case should not happen. filenames = sorted(filenames) for filename in filenames: - # filename is like 'pip-20.2.3-py2.py3-none-any.whl' + # filename is like 'pip-21.2.4-py3-none-any.whl' if not filename.endswith(".whl"): continue for name in _PACKAGE_NAMES: @@ -51,7 +51,7 @@ def _find_packages(path): else: continue - # Extract '20.2.2' from 'pip-20.2.2-py2.py3-none-any.whl' + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' version = filename.removeprefix(prefix).partition('-')[0] wheel_path = os.path.join(path, filename) packages[name] = _Package(version, None, wheel_path) diff --git a/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl deleted file mode 100644 index d417df63d9ec19..00000000000000 Binary files a/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-21.2.4-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.2.4-py3-none-any.whl new file mode 100644 index 00000000000000..46d3012c59b174 Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-21.2.4-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-58.1.0-py3-none-any.whl similarity index 80% rename from Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-58.1.0-py3-none-any.whl index af8f8ba21003b1..18c8c22958f1f1 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-58.1.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 0776761ae6e735..461d276eed862a 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -235,11 +235,18 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) # If another member with the same value was already defined, the # new member becomes an alias to the existing one. - for name, canonical_member in enum_class._member_map_.items(): - if canonical_member._value_ == enum_member._value_: - enum_member = canonical_member - break - else: + try: + try: + # try to do a fast lookup to avoid the quadratic loop + enum_member = enum_class._value2member_map_[value] + except TypeError: + for name, canonical_member in enum_class._member_map_.items(): + if canonical_member._value_ == value: + enum_member = canonical_member + break + else: + raise KeyError + except KeyError: # this could still be an alias if the value is multi-bit and the # class is a flag class if ( @@ -301,7 +308,7 @@ class _EnumDict(dict): """ def __init__(self): super().__init__() - self._member_names = [] + self._member_names = {} # use a dict to keep insertion order self._last_values = [] self._ignore = [] self._auto_called = False @@ -365,7 +372,7 @@ def __setitem__(self, key, value): ) self._auto_called = True value = value.value - self._member_names.append(key) + self._member_names[key] = None self._last_values.append(value) super().__setitem__(key, value) diff --git a/Lib/gzip.py b/Lib/gzip.py index 0dddb51553fabd..ac1781042b2640 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -398,10 +398,6 @@ def readline(self, size=-1): self._check_not_closed() return self._buffer.readline(size) - def __iter__(self): - self._check_not_closed() - return self._buffer.__iter__() - def _read_exact(fp, n): '''Read exactly *n* bytes from `fp` diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 9e49effca1fcc5..bef0f4fe4bf776 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -328,13 +328,6 @@ def parse_starttag(self, i): end = rawdata[k:endpos].strip() if end not in (">", "/>"): - lineno, offset = self.getpos() - if "\n" in self.__starttag_text: - lineno = lineno + self.__starttag_text.count("\n") - offset = len(self.__starttag_text) \ - - self.__starttag_text.rfind("\n") - else: - offset = offset + len(self.__starttag_text) self.handle_data(rawdata[i:endpos]) return endpos if end.endswith('/>'): @@ -405,7 +398,7 @@ def parse_endtag(self, i): tagname = namematch.group(1).lower() # consume and ignore other stuff between the name and the > # Note: this is not 100% correct, since we might have things like - # , but looking for > after tha name should cover + # , but looking for > after the name should cover # most of the cases and is much simpler gtpos = rawdata.find('>', namematch.end()) self.handle_endtag(tagname) diff --git a/Lib/http/client.py b/Lib/http/client.py index 08cf2ed9b3716b..a6ab135b2c3879 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -70,6 +70,7 @@ import email.parser import email.message +import errno import http import io import re @@ -939,7 +940,12 @@ def connect(self): sys.audit("http.client.connect", self, self.host, self.port) self.sock = self._create_connection( (self.host,self.port), self.timeout, self.source_address) - self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # Might fail in OSs that don't implement TCP_NODELAY + try: + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + except OSError as e: + if e.errno != errno.ENOPROTOOPT: + raise if self._tunnel_host: self._tunnel() diff --git a/Lib/idlelib/ChangeLog b/Lib/idlelib/ChangeLog index d7d7e1efdb1d30..c8960cfa535d03 100644 --- a/Lib/idlelib/ChangeLog +++ b/Lib/idlelib/ChangeLog @@ -1175,7 +1175,7 @@ Wed Mar 10 05:18:02 1999 Guido van Rossum classes in selected module methods of selected class - Sinlge clicking in a directory, module or class item updates the next + Single clicking in a directory, module or class item updates the next column with info about the selected item. Double clicking in a module, class or method item opens the file (and selects the clicked item if it is a class or method). diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index bb7ee035c4fefb..032d31225315fb 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -9,6 +9,12 @@ import string import sys +# Modified keyword list is used in fetch_completions. +completion_kwds = [s for s in keyword.kwlist + if s not in {'True', 'False', 'None'}] # In builtins. +completion_kwds.extend(('match', 'case')) # Context keywords. +completion_kwds.sort() + # Two types of completions; defined here for autocomplete_w import below. ATTRS, FILES = 0, 1 from idlelib import autocomplete_w @@ -177,9 +183,7 @@ def fetch_completions(self, what, mode): namespace = {**__main__.__builtins__.__dict__, **__main__.__dict__} bigl = eval("dir()", namespace) - kwds = (s for s in keyword.kwlist - if s not in {'True', 'False', 'None'}) - bigl.extend(kwds) + bigl.extend(completion_kwds) bigl.sort() if "__all__" in bigl: smalll = sorted(eval("__all__", namespace)) diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index 1373b7642a6ea9..666ff4cb848510 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -246,7 +246,7 @@ def _wrapper(parent): # htest # _object_browser_spec = { 'file': 'debugobj', 'kwds': {}, - 'msg': "Double click on items upto the lowest level.\n" + 'msg': "Double click on items up to the lowest level.\n" "Attributes of the objects and related information " "will be displayed side-by-side at each level." } @@ -255,7 +255,7 @@ def _wrapper(parent): # htest # 'file': 'pathbrowser', 'kwds': {}, 'msg': "Test for correct display of all paths in sys.path.\n" - "Toggle nested items upto the lowest level.\n" + "Toggle nested items up to the lowest level.\n" "Double clicking on an item prints a traceback\n" "for an exception that is ignored." } @@ -341,7 +341,7 @@ def _wrapper(parent): # htest # 'file': 'tree', 'kwds': {}, 'msg': "The canvas is scrollable.\n" - "Click on folders upto to the lowest level." + "Click on folders up to to the lowest level." } _undo_delegator_spec = { diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py index db583553838fb3..8304734b847a83 100644 --- a/Lib/idlelib/idle_test/mock_tk.py +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -79,7 +79,7 @@ def tearDownClass(cls): --- For 'ask' functions, set func.result return value before calling the method that uses the message function. When messagebox functions are the - only gui alls in a method, this replacement makes the method gui-free, + only GUI calls in a method, this replacement makes the method GUI-free, """ askokcancel = Mbox_func() # True or False askquestion = Mbox_func() # 'yes' or 'no' diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 642bb5db64dc34..a811363c18d04e 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -218,6 +218,11 @@ def make_acw(): return self.dummy_acw() self.assertTrue(acp.open_completions(ac.TAB)) self.text.delete('1.0', 'end') + def test_completion_kwds(self): + self.assertIn('and', ac.completion_kwds) + self.assertIn('case', ac.completion_kwds) + self.assertNotIn('None', ac.completion_kwds) + def test_fetch_completions(self): # Test that fetch_completions returns 2 lists: # For attribute completion, a large list containing all variables, and diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py index ba582bb3ca51b4..b3a3bfb88f9c31 100644 --- a/Lib/idlelib/idle_test/test_multicall.py +++ b/Lib/idlelib/idle_test/test_multicall.py @@ -37,7 +37,7 @@ def test_init(self): def test_yview(self): # Added for tree.wheel_event - # (it depends on yview to not be overriden) + # (it depends on yview to not be overridden) mc = self.mc self.assertIs(mc.yview, Text.yview) mctext = self.mc(self.root) diff --git a/Lib/idlelib/idle_test/test_pyparse.py b/Lib/idlelib/idle_test/test_pyparse.py index fb5726db1d821e..384db566ac76cd 100644 --- a/Lib/idlelib/idle_test/test_pyparse.py +++ b/Lib/idlelib/idle_test/test_pyparse.py @@ -284,7 +284,7 @@ def test_get_num_lines_in_stmt(self): tests = ( TestInfo('[x for x in a]\n', 1), # Closed on one line. TestInfo('[x\nfor x in a\n', 2), # Not closed. - TestInfo('[x\\\nfor x in a\\\n', 2), # "", uneeded backslashes. + TestInfo('[x\\\nfor x in a\\\n', 2), # "", unneeded backslashes. TestInfo('[x\nfor x in a\n]\n', 3), # Closed on multi-line. TestInfo('\n"""Docstring comment L1"""\nL2\nL3\nL4\n', 1), TestInfo('\n"""Docstring comment L1\nL2"""\nL3\nL4\n', 1), diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 4e7440038ac997..6c333b0bc3b818 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -66,6 +66,13 @@ HOST = '127.0.0.1' # python execution server on localhost loopback PORT = 0 # someday pass in host, port for remote debug capability +try: # In case IDLE started with -n. + eof = 'Ctrl-D (end-of-file)' + exit.eof = eof + quit.eof = eof +except NameError: # In case python started with -S. + pass + # Override warnings module to write to warning_stream. Initialize to send IDLE # internal warnings to the console. ScriptBinding.check_syntax() will # temporarily redirect the stream to the shell window to display warnings when diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 3836727691229e..47c4cbdcb8c3f9 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -40,6 +40,13 @@ LOCALHOST = '127.0.0.1' +try: + eof = 'Ctrl-D (end-of-file)' + exit.eof = eof + quit.eof = eof +except NameError: # In case subprocess started with -S (maybe in future). + pass + def idle_formatwarning(message, category, filename, lineno, line=None): """Format warnings the IDLE way.""" diff --git a/Lib/importlib/_adapters.py b/Lib/importlib/_adapters.py index 9907b148b396d2..ea363d86a564b5 100644 --- a/Lib/importlib/_adapters.py +++ b/Lib/importlib/_adapters.py @@ -41,8 +41,8 @@ def _io_wrapper(file, mode='r', *args, **kwargs): class CompatibilityFiles: """ - Adapter for an existing or non-existant resource reader - to provide a compability .files(). + Adapter for an existing or non-existent resource reader + to provide a compatibility .files(). """ class SpecPath(abc.Traversable): @@ -83,7 +83,7 @@ def open(self, mode='r', *args, **kwargs): class ChildPath(abc.Traversable): """ Path tied to a resource reader child. - Can be read but doesn't expose any meaningfull children. + Can be read but doesn't expose any meaningful children. """ def __init__(self, reader, name): diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index e64f7ad151755d..afb95f4e1df869 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -421,7 +421,10 @@ def has_location(self, value): def spec_from_loader(name, loader, *, origin=None, is_package=None): """Return a module spec based on various loader methods.""" - if hasattr(loader, 'get_filename'): + if origin is None: + origin = getattr(loader, '_ORIGIN', None) + + if not origin and hasattr(loader, 'get_filename'): if _bootstrap_external is None: raise NotImplementedError spec_from_file_location = _bootstrap_external.spec_from_file_location @@ -467,12 +470,9 @@ def _spec_from_module(module, loader=None, origin=None): except AttributeError: location = None if origin is None: - if location is None: - try: - origin = loader._ORIGIN - except AttributeError: - origin = None - else: + if loader is not None: + origin = getattr(loader, '_ORIGIN', None) + if not origin and location is not None: origin = location try: cached = module.__cached__ @@ -484,7 +484,7 @@ def _spec_from_module(module, loader=None, origin=None): submodule_search_locations = None spec = ModuleSpec(name, loader, origin=origin) - spec._set_fileattr = False if location is None else True + spec._set_fileattr = False if location is None else (origin == location) spec.cached = cached spec.submodule_search_locations = submodule_search_locations return spec @@ -507,9 +507,9 @@ def _init_module_attrs(spec, module, *, override=False): if spec.submodule_search_locations is not None: if _bootstrap_external is None: raise NotImplementedError - _NamespaceLoader = _bootstrap_external._NamespaceLoader + NamespaceLoader = _bootstrap_external.NamespaceLoader - loader = _NamespaceLoader.__new__(_NamespaceLoader) + loader = NamespaceLoader.__new__(NamespaceLoader) loader._path = spec.submodule_search_locations spec.loader = loader # While the docs say that module.__file__ is not set for @@ -541,6 +541,7 @@ def _init_module_attrs(spec, module, *, override=False): # __path__ if override or getattr(module, '__path__', None) is None: if spec.submodule_search_locations is not None: + # XXX We should extend __path__ if it's already a list. try: module.__path__ = spec.submodule_search_locations except AttributeError: @@ -825,11 +826,128 @@ def module_repr(m): return ''.format(m.__name__, FrozenImporter._ORIGIN) @classmethod - def find_spec(cls, fullname, path=None, target=None): - if _imp.is_frozen(fullname): - return spec_from_loader(fullname, cls, origin=cls._ORIGIN) + def _fix_up_module(cls, module): + spec = module.__spec__ + state = spec.loader_state + if state is None: + # The module is missing FrozenImporter-specific values. + + # Fix up the spec attrs. + origname = vars(module).pop('__origname__', None) + assert origname, 'see PyImport_ImportFrozenModuleObject()' + ispkg = hasattr(module, '__path__') + assert _imp.is_frozen_package(module.__name__) == ispkg, ispkg + filename, pkgdir = cls._resolve_filename(origname, spec.name, ispkg) + spec.loader_state = type(sys.implementation)( + filename=filename, + origname=origname, + ) + __path__ = spec.submodule_search_locations + if ispkg: + assert __path__ == [], __path__ + if pkgdir: + spec.submodule_search_locations.insert(0, pkgdir) + else: + assert __path__ is None, __path__ + + # Fix up the module attrs (the bare minimum). + assert not hasattr(module, '__file__'), module.__file__ + if filename: + try: + module.__file__ = filename + except AttributeError: + pass + if ispkg: + if module.__path__ != __path__: + assert module.__path__ == [], module.__path__ + module.__path__.extend(__path__) else: + # These checks ensure that _fix_up_module() is only called + # in the right places. + __path__ = spec.submodule_search_locations + ispkg = __path__ is not None + # Check the loader state. + assert sorted(vars(state)) == ['filename', 'origname'], state + if state.origname: + # The only frozen modules with "origname" set are stdlib modules. + (__file__, pkgdir, + ) = cls._resolve_filename(state.origname, spec.name, ispkg) + assert state.filename == __file__, (state.filename, __file__) + if pkgdir: + assert __path__ == [pkgdir], (__path__, pkgdir) + else: + assert __path__ == ([] if ispkg else None), __path__ + else: + __file__ = None + assert state.filename is None, state.filename + assert __path__ == ([] if ispkg else None), __path__ + # Check the file attrs. + if __file__: + assert hasattr(module, '__file__') + assert module.__file__ == __file__, (module.__file__, __file__) + else: + assert not hasattr(module, '__file__'), module.__file__ + if ispkg: + assert hasattr(module, '__path__') + assert module.__path__ == __path__, (module.__path__, __path__) + else: + assert not hasattr(module, '__path__'), module.__path__ + assert not spec.has_location + + @classmethod + def _resolve_filename(cls, fullname, alias=None, ispkg=False): + if not fullname or not getattr(sys, '_stdlib_dir', None): + return None, None + try: + sep = cls._SEP + except AttributeError: + sep = cls._SEP = '\\' if sys.platform == 'win32' else '/' + + if fullname != alias: + if fullname.startswith('<'): + fullname = fullname[1:] + if not ispkg: + fullname = f'{fullname}.__init__' + else: + ispkg = False + relfile = fullname.replace('.', sep) + if ispkg: + pkgdir = f'{sys._stdlib_dir}{sep}{relfile}' + filename = f'{pkgdir}{sep}__init__.py' + else: + pkgdir = None + filename = f'{sys._stdlib_dir}{sep}{relfile}.py' + return filename, pkgdir + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + info = _call_with_frames_removed(_imp.find_frozen, fullname) + if info is None: return None + # We get the marshaled data in exec_module() (the loader + # part of the importer), instead of here (the finder part). + # The loader is the usual place to get the data that will + # be loaded into the module. (For example, see _LoaderBasics + # in _bootstra_external.py.) Most importantly, this importer + # is simpler if we wait to get the data. + # However, getting as much data in the finder as possible + # to later load the module is okay, and sometimes important. + # (That's why ModuleSpec.loader_state exists.) This is + # especially true if it avoids throwing away expensive data + # the loader would otherwise duplicate later and can be done + # efficiently. In this case it isn't worth it. + _, ispkg, origname = info + spec = spec_from_loader(fullname, cls, + origin=cls._ORIGIN, + is_package=ispkg) + filename, pkgdir = cls._resolve_filename(origname, fullname, ispkg) + spec.loader_state = type(sys.implementation)( + filename=filename, + origname=origname, + ) + if pkgdir: + spec.submodule_search_locations.insert(0, pkgdir) + return spec @classmethod def find_module(cls, fullname, path=None): @@ -845,14 +963,21 @@ def find_module(cls, fullname, path=None): @staticmethod def create_module(spec): - """Use default semantics for module creation.""" + """Set __file__, if able.""" + module = _new_module(spec.name) + try: + filename = spec.loader_state.filename + except AttributeError: + pass + else: + if filename: + module.__file__ = filename + return module @staticmethod def exec_module(module): - name = module.__spec__.name - if not _imp.is_frozen(name): - raise ImportError('{!r} is not a frozen module'.format(name), - name=name) + spec = module.__spec__ + name = spec.name code = _call_with_frames_removed(_imp.get_frozen_object, name) exec(code, module.__dict__) @@ -864,7 +989,16 @@ def load_module(cls, fullname): """ # Warning about deprecation implemented in _load_module_shim(). - return _load_module_shim(cls, fullname) + module = _load_module_shim(cls, fullname) + info = _imp.find_frozen(fullname) + assert info is not None + _, ispkg, origname = info + module.__origname__ = origname + vars(module).pop('__file__', None) + if ispkg: + module.__path__ = [] + cls._fix_up_module(module) + return module @classmethod @_requires_frozen @@ -1204,6 +1338,8 @@ def _setup(sys_module, _imp_module): continue spec = _spec_from_module(module, loader) _init_module_attrs(spec, module) + if loader is FrozenImporter: + loader._fix_up_module(module) # Directly load built-in modules needed during bootstrap. self_module = sys.modules[__name__] diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index d624b73f38eb66..ef4f23a4b499f1 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -363,6 +363,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) # Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) # Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) +# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) # # MAGIC must change whenever the bytecode emitted by the compiler may no @@ -372,7 +373,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3460).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3461).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' @@ -1278,8 +1279,10 @@ def append(self, item): self._path.append(item) -# We use this exclusively in module_from_spec() for backward-compatibility. -class _NamespaceLoader: +# This class is actually exposed publicly in a namespace package's __loader__ +# attribute, so it should be available through a non-private name. +# https://bugs.python.org/issue35673 +class NamespaceLoader: def __init__(self, name, path, path_finder): self._path = _NamespacePath(name, path, path_finder) @@ -1290,7 +1293,7 @@ def module_repr(module): The method is deprecated. The import machinery does the job itself. """ - _warnings.warn("_NamespaceLoader.module_repr() is deprecated and " + _warnings.warn("NamespaceLoader.module_repr() is deprecated and " "slated for removal in Python 3.12", DeprecationWarning) return ''.format(module.__name__) @@ -1326,6 +1329,10 @@ def get_resource_reader(self, module): return NamespaceReader(self._path) +# We use this exclusively in module_from_spec() for backward-compatibility. +_NamespaceLoader = NamespaceLoader + + # Finders ##################################################################### class PathFinder: diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 0b4a3f80717502..1d6843b2ddd447 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -213,7 +213,7 @@ def source_to_code(data, path=''): exec_module = _bootstrap_external._LoaderBasics.exec_module load_module = _bootstrap_external._LoaderBasics.load_module -_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter) +_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.NamespaceLoader) class ExecutionLoader(InspectLoader): diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 9a7757fb6e4494..d9a19a13f7b275 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -12,6 +12,7 @@ from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import NamespaceLoader def all_suffixes(): diff --git a/Lib/inspect.py b/Lib/inspect.py index 1b2fc918c81313..a956acf310f85b 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -47,7 +47,6 @@ import tokenize import token import types -import warnings import functools import builtins from operator import attrgetter @@ -1214,37 +1213,6 @@ def getargs(co): varkw = co.co_varnames[nargs] return Arguments(args + kwonlyargs, varargs, varkw) -ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def getargspec(func): - """Get the names and default values of a function's parameters. - - A tuple of four things is returned: (args, varargs, keywords, defaults). - 'args' is a list of the argument names, including keyword-only argument names. - 'varargs' and 'keywords' are the names of the * and ** parameters or None. - 'defaults' is an n-tuple of the default values of the last n parameters. - - This function is deprecated, as it does not support annotations or - keyword-only parameters and will raise ValueError if either is present - on the supplied callable. - - For a more structured introspection API, use inspect.signature() instead. - - Alternatively, use getfullargspec() for an API with a similar namedtuple - based interface, but full support for annotations and keyword-only - parameters. - - Deprecated since Python 3.5, use `inspect.getfullargspec()`. - """ - warnings.warn("inspect.getargspec() is deprecated since Python 3.0, " - "use inspect.signature() or inspect.getfullargspec()", - DeprecationWarning, stacklevel=2) - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - getfullargspec(func) - if kwonlyargs or ann: - raise ValueError("Function has keyword-only parameters or annotations" - ", use inspect.signature() API which can support them") - return ArgSpec(args, varargs, varkw, defaults) FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') @@ -1369,63 +1337,6 @@ def _formatannotation(annotation): return formatannotation(annotation, module) return _formatannotation -def formatargspec(args, varargs=None, varkw=None, defaults=None, - kwonlyargs=(), kwonlydefaults={}, annotations={}, - formatarg=str, - formatvarargs=lambda name: '*' + name, - formatvarkw=lambda name: '**' + name, - formatvalue=lambda value: '=' + repr(value), - formatreturns=lambda text: ' -> ' + text, - formatannotation=formatannotation): - """Format an argument spec from the values returned by getfullargspec. - - The first seven arguments are (args, varargs, varkw, defaults, - kwonlyargs, kwonlydefaults, annotations). The other five arguments - are the corresponding optional formatting functions that are called to - turn names and values into strings. The last argument is an optional - function to format the sequence of arguments. - - Deprecated since Python 3.5: use the `signature` function and `Signature` - objects. - """ - - from warnings import warn - - warn("`formatargspec` is deprecated since Python 3.5. Use `signature` and " - "the `Signature` object directly", - DeprecationWarning, - stacklevel=2) - - def formatargandannotation(arg): - result = formatarg(arg) - if arg in annotations: - result += ': ' + formatannotation(annotations[arg]) - return result - specs = [] - if defaults: - firstdefault = len(args) - len(defaults) - for i, arg in enumerate(args): - spec = formatargandannotation(arg) - if defaults and i >= firstdefault: - spec = spec + formatvalue(defaults[i - firstdefault]) - specs.append(spec) - if varargs is not None: - specs.append(formatvarargs(formatargandannotation(varargs))) - else: - if kwonlyargs: - specs.append('*') - if kwonlyargs: - for kwonlyarg in kwonlyargs: - spec = formatargandannotation(kwonlyarg) - if kwonlydefaults and kwonlyarg in kwonlydefaults: - spec += formatvalue(kwonlydefaults[kwonlyarg]) - specs.append(spec) - if varkw is not None: - specs.append(formatvarkw(formatargandannotation(varkw))) - result = '(' + ', '.join(specs) + ')' - if 'return' in annotations: - result += formatreturns(formatannotation(annotations['return'])) - return result def formatargvalues(args, varargs, varkw, locals, formatarg=str, @@ -2932,30 +2843,6 @@ def __init__(self, parameters=None, *, return_annotation=_empty, self._parameters = types.MappingProxyType(params) self._return_annotation = return_annotation - @classmethod - def from_function(cls, func): - """Constructs Signature for the given python function. - - Deprecated since Python 3.5, use `Signature.from_callable()`. - """ - - warnings.warn("inspect.Signature.from_function() is deprecated since " - "Python 3.5, use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_function(cls, func) - - @classmethod - def from_builtin(cls, func): - """Constructs Signature for the given builtin function. - - Deprecated since Python 3.5, use `Signature.from_callable()`. - """ - - warnings.warn("inspect.Signature.from_builtin() is deprecated since " - "Python 3.5, use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_builtin(cls, func) - @classmethod def from_callable(cls, obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False): diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py index d1cd10d327587c..fe547b2228072a 100644 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ b/Lib/lib2to3/fixes/fix_metaclass.py @@ -51,7 +51,7 @@ def fixup_parse_tree(cls_node): # already in the preferred format, do nothing return - # !%@#! oneliners have no suite node, we have to fake one up + # !%@#! one-liners have no suite node, we have to fake one up for i, node in enumerate(cls_node.children): if node.type == token.COLON: break diff --git a/Lib/lib2to3/fixes/fix_paren.py b/Lib/lib2to3/fixes/fix_paren.py index b205aa7e1e93fb..df3da5f5232c9c 100644 --- a/Lib/lib2to3/fixes/fix_paren.py +++ b/Lib/lib2to3/fixes/fix_paren.py @@ -1,4 +1,4 @@ -"""Fixer that addes parentheses where they are required +"""Fixer that adds parentheses where they are required This converts ``[x for x in 1, 2]`` to ``[x for x in (1, 2)]``.""" diff --git a/Lib/linecache.py b/Lib/linecache.py index 513b17e999880b..23191d6501d2a8 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -154,7 +154,7 @@ def lazycache(filename, module_globals): :return: True if a lazy load is registered in the cache, otherwise False. To register such a load a module loader with a - get_source method must be found, the filename must be a cachable + get_source method must be found, the filename must be a cacheable filename, and the filename must not be already cached. """ if filename in cache: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index f1a2e3b69986e9..b613bec1c42705 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -187,14 +187,17 @@ def shouldRollover(self, record): Basically, see if the supplied record would cause the file to exceed the size limit we have. """ + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? msg = "%s\n" % self.format(record) self.stream.seek(0, 2) #due to non-posix-compliant Windows feature if self.stream.tell() + len(msg) >= self.maxBytes: - return 1 - return 0 + return True + return False class TimedRotatingFileHandler(BaseRotatingHandler): """ @@ -345,10 +348,13 @@ def shouldRollover(self, record): record is not used, as we are just comparing times, but it is needed so the method signatures are the same """ + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False t = int(time.time()) if t >= self.rolloverAt: - return 1 - return 0 + return True + return False def getFilesToDelete(self): """ diff --git a/Lib/lzma.py b/Lib/lzma.py index 9abf06d91db184..800f52198fbb79 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -221,10 +221,6 @@ def readline(self, size=-1): self._check_can_read() return self._buffer.readline(size) - def __iter__(self): - self._check_can_read() - return self._buffer.__iter__() - def write(self, data): """Write a bytes object to the file. diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 3ba91771ed03dd..e14d0719bed489 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -552,8 +552,10 @@ def _default_mime_types(): '.h' : 'text/plain', '.ksh' : 'text/plain', '.pl' : 'text/plain', + '.srt' : 'text/plain', '.rtx' : 'text/richtext', '.tsv' : 'text/tab-separated-values', + '.vtt' : 'text/vtt', '.py' : 'text/x-python', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', diff --git a/Lib/opcode.py b/Lib/opcode.py index 5d356746888757..5377ec32bf1535 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -225,13 +225,22 @@ def jabs_op(name, op): "BINARY_ADD_FLOAT", "BINARY_ADD_UNICODE", "BINARY_ADD_UNICODE_INPLACE_FAST", + "BINARY_MULTIPLY_ADAPTIVE", + "BINARY_MULTIPLY_INT", + "BINARY_MULTIPLY_FLOAT", "BINARY_SUBSCR_ADAPTIVE", "BINARY_SUBSCR_LIST_INT", "BINARY_SUBSCR_TUPLE_INT", "BINARY_SUBSCR_DICT", + "CALL_FUNCTION_ADAPTIVE", + "CALL_FUNCTION_BUILTIN_O", + "CALL_FUNCTION_BUILTIN_FAST", + "CALL_FUNCTION_LEN", + "CALL_FUNCTION_ISINSTANCE", + "CALL_FUNCTION_PY_SIMPLE", "JUMP_ABSOLUTE_QUICK", "LOAD_ATTR_ADAPTIVE", - "LOAD_ATTR_SPLIT_KEYS", + "LOAD_ATTR_INSTANCE_VALUE", "LOAD_ATTR_WITH_HINT", "LOAD_ATTR_SLOT", "LOAD_ATTR_MODULE", @@ -242,8 +251,9 @@ def jabs_op(name, op): "LOAD_METHOD_CACHED", "LOAD_METHOD_CLASS", "LOAD_METHOD_MODULE", + "LOAD_METHOD_NO_DICT", "STORE_ATTR_ADAPTIVE", - "STORE_ATTR_SPLIT_KEYS", + "STORE_ATTR_INSTANCE_VALUE", "STORE_ATTR_SLOT", "STORE_ATTR_WITH_HINT", # Super instructions diff --git a/Lib/operator.py b/Lib/operator.py index 72105be05f1c24..30116c1189a499 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -10,7 +10,7 @@ This is the pure Python implementation of the module. """ -__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', +__all__ = ['abs', 'add', 'and_', 'attrgetter', 'call', 'concat', 'contains', 'countOf', 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', diff --git a/Lib/pickle.py b/Lib/pickle.py index 5ab312f2acaee6..e7f30f226101f5 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1173,7 +1173,7 @@ def __init__(self, file, *, fix_imports=True, used in Python 3. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python 2; these default to 'ASCII' and 'strict', respectively. *encoding* can be - 'bytes' to read theses 8-bit string instances as bytes objects. + 'bytes' to read these 8-bit string instances as bytes objects. """ self._buffers = iter(buffers) if buffers is not None else None self._file_readline = file.readline diff --git a/Lib/platform.py b/Lib/platform.py index 240f701754d4ac..9e9b42238fb76c 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1261,7 +1261,7 @@ def platform(aliased=0, terse=0): def _parse_os_release(lines): # These fields are mandatory fields with well-known defaults - # in pratice all Linux distributions override NAME, ID, and PRETTY_NAME. + # in practice all Linux distributions override NAME, ID, and PRETTY_NAME. info = { "NAME": "Linux", "ID": "linux", diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 0f9b59025cee38..388614e51b1847 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -190,7 +190,7 @@ def main(): ) args = parser.parse_args() if args.filenames == ['-']: - filenames = sys.stdin.readlines() + filenames = [filename.rstrip('\n') for filename in sys.stdin.readlines()] else: filenames = args.filenames for filename in filenames: diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 34a608760338e3..3a2ff218f8319a 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -542,7 +542,7 @@ def repr_string(self, x, level): # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + self.escape(test) + testrepr[0] return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)', - r'\1', + r'\1', self.escape(testrepr)) repr_str = repr_string @@ -567,49 +567,48 @@ class HTMLDoc(Doc): def page(self, title, contents): """Format an HTML page.""" return '''\ - -Python: %s - - + + + + +Python: %s + %s ''' % (title, contents) - def heading(self, title, fgcol, bgcol, extras=''): + def heading(self, title, extras=''): """Format a page heading.""" return ''' - - -
 
- 
%s		%s
- ''' % (bgcol, fgcol, title, fgcol, extras or ' ') - - def section(self, title, fgcol, bgcol, contents, width=6, + + + +
 
%s		%s
+ ''' % (title, extras or ' ') + + def section(self, title, cls, contents, width=6, prelude='', marginalia=None, gap=' '): """Format a section with a heading.""" if marginalia is None: - marginalia = '' + ' ' * width + '' + marginalia = '' + ' ' * width + '' result = '''

- - - - ''' % (bgcol, fgcol, title) +
 
-%s
+ + + ''' % (cls, title) if prelude: result = result + ''' - - -''' % (bgcol, marginalia, prelude, gap) + + +''' % (cls, marginalia, cls, prelude, gap) else: result = result + ''' -''' % (bgcol, marginalia, gap) +''' % (cls, marginalia, gap) - return result + '\n
 
%s
%s%s
%s
%s%s
%s
%s%s
%s%s%s
' % contents + return result + '\n%s' % contents def bigsection(self, title, *args): """Format a section with a big heading.""" - title = '%s' % title + title = '%s' % title return self.section(title, *args) def preformat(self, text): @@ -618,19 +617,19 @@ def preformat(self, text): return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', ' ', ' ', '\n', '
\n') - def multicolumn(self, list, format, cols=4): + def multicolumn(self, list, format): """Format a list of items into a multi-column list.""" result = '' - rows = (len(list)+cols-1)//cols - for col in range(cols): - result = result + '' % (100//cols) + rows = (len(list) + 3) // 4 + for col in range(4): + result = result + '' for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '
\n' result = result + '' - return '%s
' % result + return '%s
' % result - def grey(self, text): return '%s' % text + def grey(self, text): return '%s' % text def namelink(self, name, *dicts): """Make a link for an identifier, given name-to-URL mappings.""" @@ -719,14 +718,14 @@ def formattree(self, tree, modname, parent=None): for entry in tree: if type(entry) is type(()): c, bases = entry - result = result + '

' + result = result + '
' result = result + self.classlink(c, modname) if bases and bases != (parent,): parents = [] for base in bases: parents.append(self.classlink(base, modname)) result = result + '(' + ', '.join(parents) + ')' - result = result + '\n
' + result = result + '\n' elif type(entry) is type([]): result = result + '
\n%s
\n' % self.formattree( entry, modname, c) @@ -743,10 +742,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): links = [] for i in range(len(parts)-1): links.append( - '%s' % + '%s' % ('.'.join(parts[:i+1]), parts[i])) linkedname = '.'.join(links + parts[-1:]) - head = '%s' % linkedname + head = '%s' % linkedname try: path = inspect.getabsfile(object) url = urllib.parse.quote(path) @@ -768,9 +767,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): docloc = '
Module Reference' % locals() else: docloc = '' - result = self.heading( - head, '#ffffff', '#7799ee', - 'index
' + filelink + docloc) + result = self.heading(head, 'index
' + filelink + docloc) modules = inspect.getmembers(object, inspect.ismodule) @@ -805,7 +802,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) - doc = doc and '%s' % doc + doc = doc and '%s' % doc result = result + '

%s

\n' % doc if hasattr(object, '__path__'): @@ -815,12 +812,12 @@ def docmodule(self, object, name=None, mod=None, *ignored): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) result = result + self.bigsection( - 'Package Contents', '#ffffff', '#aa55cc', contents) + 'Package Contents', 'pkg-content', contents) elif modules: contents = self.multicolumn( modules, lambda t: self.modulelink(t[1])) result = result + self.bigsection( - 'Modules', '#ffffff', '#aa55cc', contents) + 'Modules', 'pkg-content', contents) if classes: classlist = [value for (key, value) in classes] @@ -829,27 +826,25 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in classes: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Classes', '#ffffff', '#ee77aa', ' '.join(contents)) + 'Classes', 'index', ' '.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Functions', '#ffffff', '#eeaa77', ' '.join(contents)) + 'Functions', 'functions', ' '.join(contents)) if data: contents = [] for key, value in data: contents.append(self.document(value, key)) result = result + self.bigsection( - 'Data', '#ffffff', '#55aa55', '
\n'.join(contents)) + 'Data', 'data', '
\n'.join(contents)) if hasattr(object, '__author__'): contents = self.markup(str(object.__author__), self.preformat) - result = result + self.bigsection( - 'Author', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Author', 'author', contents) if hasattr(object, '__credits__'): contents = self.markup(str(object.__credits__), self.preformat) - result = result + self.bigsection( - 'Credits', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Credits', 'credits', contents) return result @@ -923,7 +918,7 @@ def spilldata(msg, attrs, predicate): else: doc = self.markup(getdoc(value), self.preformat, funcs, classes, mdict) - doc = '
%s' % doc + doc = '
%s' % doc push('
%s%s
\n' % (base, doc)) push('\n') return attrs @@ -1011,9 +1006,9 @@ def spilldata(msg, attrs, predicate): if decl: doc = decl + (doc or '') doc = self.markup(doc, self.preformat, funcs, classes, mdict) - doc = doc and '%s
 ' % doc + doc = doc and '%s
 ' % doc - return self.section(title, '#000000', '#ffc8d8', contents, 3, doc) + return self.section(title, 'title', contents, 3, doc) def formatvalue(self, object): """Format an argument default value as text.""" @@ -1074,14 +1069,14 @@ def docroutine(self, object, name=None, mod=None, argspec = '(...)' decl = asyncqualifier + title + self.escape(argspec) + (note and - self.grey('%s' % note)) + self.grey('%s' % note)) if skipdocs: return '
%s
\n' % decl else: doc = self.markup( getdoc(object), self.preformat, funcs, classes, methods) - doc = doc and '
%s
' % doc + doc = doc and '
%s
' % doc return '
%s
%s
\n' % (decl, doc) def docdata(self, object, name=None, mod=None, cl=None): @@ -1093,7 +1088,7 @@ def docdata(self, object, name=None, mod=None, cl=None): push('
%s
\n' % name) doc = self.markup(getdoc(object), self.preformat) if doc: - push('
%s
\n' % doc) + push('
%s
\n' % doc) push('
\n') return ''.join(results) @@ -1118,7 +1113,7 @@ def index(self, dir, shadowed=None): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) - return self.bigsection(dir, '#ffffff', '#ee77aa', contents) + return self.bigsection(dir, 'index', contents) # -------------------------------------------- text documentation generator @@ -2446,10 +2441,12 @@ def page(self, title, contents): '' % css_path) return '''\ - -Pydoc: %s - -%s%s
%s
+ + + + +Pydoc: %s +%s%s
%s
''' % (title, css_link, html_navbar(), contents) @@ -2489,22 +2486,21 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'Index of Modules', - '#ffffff', '#7799ee') + 'Index of Modules' + ) names = [name for name in sys.builtin_module_names if name != '__main__'] contents = html.multicolumn(names, bltinlink) contents = [heading, '

' + html.bigsection( - 'Built-in Modules', '#ffffff', '#ee77aa', contents)] + 'Built-in Modules', 'index', contents)] seen = {} for dir in sys.path: contents.append(html.index(dir, seen)) contents.append( - '

pydoc by Ka-Ping Yee' - '<ping@lfw.org>') + '

pydoc by Ka-Ping Yee' + '<ping@lfw.org>

') return 'Index of Modules', ''.join(contents) def html_search(key): @@ -2529,12 +2525,12 @@ def bltinlink(name): results = [] heading = html.heading( - 'Search Results', - '#ffffff', '#7799ee') + 'Search Results', + ) for name, desc in search_result: results.append(bltinlink(name) + desc) contents = heading + html.bigsection( - 'key = %s' % key, '#ffffff', '#ee77aa', '
'.join(results)) + 'key = %s' % key, 'index', '
'.join(results)) return 'Search Results', contents def html_topics(): @@ -2544,20 +2540,20 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.topics.keys()) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Topics', '#ffffff', '#ee77aa', contents) + 'Topics', 'index', contents) return 'Topics', contents def html_keywords(): """Index of keywords.""" heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.keywords.keys()) def bltinlink(name): @@ -2565,7 +2561,7 @@ def bltinlink(name): contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Keywords', '#ffffff', '#ee77aa', contents) + 'Keywords', 'index', contents) return 'Keywords', contents def html_topicpage(topic): @@ -2578,10 +2574,10 @@ def html_topicpage(topic): else: title = 'TOPIC' heading = html.heading( - '%s' % title, - '#ffffff', '#7799ee') + '%s' % title, + ) contents = '
%s
' % html.markup(contents) - contents = html.bigsection(topic , '#ffffff','#ee77aa', contents) + contents = html.bigsection(topic , 'index', contents) if xrefs: xrefs = sorted(xrefs.split()) @@ -2589,8 +2585,7 @@ def bltinlink(name): return '%s' % (name, name) xrefs = html.multicolumn(xrefs, bltinlink) - xrefs = html.section('Related help topics: ', - '#ffffff', '#ee77aa', xrefs) + xrefs = html.section('Related help topics: ', 'index', xrefs) return ('%s %s' % (title, topic), ''.join((heading, contents, xrefs))) @@ -2604,12 +2599,11 @@ def html_getobj(url): def html_error(url, exc): heading = html.heading( - 'Error', - '#ffffff', '#7799ee') + 'Error', + ) contents = '
'.join(html.escape(line) for line in format_exception_only(type(exc), exc)) - contents = heading + html.bigsection(url, '#ffffff', '#bb0000', - contents) + contents = heading + html.bigsection(url, 'error', contents) return "Error - %s" % url, contents def get_html_page(url): diff --git a/Lib/pydoc_data/_pydoc.css b/Lib/pydoc_data/_pydoc.css index f036ef37a5aba7..a6aa2e4c1a021e 100644 --- a/Lib/pydoc_data/_pydoc.css +++ b/Lib/pydoc_data/_pydoc.css @@ -4,3 +4,109 @@ Contents of this file are subject to change without notice. */ + +body { + background-color: #f0f0f8; +} + +table.heading tr { + background-color: #7799ee; +} + +.decor { + color: #ffffff; +} + +.title-decor { + background-color: #ffc8d8; + color: #000000; +} + +.pkg-content-decor { + background-color: #aa55cc; +} + +.index-decor { + background-color: #ee77aa; +} + +.functions-decor { + background-color: #eeaa77; +} + +.data-decor { + background-color: #55aa55; +} + +.author-decor { + background-color: #7799ee; +} + +.credits-decor { + background-color: #7799ee; +} + +.error-decor { + background-color: #bb0000; +} + +.grey { + color: #909090; +} + +.white { + color: #ffffff; +} + +.repr { + color: #c040c0; +} + +table.heading tr td.title { + vertical-align: bottom; +} + +table.heading tr td.extra { + vertical-align: bottom; + text-align: right; +} + +.heading-text { + font-family: helvetica, arial; +} + +.bigsection { + font-size: larger; +} + +.title { + font-size: x-large; +} + +.code { + font-family: monospace; +} + +table { + width: 100%; + border-spacing : 0; + border-collapse : collapse; + border: 0; +} + +td { + padding: 2; +} + +td.section-title { + vertical-align: bottom; +} + +td.multicolumn { + width: 25%; + vertical-align: bottom; +} + +td.singlecolumn { + width: 100%; +} diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 40f7a50128619f..eb523370c58d77 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Apr 5 17:39:41 2021 +# Autogenerated by Sphinx on Tue Oct 5 13:43:52 2021 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -551,13 +551,65 @@ 'exception.\n' ' That new exception causes the old one to be lost.\n' '\n' - '[2] A string literal appearing as the first statement in the ' + '[2] In pattern matching, a sequence is defined as one of the\n' + ' following:\n' + '\n' + ' * a class that inherits from "collections.abc.Sequence"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Sequence"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_SEQUENCE"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The following standard library classes are sequences:\n' + '\n' + ' * "array.array"\n' + '\n' + ' * "collections.deque"\n' + '\n' + ' * "list"\n' + '\n' + ' * "memoryview"\n' + '\n' + ' * "range"\n' + '\n' + ' * "tuple"\n' + '\n' + ' Note:\n' + '\n' + ' Subject values of type "str", "bytes", and "bytearray" do ' + 'not\n' + ' match sequence patterns.\n' + '\n' + '[3] In pattern matching, a mapping is defined as one of the ' + 'following:\n' + '\n' + ' * a class that inherits from "collections.abc.Mapping"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Mapping"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_MAPPING"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The standard library classes "dict" and ' + '"types.MappingProxyType"\n' + ' are mappings.\n' + '\n' + '[4] A string literal appearing as the first statement in the ' 'function\n' ' body is transformed into the function’s "__doc__" attribute ' 'and\n' ' therefore the function’s *docstring*.\n' '\n' - '[3] A string literal appearing as the first statement in the class\n' + '[5] A string literal appearing as the first statement in the class\n' ' body is transformed into the namespace’s "__doc__" item and\n' ' therefore the class’s *docstring*.\n', 'atom-identifiers': 'Identifiers (Names)\n' @@ -884,32 +936,6 @@ '*instance* of the\n' ' owner class.\n' '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is ' - 'created. The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' Note:\n' - '\n' - ' "__set_name__()" is only called implicitly as part ' - 'of the "type"\n' - ' constructor, so it will need to be called ' - 'explicitly with the\n' - ' appropriate parameters when a descriptor is added ' - 'to a class\n' - ' after initial creation:\n' - '\n' - ' class A:\n' - ' pass\n' - ' descr = custom_descriptor()\n' - ' A.attr = descr\n' - " descr.__set_name__(A, 'attr')\n" - '\n' - ' See Creating the class object for more details.\n' - '\n' - ' New in version 3.6.\n' - '\n' 'The attribute "__objclass__" is interpreted by the ' '"inspect" module as\n' 'specifying the class where this object was defined ' @@ -988,9 +1014,9 @@ '\n' 'For instance bindings, the precedence of descriptor ' 'invocation depends\n' - 'on the which descriptor methods are defined. A ' - 'descriptor can define\n' - 'any combination of "__get__()", "__set__()" and ' + 'on which descriptor methods are defined. A descriptor ' + 'can define any\n' + 'combination of "__get__()", "__set__()" and ' '"__delete__()". If it\n' 'does not define "__get__()", then accessing the ' 'attribute will return\n' @@ -1261,6 +1287,10 @@ 'In the latter case, sequence repetition is performed; a negative\n' 'repetition factor yields an empty sequence.\n' '\n' + 'This operation can be customized using the special "__mul__()" ' + 'and\n' + '"__rmul__()" methods.\n' + '\n' 'The "@" (at) operator is intended to be used for matrix\n' 'multiplication. No builtin Python types implement this operator.\n' '\n' @@ -1276,6 +1306,10 @@ 'result. Division by zero raises the "ZeroDivisionError" ' 'exception.\n' '\n' + 'This operation can be customized using the special "__truediv__()" ' + 'and\n' + '"__floordiv__()" methods.\n' + '\n' 'The "%" (modulo) operator yields the remainder from the division ' 'of\n' 'the first argument by the second. The numeric arguments are ' @@ -1307,6 +1341,10 @@ 'string formatting is described in the Python Library Reference,\n' 'section printf-style String Formatting.\n' '\n' + 'The *modulo* operation can be customized using the special ' + '"__mod__()"\n' + 'method.\n' + '\n' 'The floor division operator, the modulo operator, and the ' '"divmod()"\n' 'function are not defined for complex numbers. Instead, convert to ' @@ -1321,9 +1359,16 @@ 'and then added together. In the latter case, the sequences are\n' 'concatenated.\n' '\n' + 'This operation can be customized using the special "__add__()" ' + 'and\n' + '"__radd__()" methods.\n' + '\n' 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' - 'The numeric arguments are first converted to a common type.\n', + 'The numeric arguments are first converted to a common type.\n' + '\n' + 'This operation can be customized using the special "__sub__()" ' + 'method.\n', 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' @@ -1336,14 +1381,18 @@ '\n' 'The "&" operator yields the bitwise AND of its arguments, which ' 'must\n' - 'be integers.\n' + 'be integers or one of them must be a custom object overriding\n' + '"__and__()" or "__rand__()" special methods.\n' '\n' 'The "^" operator yields the bitwise XOR (exclusive OR) of its\n' - 'arguments, which must be integers.\n' + 'arguments, which must be integers or one of them must be a ' + 'custom\n' + 'object overriding "__xor__()" or "__rxor__()" special methods.\n' '\n' 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' - 'which must be integers.\n', + 'which must be integers or one of them must be a custom object\n' + 'overriding "__or__()" or "__ror__()" special methods.\n', 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' @@ -1360,6 +1409,10 @@ 'through their "__code__" attribute. See also the ' '"code" module.\n' '\n' + 'Accessing "__code__" raises an auditing event ' + '"object.__getattr__"\n' + 'with arguments "obj" and ""__code__"".\n' + '\n' 'A code object can be executed or evaluated by passing ' 'it (instead of a\n' 'source string) to the "exec()" or "eval()" built-in ' @@ -1704,7 +1757,7 @@ 'original global namespace. (Usually, the suite contains mostly\n' 'function definitions.) When the class’s suite finishes execution, ' 'its\n' - 'execution frame is discarded but its local namespace is saved. [3] ' + 'execution frame is discarded but its local namespace is saved. [5] ' 'A\n' 'class object is then created using the inheritance list for the ' 'base\n' @@ -1785,7 +1838,11 @@ ' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n' ' | "is" ["not"] | ["not"] "in"\n' '\n' - 'Comparisons yield boolean values: "True" or "False".\n' + 'Comparisons yield boolean values: "True" or "False". Custom ' + '*rich\n' + 'comparison methods* may return non-boolean values. In this ' + 'case Python\n' + 'will call "bool()" on such value in boolean contexts.\n' '\n' 'Comparisons can be chained arbitrarily, e.g., "x < y <= z" ' 'is\n' @@ -2381,11 +2438,11 @@ 'resulting\n' 'object is “compatible” with the exception. An object is ' 'compatible\n' - 'with an exception if it is the class or a base class of the ' - 'exception\n' - 'object, or a tuple containing an item that is the class or a ' - 'base\n' - 'class of the exception object.\n' + 'with an exception if the object is the class or a base class of ' + 'the\n' + 'exception object, or a tuple containing an item that is the ' + 'class or a\n' + 'base class of the exception object.\n' '\n' 'If no except clause matches the exception, the search for an ' 'exception\n' @@ -2694,7 +2751,7 @@ ' subject_expr ::= star_named_expression "," ' 'star_named_expressions?\n' ' | named_expression\n' - " case_block ::= 'case' patterns [guard] ':' block\n" + ' case_block ::= \'case\' patterns [guard] ":" block\n' '\n' 'Note:\n' '\n' @@ -2764,7 +2821,7 @@ 'have\n' ' happened.\n' '\n' - ' * If the guard evaluates as truthy or missing, the "block" ' + ' * If the guard evaluates as true or is missing, the "block" ' 'inside\n' ' "case_block" is executed.\n' '\n' @@ -2825,12 +2882,12 @@ '\n' '2. If the pattern succeeded, evaluate the "guard".\n' '\n' - ' * If the "guard" condition evaluates to “truthy”, the case ' - 'block is\n' + ' * If the "guard" condition evaluates as true, the case block ' + 'is\n' ' selected.\n' '\n' - ' * If the "guard" condition evaluates to “falsy”, the case ' - 'block is\n' + ' * If the "guard" condition evaluates as false, the case block ' + 'is\n' ' not selected.\n' '\n' ' * If the "guard" raises an exception during evaluation, the\n' @@ -3001,7 +3058,7 @@ '\n' 'A single underscore "_" is not a capture pattern (this is what ' '"!\'_\'"\n' - 'expresses). And is instead treated as a "wildcard_pattern".\n' + 'expresses). It is instead treated as a "wildcard_pattern".\n' '\n' 'In a given pattern, a given name can only be bound once. E.g. ' '"case\n' @@ -3029,7 +3086,10 @@ '\n' " wildcard_pattern ::= '_'\n" '\n' - '"_" is a soft keyword.\n' + '"_" is a soft keyword within any pattern, but only within ' + 'patterns.\n' + 'It is an identifier, as usual, even within "match" subject\n' + 'expressions, "guard"s, and "case" blocks.\n' '\n' 'In simple terms, "_" will always succeed.\n' '\n' @@ -3073,7 +3133,7 @@ 'additional\n' 'syntax. Syntax:\n' '\n' - " group_pattern ::= '(' pattern ')'\n" + ' group_pattern ::= "(" pattern ")"\n' '\n' 'In simple terms "(P)" has the same effect as "P".\n' '\n' @@ -3120,8 +3180,9 @@ 'pattern\n' 'against a subject value:\n' '\n' - '1. If the subject value is not an instance of a\n' - ' "collections.abc.Sequence" the sequence pattern fails.\n' + '1. If the subject value is not a sequence [2], the sequence ' + 'pattern\n' + ' fails.\n' '\n' '2. If the subject value is an instance of "str", "bytes" or\n' ' "bytearray" the sequence pattern fails.\n' @@ -3176,7 +3237,7 @@ 'the\n' 'following happens:\n' '\n' - '* "isinstance(, collections.abc.Sequence)"\n' + '* check "" is a sequence\n' '\n' '* "len(subject) == "\n' '\n' @@ -3210,18 +3271,19 @@ 'double star pattern must be the last subpattern in the mapping\n' 'pattern.\n' '\n' - 'Duplicate key values in mapping patterns are disallowed. (If all ' - 'key\n' - 'patterns are literal patterns this is considered a syntax ' - 'error;\n' - 'otherwise this is a runtime error and will raise "ValueError".)\n' + 'Duplicate keys in mapping patterns are disallowed. Duplicate ' + 'literal\n' + 'keys will raise a "SyntaxError". Two keys that otherwise have ' + 'the same\n' + 'value will raise a "ValueError" at runtime.\n' '\n' 'The following is the logical flow for matching a mapping ' 'pattern\n' 'against a subject value:\n' '\n' - '1. If the subject value is not an instance of\n' - ' "collections.abc.Mapping", the mapping pattern fails.\n' + '1. If the subject value is not a mapping [3],the mapping ' + 'pattern\n' + ' fails.\n' '\n' '2. If every key given in the mapping pattern is present in the ' 'subject\n' @@ -3231,7 +3293,10 @@ '\n' '3. If duplicate keys are detected in the mapping pattern, the ' 'pattern\n' - ' is considered invalid and "ValueError" is raised.\n' + ' is considered invalid. A "SyntaxError" is raised for ' + 'duplicate\n' + ' literal values; or a "ValueError" for named keys of the same ' + 'value.\n' '\n' 'Note:\n' '\n' @@ -3247,7 +3312,7 @@ 'the\n' 'following happens:\n' '\n' - '* "isinstance(, collections.abc.Mapping)"\n' + '* check "" is a mapping\n' '\n' '* "KEY1 in "\n' '\n' @@ -3293,7 +3358,9 @@ ' For a number of built-in types (specified below), a single\n' ' positional subpattern is accepted which will match the ' 'entire\n' - ' subject; for these types no keyword patterns are accepted.\n' + ' subject; for these types keyword patterns also work as for ' + 'other\n' + ' types.\n' '\n' ' If only keyword patterns are present, they are processed as\n' ' follows, one by one:\n' @@ -3324,15 +3391,14 @@ 'class\n' ' "name_or_attr" before matching:\n' '\n' - ' I. The equivalent of "getattr(cls, "__match_args__", ())" ' - 'is\n' + ' I. The equivalent of "getattr(cls, "__match_args__", ())" is\n' ' called.\n' '\n' ' * If this raises an exception, the exception bubbles up.\n' '\n' - ' * If the returned value is not a list or tuple, the ' - 'conversion\n' - ' fails and "TypeError" is raised.\n' + ' * If the returned value is not a tuple, the conversion ' + 'fails and\n' + ' "TypeError" is raised.\n' '\n' ' * If there are more positional patterns than\n' ' "len(cls.__match_args__)", "TypeError" is raised.\n' @@ -3426,7 +3492,6 @@ ' decorators ::= decorator+\n' ' decorator ::= "@" assignment_expression ' 'NEWLINE\n' - ' dotted_name ::= identifier ("." identifier)*\n' ' parameter_list ::= defparameter ("," ' 'defparameter)* "," "/" ["," [parameter_list_no_posonly]]\n' ' | parameter_list_no_posonly\n' @@ -3451,7 +3516,7 @@ '\n' 'The function definition does not execute the function body; this ' 'gets\n' - 'executed only when the function is called. [2]\n' + 'executed only when the function is called. [4]\n' '\n' 'A function definition may be wrapped by one or more *decorator*\n' 'expressions. Decorator expressions are evaluated when the ' @@ -3526,7 +3591,7 @@ 'Calls.\n' 'A function call always assigns values to all parameters ' 'mentioned in\n' - 'the parameter list, either from position arguments, from ' + 'the parameter list, either from positional arguments, from ' 'keyword\n' 'arguments, or from default values. If the form “"*identifier"” ' 'is\n' @@ -3538,8 +3603,14 @@ 'new\n' 'empty mapping of the same type. Parameters after “"*"” or\n' '“"*identifier"” are keyword-only parameters and may only be ' - 'passed\n' - 'used keyword arguments.\n' + 'passed by\n' + 'keyword arguments. Parameters before “"/"” are positional-only\n' + 'parameters and may only be passed by positional arguments.\n' + '\n' + 'Changed in version 3.8: The "/" function parameter syntax may be ' + 'used\n' + 'to indicate positional-only parameters. See **PEP 570** for ' + 'details.\n' '\n' 'Parameters may have an *annotation* of the form “": ' 'expression"”\n' @@ -3552,11 +3623,20 @@ 'parameter list. These annotations can be any valid Python ' 'expression.\n' 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as string values ' - 'in a\n' + 'function. The annotation values are available as values of a\n' 'dictionary keyed by the parameters’ names in the ' '"__annotations__"\n' - 'attribute of the function object.\n' + 'attribute of the function object. If the "annotations" import ' + 'from\n' + '"__future__" is used, annotations are preserved as strings at ' + 'runtime\n' + 'which enables postponed evaluation. Otherwise, they are ' + 'evaluated\n' + 'when the function definition is executed. In this case ' + 'annotations\n' + 'may be evaluated in a different order than they appear in the ' + 'source\n' + 'code.\n' '\n' 'It is also possible to create anonymous functions (functions not ' 'bound\n' @@ -3641,7 +3721,7 @@ 'function definitions.) When the class’s suite finishes ' 'execution, its\n' 'execution frame is discarded but its local namespace is saved. ' - '[3] A\n' + '[5] A\n' 'class object is then created using the inheritance list for the ' 'base\n' 'classes and the saved local namespace for the attribute ' @@ -3845,13 +3925,65 @@ 'exception.\n' ' That new exception causes the old one to be lost.\n' '\n' - '[2] A string literal appearing as the first statement in the ' + '[2] In pattern matching, a sequence is defined as one of the\n' + ' following:\n' + '\n' + ' * a class that inherits from "collections.abc.Sequence"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Sequence"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_SEQUENCE"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The following standard library classes are sequences:\n' + '\n' + ' * "array.array"\n' + '\n' + ' * "collections.deque"\n' + '\n' + ' * "list"\n' + '\n' + ' * "memoryview"\n' + '\n' + ' * "range"\n' + '\n' + ' * "tuple"\n' + '\n' + ' Note:\n' + '\n' + ' Subject values of type "str", "bytes", and "bytearray" do ' + 'not\n' + ' match sequence patterns.\n' + '\n' + '[3] In pattern matching, a mapping is defined as one of the ' + 'following:\n' + '\n' + ' * a class that inherits from "collections.abc.Mapping"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Mapping"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_MAPPING"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The standard library classes "dict" and ' + '"types.MappingProxyType"\n' + ' are mappings.\n' + '\n' + '[4] A string literal appearing as the first statement in the ' 'function\n' ' body is transformed into the function’s "__doc__" attribute ' 'and\n' ' therefore the function’s *docstring*.\n' '\n' - '[3] A string literal appearing as the first statement in the ' + '[5] A string literal appearing as the first statement in the ' 'class\n' ' body is transformed into the namespace’s "__doc__" item and\n' ' therefore the class’s *docstring*.\n', @@ -3989,13 +4121,13 @@ '\n' ' If "__new__()" is invoked during object construction and ' 'it returns\n' - ' an instance or subclass of *cls*, then the new ' - 'instance’s\n' - ' "__init__()" method will be invoked like ' - '"__init__(self[, ...])",\n' - ' where *self* is the new instance and the remaining ' - 'arguments are\n' - ' the same as were passed to the object constructor.\n' + ' an instance of *cls*, then the new instance’s ' + '"__init__()" method\n' + ' will be invoked like "__init__(self[, ...])", where ' + '*self* is the\n' + ' new instance and the remaining arguments are the same as ' + 'were\n' + ' passed to the object constructor.\n' '\n' ' If "__new__()" does not return an instance of *cls*, ' 'then the new\n' @@ -4703,13 +4835,18 @@ '\n' 'If a file ".pdbrc" exists in the user’s home directory or in ' 'the\n' - 'current directory, it is read in and executed as if it had been ' - 'typed\n' - 'at the debugger prompt. This is particularly useful for ' - 'aliases. If\n' - 'both files exist, the one in the home directory is read first ' - 'and\n' - 'aliases defined there can be overridden by the local file.\n' + 'current directory, it is read with "\'utf-8\'" encoding and ' + 'executed as\n' + 'if it had been typed at the debugger prompt. This is ' + 'particularly\n' + 'useful for aliases. If both files exist, the one in the home\n' + 'directory is read first and aliases defined there can be ' + 'overridden by\n' + 'the local file.\n' + '\n' + 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' + 'encoding.\n' + 'Previously, it was read with the system locale encoding.\n' '\n' 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' 'continue debugging, such as "continue" or "next". Previously, ' @@ -6075,19 +6212,19 @@ 'complex\n' 'types. For integers, when binary, octal, or hexadecimal ' 'output is\n' - 'used, this option adds the prefix respective "\'0b\'", ' - '"\'0o\'", or "\'0x\'"\n' - 'to the output value. For float and complex the alternate ' - 'form causes\n' - 'the result of the conversion to always contain a ' - 'decimal-point\n' - 'character, even if no digits follow it. Normally, a ' - 'decimal-point\n' - 'character appears in the result of these conversions only ' - 'if a digit\n' - 'follows it. In addition, for "\'g\'" and "\'G\'" ' - 'conversions, trailing\n' - 'zeros are not removed from the result.\n' + 'used, this option adds the respective prefix "\'0b\'", ' + '"\'0o\'", "\'0x\'",\n' + 'or "\'0X\'" to the output value. For float and complex the ' + 'alternate\n' + 'form causes the result of the conversion to always contain ' + 'a decimal-\n' + 'point character, even if no digits follow it. Normally, a ' + 'decimal-\n' + 'point character appears in the result of these conversions ' + 'only if a\n' + 'digit follows it. In addition, for "\'g\'" and "\'G\'" ' + 'conversions,\n' + 'trailing zeros are not removed from the result.\n' '\n' 'The "\',\'" option signals the use of a comma for a ' 'thousands separator.\n' @@ -6204,8 +6341,12 @@ '+-----------+------------------------------------------------------------+\n' ' | "\'X\'" | Hex format. Outputs the number in base ' '16, using upper- |\n' - ' | | case letters for the digits above ' - '9. |\n' + ' | | case letters for the digits above 9. In ' + 'case "\'#\'" is |\n' + ' | | specified, the prefix "\'0x\'" will be ' + 'upper-cased to "\'0X\'" |\n' + ' | | as ' + 'well. |\n' ' ' '+-----------+------------------------------------------------------------+\n' ' | "\'n\'" | Number. This is the same as "\'d\'", ' @@ -6562,7 +6703,6 @@ ' decorators ::= decorator+\n' ' decorator ::= "@" assignment_expression ' 'NEWLINE\n' - ' dotted_name ::= identifier ("." identifier)*\n' ' parameter_list ::= defparameter ("," ' 'defparameter)* "," "/" ["," [parameter_list_no_posonly]]\n' ' | parameter_list_no_posonly\n' @@ -6587,7 +6727,7 @@ '\n' 'The function definition does not execute the function body; this ' 'gets\n' - 'executed only when the function is called. [2]\n' + 'executed only when the function is called. [4]\n' '\n' 'A function definition may be wrapped by one or more *decorator*\n' 'expressions. Decorator expressions are evaluated when the ' @@ -6662,7 +6802,7 @@ 'Calls.\n' 'A function call always assigns values to all parameters ' 'mentioned in\n' - 'the parameter list, either from position arguments, from ' + 'the parameter list, either from positional arguments, from ' 'keyword\n' 'arguments, or from default values. If the form “"*identifier"” ' 'is\n' @@ -6674,8 +6814,14 @@ 'new\n' 'empty mapping of the same type. Parameters after “"*"” or\n' '“"*identifier"” are keyword-only parameters and may only be ' - 'passed\n' - 'used keyword arguments.\n' + 'passed by\n' + 'keyword arguments. Parameters before “"/"” are positional-only\n' + 'parameters and may only be passed by positional arguments.\n' + '\n' + 'Changed in version 3.8: The "/" function parameter syntax may be ' + 'used\n' + 'to indicate positional-only parameters. See **PEP 570** for ' + 'details.\n' '\n' 'Parameters may have an *annotation* of the form “": ' 'expression"”\n' @@ -6688,11 +6834,20 @@ 'parameter list. These annotations can be any valid Python ' 'expression.\n' 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as string values ' - 'in a\n' + 'function. The annotation values are available as values of a\n' 'dictionary keyed by the parameters’ names in the ' '"__annotations__"\n' - 'attribute of the function object.\n' + 'attribute of the function object. If the "annotations" import ' + 'from\n' + '"__future__" is used, annotations are preserved as strings at ' + 'runtime\n' + 'which enables postponed evaluation. Otherwise, they are ' + 'evaluated\n' + 'when the function definition is executed. In this case ' + 'annotations\n' + 'may be evaluated in a different order than they appear in the ' + 'source\n' + 'code.\n' '\n' 'It is also possible to create anonymous functions (functions not ' 'bound\n' @@ -6909,8 +7064,8 @@ '\n' 'A non-normative HTML file listing all valid identifier ' 'characters for\n' - 'Unicode 4.1 can be found at\n' - 'https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt\n' + 'Unicode 14.0.0 can be found at\n' + 'https://www.unicode.org/Public/14.0.0/ucd/DerivedCoreProperties.txt\n' '\n' '\n' 'Keywords\n' @@ -7051,7 +7206,7 @@ ' | "from" relative_module "import" "(" ' 'identifier ["as" identifier]\n' ' ("," identifier ["as" identifier])* [","] ")"\n' - ' | "from" module "import" "*"\n' + ' | "from" relative_module "import" "*"\n' ' module ::= (identifier ".")* identifier\n' ' relative_module ::= "."* module | "."+\n' '\n' @@ -7395,10 +7550,7 @@ 'lambda': 'Lambdas\n' '*******\n' '\n' - ' lambda_expr ::= "lambda" [parameter_list] ":" ' - 'expression\n' - ' lambda_expr_nocond ::= "lambda" [parameter_list] ":" ' - 'expression_nocond\n' + ' lambda_expr ::= "lambda" [parameter_list] ":" expression\n' '\n' 'Lambda expressions (sometimes called lambda forms) are used to ' 'create\n' @@ -7715,11 +7867,11 @@ 'instance, to\n' ' evaluate the expression "x + y", where *x* is an ' 'instance of a\n' - ' class that has an "__add__()" method, "x.__add__(y)" is ' - 'called.\n' - ' The "__divmod__()" method should be the equivalent to ' - 'using\n' - ' "__floordiv__()" and "__mod__()"; it should not be ' + ' class that has an "__add__()" method, ' + '"type(x).__add__(x, y)" is\n' + ' called. The "__divmod__()" method should be the ' + 'equivalent to\n' + ' using "__floordiv__()" and "__mod__()"; it should not be ' 'related to\n' ' "__truediv__()". Note that "__pow__()" should be ' 'defined to accept\n' @@ -7760,9 +7912,9 @@ 'expression "x -\n' ' y", where *y* is an instance of a class that has an ' '"__rsub__()"\n' - ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' - 'returns\n' - ' *NotImplemented*.\n' + ' method, "type(y).__rsub__(y, x)" is called if ' + '"type(x).__sub__(x,\n' + ' y)" returns *NotImplemented*.\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -8009,8 +8161,8 @@ '\n' 'The following table summarizes the operator precedence ' 'in Python, from\n' - 'lowest precedence (least binding) to highest precedence ' - '(most\n' + 'highest precedence (most binding) to lowest precedence ' + '(least\n' 'binding). Operators in the same box have the same ' 'precedence. Unless\n' 'the syntax is explicitly given, operators are binary. ' @@ -8029,71 +8181,71 @@ '| Operator | ' 'Description |\n' '|=================================================|=======================================|\n' - '| ":=" | ' - 'Assignment expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "lambda" | ' - 'Lambda expression |\n' + '| "(expressions...)", "[expressions...]", "{key: | ' + 'Binding or parenthesized expression, |\n' + '| value...}", "{expressions...}" | list ' + 'display, dictionary display, set |\n' + '| | ' + 'display |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "if" – "else" | ' - 'Conditional expression |\n' + '| "x[index]", "x[index:index]", | ' + 'Subscription, slicing, call, |\n' + '| "x(arguments...)", "x.attribute" | ' + 'attribute reference |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "or" | ' - 'Boolean OR |\n' + '| "await" "x" | ' + 'Await expression |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "and" | ' - 'Boolean AND |\n' + '| "**" | ' + 'Exponentiation [5] |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "not" "x" | ' - 'Boolean NOT |\n' + '| "+x", "-x", "~x" | ' + 'Positive, negative, bitwise NOT |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "in", "not in", "is", "is not", "<", "<=", ">", | ' - 'Comparisons, including membership |\n' - '| ">=", "!=", "==" | ' - 'tests and identity tests |\n' + '| "*", "@", "/", "//", "%" | ' + 'Multiplication, matrix |\n' + '| | ' + 'multiplication, division, floor |\n' + '| | ' + 'division, remainder [6] |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "|" | ' - 'Bitwise OR |\n' + '| "+", "-" | ' + 'Addition and subtraction |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "^" | ' - 'Bitwise XOR |\n' + '| "<<", ">>" | ' + 'Shifts |\n' '+-------------------------------------------------+---------------------------------------+\n' '| "&" | ' 'Bitwise AND |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "<<", ">>" | ' - 'Shifts |\n' + '| "^" | ' + 'Bitwise XOR |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "+", "-" | ' - 'Addition and subtraction |\n' + '| "|" | ' + 'Bitwise OR |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "*", "@", "/", "//", "%" | ' - 'Multiplication, matrix |\n' - '| | ' - 'multiplication, division, floor |\n' - '| | ' - 'division, remainder [5] |\n' + '| "in", "not in", "is", "is not", "<", "<=", ">", | ' + 'Comparisons, including membership |\n' + '| ">=", "!=", "==" | ' + 'tests and identity tests |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "+x", "-x", "~x" | ' - 'Positive, negative, bitwise NOT |\n' + '| "not" "x" | ' + 'Boolean NOT |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "**" | ' - 'Exponentiation [6] |\n' + '| "and" | ' + 'Boolean AND |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "await" "x" | ' - 'Await expression |\n' + '| "or" | ' + 'Boolean OR |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "x[index]", "x[index:index]", | ' - 'Subscription, slicing, call, |\n' - '| "x(arguments...)", "x.attribute" | ' - 'attribute reference |\n' + '| "if" – "else" | ' + 'Conditional expression |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "(expressions...)", "[expressions...]", "{key: | ' - 'Binding or parenthesized expression, |\n' - '| value...}", "{expressions...}" | list ' - 'display, dictionary display, set |\n' - '| | ' - 'display |\n' + '| "lambda" | ' + 'Lambda expression |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| ":=" | ' + 'Assignment expression |\n' '+-------------------------------------------------+---------------------------------------+\n' '\n' '-[ Footnotes ]-\n' @@ -8174,14 +8326,14 @@ 'Check their\n' ' documentation for more info.\n' '\n' - '[5] The "%" operator is also used for string formatting; ' - 'the same\n' - ' precedence applies.\n' - '\n' - '[6] The power operator "**" binds less tightly than an ' + '[5] The power operator "**" binds less tightly than an ' 'arithmetic or\n' ' bitwise unary operator on its right, that is, ' - '"2**-1" is "0.5".\n', + '"2**-1" is "0.5".\n' + '\n' + '[6] The "%" operator is also used for string formatting; ' + 'the same\n' + ' precedence applies.\n', 'pass': 'The "pass" statement\n' '********************\n' '\n' @@ -8229,7 +8381,10 @@ '"ZeroDivisionError".\n' 'Raising a negative number to a fractional power results in a ' '"complex"\n' - 'number. (In earlier versions it raised a "ValueError".)\n', + 'number. (In earlier versions it raised a "ValueError".)\n' + '\n' + 'This operation can be customized using the special "__pow__()" ' + 'method.\n', 'raise': 'The "raise" statement\n' '*********************\n' '\n' @@ -8266,12 +8421,18 @@ '\n' 'The "from" clause is used for exception chaining: if given, the ' 'second\n' - '*expression* must be another exception class or instance, which ' - 'will\n' - 'then be attached to the raised exception as the "__cause__" ' - 'attribute\n' - '(which is writable). If the raised exception is not handled, both\n' - 'exceptions will be printed:\n' + '*expression* must be another exception class or instance. If the\n' + 'second expression is an exception instance, it will be attached to ' + 'the\n' + 'raised exception as the "__cause__" attribute (which is writable). ' + 'If\n' + 'the expression is an exception class, the class will be ' + 'instantiated\n' + 'and the resulting exception instance will be attached to the ' + 'raised\n' + 'exception as the "__cause__" attribute. If the raised exception is ' + 'not\n' + 'handled, both exceptions will be printed:\n' '\n' ' >>> try:\n' ' ... print(1 / 0)\n' @@ -8623,6 +8784,10 @@ 'the\n' 'second argument.\n' '\n' + 'This operation can be customized using the special ' + '"__lshift__()" and\n' + '"__rshift__()" methods.\n' + '\n' 'A right shift by *n* bits is defined as floor division by ' '"pow(2,n)".\n' 'A left shift by *n* bits is defined as multiplication with ' @@ -8837,13 +9002,13 @@ '\n' ' If "__new__()" is invoked during object construction and ' 'it returns\n' - ' an instance or subclass of *cls*, then the new ' - 'instance’s\n' - ' "__init__()" method will be invoked like "__init__(self[, ' - '...])",\n' - ' where *self* is the new instance and the remaining ' - 'arguments are\n' - ' the same as were passed to the object constructor.\n' + ' an instance of *cls*, then the new instance’s ' + '"__init__()" method\n' + ' will be invoked like "__init__(self[, ...])", where ' + '*self* is the\n' + ' new instance and the remaining arguments are the same as ' + 'were\n' + ' passed to the object constructor.\n' '\n' ' If "__new__()" does not return an instance of *cls*, then ' 'the new\n' @@ -9511,32 +9676,6 @@ 'of the\n' ' owner class.\n' '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is created. ' - 'The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' Note:\n' - '\n' - ' "__set_name__()" is only called implicitly as part of ' - 'the "type"\n' - ' constructor, so it will need to be called explicitly ' - 'with the\n' - ' appropriate parameters when a descriptor is added to a ' - 'class\n' - ' after initial creation:\n' - '\n' - ' class A:\n' - ' pass\n' - ' descr = custom_descriptor()\n' - ' A.attr = descr\n' - " descr.__set_name__(A, 'attr')\n" - '\n' - ' See Creating the class object for more details.\n' - '\n' - ' New in version 3.6.\n' - '\n' 'The attribute "__objclass__" is interpreted by the "inspect" ' 'module as\n' 'specifying the class where this object was defined (setting ' @@ -9613,10 +9752,10 @@ '\n' 'For instance bindings, the precedence of descriptor ' 'invocation depends\n' - 'on the which descriptor methods are defined. A descriptor ' - 'can define\n' - 'any combination of "__get__()", "__set__()" and ' - '"__delete__()". If it\n' + 'on which descriptor methods are defined. A descriptor can ' + 'define any\n' + 'combination of "__get__()", "__set__()" and "__delete__()". ' + 'If it\n' 'does not define "__get__()", then accessing the attribute ' 'will return\n' 'the descriptor object itself unless there is a value in the ' @@ -9826,6 +9965,38 @@ '\n' ' New in version 3.6.\n' '\n' + 'When a class is created, "type.__new__()" scans the class ' + 'variables\n' + 'and makes callbacks to those with a "__set_name__()" hook.\n' + '\n' + 'object.__set_name__(self, owner, name)\n' + '\n' + ' Automatically called at the time the owning class *owner* ' + 'is\n' + ' created. The object has been assigned to *name* in that ' + 'class:\n' + '\n' + ' class A:\n' + ' x = C() # Automatically calls: x.__set_name__(A, ' + "'x')\n" + '\n' + ' If the class variable is assigned after the class is ' + 'created,\n' + ' "__set_name__()" will not be called automatically. If ' + 'needed,\n' + ' "__set_name__()" can be called directly:\n' + '\n' + ' class A:\n' + ' pass\n' + '\n' + ' c = C()\n' + ' A.x = c # The hook is not called\n' + " c.__set_name__(A, 'x') # Manually invoke the hook\n" + '\n' + ' See Creating the class object for more details.\n' + '\n' + ' New in version 3.6.\n' + '\n' '\n' 'Metaclasses\n' '-----------\n' @@ -10021,22 +10192,21 @@ 'When using the default metaclass "type", or any metaclass ' 'that\n' 'ultimately calls "type.__new__", the following additional\n' - 'customisation steps are invoked after creating the class ' + 'customization steps are invoked after creating the class ' 'object:\n' '\n' - '* first, "type.__new__" collects all of the descriptors in ' - 'the class\n' - ' namespace that define a "__set_name__()" method;\n' + '1. The "type.__new__" method collects all of the attributes ' + 'in the\n' + ' class namespace that define a "__set_name__()" method;\n' '\n' - '* second, all of these "__set_name__" methods are called ' - 'with the\n' - ' class being defined and the assigned name of that ' - 'particular\n' - ' descriptor;\n' + '2. Those "__set_name__" methods are called with the class ' + 'being\n' + ' defined and the assigned name of that particular ' + 'attribute;\n' '\n' - '* finally, the "__init_subclass__()" hook is called on the ' - 'immediate\n' - ' parent of the new class in its method resolution order.\n' + '3. The "__init_subclass__()" hook is called on the immediate ' + 'parent of\n' + ' the new class in its method resolution order.\n' '\n' 'After the class object is created, it is passed to the ' 'class\n' @@ -10437,11 +10607,11 @@ 'to\n' ' evaluate the expression "x + y", where *x* is an instance ' 'of a\n' - ' class that has an "__add__()" method, "x.__add__(y)" is ' - 'called.\n' - ' The "__divmod__()" method should be the equivalent to ' - 'using\n' - ' "__floordiv__()" and "__mod__()"; it should not be ' + ' class that has an "__add__()" method, "type(x).__add__(x, ' + 'y)" is\n' + ' called. The "__divmod__()" method should be the ' + 'equivalent to\n' + ' using "__floordiv__()" and "__mod__()"; it should not be ' 'related to\n' ' "__truediv__()". Note that "__pow__()" should be defined ' 'to accept\n' @@ -10482,9 +10652,9 @@ 'expression "x -\n' ' y", where *y* is an instance of a class that has an ' '"__rsub__()"\n' - ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' - 'returns\n' - ' *NotImplemented*.\n' + ' method, "type(y).__rsub__(y, x)" is called if ' + '"type(x).__sub__(x,\n' + ' y)" returns *NotImplemented*.\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -10677,17 +10847,16 @@ '\n' 'object.__match_args__\n' '\n' - ' This class variable can be assigned a tuple or list of ' - 'strings.\n' - ' When this class is used in a class pattern with ' - 'positional\n' - ' arguments, each positional argument will be converted ' - 'into a\n' - ' keyword argument, using the corresponding value in ' - '*__match_args__*\n' - ' as the keyword. The absence of this attribute is ' - 'equivalent to\n' - ' setting it to "()".\n' + ' This class variable can be assigned a tuple of strings. ' + 'When this\n' + ' class is used in a class pattern with positional ' + 'arguments, each\n' + ' positional argument will be converted into a keyword ' + 'argument,\n' + ' using the corresponding value in *__match_args__* as the ' + 'keyword.\n' + ' The absence of this attribute is equivalent to setting it ' + 'to "()".\n' '\n' 'For example, if "MyClass.__match_args__" is "("left", ' '"center",\n' @@ -12078,10 +12247,10 @@ 'exception. For an except clause with an expression, that expression\n' 'is evaluated, and the clause matches the exception if the resulting\n' 'object is “compatible” with the exception. An object is compatible\n' - 'with an exception if it is the class or a base class of the ' - 'exception\n' - 'object, or a tuple containing an item that is the class or a base\n' - 'class of the exception object.\n' + 'with an exception if the object is the class or a base class of the\n' + 'exception object, or a tuple containing an item that is the class or ' + 'a\n' + 'base class of the exception object.\n' '\n' 'If no except clause matches the exception, the search for an ' 'exception\n' @@ -12680,7 +12849,13 @@ '| |\n' ' | | and "\'return\'" for the ' 'return | |\n' - ' | | annotation, if provided. ' + ' | | annotation, if provided. For ' + '| |\n' + ' | | more information on working ' + '| |\n' + ' | | with this attribute, see ' + '| |\n' + ' | | Annotations Best Practices. ' '| |\n' ' ' '+---------------------------+---------------------------------+-------------+\n' @@ -12905,20 +13080,34 @@ ' Attribute assignment updates the module’s namespace dictionary,\n' ' e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n' '\n' - ' Predefined (writable) attributes: "__name__" is the module’s ' - 'name;\n' - ' "__doc__" is the module’s documentation string, or "None" if\n' - ' unavailable; "__annotations__" (optional) is a dictionary\n' - ' containing *variable annotations* collected during module body\n' - ' execution; "__file__" is the pathname of the file from which ' + ' Predefined (writable) attributes:\n' + '\n' + ' "__name__"\n' + ' The module’s name.\n' + '\n' + ' "__doc__"\n' + ' The module’s documentation string, or "None" if ' + 'unavailable.\n' + '\n' + ' "__file__"\n' + ' The pathname of the file from which the module was loaded, ' + 'if\n' + ' it was loaded from a file. The "__file__" attribute may ' + 'be\n' + ' missing for certain types of modules, such as C modules ' + 'that\n' + ' are statically linked into the interpreter. For ' + 'extension\n' + ' modules loaded dynamically from a shared library, it’s ' 'the\n' - ' module was loaded, if it was loaded from a file. The "__file__"\n' - ' attribute may be missing for certain types of modules, such as ' - 'C\n' - ' modules that are statically linked into the interpreter; for\n' - ' extension modules loaded dynamically from a shared library, it ' - 'is\n' - ' the pathname of the shared library file.\n' + ' pathname of the shared library file.\n' + '\n' + ' "__annotations__"\n' + ' A dictionary containing *variable annotations* collected\n' + ' during module body execution. For best practices on ' + 'working\n' + ' with "__annotations__", please see Annotations Best\n' + ' Practices.\n' '\n' ' Special read-only attribute: "__dict__" is the module’s ' 'namespace\n' @@ -12976,20 +13165,31 @@ 'instance\n' ' (see below).\n' '\n' - ' Special attributes: "__name__" is the class name; "__module__" ' - 'is\n' - ' the module name in which the class was defined; "__dict__" is ' - 'the\n' - ' dictionary containing the class’s namespace; "__bases__" is a ' - 'tuple\n' - ' containing the base classes, in the order of their occurrence ' - 'in\n' - ' the base class list; "__doc__" is the class’s documentation ' - 'string,\n' - ' or "None" if undefined; "__annotations__" (optional) is a\n' - ' dictionary containing *variable annotations* collected during ' - 'class\n' - ' body execution.\n' + ' Special attributes:\n' + '\n' + ' "__name__"\n' + ' The class name.\n' + '\n' + ' "__module__"\n' + ' The name of the module in which the class was defined.\n' + '\n' + ' "__dict__"\n' + ' The dictionary containing the class’s namespace.\n' + '\n' + ' "__bases__"\n' + ' A tuple containing the base classes, in the order of ' + 'their\n' + ' occurrence in the base class list.\n' + '\n' + ' "__doc__"\n' + ' The class’s documentation string, or "None" if undefined.\n' + '\n' + ' "__annotations__"\n' + ' A dictionary containing *variable annotations* collected\n' + ' during class body execution. For best practices on ' + 'working\n' + ' with "__annotations__", please see Annotations Best\n' + ' Practices.\n' '\n' 'Class instances\n' ' A class instance is created by calling a class object (see ' @@ -13072,6 +13272,7 @@ '\n' ' Special read-only attributes: "co_name" gives the function ' 'name;\n' + ' "co_qualname" gives the fully qualified function name;\n' ' "co_argcount" is the total number of positional arguments\n' ' (including positional-only arguments and arguments with ' 'default\n' @@ -13132,6 +13333,54 @@ ' "co_consts" is the documentation string of the function, or\n' ' "None" if undefined.\n' '\n' + ' codeobject.co_positions()\n' + '\n' + ' Returns an iterable over the source code positions of ' + 'each\n' + ' bytecode instruction in the code object.\n' + '\n' + ' The iterator returns tuples containing the "(start_line,\n' + ' end_line, start_column, end_column)". The *i-th* tuple\n' + ' corresponds to the position of the source code that ' + 'compiled\n' + ' to the *i-th* instruction. Column information is ' + '0-indexed\n' + ' utf-8 byte offsets on the given source line.\n' + '\n' + ' This positional information can be missing. A ' + 'non-exhaustive\n' + ' lists of cases where this may happen:\n' + '\n' + ' * Running the interpreter with "-X" "no_debug_ranges".\n' + '\n' + ' * Loading a pyc file compiled while using "-X"\n' + ' "no_debug_ranges".\n' + '\n' + ' * Position tuples corresponding to artificial ' + 'instructions.\n' + '\n' + ' * Line and column numbers that can’t be represented due ' + 'to\n' + ' implementation specific limitations.\n' + '\n' + ' When this occurs, some or all of the tuple elements can ' + 'be\n' + ' "None".\n' + '\n' + ' New in version 3.11.\n' + '\n' + ' Note:\n' + '\n' + ' This feature requires storing column positions in code\n' + ' objects which may result in a small increase of disk ' + 'usage\n' + ' of compiled Python files or interpreter memory usage. ' + 'To\n' + ' avoid storing the extra information and/or deactivate\n' + ' printing the extra traceback information, the "-X"\n' + ' "no_debug_ranges" command line flag or the\n' + ' "PYTHONNODEBUGRANGES" environment variable can be used.\n' + '\n' ' Frame objects\n' ' Frame objects represent execution frames. They may occur in\n' ' traceback objects (see below), and are also passed to ' @@ -13150,6 +13399,10 @@ ' gives the precise instruction (this is an index into the\n' ' bytecode string of the code object).\n' '\n' + ' Accessing "f_code" raises an auditing event ' + '"object.__getattr__"\n' + ' with arguments "obj" and ""f_code"".\n' + '\n' ' Special writable attributes: "f_trace", if not "None", is a\n' ' function called for various events during code execution ' '(this\n' @@ -13233,6 +13486,9 @@ ' the exception occurred in a "try" statement with no matching\n' ' except clause or with a finally clause.\n' '\n' + ' Accessing "tb_frame" raises an auditing event\n' + ' "object.__getattr__" with arguments "obj" and ""tb_frame"".\n' + '\n' ' Special writable attribute: "tb_next" is the next level in ' 'the\n' ' stack trace (towards the frame where the exception occurred), ' @@ -13283,9 +13539,8 @@ ' object actually returned is the wrapped object, which is not\n' ' subject to any further transformation. Static method objects ' 'are\n' - ' not themselves callable, although the objects they wrap ' - 'usually\n' - ' are. Static method objects are created by the built-in\n' + ' also callable. Static method objects are created by the ' + 'built-in\n' ' "staticmethod()" constructor.\n' '\n' ' Class method objects\n' @@ -14237,7 +14492,7 @@ '| | "s[i:i] = ' '[x]") | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at *i* ' + '| "s.pop()" or "s.pop(i)" | retrieves the item at *i* ' 'and | (2) |\n' '| | also removes it from ' '*s* | |\n' @@ -14700,7 +14955,7 @@ '| | "s[i:i] = ' '[x]") | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at ' + '| "s.pop()" or "s.pop(i)" | retrieves the item at ' '*i* and | (2) |\n' '| | also removes it from ' '*s* | |\n' @@ -14765,15 +15020,21 @@ ' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n' '\n' 'The unary "-" (minus) operator yields the negation of its numeric\n' - 'argument.\n' + 'argument; the operation can be overridden with the "__neg__()" ' + 'special\n' + 'method.\n' '\n' 'The unary "+" (plus) operator yields its numeric argument ' - 'unchanged.\n' + 'unchanged;\n' + 'the operation can be overridden with the "__pos__()" special ' + 'method.\n' '\n' 'The unary "~" (invert) operator yields the bitwise inversion of ' 'its\n' 'integer argument. The bitwise inversion of "x" is defined as\n' - '"-(x+1)". It only applies to integral numbers.\n' + '"-(x+1)". It only applies to integral numbers or to custom ' + 'objects\n' + 'that override the "__invert__()" special method.\n' '\n' 'In all three cases, if the argument does not have the proper type, ' 'a\n' diff --git a/Lib/random.py b/Lib/random.py index 3569d58cc42430..92a71e14c480bd 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -286,27 +286,17 @@ def randbytes(self, n): ## -------------------- integer methods ------------------- def randrange(self, start, stop=None, step=_ONE): - """Choose a random item from range(start, stop[, step]). + """Choose a random item from range(stop) or range(start, stop[, step]). - This fixes the problem with randint() which includes the - endpoint; in Python this is usually not what you want. + Roughly equivalent to ``choice(range(start, stop, step))`` + but supports arbitrarily large ranges and is optimized + for common cases. """ # This code is a bit messy to make it fast for the # common case while still doing adequate error checking. - try: - istart = _index(start) - except TypeError: - istart = int(start) - if istart != start: - _warn('randrange() will raise TypeError in the future', - DeprecationWarning, 2) - raise ValueError("non-integer arg 1 for randrange()") - _warn('non-integer arguments to randrange() have been deprecated ' - 'since Python 3.10 and will be removed in a subsequent ' - 'version', - DeprecationWarning, 2) + istart = _index(start) if stop is None: # We don't check for "step != 1" because it hasn't been # type checked and converted to an integer yet. @@ -316,37 +306,15 @@ def randrange(self, start, stop=None, step=_ONE): return self._randbelow(istart) raise ValueError("empty range for randrange()") - # stop argument supplied. - try: - istop = _index(stop) - except TypeError: - istop = int(stop) - if istop != stop: - _warn('randrange() will raise TypeError in the future', - DeprecationWarning, 2) - raise ValueError("non-integer stop for randrange()") - _warn('non-integer arguments to randrange() have been deprecated ' - 'since Python 3.10 and will be removed in a subsequent ' - 'version', - DeprecationWarning, 2) + # Stop argument supplied. + istop = _index(stop) width = istop - istart - try: - istep = _index(step) - except TypeError: - istep = int(step) - if istep != step: - _warn('randrange() will raise TypeError in the future', - DeprecationWarning, 2) - raise ValueError("non-integer step for randrange()") - _warn('non-integer arguments to randrange() have been deprecated ' - 'since Python 3.10 and will be removed in a subsequent ' - 'version', - DeprecationWarning, 2) + istep = _index(step) # Fast path. if istep == 1: if width > 0: return istart + self._randbelow(width) - raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width)) + raise ValueError(f"empty range in randrange({start}, {stop}, {step})") # Non-unit step argument supplied. if istep > 0: @@ -356,7 +324,7 @@ def randrange(self, start, stop=None, step=_ONE): else: raise ValueError("zero step for randrange()") if n <= 0: - raise ValueError("empty range for randrange()") + raise ValueError(f"empty range in randrange({start}, {stop}, {step})") return istart + istep * self._randbelow(n) def randint(self, a, b): diff --git a/Lib/shutil.py b/Lib/shutil.py index e544498fca7c84..949e024853c1d2 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1372,9 +1372,9 @@ def get_terminal_size(fallback=(80, 24)): # os.get_terminal_size() is unsupported size = os.terminal_size(fallback) if columns <= 0: - columns = size.columns + columns = size.columns or fallback[0] if lines <= 0: - lines = size.lines + lines = size.lines or fallback[1] return os.terminal_size((columns, lines)) diff --git a/Lib/site.py b/Lib/site.py index 939893eb5ee93b..e129f3b4851f3d 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -418,8 +418,10 @@ def setcopyright(): files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. - if hasattr(os, '__file__'): + here = getattr(sys, '_stdlib_dir', None) + if not here and hasattr(os, '__file__'): here = os.path.dirname(os.__file__) + if here: files.extend(["LICENSE.txt", "LICENSE"]) dirs.extend([os.path.join(here, os.pardir), here, os.curdir]) builtins.license = _sitebuiltins._Printer( diff --git a/Lib/sqlite3/test/test_dbapi.py b/Lib/sqlite3/test/test_dbapi.py index 732e21dd3a285e..d82543663d18bb 100644 --- a/Lib/sqlite3/test/test_dbapi.py +++ b/Lib/sqlite3/test/test_dbapi.py @@ -168,9 +168,9 @@ def test_module_constants(self): "SQLITE_TRANSACTION", "SQLITE_UPDATE", ] - if sqlite.version_info >= (3, 7, 17): + if sqlite.sqlite_version_info >= (3, 7, 17): consts += ["SQLITE_NOTICE", "SQLITE_WARNING"] - if sqlite.version_info >= (3, 8, 3): + if sqlite.sqlite_version_info >= (3, 8, 3): consts.append("SQLITE_RECURSIVE") consts += ["PARSE_DECLTYPES", "PARSE_COLNAMES"] for const in consts: @@ -616,7 +616,7 @@ def test_fetchone_no_statement(self): self.assertEqual(row, None) def test_array_size(self): - # must default ot 1 + # must default to 1 self.assertEqual(self.cu.arraysize, 1) # now set to 2 diff --git a/Lib/sqlite3/test/test_regression.py b/Lib/sqlite3/test/test_regression.py index ff356734860b64..3d71809d9c11cf 100644 --- a/Lib/sqlite3/test/test_regression.py +++ b/Lib/sqlite3/test/test_regression.py @@ -475,6 +475,17 @@ def dup(v): con.execute("drop table t") con.commit() + def test_executescript_step_through_select(self): + with managed_connect(":memory:", in_mem=True) as con: + values = [(v,) for v in range(5)] + with con: + con.execute("create table t(t)") + con.executemany("insert into t values(?)", values) + steps = [] + con.create_function("step", 1, lambda x: steps.append((x,))) + con.executescript("select step(t) from t") + self.assertEqual(steps, values) + if __name__ == "__main__": unittest.main() diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 95b48f6429d5f8..daf9f000060a35 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -184,7 +184,7 @@ def is_python_build(check_home=False): if _PYTHON_BUILD: for scheme in ('posix_prefix', 'posix_home'): - # On POSIX-y platofrms, Python will: + # On POSIX-y platforms, Python will: # - Build from .h files in 'headers' (which is only added to the # scheme when building CPython) # - Install .h files to 'include' diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 18d415adf54418..c1ee1222e09b5a 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2349,6 +2349,15 @@ def next(self): raise ReadError(str(e)) from None except SubsequentHeaderError as e: raise ReadError(str(e)) from None + except Exception as e: + try: + import zlib + if isinstance(e, zlib.error): + raise ReadError(f'zlib error: {e}') from None + else: + raise e + except ImportError: + raise e break if tarinfo is not None: diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 8570c3ba0627fa..531cbf32f1283f 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -88,6 +88,10 @@ def _infer_return_type(*args): for arg in args: if arg is None: continue + + if isinstance(arg, _os.PathLike): + arg = _os.fspath(arg) + if isinstance(arg, bytes): if return_type is str: raise TypeError("Can't mix bytes and non-bytes in " diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 125e8906d8abce..3bc5b8f3d79b02 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3773,12 +3773,19 @@ def _attach_existing_shmem_then_write(shmem_name_or_obj, binary_data): local_sms.buf[:len(binary_data)] = binary_data local_sms.close() + def _new_shm_name(self, prefix): + # Add a PID to the name of a POSIX shared memory object to allow + # running multiprocessing tests (test_multiprocessing_fork, + # test_multiprocessing_spawn, etc) in parallel. + return prefix + str(os.getpid()) + def test_shared_memory_basics(self): - sms = shared_memory.SharedMemory('test01_tsmb', create=True, size=512) + name_tsmb = self._new_shm_name('test01_tsmb') + sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) self.addCleanup(sms.unlink) # Verify attributes are readable. - self.assertEqual(sms.name, 'test01_tsmb') + self.assertEqual(sms.name, name_tsmb) self.assertGreaterEqual(sms.size, 512) self.assertGreaterEqual(len(sms.buf), sms.size) @@ -3786,24 +3793,17 @@ def test_shared_memory_basics(self): self.assertIn(sms.name, str(sms)) self.assertIn(str(sms.size), str(sms)) - # Test pickling - sms.buf[0:6] = b'pickle' - pickled_sms = pickle.dumps(sms) - sms2 = pickle.loads(pickled_sms) - self.assertEqual(sms.name, sms2.name) - self.assertEqual(bytes(sms.buf[0:6]), bytes(sms2.buf[0:6]), b'pickle') - # Modify contents of shared memory segment through memoryview. sms.buf[0] = 42 self.assertEqual(sms.buf[0], 42) # Attach to existing shared memory segment. - also_sms = shared_memory.SharedMemory('test01_tsmb') + also_sms = shared_memory.SharedMemory(name_tsmb) self.assertEqual(also_sms.buf[0], 42) also_sms.close() # Attach to existing shared memory segment but specify a new size. - same_sms = shared_memory.SharedMemory('test01_tsmb', size=20*sms.size) + same_sms = shared_memory.SharedMemory(name_tsmb, size=20*sms.size) self.assertLess(same_sms.size, 20*sms.size) # Size was ignored. same_sms.close() @@ -3821,7 +3821,7 @@ def test_shared_memory_basics(self): 'multiprocessing.shared_memory._make_filename') as mock_make_filename: NAME_PREFIX = shared_memory._SHM_NAME_PREFIX - names = ['test01_fn', 'test02_fn'] + names = [self._new_shm_name('test01_fn'), self._new_shm_name('test02_fn')] # Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary # because some POSIX compliant systems require name to start with / names = [NAME_PREFIX + name for name in names] @@ -3843,17 +3843,17 @@ def test_shared_memory_basics(self): # manages unlinking on its own and unlink() does nothing). # True release of shared memory segment does not necessarily # happen until process exits, depending on the OS platform. + name_dblunlink = self._new_shm_name('test01_dblunlink') + sms_uno = shared_memory.SharedMemory( + name_dblunlink, + create=True, + size=5000 + ) with self.assertRaises(FileNotFoundError): - sms_uno = shared_memory.SharedMemory( - 'test01_dblunlink', - create=True, - size=5000 - ) - try: self.assertGreaterEqual(sms_uno.size, 5000) - sms_duo = shared_memory.SharedMemory('test01_dblunlink') + sms_duo = shared_memory.SharedMemory(name_dblunlink) sms_duo.unlink() # First shm_unlink() call. sms_duo.close() sms_uno.close() @@ -3865,7 +3865,7 @@ def test_shared_memory_basics(self): # Attempting to create a new shared memory segment with a # name that is already in use triggers an exception. there_can_only_be_one_sms = shared_memory.SharedMemory( - 'test01_tsmb', + name_tsmb, create=True, size=512 ) @@ -3879,7 +3879,7 @@ def test_shared_memory_basics(self): # case of MacOS/darwin, requesting a smaller size is disallowed. class OptionalAttachSharedMemory(shared_memory.SharedMemory): _flags = os.O_CREAT | os.O_RDWR - ok_if_exists_sms = OptionalAttachSharedMemory('test01_tsmb') + ok_if_exists_sms = OptionalAttachSharedMemory(name_tsmb) self.assertEqual(ok_if_exists_sms.size, sms.size) ok_if_exists_sms.close() @@ -3891,6 +3891,29 @@ class OptionalAttachSharedMemory(shared_memory.SharedMemory): sms.close() + def test_shared_memory_recreate(self): + # Test if shared memory segment is created properly, + # when _make_filename returns an existing shared memory segment name + with unittest.mock.patch( + 'multiprocessing.shared_memory._make_filename') as mock_make_filename: + + NAME_PREFIX = shared_memory._SHM_NAME_PREFIX + names = ['test01_fn', 'test02_fn'] + # Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary + # because some POSIX compliant systems require name to start with / + names = [NAME_PREFIX + name for name in names] + + mock_make_filename.side_effect = names + shm1 = shared_memory.SharedMemory(create=True, size=1) + self.addCleanup(shm1.unlink) + self.assertEqual(shm1._name, names[0]) + + mock_make_filename.side_effect = names + shm2 = shared_memory.SharedMemory(create=True, size=1) + self.addCleanup(shm2.unlink) + self.assertEqual(shm2._name, names[1]) + + def test_invalid_shared_memory_cration(self): # Test creating a shared memory segment with negative size with self.assertRaises(ValueError): sms_invalid = shared_memory.SharedMemory(create=True, size=-1) @@ -3903,6 +3926,47 @@ class OptionalAttachSharedMemory(shared_memory.SharedMemory): with self.assertRaises(ValueError): sms_invalid = shared_memory.SharedMemory(create=True) + def test_shared_memory_pickle_unpickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sms = shared_memory.SharedMemory(create=True, size=512) + self.addCleanup(sms.unlink) + sms.buf[0:6] = b'pickle' + + # Test pickling + pickled_sms = pickle.dumps(sms, protocol=proto) + + # Test unpickling + sms2 = pickle.loads(pickled_sms) + self.assertIsInstance(sms2, shared_memory.SharedMemory) + self.assertEqual(sms.name, sms2.name) + self.assertEqual(bytes(sms.buf[0:6]), b'pickle') + self.assertEqual(bytes(sms2.buf[0:6]), b'pickle') + + # Test that unpickled version is still the same SharedMemory + sms.buf[0:6] = b'newval' + self.assertEqual(bytes(sms.buf[0:6]), b'newval') + self.assertEqual(bytes(sms2.buf[0:6]), b'newval') + + sms2.buf[0:6] = b'oldval' + self.assertEqual(bytes(sms.buf[0:6]), b'oldval') + self.assertEqual(bytes(sms2.buf[0:6]), b'oldval') + + def test_shared_memory_pickle_unpickle_dead_object(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sms = shared_memory.SharedMemory(create=True, size=512) + sms.buf[0:6] = b'pickle' + pickled_sms = pickle.dumps(sms, protocol=proto) + + # Now, we are going to kill the original object. + # So, unpickled one won't be able to attach to it. + sms.close() + sms.unlink() + + with self.assertRaises(FileNotFoundError): + pickle.loads(pickled_sms) + def test_shared_memory_across_processes(self): # bpo-40135: don't define shared memory block's name in case of # the failure when we run multiprocessing tests in parallel. @@ -4084,10 +4148,11 @@ def test_shared_memory_ShareableList_basics(self): self.assertEqual(sl.count(b'adios'), 0) # Exercise creating a duplicate. - sl_copy = shared_memory.ShareableList(sl, name='test03_duplicate') + name_duplicate = self._new_shm_name('test03_duplicate') + sl_copy = shared_memory.ShareableList(sl, name=name_duplicate) try: self.assertNotEqual(sl.shm.name, sl_copy.shm.name) - self.assertEqual('test03_duplicate', sl_copy.shm.name) + self.assertEqual(name_duplicate, sl_copy.shm.name) self.assertEqual(list(sl), list(sl_copy)) self.assertEqual(sl.format, sl_copy.format) sl_copy[-1] = 77 @@ -4119,29 +4184,45 @@ def test_shared_memory_ShareableList_basics(self): empty_sl.shm.unlink() def test_shared_memory_ShareableList_pickling(self): - sl = shared_memory.ShareableList(range(10)) - self.addCleanup(sl.shm.unlink) - - serialized_sl = pickle.dumps(sl) - deserialized_sl = pickle.loads(serialized_sl) - self.assertTrue( - isinstance(deserialized_sl, shared_memory.ShareableList) - ) - self.assertTrue(deserialized_sl[-1], 9) - self.assertFalse(sl is deserialized_sl) - deserialized_sl[4] = "changed" - self.assertEqual(sl[4], "changed") - - # Verify data is not being put into the pickled representation. - name = 'a' * len(sl.shm.name) - larger_sl = shared_memory.ShareableList(range(400)) - self.addCleanup(larger_sl.shm.unlink) - serialized_larger_sl = pickle.dumps(larger_sl) - self.assertTrue(len(serialized_sl) == len(serialized_larger_sl)) - larger_sl.shm.close() - - deserialized_sl.shm.close() - sl.shm.close() + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sl = shared_memory.ShareableList(range(10)) + self.addCleanup(sl.shm.unlink) + + serialized_sl = pickle.dumps(sl, protocol=proto) + deserialized_sl = pickle.loads(serialized_sl) + self.assertIsInstance( + deserialized_sl, shared_memory.ShareableList) + self.assertEqual(deserialized_sl[-1], 9) + self.assertIsNot(sl, deserialized_sl) + + deserialized_sl[4] = "changed" + self.assertEqual(sl[4], "changed") + sl[3] = "newvalue" + self.assertEqual(deserialized_sl[3], "newvalue") + + larger_sl = shared_memory.ShareableList(range(400)) + self.addCleanup(larger_sl.shm.unlink) + serialized_larger_sl = pickle.dumps(larger_sl, protocol=proto) + self.assertEqual(len(serialized_sl), len(serialized_larger_sl)) + larger_sl.shm.close() + + deserialized_sl.shm.close() + sl.shm.close() + + def test_shared_memory_ShareableList_pickling_dead_object(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sl = shared_memory.ShareableList(range(10)) + serialized_sl = pickle.dumps(sl, protocol=proto) + + # Now, we are going to kill the original object. + # So, unpickled one won't be able to attach to it. + sl.shm.close() + sl.shm.unlink() + + with self.assertRaises(FileNotFoundError): + pickle.loads(serialized_sl) def test_shared_memory_cleaned_after_process_termination(self): cmd = '''if 1: @@ -4194,7 +4275,7 @@ def test_shared_memory_cleaned_after_process_termination(self): "shared_memory objects to clean up at shutdown", err) # -# +# Test to verify that `Finalize` works. # class _TestFinalize(BaseTestCase): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 6414f1ace3fed3..9f551d9b9748d3 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -4064,7 +4064,7 @@ def test_even_more_compare(self): self.assertEqual(t1, t1) self.assertEqual(t2, t2) - # Equal afer adjustment. + # Equal after adjustment. t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(1, "")) t2 = self.theclass(2, 1, 1, 3, 13, tzinfo=FixedOffset(3*60+13+2, "")) self.assertEqual(t1, t2) @@ -4903,7 +4903,7 @@ def test_easy(self): # OTOH, these fail! Don't enable them. The difficulty is that # the edge case tests assume that every hour is representable in # the "utc" class. This is always true for a fixed-offset tzinfo - # class (lke utc_real and utc_fake), but not for Eastern or Central. + # class (like utc_real and utc_fake), but not for Eastern or Central. # For these adjacent DST-aware time zones, the range of time offsets # tested ends up creating hours in the one that aren't representable # in the other. For the same reason, we would see failures in the diff --git a/Lib/test/decimaltestdata/abs.decTest b/Lib/test/decimaltestdata/abs.decTest index 01f73d7766648f..569b8fcd84ab62 100644 --- a/Lib/test/decimaltestdata/abs.decTest +++ b/Lib/test/decimaltestdata/abs.decTest @@ -20,7 +20,7 @@ version: 2.59 -- This set of tests primarily tests the existence of the operator. --- Additon, subtraction, rounding, and more overflows are tested +-- Addition, subtraction, rounding, and more overflows are tested -- elsewhere. precision: 9 diff --git a/Lib/test/decimaltestdata/extra.decTest b/Lib/test/decimaltestdata/extra.decTest index b630d8e3f9d45e..31291202a35e79 100644 --- a/Lib/test/decimaltestdata/extra.decTest +++ b/Lib/test/decimaltestdata/extra.decTest @@ -156,7 +156,7 @@ extr1302 fma -Inf 0E-456 sNaN148 -> NaN Invalid_operation -- max/min/max_mag/min_mag bug in 2.5.2/2.6/3.0: max(NaN, finite) gave -- incorrect answers when the finite number required rounding; similarly --- for the other thre functions +-- for the other three functions maxexponent: 999 minexponent: -999 precision: 6 diff --git a/Lib/test/gdb_sample.py b/Lib/test/gdb_sample.py index cab13fb4da594c..4188f50136fb97 100644 --- a/Lib/test/gdb_sample.py +++ b/Lib/test/gdb_sample.py @@ -1,7 +1,7 @@ # Sample script for use by test_gdb.py def foo(a, b, c): - bar(a, b, c) + bar(a=a, b=b, c=c) def bar(a, b, c): baz(a, b, c) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 29f4ede523b007..6e6ff1201d81c7 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,7 +1,6 @@ import argparse import os import sys -from test import support from test.support import os_helper diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 096b5381cd9339..1069e2da008ff7 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,5 +1,4 @@ import os -import re import sys import warnings from inspect import isabstract @@ -114,7 +113,7 @@ def get_pooled_int(value): # These checkers return False on success, True on failure def check_rc_deltas(deltas): - # Checker for reference counters and memomry blocks. + # Checker for reference counters and memory blocks. # # bpo-30776: Try to ignore false positives: # diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index fe4693bad9ca6d..23970410a28fb0 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -196,18 +196,30 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: stream = io.StringIO() orig_stdout = sys.stdout orig_stderr = sys.stderr + print_warning = support.print_warning + orig_print_warnings_stderr = print_warning.orig_stderr + + output = None try: sys.stdout = stream sys.stderr = stream + # print_warning() writes into the temporary stream to preserve + # messages order. If support.environment_altered becomes true, + # warnings will be written to sys.stderr below. + print_warning.orig_stderr = stream + result = _runtest_inner(ns, test_name, display_failure=False) if not isinstance(result, Passed): output = stream.getvalue() - orig_stderr.write(output) - orig_stderr.flush() finally: sys.stdout = orig_stdout sys.stderr = orig_stderr + print_warning.orig_stderr = orig_print_warnings_stderr + + if output is not None: + sys.stderr.write(output) + sys.stderr.flush() else: # Tell tests to be moderately quiet support.verbose = ns.verbose diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index c83e44aed05143..f02f56e98bcda2 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,4 +1,3 @@ -import collections import faulthandler import json import os @@ -70,7 +69,9 @@ def run_test_in_subprocess(testname: str, ns: Namespace) -> subprocess.Popen: kw['start_new_session'] = True return subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + # bpo-45410: Write stderr into stdout to keep + # messages order + stderr=subprocess.STDOUT, universal_newlines=True, close_fds=(os.name != 'nt'), cwd=os_helper.SAVEDCWD, @@ -114,8 +115,8 @@ def stop(self): class MultiprocessResult(NamedTuple): result: TestResult + # bpo-45410: stderr is written into stdout to keep messages order stdout: str - stderr: str error_msg: str @@ -195,11 +196,10 @@ def mp_result_error( self, test_result: TestResult, stdout: str = '', - stderr: str = '', err_msg=None ) -> MultiprocessResult: test_result.duration_sec = time.monotonic() - self.start_time - return MultiprocessResult(test_result, stdout, stderr, err_msg) + return MultiprocessResult(test_result, stdout, err_msg) def _run_process(self, test_name: str) -> tuple[int, str, str]: self.start_time = time.monotonic() @@ -223,13 +223,14 @@ def _run_process(self, test_name: str) -> tuple[int, str, str]: raise ExitThread try: - stdout, stderr = popen.communicate(timeout=self.timeout) + # bpo-45410: stderr is written into stdout + stdout, _ = popen.communicate(timeout=self.timeout) retcode = popen.returncode assert retcode is not None except subprocess.TimeoutExpired: if self._stopped: - # kill() has been called: communicate() fails - # on reading closed stdout/stderr + # kill() has been called: communicate() fails on reading + # closed stdout raise ExitThread # On timeout, kill the process @@ -238,20 +239,19 @@ def _run_process(self, test_name: str) -> tuple[int, str, str]: # None means TIMEOUT for the caller retcode = None # bpo-38207: Don't attempt to call communicate() again: on it - # can hang until all child processes using stdout and stderr + # can hang until all child processes using stdout # pipes completes. - stdout = stderr = '' + stdout = '' except OSError: if self._stopped: # kill() has been called: communicate() fails - # on reading closed stdout/stderr + # on reading closed stdout raise ExitThread raise else: stdout = stdout.strip() - stderr = stderr.rstrip() - return (retcode, stdout, stderr) + return (retcode, stdout) except: self._kill() raise @@ -261,10 +261,10 @@ def _run_process(self, test_name: str) -> tuple[int, str, str]: self.current_test_name = None def _runtest(self, test_name: str) -> MultiprocessResult: - retcode, stdout, stderr = self._run_process(test_name) + retcode, stdout = self._run_process(test_name) if retcode is None: - return self.mp_result_error(Timeout(test_name), stdout, stderr) + return self.mp_result_error(Timeout(test_name), stdout) err_msg = None if retcode != 0: @@ -282,10 +282,9 @@ def _runtest(self, test_name: str) -> MultiprocessResult: err_msg = "Failed to parse worker JSON: %s" % exc if err_msg is not None: - return self.mp_result_error(ChildError(test_name), - stdout, stderr, err_msg) + return self.mp_result_error(ChildError(test_name), stdout, err_msg) - return MultiprocessResult(result, stdout, stderr, err_msg) + return MultiprocessResult(result, stdout, err_msg) def run(self) -> None: while not self._stopped: @@ -309,10 +308,8 @@ def run(self) -> None: def _wait_completed(self) -> None: popen = self._popen - # stdout and stderr must be closed to ensure that communicate() - # does not hang + # stdout must be closed to ensure that communicate() does not hang popen.stdout.close() - popen.stderr.close() try: popen.wait(JOIN_TIMEOUT) @@ -449,8 +446,6 @@ def _process_result(self, item: QueueOutput) -> bool: if mp_result.stdout: print(mp_result.stdout, flush=True) - if mp_result.stderr and not self.ns.pgo: - print(mp_result.stderr, file=sys.stderr, flush=True) if must_stop(mp_result.result, self.ns): return True diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index f0bfcf389992d8..60c9be24617a65 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -320,7 +320,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): support.environment_altered = True restore(original) if not self.quiet and not self.pgo: - print_warning(f"{name} was modified by {self.testname}") - print(f" Before: {original}\n After: {current} ", - file=sys.stderr, flush=True) + print_warning( + f"{name} was modified by {self.testname}\n" + f" Before: {original}\n" + f" After: {current} ") return False diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ad18b50f5db99d..8578a028c78bc2 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -68,21 +68,14 @@ def print_warning(msg): orig_unraisablehook = None -def flush_std_streams(): - if sys.stdout is not None: - sys.stdout.flush() - if sys.stderr is not None: - sys.stderr.flush() - - def regrtest_unraisable_hook(unraisable): global orig_unraisablehook support.environment_altered = True - print_warning("Unraisable exception") + support.print_warning("Unraisable exception") old_stderr = sys.stderr try: - flush_std_streams() - sys.stderr = sys.__stderr__ + support.flush_std_streams() + sys.stderr = support.print_warning.orig_stderr orig_unraisablehook(unraisable) sys.stderr.flush() finally: @@ -101,11 +94,11 @@ def setup_unraisable_hook(): def regrtest_threading_excepthook(args): global orig_threading_excepthook support.environment_altered = True - print_warning(f"Uncaught thread exception: {args.exc_type.__name__}") + support.print_warning(f"Uncaught thread exception: {args.exc_type.__name__}") old_stderr = sys.stderr try: - flush_std_streams() - sys.stderr = sys.__stderr__ + support.flush_std_streams() + sys.stderr = support.print_warning.orig_stderr orig_threading_excepthook(args) sys.stderr.flush() finally: diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index dffb7d4418dfe6..d82629368dff8a 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -455,6 +455,12 @@ def test_at_fork_reinit(self): with evt._cond: self.assertFalse(evt._cond.acquire(False)) + def test_repr(self): + evt = self.eventtype() + self.assertRegex(repr(evt), r"<\w+\.Event at .*: unset>") + evt.set() + self.assertRegex(repr(evt), r"<\w+\.Event at .*: set>") + class ConditionTests(BaseTestCase): """ @@ -802,6 +808,15 @@ def test_release_unacquired(self): sem.acquire() sem.release() + def test_repr(self): + sem = self.semtype(3) + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=3>") + sem.acquire() + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=2>") + sem.release() + sem.release() + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=4>") + class BoundedSemaphoreTests(BaseSemaphoreTests): """ @@ -816,6 +831,12 @@ def test_release_unacquired(self): sem.release() self.assertRaises(ValueError, sem.release) + def test_repr(self): + sem = self.semtype(3) + self.assertRegex(repr(sem), r"<\w+\.BoundedSemaphore at .*: value=3/3>") + sem.acquire() + self.assertRegex(repr(sem), r"<\w+\.BoundedSemaphore at .*: value=2/3>") + class BarrierTests(BaseTestCase): """ @@ -1008,3 +1029,18 @@ def test_single_thread(self): b = self.barriertype(1) b.wait() b.wait() + + def test_repr(self): + b = self.barriertype(3) + self.assertRegex(repr(b), r"<\w+\.Barrier at .*: waiters=0/3>") + def f(): + b.wait(3) + bunch = Bunch(f, 2) + bunch.wait_for_started() + time.sleep(0.2) + self.assertRegex(repr(b), r"<\w+\.Barrier at .*: waiters=2/3>") + b.wait(3) + bunch.wait_for_finished() + self.assertRegex(repr(b), r"<\w+\.Barrier at .*: waiters=0/3>") + b.abort() + self.assertRegex(repr(b), r"<\w+\.Barrier at .*: broken>") diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 25283f8c538d8f..3e2c781caa11ac 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3723,7 +3723,7 @@ class MyClass: self.assertEqual(new_f, 5) self.assertEqual(some_str, 'some str') - # math.log does not have its usual reducer overriden, so the + # math.log does not have its usual reducer overridden, so the # custom reduction callback should silently direct the pickler # to the default pickling by attribute, by returning # NotImplemented @@ -3740,7 +3740,7 @@ class MyClass: def test_reducer_override_no_reference_cycle(self): # bpo-39492: reducer_override used to induce a spurious reference cycle # inside the Pickler object, that could prevent all serialized objects - # from being garbage-collected without explicity invoking gc.collect. + # from being garbage-collected without explicitly invoking gc.collect. for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b29f438e52cf82..85fd74126b5f47 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1167,11 +1167,24 @@ def run_doctest(module, verbosity=None, optionflags=0): #======================================================================= # Support for saving and restoring the imported modules. +def flush_std_streams(): + if sys.stdout is not None: + sys.stdout.flush() + if sys.stderr is not None: + sys.stderr.flush() + + def print_warning(msg): - # bpo-39983: Print into sys.__stderr__ to display the warning even - # when sys.stderr is captured temporarily by a test + # bpo-45410: Explicitly flush stdout to keep logs in order + flush_std_streams() + stream = print_warning.orig_stderr for line in msg.splitlines(): - print(f"Warning -- {line}", file=sys.__stderr__, flush=True) + print(f"Warning -- {line}", file=stream) + stream.flush() + +# bpo-39983: Store the original sys.stderr at Python startup to be able to +# log warnings even if sys.stderr is captured temporarily by a test. +print_warning.orig_stderr = sys.stderr # Flag used by saved_test_environment of test.libregrtest.save_env, diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 10f745aa6b1448..9bce29895249ae 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -81,33 +81,13 @@ def import_module(name, deprecated=False, *, required_on=()): raise unittest.SkipTest(str(msg)) -def _save_and_remove_module(name, orig_modules): - """Helper function to save and remove a module from sys.modules - - Raise ImportError if the module can't be imported. - """ - # try to import the module and raise an error if it can't be imported - if name not in sys.modules: - __import__(name) - del sys.modules[name] +def _save_and_remove_modules(names): + orig_modules = {} + prefixes = tuple(name + '.' for name in names) for modname in list(sys.modules): - if modname == name or modname.startswith(name + '.'): - orig_modules[modname] = sys.modules[modname] - del sys.modules[modname] - - -def _save_and_block_module(name, orig_modules): - """Helper function to save and block a module in sys.modules - - Return True if the module was in sys.modules, False otherwise. - """ - saved = True - try: - orig_modules[name] = sys.modules[name] - except KeyError: - saved = False - sys.modules[name] = None - return saved + if modname in names or modname.startswith(prefixes): + orig_modules[modname] = sys.modules.pop(modname) + return orig_modules @contextlib.contextmanager @@ -136,7 +116,8 @@ def import_fresh_module(name, fresh=(), blocked=(), *, this operation. *fresh* is an iterable of additional module names that are also removed - from the sys.modules cache before doing the import. + from the sys.modules cache before doing the import. If one of these + modules can't be imported, None is returned. *blocked* is an iterable of module names that are replaced with None in the module cache during the import to ensure that attempts to import @@ -160,25 +141,25 @@ def import_fresh_module(name, fresh=(), blocked=(), *, with _ignore_deprecated_imports(deprecated): # Keep track of modules saved for later restoration as well # as those which just need a blocking entry removed - orig_modules = {} - names_to_remove = [] - _save_and_remove_module(name, orig_modules) + fresh = list(fresh) + blocked = list(blocked) + names = {name, *fresh, *blocked} + orig_modules = _save_and_remove_modules(names) + for modname in blocked: + sys.modules[modname] = None + try: - for fresh_name in fresh: - _save_and_remove_module(fresh_name, orig_modules) - for blocked_name in blocked: - if not _save_and_block_module(blocked_name, orig_modules): - names_to_remove.append(blocked_name) with frozen_modules(usefrozen): - fresh_module = importlib.import_module(name) - except ImportError: - fresh_module = None + # Return None when one of the "fresh" modules can not be imported. + try: + for modname in fresh: + __import__(modname) + except ImportError: + return None + return importlib.import_module(name) finally: - for orig_name, module in orig_modules.items(): - sys.modules[orig_name] = module - for name_to_remove in names_to_remove: - del sys.modules[name_to_remove] - return fresh_module + _save_and_remove_modules(names) + sys.modules.update(orig_modules) class CleanImport(object): diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index 0632577cdb303c..92a64e8354acbc 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -157,7 +157,7 @@ class catch_threading_exception: Context manager catching threading.Thread exception using threading.excepthook. - Attributes set when an exception is catched: + Attributes set when an exception is caught: * exc_type * exc_value diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 81bce2e6204216..177a8a64a4329f 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1221,7 +1221,7 @@ def test_channel_list_interpreters_basic(self): import _xxsubinterpreters as _interpreters obj = _interpreters.channel_recv({cid}) """)) - # Test for channel that has boths ends associated to an interpreter. + # Test for channel that has both ends associated to an interpreter. send_interps = interpreters.channel_list_interpreters(cid, send=True) recv_interps = interpreters.channel_list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 8babab399543bf..21732d14604c0b 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2164,6 +2164,42 @@ def test_help_non_breaking_spaces(self): wrap\N{NO-BREAK SPACE}at non-breaking spaces ''')) + def test_help_blank(self): + # Issue 24444 + parser = ErrorRaisingArgumentParser( + prog='PROG', description='main description') + parser.add_argument( + 'foo', + help=' ') + self.assertEqual(parser.format_help(), textwrap.dedent('''\ + usage: PROG [-h] foo + + main description + + positional arguments: + foo + + options: + -h, --help show this help message and exit + ''')) + + parser = ErrorRaisingArgumentParser( + prog='PROG', description='main description') + parser.add_argument( + 'foo', choices=[], + help='%(choices)s') + self.assertEqual(parser.format_help(), textwrap.dedent('''\ + usage: PROG [-h] {} + + main description + + positional arguments: + {} + + options: + -h, --help show this help message and exit + ''')) + def test_help_alternate_prefix_chars(self): parser = self._get_parser(prefix_chars='+:/') self.assertEqual(parser.format_usage(), diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 1f4257b1191baa..e630677f796845 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1075,6 +1075,14 @@ def test_literal_eval_malformed_lineno(self): with self.assertRaisesRegex(ValueError, msg): ast.literal_eval(node) + def test_literal_eval_syntax_errors(self): + msg = "unexpected character after line continuation character" + with self.assertRaisesRegex(SyntaxError, msg): + ast.literal_eval(r''' + \ + (\ + \ ''') + def test_bad_integer(self): # issue13436: Bad error message with invalid numeric values body = [ast.ImportFrom(module='time', diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 441adeea8f8e07..b2492c1acfecef 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -720,24 +720,68 @@ async def f(): self.loop.run_until_complete(f()) def test_explicit_lock(self): - lock = asyncio.Lock() - cond = asyncio.Condition(lock) + async def f(lock=None, cond=None): + if lock is None: + lock = asyncio.Lock() + if cond is None: + cond = asyncio.Condition(lock) + self.assertIs(cond._lock, lock) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + async with cond: + self.assertTrue(lock.locked()) + self.assertTrue(cond.locked()) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + async with lock: + self.assertTrue(lock.locked()) + self.assertTrue(cond.locked()) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) - self.assertIs(cond._lock, lock) - self.assertIs(cond._loop, lock._loop) + # All should work in the same way. + self.loop.run_until_complete(f()) + self.loop.run_until_complete(f(asyncio.Lock())) + lock = asyncio.Lock() + self.loop.run_until_complete(f(lock, asyncio.Condition(lock))) def test_ambiguous_loops(self): - loop = self.new_test_loop() + loop = asyncio.new_event_loop() self.addCleanup(loop.close) - lock = asyncio.Lock() - lock._loop = loop - - async def _create_condition(): - with self.assertRaises(ValueError): - asyncio.Condition(lock) - - self.loop.run_until_complete(_create_condition()) + async def wrong_loop_in_lock(): + with self.assertRaises(TypeError): + asyncio.Lock(loop=loop) # actively disallowed since 3.10 + lock = asyncio.Lock() + lock._loop = loop # use private API for testing + async with lock: + # acquired immediately via the fast-path + # without interaction with any event loop. + cond = asyncio.Condition(lock) + # cond.acquire() will trigger waiting on the lock + # and it will discover the event loop mismatch. + with self.assertRaisesRegex( + RuntimeError, + "is bound to a different event loop", + ): + await cond.acquire() + + async def wrong_loop_in_cond(): + # Same analogy here with the condition's loop. + lock = asyncio.Lock() + async with lock: + with self.assertRaises(TypeError): + asyncio.Condition(lock, loop=loop) + cond = asyncio.Condition(lock) + cond._loop = loop + with self.assertRaisesRegex( + RuntimeError, + "is bound to a different event loop", + ): + await cond.wait() + + self.loop.run_until_complete(wrong_loop_in_lock()) + self.loop.run_until_complete(wrong_loop_in_cond()) def test_timeout_in_block(self): loop = asyncio.new_event_loop() diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 6eaa2899442184..a6ea24ceceda2a 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -711,7 +711,7 @@ def test_read_all_from_pipe_reader(self): # See asyncio issue 168. This test is derived from the example # subprocess_attach_read_pipe.py, but we configure the # StreamReader's limit so that twice it is less than the size - # of the data writter. Also we must explicitly attach a child + # of the data writer. Also we must explicitly attach a child # watcher to the event loop. code = """\ diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3cf88188ecf3b1..14fa6dd76f9ca8 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -228,7 +228,7 @@ def prepare_broken_pipe_test(self): # buffer large enough to feed the whole pipe buffer large_data = b'x' * support.PIPE_MAX_SIZE - # the program ends before the stdin can be feeded + # the program ends before the stdin can be fed proc = self.loop.run_until_complete( asyncio.create_subprocess_exec( sys.executable, '-c', 'pass', diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 92b1a43500311d..362fbf8df08ca1 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -3269,7 +3269,7 @@ def test_run_coroutine_threadsafe_with_timeout(self): self.assertTrue(task.done()) def test_run_coroutine_threadsafe_task_cancelled(self): - """Test coroutine submission from a tread to an event loop + """Test coroutine submission from a thread to an event loop when the task is cancelled.""" callback = lambda: self.target(cancel=True) future = self.loop.run_in_executor(None, callback) @@ -3277,7 +3277,7 @@ def test_run_coroutine_threadsafe_task_cancelled(self): self.loop.run_until_complete(future) def test_run_coroutine_threadsafe_task_factory_exception(self): - """Test coroutine submission from a tread to an event loop + """Test coroutine submission from a thread to an event loop when the task factory raise an exception.""" def task_factory(loop, coro): diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index c5ce26323b5f9e..d99b3b7ed7d36d 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -18,7 +18,7 @@ class AuditTest(unittest.TestCase): def do_test(self, *args): with subprocess.Popen( - [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + [sys.executable, "-Xutf8", AUDIT_TESTS_PY, *args], encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -32,7 +32,7 @@ def do_test(self, *args): def run_python(self, *args): events = [] with subprocess.Popen( - [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + [sys.executable, "-Xutf8", AUDIT_TESTS_PY, *args], encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 418492432a1676..217f2945468844 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -788,5 +788,15 @@ def test_decode(self): output = self.get_output('-d', os_helper.TESTFN) self.assertEqual(output.rstrip(), b'a\xffb') + def test_prints_usage_with_help_flag(self): + output = self.get_output('-h') + self.assertIn(b'usage: ', output) + self.assertIn(b'-d, -u: decode', output) + + def test_prints_usage_with_invalid_flag(self): + output = script_helper.assert_python_failure('-m', 'base64', '-x').err + self.assertIn(b'usage: ', output) + self.assertIn(b'-d, -u: decode', output) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index bd8353d038b6fe..6dc4fa555021cc 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1861,7 +1861,7 @@ def test_warning_notimplemented(self): # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - # For the time being, it will continue to evaluate as truthy, but + # For the time being, it will continue to evaluate as a true value, but # issue a deprecation warning (with the eventual intent to make it # a TypeError). self.assertWarns(DeprecationWarning, bool, NotImplemented) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index db029ef731c382..d9549422866764 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -264,7 +264,7 @@ def test_return_result_with_error(self): def test_getitem_with_error(self): # Test _Py_CheckSlotResult(). Raise an exception and then calls - # PyObject_GetItem(): check that the assertion catchs the bug. + # PyObject_GetItem(): check that the assertion catches the bug. # PyObject_GetItem() must not be called with an exception set. code = textwrap.dedent(""" import _testcapi @@ -766,6 +766,37 @@ def test_mutate_exception(self): self.assertFalse(hasattr(binascii.Error, "foobar")) + def test_module_state_shared_in_global(self): + """ + bpo-44050: Extension module state should be shared between interpreters + when it doesn't support sub-interpreters. + """ + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + script = textwrap.dedent(f""" + import importlib.machinery + import importlib.util + import os + + fullname = '_test_module_state_shared' + origin = importlib.util.find_spec('_testmultiphase').origin + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + attr_id = str(id(module.Error)).encode() + + os.write({w}, attr_id) + """) + exec(script) + main_attr_id = os.read(r, 100) + + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + subinterp_attr_id = os.read(r, 100) + self.assertEqual(main_attr_id, subinterp_attr_id) + class TestThreadState(unittest.TestCase): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index d93e98f372532f..1dc8c45885cbef 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -83,8 +83,17 @@ def get_xoptions(*args): opts = get_xoptions() self.assertEqual(opts, {}) - opts = get_xoptions('-Xa', '-Xb=c,d=e') - self.assertEqual(opts, {'a': True, 'b': 'c,d=e'}) + opts = get_xoptions('-Xno_debug_ranges', '-Xdev=1234') + self.assertEqual(opts, {'no_debug_ranges': True, 'dev': '1234'}) + + @unittest.skipIf(interpreter_requires_environment(), + 'Cannot run -E tests when PYTHON env vars are required.') + def test_unknown_xoptions(self): + rc, out, err = assert_python_failure('-X', 'blech') + self.assertIn(b'Unknown value for option -X', err) + msg = b'Fatal Python error: Unknown value for option -X' + self.assertEqual(err.splitlines().count(msg), 1) + self.assertEqual(b'', out) def test_showrefcount(self): def run_python(*args): diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index f1a149f19b7d28..506b51c428fb57 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -114,7 +114,7 @@ def check_partial(self, input, partialresults): q = Queue(b"") r = codecs.getreader(self.encoding)(q) result = "" - for (c, partialresult) in zip(input.encode(self.encoding), partialresults): + for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True): q.write(bytes([c])) result += r.read() self.assertEqual(result, partialresult) @@ -125,7 +125,7 @@ def check_partial(self, input, partialresults): # do the check again, this time using an incremental decoder d = codecs.getincrementaldecoder(self.encoding)() result = "" - for (c, partialresult) in zip(input.encode(self.encoding), partialresults): + for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True): result += d.decode(bytes([c])) self.assertEqual(result, partialresult) # check that there's nothing left in the buffers @@ -135,7 +135,7 @@ def check_partial(self, input, partialresults): # Check whether the reset method works properly d.reset() result = "" - for (c, partialresult) in zip(input.encode(self.encoding), partialresults): + for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True): result += d.decode(bytes([c])) self.assertEqual(result, partialresult) # check that there's nothing left in the buffers @@ -2353,7 +2353,11 @@ def test_unicode_escape(self): (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10)) -class UnicodeEscapeTest(unittest.TestCase): +class UnicodeEscapeTest(ReadTest, unittest.TestCase): + encoding = "unicode-escape" + + test_lone_surrogates = None + def test_empty(self): self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0)) @@ -2440,8 +2444,50 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) + def test_partial(self): + self.check_partial( + "\x00\t\n\r\\\xff\uffff\U00010000", + [ + '', + '', + '', + '\x00', + '\x00', + '\x00\t', + '\x00\t', + '\x00\t\n', + '\x00\t\n', + '\x00\t\n\r', + '\x00\t\n\r', + '\x00\t\n\r\\', + '\x00\t\n\r\\', + '\x00\t\n\r\\', + '\x00\t\n\r\\', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff\U00010000', + ] + ) + +class RawUnicodeEscapeTest(ReadTest, unittest.TestCase): + encoding = "raw-unicode-escape" + + test_lone_surrogates = None -class RawUnicodeEscapeTest(unittest.TestCase): def test_empty(self): self.assertEqual(codecs.raw_unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.raw_unicode_escape_decode(b""), ("", 0)) @@ -2490,6 +2536,35 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) + def test_partial(self): + self.check_partial( + "\x00\t\n\r\\\xff\uffff\U00010000", + [ + '\x00', + '\x00\t', + '\x00\t\n', + '\x00\t\n\r', + '\x00\t\n\r', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff\U00010000', + ] + ) + class EscapeEncodeTest(unittest.TestCase): diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index b9be4c03142b6d..1bfd44f9547881 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1594,7 +1594,7 @@ def assertSameSet(self, s1, s2): self.assertSetEqual(set(s1), set(s2)) def test_Set_from_iterable(self): - """Verify _from_iterable overriden to an instance method works.""" + """Verify _from_iterable overridden to an instance method works.""" class SetUsingInstanceFromIterable(MutableSet): def __init__(self, values, created_by): if not created_by: diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 43b8507771e25a..bc8e4e4e2918fb 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -1,6 +1,7 @@ """Unit tests for contextlib.py, and other context managers.""" import io +import os import sys import tempfile import threading @@ -799,6 +800,40 @@ def suppress_exc(*exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) + def test_exit_exception_explicit_none_context(self): + # Ensure ExitStack chaining matches actual nested `with` statements + # regarding explicit __context__ = None. + + class MyException(Exception): + pass + + @contextmanager + def my_cm(): + try: + yield + except BaseException: + exc = MyException() + try: + raise exc + finally: + exc.__context__ = None + + @contextmanager + def my_cm_with_exit_stack(): + with self.exit_stack() as stack: + stack.enter_context(my_cm()) + yield stack + + for cm in (my_cm, my_cm_with_exit_stack): + with self.subTest(): + try: + with cm(): + raise IndexError() + except MyException as exc: + self.assertIsNone(exc.__context__) + else: + self.fail("Expected IndexError, but no exception was raised") + def test_exit_exception_non_suppressing(self): # http://bugs.python.org/issue19092 def raise_exc(exc): @@ -1080,5 +1115,47 @@ def test_cm_is_reentrant(self): 1/0 self.assertTrue(outer_continued) + +class TestChdir(unittest.TestCase): + def test_simple(self): + old_cwd = os.getcwd() + target = os.path.join(os.path.dirname(__file__), 'data') + self.assertNotEqual(old_cwd, target) + + with chdir(target): + self.assertEqual(os.getcwd(), target) + self.assertEqual(os.getcwd(), old_cwd) + + def test_reentrant(self): + old_cwd = os.getcwd() + target1 = os.path.join(os.path.dirname(__file__), 'data') + target2 = os.path.join(os.path.dirname(__file__), 'ziptestdata') + self.assertNotIn(old_cwd, (target1, target2)) + chdir1, chdir2 = chdir(target1), chdir(target2) + + with chdir1: + self.assertEqual(os.getcwd(), target1) + with chdir2: + self.assertEqual(os.getcwd(), target2) + with chdir1: + self.assertEqual(os.getcwd(), target1) + self.assertEqual(os.getcwd(), target2) + self.assertEqual(os.getcwd(), target1) + self.assertEqual(os.getcwd(), old_cwd) + + def test_exception(self): + old_cwd = os.getcwd() + target = os.path.join(os.path.dirname(__file__), 'data') + self.assertNotEqual(old_cwd, target) + + try: + with chdir(target): + self.assertEqual(os.getcwd(), target) + raise RuntimeError("boom") + except RuntimeError as re: + self.assertEqual(str(re), "boom") + self.assertEqual(os.getcwd(), old_cwd) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index c738bf3c0bdfeb..c16c7ecd19a259 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -646,6 +646,41 @@ async def suppress_exc(*exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) + @_async_test + async def test_async_exit_exception_explicit_none_context(self): + # Ensure AsyncExitStack chaining matches actual nested `with` statements + # regarding explicit __context__ = None. + + class MyException(Exception): + pass + + @asynccontextmanager + async def my_cm(): + try: + yield + except BaseException: + exc = MyException() + try: + raise exc + finally: + exc.__context__ = None + + @asynccontextmanager + async def my_cm_with_exit_stack(): + async with self.exit_stack() as stack: + await stack.enter_async_context(my_cm()) + yield stack + + for cm in (my_cm, my_cm_with_exit_stack): + with self.subTest(): + try: + async with cm(): + raise IndexError() + except MyException as exc: + self.assertIsNone(exc.__context__) + else: + self.fail("Expected IndexError, but no exception was raised") + @_async_test async def test_instance_bypass_async(self): class Example(object): pass diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 09e72a71f1db92..95a19dd46cb4ff 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -44,6 +44,8 @@ def _test_arg_valid(self, ctor, arg): quoting=csv.QUOTE_ALL, quotechar='') self.assertRaises(TypeError, ctor, arg, quoting=csv.QUOTE_ALL, quotechar=None) + self.assertRaises(TypeError, ctor, arg, + quoting=csv.QUOTE_NONE, quotechar='') def test_reader_arg_valid(self): self._test_arg_valid(csv.reader, []) @@ -217,6 +219,17 @@ def test_write_escape(self): self._write_test(['C\\', '6', '7', 'X"'], 'C\\\\,6,7,"X"""', escapechar='\\', quoting=csv.QUOTE_MINIMAL) + def test_write_lineterminator(self): + for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': + with self.subTest(lineterminator=lineterminator): + with StringIO() as sio: + writer = csv.writer(sio, lineterminator=lineterminator) + writer.writerow(['a', 'b']) + writer.writerow([1, 2]) + self.assertEqual(sio.getvalue(), + f'a,b{lineterminator}' + f'1,2{lineterminator}') + def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') self._write_test(iter(['a', 1, None]), 'a,1,') @@ -286,14 +299,10 @@ def test_read_oddinputs(self): self._read_test([''], [[]]) self.assertRaises(csv.Error, self._read_test, ['"ab"c'], None, strict = 1) - # cannot handle null bytes for the moment - self.assertRaises(csv.Error, self._read_test, - ['ab\0c'], None, strict = 1) self._read_test(['"ab"c'], [['abc']], doublequote = 0) self.assertRaises(csv.Error, self._read_test, - [b'ab\0c'], None) - + [b'abc'], None) def test_read_eol(self): self._read_test(['a,b'], [['a','b']]) @@ -313,6 +322,18 @@ def test_read_eof(self): self.assertRaises(csv.Error, self._read_test, ['^'], [], escapechar='^', strict=True) + def test_read_nul(self): + self._read_test(['\0'], [['\0']]) + self._read_test(['a,\0b,c'], [['a', '\0b', 'c']]) + self._read_test(['a,b\0,c'], [['a', 'b\0', 'c']]) + self._read_test(['a,b\\\0,c'], [['a', 'b\0', 'c']], escapechar='\\') + self._read_test(['a,"\0b",c'], [['a', '\0b', 'c']]) + + def test_read_delimiter(self): + self._read_test(['a,b,c'], [['a', 'b', 'c']]) + self._read_test(['a;b;c'], [['a', 'b', 'c']], delimiter=';') + self._read_test(['a\0b\0c'], [['a', 'b', 'c']], delimiter='\0') + def test_read_escape(self): self._read_test(['a,\\b,c'], [['a', 'b', 'c']], escapechar='\\') self._read_test(['a,b\\,c'], [['a', 'b,c']], escapechar='\\') @@ -320,6 +341,10 @@ def test_read_escape(self): self._read_test(['a,"b,\\c"'], [['a', 'b,c']], escapechar='\\') self._read_test(['a,"b,c\\""'], [['a', 'b,c"']], escapechar='\\') self._read_test(['a,"b,c"\\'], [['a', 'b,c\\']], escapechar='\\') + self._read_test(['a,^b,c'], [['a', 'b', 'c']], escapechar='^') + self._read_test(['a,\0b,c'], [['a', 'b', 'c']], escapechar='\0') + self._read_test(['a,\\b,c'], [['a', '\\b', 'c']], escapechar=None) + self._read_test(['a,\\b,c'], [['a', '\\b', 'c']]) def test_read_quoting(self): self._read_test(['1,",3,",5'], [['1', ',3,', '5']]) @@ -334,6 +359,8 @@ def test_read_quoting(self): self.assertRaises(ValueError, self._read_test, ['abc,3'], [[]], quoting=csv.QUOTE_NONNUMERIC) + self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') + self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size @@ -887,6 +914,12 @@ class mydialect(csv.Dialect): self.assertEqual(d.quotechar, '"') self.assertTrue(d.doublequote) + mydialect.quotechar = "" + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + '"quotechar" must be a 1-character string') + mydialect.quotechar = "''" with self.assertRaises(csv.Error) as cm: mydialect() @@ -897,7 +930,7 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be string, not int') + '"quotechar" must be string or None, not int') def test_delimiter(self): class mydialect(csv.Dialect): @@ -934,6 +967,39 @@ class mydialect(csv.Dialect): self.assertEqual(str(cm.exception), '"delimiter" must be string, not int') + mydialect.delimiter = None + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + '"delimiter" must be string, not NoneType') + + def test_escapechar(self): + class mydialect(csv.Dialect): + delimiter = ";" + escapechar = '\\' + doublequote = False + skipinitialspace = True + lineterminator = '\r\n' + quoting = csv.QUOTE_NONE + d = mydialect() + self.assertEqual(d.escapechar, "\\") + + mydialect.escapechar = "" + with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + mydialect() + + mydialect.escapechar = "**" + with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + mydialect() + + mydialect.escapechar = b"*" + with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not bytes'): + mydialect() + + mydialect.escapechar = 4 + with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not int'): + mydialect() + def test_lineterminator(self): class mydialect(csv.Dialect): delimiter = ";" @@ -1045,6 +1111,12 @@ class TestSniffer(unittest.TestCase): a,b """) + sample14 = """\ +abc\0def +ghijkl\0mno +ghi\0jkl +""" + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1113,6 +1185,8 @@ def test_delimiters(self): dialect = sniffer.sniff(self.sample9) self.assertEqual(dialect.delimiter, '+') self.assertEqual(dialect.quotechar, "'") + dialect = sniffer.sniff(self.sample14) + self.assertEqual(dialect.delimiter, '\0') def test_doublequote(self): sniffer = csv.Sniffer() diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 33c9fcd1656219..bdcb4a2cfd1a07 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1907,7 +1907,7 @@ class Parent(Generic[T]): # Check MRO resolution. self.assertEqual(Child.__mro__, (Child, Parent, Generic, object)) - def test_dataclassses_pickleable(self): + def test_dataclasses_pickleable(self): global P, Q, R @dataclass class P: @@ -3695,7 +3695,7 @@ class B: with self.assertRaisesRegex(TypeError, msg): B(3, 4, 5) - # Explicitely make a field that follows KW_ONLY be non-keyword-only. + # Explicitly make a field that follows KW_ONLY be non-keyword-only. @dataclass class C: a: int diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index bdb9f02e5756a2..7f9094fa7bd4e6 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -1,59 +1,57 @@ import unittest import sys -from test.support import run_unittest from test.support.import_helper import import_fresh_module TESTS = 'test.datetimetester' -try: - pure_tests = import_fresh_module(TESTS, fresh=['datetime', '_strptime'], - blocked=['_datetime']) - fast_tests = import_fresh_module(TESTS, fresh=['datetime', - '_datetime', '_strptime']) -finally: - # XXX: import_fresh_module() is supposed to leave sys.module cache untouched, - # XXX: but it does not, so we have to cleanup ourselves. - for modname in ['datetime', '_datetime', '_strptime']: - sys.modules.pop(modname, None) -test_modules = [pure_tests, fast_tests] -test_suffixes = ["_Pure", "_Fast"] -# XXX(gb) First run all the _Pure tests, then all the _Fast tests. You might -# not believe this, but in spite of all the sys.modules trickery running a _Pure -# test last will leave a mix of pure and native datetime stuff lying around. -all_test_classes = [] +def load_tests(loader, tests, pattern): + try: + pure_tests = import_fresh_module(TESTS, fresh=['datetime', '_strptime'], + blocked=['_datetime']) + fast_tests = import_fresh_module(TESTS, fresh=['datetime', + '_datetime', '_strptime']) + finally: + # XXX: import_fresh_module() is supposed to leave sys.module cache untouched, + # XXX: but it does not, so we have to cleanup ourselves. + for modname in ['datetime', '_datetime', '_strptime']: + sys.modules.pop(modname, None) -for module, suffix in zip(test_modules, test_suffixes): - test_classes = [] - for name, cls in module.__dict__.items(): - if not isinstance(cls, type): - continue - if issubclass(cls, unittest.TestCase): - test_classes.append(cls) - elif issubclass(cls, unittest.TestSuite): - suit = cls() - test_classes.extend(type(test) for test in suit) - test_classes = sorted(set(test_classes), key=lambda cls: cls.__qualname__) - for cls in test_classes: - cls.__name__ += suffix - cls.__qualname__ += suffix - @classmethod - def setUpClass(cls_, module=module): - cls_._save_sys_modules = sys.modules.copy() - sys.modules[TESTS] = module - sys.modules['datetime'] = module.datetime_module - sys.modules['_strptime'] = module._strptime - @classmethod - def tearDownClass(cls_): - sys.modules.clear() - sys.modules.update(cls_._save_sys_modules) - cls.setUpClass = setUpClass - cls.tearDownClass = tearDownClass - all_test_classes.extend(test_classes) + test_modules = [pure_tests, fast_tests] + test_suffixes = ["_Pure", "_Fast"] + # XXX(gb) First run all the _Pure tests, then all the _Fast tests. You might + # not believe this, but in spite of all the sys.modules trickery running a _Pure + # test last will leave a mix of pure and native datetime stuff lying around. + for module, suffix in zip(test_modules, test_suffixes): + test_classes = [] + for name, cls in module.__dict__.items(): + if not isinstance(cls, type): + continue + if issubclass(cls, unittest.TestCase): + test_classes.append(cls) + elif issubclass(cls, unittest.TestSuite): + suit = cls() + test_classes.extend(type(test) for test in suit) + test_classes = sorted(set(test_classes), key=lambda cls: cls.__qualname__) + for cls in test_classes: + cls.__name__ += suffix + cls.__qualname__ += suffix + @classmethod + def setUpClass(cls_, module=module): + cls_._save_sys_modules = sys.modules.copy() + sys.modules[TESTS] = module + sys.modules['datetime'] = module.datetime_module + sys.modules['_strptime'] = module._strptime + @classmethod + def tearDownClass(cls_): + sys.modules.clear() + sys.modules.update(cls_._save_sys_modules) + cls.setUpClass = setUpClass + cls.tearDownClass = tearDownClass + tests.addTests(loader.loadTestsFromTestCase(cls)) + return tests -def test_main(): - run_unittest(*all_test_classes) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index cf0758653ff380..f21eebc6530c76 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -1,24 +1,21 @@ """Test script for the dbm.open function based on testdumbdbm.py""" import unittest -import glob +import dbm import os from test.support import import_helper from test.support import os_helper -# Skip tests if dbm module doesn't exist. -dbm = import_helper.import_module('dbm') - try: from dbm import ndbm except ImportError: ndbm = None -_fname = os_helper.TESTFN +dirname = os_helper.TESTFN +_fname = os.path.join(dirname, os_helper.TESTFN) # -# Iterates over every database module supported by dbm currently available, -# setting dbm to use each in turn, and yielding that module +# Iterates over every database module supported by dbm currently available. # def dbm_iterator(): for name in dbm._names: @@ -32,11 +29,12 @@ def dbm_iterator(): # # Clean up all scratch databases we might have created during testing # -def delete_files(): - # we don't know the precise name the underlying database uses - # so we use glob to locate all names - for f in glob.glob(glob.escape(_fname) + "*"): - os_helper.unlink(f) +def cleaunup_test_dir(): + os_helper.rmtree(dirname) + +def setup_test_dir(): + cleaunup_test_dir() + os.mkdir(dirname) class AnyDBMTestCase: @@ -144,86 +142,76 @@ def read_helper(self, f): for key in self._dict: self.assertEqual(self._dict[key], f[key.encode("ascii")]) - def tearDown(self): - delete_files() + def test_keys(self): + with dbm.open(_fname, 'c') as d: + self.assertEqual(d.keys(), []) + a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] + for k, v in a: + d[k] = v + self.assertEqual(sorted(d.keys()), sorted(k for (k, v) in a)) + for k, v in a: + self.assertIn(k, d) + self.assertEqual(d[k], v) + self.assertNotIn(b'xxx', d) + self.assertRaises(KeyError, lambda: d[b'xxx']) def setUp(self): + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) dbm._defaultmod = self.module - delete_files() + self.addCleanup(cleaunup_test_dir) + setup_test_dir() class WhichDBTestCase(unittest.TestCase): def test_whichdb(self): + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) _bytes_fname = os.fsencode(_fname) - for path in [_fname, os_helper.FakePath(_fname), - _bytes_fname, os_helper.FakePath(_bytes_fname)]: - for module in dbm_iterator(): - # Check whether whichdb correctly guesses module name - # for databases opened with "module" module. - # Try with empty files first - name = module.__name__ - if name == 'dbm.dumb': - continue # whichdb can't support dbm.dumb - delete_files() - f = module.open(path, 'c') - f.close() + fnames = [_fname, os_helper.FakePath(_fname), + _bytes_fname, os_helper.FakePath(_bytes_fname)] + for module in dbm_iterator(): + # Check whether whichdb correctly guesses module name + # for databases opened with "module" module. + name = module.__name__ + setup_test_dir() + dbm._defaultmod = module + # Try with empty files first + with module.open(_fname, 'c'): pass + for path in fnames: self.assertEqual(name, self.dbm.whichdb(path)) - # Now add a key - f = module.open(path, 'w') + # Now add a key + with module.open(_fname, 'w') as f: f[b"1"] = b"1" # and test that we can find it self.assertIn(b"1", f) # and read it self.assertEqual(f[b"1"], b"1") - f.close() + for path in fnames: self.assertEqual(name, self.dbm.whichdb(path)) @unittest.skipUnless(ndbm, reason='Test requires ndbm') def test_whichdb_ndbm(self): # Issue 17198: check that ndbm which is referenced in whichdb is defined - db_file = '{}_ndbm.db'.format(_fname) - with open(db_file, 'w'): - self.addCleanup(os_helper.unlink, db_file) - db_file_bytes = os.fsencode(db_file) - self.assertIsNone(self.dbm.whichdb(db_file[:-3])) - self.assertIsNone(self.dbm.whichdb(os_helper.FakePath(db_file[:-3]))) - self.assertIsNone(self.dbm.whichdb(db_file_bytes[:-3])) - self.assertIsNone(self.dbm.whichdb(os_helper.FakePath(db_file_bytes[:-3]))) - - def tearDown(self): - delete_files() + with open(_fname + '.db', 'wb'): pass + _bytes_fname = os.fsencode(_fname) + fnames = [_fname, os_helper.FakePath(_fname), + _bytes_fname, os_helper.FakePath(_bytes_fname)] + for path in fnames: + self.assertIsNone(self.dbm.whichdb(path)) def setUp(self): - delete_files() - self.filename = os_helper.TESTFN - self.d = dbm.open(self.filename, 'c') - self.d.close() + self.addCleanup(cleaunup_test_dir) + setup_test_dir() self.dbm = import_helper.import_fresh_module('dbm') - def test_keys(self): - self.d = dbm.open(self.filename, 'c') - self.assertEqual(self.d.keys(), []) - a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] - for k, v in a: - self.d[k] = v - self.assertEqual(sorted(self.d.keys()), sorted(k for (k, v) in a)) - for k, v in a: - self.assertIn(k, self.d) - self.assertEqual(self.d[k], v) - self.assertNotIn(b'xxx', self.d) - self.assertRaises(KeyError, lambda: self.d[b'xxx']) - self.d.close() - - -def load_tests(loader, tests, pattern): - classes = [] - for mod in dbm_iterator(): - classes.append(type("TestCase-" + mod.__name__, - (AnyDBMTestCase, unittest.TestCase), - {'module': mod})) - for c in classes: - tests.addTest(loader.loadTestsFromTestCase(c)) - return tests + +for mod in dbm_iterator(): + assert mod.__name__.startswith('dbm.') + suffix = mod.__name__[4:] + testname = f'TestCase_{suffix}' + globals()[testname] = type(testname, + (AnyDBMTestCase, unittest.TestCase), + {'module': mod}) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 99263bb13b0d14..b6173a5ffec96c 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -62,7 +62,7 @@ C = import_fresh_module('decimal', fresh=['_decimal']) P = import_fresh_module('decimal', blocked=['_decimal']) -orig_sys_decimal = sys.modules['decimal'] +import decimal as orig_sys_decimal # fractions module must import the correct decimal module. cfractions = import_fresh_module('fractions', fresh=['fractions']) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 9b919e7824445e..a5404b30d2459a 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5500,17 +5500,19 @@ class A: class B(A): pass + #Shrink keys by repeatedly creating instances + [(A(), B()) for _ in range(20)] + a, b = A(), B() self.assertEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(b))) self.assertLess(sys.getsizeof(vars(a)), sys.getsizeof({"a":1})) - # Initial hash table can contain at most 5 elements. + # Initial hash table can contain only one or two elements. # Set 6 attributes to cause internal resizing. a.x, a.y, a.z, a.w, a.v, a.u = range(6) self.assertNotEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(b))) a2 = A() - self.assertEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(a2))) - self.assertLess(sys.getsizeof(vars(a)), sys.getsizeof({"a":1})) - b.u, b.v, b.w, b.t, b.s, b.r = range(6) + self.assertGreater(sys.getsizeof(vars(a)), sys.getsizeof(vars(a2))) + self.assertLess(sys.getsizeof(vars(a2)), sys.getsizeof({"a":1})) self.assertLess(sys.getsizeof(vars(b)), sys.getsizeof({"a":1})) @@ -5723,7 +5725,7 @@ class A(metaclass=M): def test_incomplete_super(self): """ - Attrubute lookup on a super object must be aware that + Attribute lookup on a super object must be aware that its target type can be uninitialized (type->tp_mro == NULL). """ class M(DebugHelperMeta): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 666cd81e68d81a..b43c83abd0eeb5 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -994,8 +994,8 @@ class C: @support.cpython_only def test_splittable_setdefault(self): - """split table must be combined when setdefault() - breaks insertion order""" + """split table must keep correct insertion + order when attributes are adding using setdefault()""" a, b = self.make_shared_key_dict(2) a['a'] = 1 @@ -1005,7 +1005,6 @@ def test_splittable_setdefault(self): size_b = sys.getsizeof(b) b['a'] = 1 - self.assertGreater(size_b, size_a) self.assertEqual(list(a), ['x', 'y', 'z', 'a', 'b']) self.assertEqual(list(b), ['x', 'y', 'z', 'b', 'a']) @@ -1020,7 +1019,6 @@ def test_splittable_del(self): with self.assertRaises(KeyError): del a['y'] - self.assertGreater(sys.getsizeof(a), orig_size) self.assertEqual(list(a), ['x', 'z']) self.assertEqual(list(b), ['x', 'y', 'z']) @@ -1031,16 +1029,12 @@ def test_splittable_del(self): @support.cpython_only def test_splittable_pop(self): - """split table must be combined when d.pop(k)""" a, b = self.make_shared_key_dict(2) - orig_size = sys.getsizeof(a) - - a.pop('y') # split table is combined + a.pop('y') with self.assertRaises(KeyError): a.pop('y') - self.assertGreater(sys.getsizeof(a), orig_size) self.assertEqual(list(a), ['x', 'z']) self.assertEqual(list(b), ['x', 'y', 'z']) @@ -1051,7 +1045,7 @@ def test_splittable_pop(self): @support.cpython_only def test_splittable_pop_pending(self): - """pop a pending key in a splitted table should not crash""" + """pop a pending key in a split table should not crash""" a, b = self.make_shared_key_dict(2) a['a'] = 4 @@ -1074,36 +1068,6 @@ def test_splittable_popitem(self): self.assertEqual(list(a), ['x', 'y']) self.assertEqual(list(b), ['x', 'y', 'z']) - @support.cpython_only - def test_splittable_setattr_after_pop(self): - """setattr() must not convert combined table into split table.""" - # Issue 28147 - import _testcapi - - class C: - pass - a = C() - - a.a = 1 - self.assertTrue(_testcapi.dict_hassplittable(a.__dict__)) - - # dict.pop() convert it to combined table - a.__dict__.pop('a') - self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) - - # But C should not convert a.__dict__ to split table again. - a.a = 1 - self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) - - # Same for popitem() - a = C() - a.a = 2 - self.assertTrue(_testcapi.dict_hassplittable(a.__dict__)) - a.__dict__.popitem() - self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) - a.a = 3 - self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) - def test_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): data = {1:"a", 2:"b", 3:"c"} @@ -1398,7 +1362,7 @@ def test_reversed(self): self.assertRaises(StopIteration, next, r) def test_reverse_iterator_for_empty_dict(self): - # bpo-38525: revered iterator should work properly + # bpo-38525: reversed iterator should work properly # empty dict is directly used for reference count test self.assertEqual(list(reversed({})), []) diff --git a/Lib/test/test_dict_version.py b/Lib/test/test_dict_version.py index 8cdccad0d79ab6..243084c75c42bc 100644 --- a/Lib/test/test_dict_version.py +++ b/Lib/test/test_dict_version.py @@ -1,5 +1,5 @@ """ -Test implementation of the PEP 509: dictionary versionning. +Test implementation of the PEP 509: dictionary versioning. """ import unittest from test.support import import_helper diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 0899db66d82410..4755e3f3a17247 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1046,7 +1046,7 @@ def _prepare_test_cases(): Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=34, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=21, argval=42, argrepr='to 42', offset=36, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=38, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=26, argval=52, argrepr='to 52', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=5, argval=52, argrepr='to 52', offset=40, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=4, argval=8, argrepr='to 8', offset=42, starts_line=7, is_jump_target=True, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=44, starts_line=10, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=46, starts_line=None, is_jump_target=False, positions=None), @@ -1071,7 +1071,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=84, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=86, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=46, argval=92, argrepr='to 92', offset=88, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=52, argval=104, argrepr='to 104', offset=90, starts_line=17, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=6, argval=104, argrepr='to 104', offset=90, starts_line=17, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=92, starts_line=11, is_jump_target=True, positions=None), Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=28, argval=56, argrepr='to 56', offset=94, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=96, starts_line=19, is_jump_target=True, positions=None), diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 571dc78bf5076e..3524a0a797c41a 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -3114,20 +3114,11 @@ def test_no_trailing_whitespace_stripping(): patches that contain trailing whitespace. More info on Issue 24746. """ -###################################################################### -## Main -###################################################################### - -def test_main(): - # Check the doctest cases in doctest itself: - ret = support.run_doctest(doctest, verbosity=True) - # Check the doctest cases defined here: - from test import test_doctest - support.run_doctest(test_doctest, verbosity=True) - - # Run unittests - support.run_unittest(__name__) +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(doctest)) + tests.addTest(doctest.DocTestSuite()) + return tests def test_coverage(coverdir): @@ -3140,8 +3131,9 @@ def test_coverage(coverdir): r.write_results(show_missing=True, summary=True, coverdir=coverdir) + if __name__ == '__main__': if '-c' in sys.argv: test_coverage('/tmp/doctest.cover') else: - test_main() + unittest.main() diff --git a/Lib/test/test_doctest2.py b/Lib/test/test_doctest2.py index 347a143641071d..ab8a0696736e23 100644 --- a/Lib/test/test_doctest2.py +++ b/Lib/test/test_doctest2.py @@ -13,7 +13,6 @@ import sys import unittest -from test import support if sys.flags.optimize >= 2: raise unittest.SkipTest("Cannot test docstrings with -O2") @@ -107,17 +106,21 @@ def clsm(cls, val): """ return val -def test_main(): - from test import test_doctest2 - EXPECTED = 19 - f, t = support.run_doctest(test_doctest2) - if t != EXPECTED: - raise support.TestFailed("expected %d tests to run, not %d" % - (EXPECTED, t)) + +class Test(unittest.TestCase): + def test_testmod(self): + import doctest, sys + EXPECTED = 19 + f, t = doctest.testmod(sys.modules[__name__]) + if f: + self.fail("%d of %d doctests failed" % (f, t)) + if t != EXPECTED: + self.fail("expected %d tests to run, not %d" % (EXPECTED, t)) + # Pollute the namespace with a bunch of imported functions and classes, # to make sure they don't get tested. from doctest import * if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 77252502883112..9a06be45855021 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -90,7 +90,17 @@ def test_valid_get_response(self): response = self.client.getresponse() self.assertEqual(response.status, 200) - self.assertEqual(response.getheader("Content-type"), "text/html") + self.assertEqual(response.getheader("Content-type"), "text/html; charset=UTF-8") + + # Server raises an exception if we don't start to read the data + response.read() + + def test_get_css(self): + self.client.request("GET", "/pydoc.css") + response = self.client.getresponse() + + self.assertEqual(response.status, 200) + self.assertEqual(response.getheader("Content-type"), "text/css; charset=UTF-8") # Server raises an exception if we don't start to read the data response.read() diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 1db73cc2d22207..3957077f5d6123 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -34,7 +34,7 @@ def normalize_trace_output(output): return "\n".join(result) except (IndexError, ValueError): raise AssertionError( - "tracer produced unparseable output:\n{}".format(output) + "tracer produced unparsable output:\n{}".format(output) ) diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 4001f716471dc2..54ffcdc544e8bb 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3009,6 +3009,7 @@ def test_parsedate_returns_None_for_invalid_strings(self): self.assertIsNone(utils.parsedate_tz('0')) self.assertIsNone(utils.parsedate('A Complete Waste of Time')) self.assertIsNone(utils.parsedate_tz('A Complete Waste of Time')) + self.assertIsNone(utils.parsedate_tz('Wed, 3 Apr 2002 12.34.56.78+0800')) # Not a part of the spec but, but this has historically worked: self.assertIsNone(utils.parsedate(None)) self.assertIsNone(utils.parsedate_tz(None)) diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 920a3d6a9cb91b..4c754bf40fc300 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -433,7 +433,7 @@ class TestEmailMessageBase: --=== Content-Type: text/plain - Your message has bounced, ser. + Your message has bounced, sir. --=== Content-Type: message/rfc822 diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e1b466a7b56b18..4186f011e2388a 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -13,7 +13,6 @@ import shutil import subprocess import sys -import sysconfig import tempfile import textwrap @@ -54,14 +53,15 @@ def remove_python_envvars(): class EmbeddingTestsMixin: def setUp(self): exename = "_testembed" + builddir = os.path.dirname(sys.executable) if MS_WINDOWS: ext = ("_d" if debug_build(sys.executable) else "") + ".exe" exename += ext - exepath = os.path.dirname(sys.executable) + exepath = builddir else: - exepath = os.path.join(support.REPO_ROOT, "Programs") + exepath = os.path.join(builddir, 'Programs') self.test_exe = exe = os.path.join(exepath, exename) - if not os.path.exists(exe): + if exepath != support.REPO_ROOT or not os.path.exists(exe): self.skipTest("%r doesn't exist" % exe) # This is needed otherwise we get a fatal error: # "Py_Initialize: Unable to get the locale encoding @@ -250,7 +250,7 @@ def test_forced_io_encoding(self): def test_pre_initialization_api(self): """ - Checks some key parts of the C-API that need to work before the runtine + Checks some key parts of the C-API that need to work before the runtime is initialized (via Py_Initialize()). """ env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) @@ -274,7 +274,7 @@ def test_pre_initialization_sys_options(self): "test_pre_initialization_sys_options", env=env) expected_output = ( "sys.warnoptions: ['once', 'module', 'default']\n" - "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" + "sys._xoptions: {'dev': '2', 'utf8': '1'}\n" "warnings.filters[:3]: ['default', 'module', 'once']\n" ) self.assertIn(expected_output, out) @@ -406,6 +406,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'module_search_paths': GET_DEFAULT_CONFIG, 'module_search_paths_set': 1, 'platlibdir': sys.platlibdir, + 'stdlib_dir': GET_DEFAULT_CONFIG, 'site_import': 1, 'bytes_warning': 0, @@ -434,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': False, + 'use_frozen_modules': 1, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -515,6 +516,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix', 'program_name', 'home', + 'stdlib_dir', # program_full_path and module_search_path are copied indirectly from # the core configuration in check_path_config(). ] @@ -590,7 +592,6 @@ def _get_expected_config(self): def get_expected_config(self, expected_preconfig, expected, expected_pathconfig, env, api, modify_path_cb=None): - cls = self.__class__ configs = self._get_expected_config() pre_config = configs['pre_config'] @@ -820,15 +821,14 @@ def test_init_from_config(self): 'argv': ['-c', 'arg2'], 'orig_argv': ['python3', '-W', 'cmdline_warnoption', - '-X', 'cmdline_xoption', + '-X', 'dev', '-c', 'pass', 'arg2'], 'parse_argv': 2, 'xoptions': [ - 'config_xoption1=3', - 'config_xoption2=', - 'config_xoption3', - 'cmdline_xoption', + 'dev=3', + 'utf8', + 'dev', ], 'warnoptions': [ 'cmdline_warnoption', @@ -1046,9 +1046,8 @@ def test_init_sys_add(self): config = { 'faulthandler': 1, 'xoptions': [ - 'config_xoption', - 'cmdline_xoption', - 'sysadd_xoption', + 'dev', + 'utf8', 'faulthandler', ], 'warnoptions': [ @@ -1058,9 +1057,12 @@ def test_init_sys_add(self): ], 'orig_argv': ['python3', '-W', 'ignore:::cmdline_warnoption', - '-X', 'cmdline_xoption'], + '-X', 'utf8'], } - self.check_all_configs("test_init_sys_add", config, api=API_PYTHON) + preconfig = {'utf8_mode': 1} + self.check_all_configs("test_init_sys_add", config, + expected_preconfig=preconfig, + api=API_PYTHON) def test_init_run_main(self): code = ('import _testinternalcapi, json; ' @@ -1142,6 +1144,10 @@ def test_init_setpath(self): 'base_prefix': '', 'exec_prefix': '', 'base_exec_prefix': '', + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': '', + 'use_frozen_modules': -1, } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1162,7 +1168,11 @@ def test_init_setpath_config(self): 'base_prefix': '', 'exec_prefix': '', 'base_exec_prefix': '', - # overriden by PyConfig + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': '', + 'use_frozen_modules': -1, + # overridden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', 'executable': 'conf_executable', @@ -1240,7 +1250,11 @@ def test_init_setpythonhome(self): self.fail(f"Unable to find home in {paths!r}") prefix = exec_prefix = home - ver = sys.version_info + if MS_WINDOWS: + stdlib = os.path.join(home, sys.platlibdir) + else: + version = f'{sys.version_info.major}.{sys.version_info.minor}' + stdlib = os.path.join(home, sys.platlibdir, f'python{version}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1251,8 +1265,11 @@ def test_init_setpythonhome(self): 'exec_prefix': exec_prefix, 'base_exec_prefix': exec_prefix, 'pythonpath_env': paths_str, + 'stdlib_dir': stdlib, } self.default_program_name(config) + if not config['executable']: + config['use_frozen_modules'] = -1 env = {'TESTHOME': home, 'PYTHONPATH': paths_str} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) @@ -1288,6 +1305,10 @@ def test_init_pybuilddir(self): 'base_executable': executable, 'executable': executable, 'module_search_paths': module_search_paths, + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': None, + 'use_frozen_modules': -1, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1345,6 +1366,8 @@ def test_init_pyvenv_cfg(self): if MS_WINDOWS: config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home + config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib') + config['use_frozen_modules'] = 1 ver = sys.version_info dll = f'python{ver.major}' @@ -1353,6 +1376,11 @@ def test_init_pyvenv_cfg(self): dll += '.DLL' dll = os.path.join(os.path.dirname(executable), dll) path_config['python3_dll'] = dll + else: + # The current getpath.c doesn't determine the stdlib dir + # in this case. + config['stdlib_dir'] = None + config['use_frozen_modules'] = -1 env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1532,8 +1560,7 @@ class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): # "Set up a preliminary stderr printer until we have enough # infrastructure for the io module in place." - def get_stdout_fd(self): - return sys.__stdout__.fileno() + STDOUT_FD = 1 def create_printer(self, fd): ctypes = import_helper.import_module('ctypes') @@ -1545,7 +1572,7 @@ def create_printer(self, fd): def test_write(self): message = "unicode:\xe9-\u20ac-\udc80!\n" - stdout_fd = self.get_stdout_fd() + stdout_fd = self.STDOUT_FD stdout_fd_copy = os.dup(stdout_fd) self.addCleanup(os.close, stdout_fd_copy) @@ -1566,7 +1593,7 @@ def test_write(self): self.assertEqual(data, message.encode('utf8', 'backslashreplace')) def test_methods(self): - fd = self.get_stdout_fd() + fd = self.STDOUT_FD printer = self.create_printer(fd) self.assertEqual(printer.fileno(), fd) self.assertEqual(printer.isatty(), os.isatty(fd)) @@ -1574,7 +1601,7 @@ def test_methods(self): printer.close() # noop def test_disallow_instantiation(self): - fd = self.get_stdout_fd() + fd = self.STDOUT_FD printer = self.create_printer(fd) support.check_disallow_instantiation(self, type(printer)) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 70d10ebc66e98b..0f8a8f134b61f0 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -209,6 +209,10 @@ def check(self, src, lineno, offset, encoding='utf-8'): src = src.decode(encoding, 'replace') line = src.split('\n')[lineno-1] self.assertIn(line, cm.exception.text) + + def test_error_offset_continuation_characters(self): + check = self.check + check('"\\\n"(1 for c in I,\\\n\\', 2, 2) def testSyntaxErrorOffset(self): check = self.check @@ -223,7 +227,7 @@ def testSyntaxErrorOffset(self): check('x = "a', 1, 5) check('lambda x: x = 2', 1, 1) check('f{a + b + c}', 1, 1) - check('[file for str(file) in []\n])', 2, 2) + check('[file for str(file) in []\n])', 1, 11) check('a = « hello » « world »', 1, 5) check('[\nfile\nfor str(file)\nin\n[]\n]', 3, 5) check('[file for\n str(file) in []]', 2, 2) @@ -1840,7 +1844,7 @@ def f(): with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) - self.assertNotIn("a1", err.getvalue()) + self.assertNotRegex(err.getvalue(), r"NameError.*a1") def test_name_error_with_custom_exceptions(self): def f(): @@ -2281,7 +2285,7 @@ def test_range_of_offsets(self): abcdefg SyntaxError: bad bad """)), - # End offset pass the source lenght + # End offset pass the source length (("bad.py", 1, 2, "abcdefg", 1, 100), dedent( """ diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py index 3d212b9202f90a..0b4a12bcf40948 100644 --- a/Lib/test/test_frozen.py +++ b/Lib/test/test_frozen.py @@ -10,6 +10,7 @@ # Invalid marshalled data in frozen.c could case the interpreter to # crash when __hello__ is imported. +import importlib.machinery import sys import unittest from test.support import captured_stdout, import_helper @@ -26,6 +27,30 @@ def test_frozen(self): __hello__.main() self.assertEqual(out.getvalue(), 'Hello world!\n') + def test_frozen_submodule_in_unfrozen_package(self): + with import_helper.CleanImport('__phello__', '__phello__.spam'): + with import_helper.frozen_modules(enabled=False): + import __phello__ + with import_helper.frozen_modules(enabled=True): + import __phello__.spam as spam + self.assertIs(spam, __phello__.spam) + self.assertIsNot(__phello__.__spec__.loader, + importlib.machinery.FrozenImporter) + self.assertIs(spam.__spec__.loader, + importlib.machinery.FrozenImporter) + + def test_unfrozen_submodule_in_frozen_package(self): + with import_helper.CleanImport('__phello__', '__phello__.spam'): + with import_helper.frozen_modules(enabled=True): + import __phello__ + with import_helper.frozen_modules(enabled=False): + import __phello__.spam as spam + self.assertIs(spam, __phello__.spam) + self.assertIs(__phello__.__spec__.loader, + importlib.machinery.FrozenImporter) + self.assertIsNot(spam.__spec__.loader, + importlib.machinery.FrozenImporter) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index fece8256a3e261..bdb4ddcc60cac2 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2437,6 +2437,48 @@ def _(self, arg: str): self.assertEqual(a.t(''), "str") self.assertEqual(a.t(0.0), "base") + def test_staticmethod_type_ann_register(self): + class A: + @functools.singledispatchmethod + @staticmethod + def t(arg): + return arg + @t.register + @staticmethod + def _(arg: int): + return isinstance(arg, int) + @t.register + @staticmethod + def _(arg: str): + return isinstance(arg, str) + a = A() + + self.assertTrue(A.t(0)) + self.assertTrue(A.t('')) + self.assertEqual(A.t(0.0), 0.0) + + def test_classmethod_type_ann_register(self): + class A: + def __init__(self, arg): + self.arg = arg + + @functools.singledispatchmethod + @classmethod + def t(cls, arg): + return cls("base") + @t.register + @classmethod + def _(cls, arg: int): + return cls("int") + @t.register + @classmethod + def _(cls, arg: str): + return cls("str") + + self.assertEqual(A.t(0).arg, "int") + self.assertEqual(A.t('').arg, "str") + self.assertEqual(A.t(0.0).arg, "base") + def test_invalid_registrations(self): msg_prefix = "Invalid first argument to `register()`: " msg_suffix = ( diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index 4ef11232a33681..5a3944e69e640e 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -329,7 +329,7 @@ def test_annotations(self): def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected - # result explicitely. + # result explicitly. self.assertAnnotationEqual("f'{x=!r}'", expected="f'x={x!r}'") self.assertAnnotationEqual("f'{x=:}'", expected="f'x={x:}'") self.assertAnnotationEqual("f'{x=:.2f}'", expected="f'x={x:.2f}'") diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 6c28b2b677ceeb..52948f1c7bde51 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -444,7 +444,7 @@ def __getattr__(self, someattribute): # 0, thus mutating the trash graph as a side effect of merely asking # whether __del__ exists. This used to (before 2.3b1) crash Python. # Now __getattr__ isn't called. - self.assertEqual(gc.collect(), 4) + self.assertEqual(gc.collect(), 2) self.assertEqual(len(gc.garbage), garbagelen) def test_boom2(self): @@ -471,7 +471,7 @@ def __getattr__(self, someattribute): # there isn't a second time, so this simply cleans up the trash cycle. # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get # reclaimed this way. - self.assertEqual(gc.collect(), 4) + self.assertEqual(gc.collect(), 2) self.assertEqual(len(gc.garbage), garbagelen) def test_boom_new(self): @@ -491,7 +491,7 @@ def __getattr__(self, someattribute): gc.collect() garbagelen = len(gc.garbage) del a, b - self.assertEqual(gc.collect(), 4) + self.assertEqual(gc.collect(), 2) self.assertEqual(len(gc.garbage), garbagelen) def test_boom2_new(self): @@ -513,7 +513,7 @@ def __getattr__(self, someattribute): gc.collect() garbagelen = len(gc.garbage) del a, b - self.assertEqual(gc.collect(), 4) + self.assertEqual(gc.collect(), 2) self.assertEqual(len(gc.garbage), garbagelen) def test_get_referents(self): @@ -943,8 +943,8 @@ def getstats(): A() t = gc.collect() c, nc = getstats() - self.assertEqual(t, 2*N) # instance object & its dict - self.assertEqual(c - oldc, 2*N) + self.assertEqual(t, N) # instance objects + self.assertEqual(c - oldc, N) self.assertEqual(nc - oldnc, 0) # But Z() is not actually collected. @@ -964,8 +964,8 @@ def getstats(): Z() t = gc.collect() c, nc = getstats() - self.assertEqual(t, 2*N) - self.assertEqual(c - oldc, 2*N) + self.assertEqual(t, N) + self.assertEqual(c - oldc, N) self.assertEqual(nc - oldnc, 0) # The A() trash should have been reclaimed already but the @@ -974,8 +974,8 @@ def getstats(): zs.clear() t = gc.collect() c, nc = getstats() - self.assertEqual(t, 4) - self.assertEqual(c - oldc, 4) + self.assertEqual(t, 2) + self.assertEqual(c - oldc, 2) self.assertEqual(nc - oldnc, 0) gc.enable() @@ -1128,8 +1128,7 @@ def test_collect_generation(self): @cpython_only def test_collect_garbage(self): self.preclean() - # Each of these cause four objects to be garbage: Two - # Uncollectables and their instance dicts. + # Each of these cause two objects to be garbage: Uncollectable() Uncollectable() C1055820(666) @@ -1138,8 +1137,8 @@ def test_collect_garbage(self): if v[1] != "stop": continue info = v[2] - self.assertEqual(info["collected"], 2) - self.assertEqual(info["uncollectable"], 8) + self.assertEqual(info["collected"], 1) + self.assertEqual(info["uncollectable"], 4) # We should now have the Uncollectables in gc.garbage self.assertEqual(len(gc.garbage), 4) @@ -1156,7 +1155,7 @@ def test_collect_garbage(self): continue info = v[2] self.assertEqual(info["collected"], 0) - self.assertEqual(info["uncollectable"], 4) + self.assertEqual(info["uncollectable"], 2) # Uncollectables should be gone self.assertEqual(len(gc.garbage), 0) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 22a8cf3c25fc0f..2805eaf9f95674 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -724,18 +724,36 @@ def test_two_abs_args(self): ' 3 def foo(a, b, c):\n', bt) +SAMPLE_WITH_C_CALL = """ + +from _testcapi import pyobject_fastcall + +def foo(a, b, c): + bar(a, b, c) + +def bar(a, b, c): + pyobject_fastcall(baz, (a, b, c)) + +def baz(*args): + id(42) + +foo(1, 2, 3) + +""" + + class StackNavigationTests(DebuggerTests): @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") def test_pyup_command(self): 'Verify that the "py-up" command works' - bt = self.get_stack_trace(script=self.get_sample_script(), + bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, cmds_after_breakpoint=['py-up', 'py-up']) self.assertMultilineMatches(bt, r'''^.* -#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) - baz\(a, b, c\) +#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) +#[0-9]+ $''') @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") @@ -759,14 +777,13 @@ def test_up_at_top(self): "Python was compiled with optimizations") def test_up_then_down(self): 'Verify "py-up" followed by "py-down"' - bt = self.get_stack_trace(script=self.get_sample_script(), + bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, cmds_after_breakpoint=['py-up', 'py-up', 'py-down']) self.assertMultilineMatches(bt, r'''^.* -#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) - baz\(a, b, c\) -#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 10, in baz \(args=\(1, 2, 3\)\) - id\(42\) +#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) +#[0-9]+ +#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) $''') class PyBtTests(DebuggerTests): @@ -785,7 +802,7 @@ def test_bt(self): File ".*gdb_sample.py", line 7, in bar baz\(a, b, c\) File ".*gdb_sample.py", line 4, in foo - bar\(a, b, c\) + bar\(a=a, b=b, c=c\) File ".*gdb_sample.py", line 12, in foo\(1, 2, 3\) ''') @@ -801,7 +818,7 @@ def test_bt_full(self): #[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) baz\(a, b, c\) #[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\) - bar\(a, b, c\) + bar\(a=a, b=b, c=c\) #[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 12, in \(\) foo\(1, 2, 3\) ''') @@ -956,13 +973,12 @@ def __init__(self): self.assertRegex(gdb_output, r" +.*$''') def setUpModule(): diff --git a/Lib/test/test_hash.py b/Lib/test/test_hash.py index c68e0d8f815377..cf9db66a29ae11 100644 --- a/Lib/test/test_hash.py +++ b/Lib/test/test_hash.py @@ -42,8 +42,8 @@ def pysiphash(uint64): def skip_unless_internalhash(test): """Skip decorator for tests that depend on SipHash24 or FNV""" - ok = sys.hash_info.algorithm in {"fnv", "siphash24"} - msg = "Requires SipHash24 or FNV" + ok = sys.hash_info.algorithm in {"fnv", "siphash13", "siphash24"} + msg = "Requires SipHash13, SipHash24 or FNV" return test if ok else unittest.skip(msg)(test) @@ -206,6 +206,19 @@ class StringlikeHashRandomizationTests(HashRandomizationTests): # seed 42, 'abc' [-678966196, 573763426263223372, -820489388, -4282905804826039665], ], + 'siphash13': [ + # NOTE: PyUCS2 layout depends on endianness + # seed 0, 'abc' + [69611762, -4594863902769663758, 69611762, -4594863902769663758], + # seed 42, 'abc' + [-975800855, 3869580338025362921, -975800855, 3869580338025362921], + # seed 42, 'abcdefghijk' + [-595844228, 7764564197781545852, -595844228, 7764564197781545852], + # seed 0, 'äú∑ℇ' + [-1093288643, -2810468059467891395, -1041341092, 4925090034378237276], + # seed 42, 'äú∑ℇ' + [-585999602, -2845126246016066802, -817336969, -2219421378907968137], + ], 'siphash24': [ # NOTE: PyUCS2 layout depends on endianness # seed 0, 'abc' diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index fdf15efde12ef2..9450104d0b9a75 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -1920,14 +1920,5 @@ def test_session_cookies(self): self.assertNotEqual(counter["session_before"], 0) -def test_main(verbose=None): - test.support.run_unittest( - DateTimeTests, - HeaderTests, - CookieTests, - FileCookieJarTests, - LWPCookieTests, - ) - if __name__ == "__main__": - test_main(verbose=True) + unittest.main() diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 99312cc1625c42..1a21025fe6eaff 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -351,8 +351,8 @@ def test_issue_35321(self): self.assertEqual(_frozen_importlib.__spec__.origin, "frozen") def test_source_hash(self): - self.assertEqual(_imp.source_hash(42, b'hi'), b'\xc6\xe7Z\r\x03:}\xab') - self.assertEqual(_imp.source_hash(43, b'hi'), b'\x85\x9765\xf8\x9a\x8b9') + self.assertEqual(_imp.source_hash(42, b'hi'), b'\xfb\xd9G\x05\xaf$\x9b~') + self.assertEqual(_imp.source_hash(43, b'hi'), b'\xd0/\x87C\xccC\xff\xe2') def test_pyc_invalidation_mode_from_cmdline(self): cases = [ diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index fbc3fc0d547ff7..66080b2ade0098 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -3,44 +3,180 @@ machinery = util.import_importlib('importlib.machinery') +import _imp +import marshal +import os.path import unittest import warnings -from test.support import import_helper +from test.support import import_helper, REPO_ROOT, STDLIB_DIR + + +def resolve_stdlib_file(name, ispkg=False): + assert name + if ispkg: + return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py') + else: + return os.path.join(STDLIB_DIR, *name.split('.')) + '.py' class FindSpecTests(abc.FinderTests): """Test finding frozen modules.""" - def find(self, name, path=None): + def find(self, name, **kwargs): finder = self.machinery.FrozenImporter with import_helper.frozen_modules(): - return finder.find_spec(name, path) + return finder.find_spec(name, **kwargs) - def test_module(self): - name = '__hello__' - spec = self.find(name) + def check_basic(self, spec, name, ispkg=False): + self.assertEqual(spec.name, name) + self.assertIs(spec.loader, self.machinery.FrozenImporter) self.assertEqual(spec.origin, 'frozen') + self.assertFalse(spec.has_location) + if ispkg: + self.assertIsNotNone(spec.submodule_search_locations) + else: + self.assertIsNone(spec.submodule_search_locations) + self.assertIsNotNone(spec.loader_state) + + def check_loader_state(self, spec, origname=None, filename=None): + if not filename: + if not origname: + origname = spec.name + filename = resolve_stdlib_file(origname) + + actual = dict(vars(spec.loader_state)) + + # Check the rest of spec.loader_state. + expected = dict( + origname=origname, + filename=filename if origname else None, + ) + self.assertDictEqual(actual, expected) + + def check_search_locations(self, spec): + """This is only called when testing packages.""" + missing = object() + filename = getattr(spec.loader_state, 'filename', missing) + origname = getattr(spec.loader_state, 'origname', None) + if not origname or filename is missing: + # We deal with this in check_loader_state(). + return + if not filename: + expected = [] + elif origname != spec.name and not origname.startswith('<'): + expected = [] + else: + expected = [os.path.dirname(filename)] + self.assertListEqual(spec.submodule_search_locations, expected) - def test_package(self): - spec = self.find('__phello__') - self.assertIsNotNone(spec) - - def test_module_in_package(self): - spec = self.find('__phello__.spam', ['__phello__']) - self.assertIsNotNone(spec) + def test_module(self): + modules = [ + '__hello__', + '__phello__.spam', + '__phello__.ham.eggs', + ] + for name in modules: + with self.subTest(f'{name} -> {name}'): + spec = self.find(name) + self.check_basic(spec, name) + self.check_loader_state(spec) + modules = { + '__hello_alias__': '__hello__', + '_frozen_importlib': 'importlib._bootstrap', + } + for name, origname in modules.items(): + with self.subTest(f'{name} -> {origname}'): + spec = self.find(name) + self.check_basic(spec, name) + self.check_loader_state(spec, origname) + modules = [ + '__phello__.__init__', + '__phello__.ham.__init__', + ] + for name in modules: + origname = '<' + name.rpartition('.')[0] + filename = resolve_stdlib_file(name) + with self.subTest(f'{name} -> {origname}'): + spec = self.find(name) + self.check_basic(spec, name) + self.check_loader_state(spec, origname, filename) + modules = { + '__hello_only__': ('Tools', 'freeze', 'flag.py'), + } + for name, path in modules.items(): + origname = None + filename = os.path.join(REPO_ROOT, *path) + with self.subTest(f'{name} -> {filename}'): + spec = self.find(name) + self.check_basic(spec, name) + self.check_loader_state(spec, origname, filename) - # No frozen package within another package to test with. + def test_package(self): + packages = [ + '__phello__', + '__phello__.ham', + ] + for name in packages: + filename = resolve_stdlib_file(name, ispkg=True) + with self.subTest(f'{name} -> {name}'): + spec = self.find(name) + self.check_basic(spec, name, ispkg=True) + self.check_loader_state(spec, name, filename) + self.check_search_locations(spec) + packages = { + '__phello_alias__': '__hello__', + } + for name, origname in packages.items(): + filename = resolve_stdlib_file(origname, ispkg=False) + with self.subTest(f'{name} -> {origname}'): + spec = self.find(name) + self.check_basic(spec, name, ispkg=True) + self.check_loader_state(spec, origname, filename) + self.check_search_locations(spec) + + # These are covered by test_module() and test_package(). + test_module_in_package = None test_package_in_package = None # No easy way to test. test_package_over_module = None + def test_path_ignored(self): + for name in ('__hello__', '__phello__', '__phello__.spam'): + actual = self.find(name) + for path in (None, object(), '', 'eggs', [], [''], ['eggs']): + with self.subTest((name, path)): + spec = self.find(name, path=path) + self.assertEqual(spec, actual) + + def test_target_ignored(self): + imported = ('__hello__', '__phello__') + with import_helper.CleanImport(*imported, usefrozen=True): + import __hello__ as match + import __phello__ as nonmatch + name = '__hello__' + actual = self.find(name) + for target in (None, match, nonmatch, object(), 'not-a-module-object'): + with self.subTest(target): + spec = self.find(name, target=target) + self.assertEqual(spec, actual) + def test_failure(self): spec = self.find('') self.assertIsNone(spec) + def test_not_using_frozen(self): + finder = self.machinery.FrozenImporter + with import_helper.frozen_modules(enabled=False): + # both frozen and not frozen + spec1 = finder.find_spec('__hello__') + # only frozen + spec2 = finder.find_spec('__hello_only__') + self.assertIsNone(spec1) + self.assertIsNone(spec2) + (Frozen_FindSpecTests, Source_FindSpecTests diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index cfa5e5bb31bea9..f1ccb8a188aca7 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -3,8 +3,11 @@ machinery = util.import_importlib('importlib.machinery') -from test.support import captured_stdout, import_helper +from test.support import captured_stdout, import_helper, STDLIB_DIR +import _imp import contextlib +import marshal +import os.path import types import unittest import warnings @@ -28,9 +31,17 @@ def fresh(name, *, oldapi=False): yield +def resolve_stdlib_file(name, ispkg=False): + assert name + if ispkg: + return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py') + else: + return os.path.join(STDLIB_DIR, *name.split('.')) + '.py' + + class ExecModuleTests(abc.LoaderTests): - def exec_module(self, name): + def exec_module(self, name, origname=None): with import_helper.frozen_modules(): is_package = self.machinery.FrozenImporter.is_package(name) spec = self.machinery.ModuleSpec( @@ -38,6 +49,10 @@ def exec_module(self, name): self.machinery.FrozenImporter, origin='frozen', is_package=is_package, + loader_state=types.SimpleNamespace( + origname=origname or name, + filename=resolve_stdlib_file(origname or name, is_package), + ), ) module = types.ModuleType(name) module.__spec__ = spec @@ -61,6 +76,7 @@ def test_module(self): self.assertEqual(getattr(module, attr), value) self.assertEqual(output, 'Hello world!\n') self.assertTrue(hasattr(module, '__spec__')) + self.assertEqual(module.__spec__.loader_state.origname, name) def test_package(self): name = '__phello__' @@ -73,6 +89,7 @@ def test_package(self): name=name, attr=attr, given=attr_value, expected=value)) self.assertEqual(output, 'Hello world!\n') + self.assertEqual(module.__spec__.loader_state.origname, name) def test_lacking_parent(self): name = '__phello__.spam' @@ -128,36 +145,41 @@ def load_module(self, name): def test_module(self): module, stdout = self.load_module('__hello__') + filename = resolve_stdlib_file('__hello__') check = {'__name__': '__hello__', '__package__': '', '__loader__': self.machinery.FrozenImporter, + '__file__': filename, } for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) + self.assertEqual(getattr(module, attr, None), value) self.assertEqual(stdout.getvalue(), 'Hello world!\n') - self.assertFalse(hasattr(module, '__file__')) def test_package(self): module, stdout = self.load_module('__phello__') + filename = resolve_stdlib_file('__phello__', ispkg=True) + pkgdir = os.path.dirname(filename) check = {'__name__': '__phello__', '__package__': '__phello__', - '__path__': [], + '__path__': [pkgdir], '__loader__': self.machinery.FrozenImporter, + '__file__': filename, } for attr, value in check.items(): - attr_value = getattr(module, attr) + attr_value = getattr(module, attr, None) self.assertEqual(attr_value, value, "for __phello__.%s, %r != %r" % (attr, attr_value, value)) self.assertEqual(stdout.getvalue(), 'Hello world!\n') - self.assertFalse(hasattr(module, '__file__')) def test_lacking_parent(self): with util.uncache('__phello__'): module, stdout = self.load_module('__phello__.spam') + filename = resolve_stdlib_file('__phello__.spam') check = {'__name__': '__phello__.spam', '__package__': '__phello__', '__loader__': self.machinery.FrozenImporter, + '__file__': filename, } for attr, value in check.items(): attr_value = getattr(module, attr) @@ -165,7 +187,6 @@ def test_lacking_parent(self): "for __phello__.spam.%s, %r != %r" % (attr, attr_value, value)) self.assertEqual(stdout.getvalue(), 'Hello world!\n') - self.assertFalse(hasattr(module, '__file__')) def test_module_reuse(self): with fresh('__hello__', oldapi=True): diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 3fe3ddc5898448..f80283233f9770 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -1,5 +1,7 @@ import contextlib import importlib +import importlib.abc +import importlib.machinery import os import sys import unittest @@ -342,6 +344,11 @@ def test_path_indexable(self): expected_path = os.path.join(self.root, 'portion1', 'foo') self.assertEqual(foo.__path__[0], expected_path) + def test_loader_abc(self): + import foo + self.assertTrue(isinstance(foo.__loader__, importlib.abc.Loader)) + self.assertTrue(isinstance(foo.__loader__, importlib.machinery.NamespaceLoader)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 0e9191e5073cc6..516efac45e523f 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -43,7 +43,7 @@ # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, # isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargvalues, formatargspec, formatargvalues, +# getclasstree, getargvalues, formatargvalues, # currentframe, stack, trace, isdatadescriptor # NOTE: There are some additional tests relating to interaction with @@ -844,24 +844,11 @@ class D(B, C): pass got = inspect.getmro(D) self.assertEqual(expected, got) - def assertArgSpecEquals(self, routine, args_e, varargs_e=None, - varkw_e=None, defaults_e=None, formatted=None): - with self.assertWarns(DeprecationWarning): - args, varargs, varkw, defaults = inspect.getargspec(routine) - self.assertEqual(args, args_e) - self.assertEqual(varargs, varargs_e) - self.assertEqual(varkw, varkw_e) - self.assertEqual(defaults, defaults_e) - if formatted is not None: - with self.assertWarns(DeprecationWarning): - self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults), - formatted) - def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, posonlyargs_e=[], kwonlyargs_e=[], kwonlydefaults_e=None, - ann_e={}, formatted=None): + ann_e={}): args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ inspect.getfullargspec(routine) self.assertEqual(args, args_e) @@ -871,58 +858,30 @@ def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, self.assertEqual(kwonlyargs, kwonlyargs_e) self.assertEqual(kwonlydefaults, kwonlydefaults_e) self.assertEqual(ann, ann_e) - if formatted is not None: - with self.assertWarns(DeprecationWarning): - self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults, - kwonlyargs, kwonlydefaults, ann), - formatted) - - def test_getargspec(self): - self.assertArgSpecEquals(mod.eggs, ['x', 'y'], formatted='(x, y)') - - self.assertArgSpecEquals(mod.spam, - ['a', 'b', 'c', 'd', 'e', 'f'], - 'g', 'h', (3, 4, 5), - '(a, b, c, d=3, e=4, f=5, *g, **h)') - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyworded, []) - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.annotated, []) - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyword_only_arg, []) - def test_getfullargspec(self): self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', kwonlyargs_e=['arg2'], - kwonlydefaults_e={'arg2':1}, - formatted='(*arg1, arg2=1)') + kwonlydefaults_e={'arg2':1}) self.assertFullArgSpecEquals(mod2.annotated, ['arg1'], - ann_e={'arg1' : list}, - formatted='(arg1: list)') + ann_e={'arg1' : list}) self.assertFullArgSpecEquals(mod2.keyword_only_arg, [], - kwonlyargs_e=['arg'], - formatted='(*, arg)') + kwonlyargs_e=['arg']) self.assertFullArgSpecEquals(mod2.all_markers, ['a', 'b', 'c', 'd'], - kwonlyargs_e=['e', 'f'], - formatted='(a, b, c, d, *, e, f)') + kwonlyargs_e=['e', 'f']) self.assertFullArgSpecEquals(mod2.all_markers_with_args_and_kwargs, ['a', 'b', 'c', 'd'], varargs_e='args', varkw_e='kwargs', - kwonlyargs_e=['e', 'f'], - formatted='(a, b, c, d, *args, e, f, **kwargs)') + kwonlyargs_e=['e', 'f']) self.assertFullArgSpecEquals(mod2.all_markers_with_defaults, ['a', 'b', 'c', 'd'], defaults_e=(1,2,3), kwonlyargs_e=['e', 'f'], - kwonlydefaults_e={'e': 4, 'f': 5}, - formatted='(a, b=1, c=2, d=3, *, e=4, f=5)') + kwonlydefaults_e={'e': 4, 'f': 5}) def test_argspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @@ -930,39 +889,9 @@ def test_argspec_api_ignores_wrapped(self): def ham(x, y): pass # Basic check - self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') - self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') + self.assertFullArgSpecEquals(ham, ['x', 'y']) self.assertFullArgSpecEquals(functools.partial(ham), - ['x', 'y'], formatted='(x, y)') - # Other variants - def check_method(f): - self.assertArgSpecEquals(f, ['self', 'x', 'y'], - formatted='(self, x, y)') - class C: - @functools.wraps(mod.spam) - def ham(self, x, y): - pass - pham = functools.partialmethod(ham) - @functools.wraps(mod.spam) - def __call__(self, x, y): - pass - check_method(C()) - check_method(C.ham) - check_method(C().ham) - check_method(C.pham) - check_method(C().pham) - - class C_new: - @functools.wraps(mod.spam) - def __new__(self, x, y): - pass - check_method(C_new) - - class C_init: - @functools.wraps(mod.spam) - def __init__(self, x, y): - pass - check_method(C_init) + ['x', 'y']) def test_getfullargspec_signature_attr(self): def test(): @@ -970,7 +899,7 @@ def test(): spam_param = inspect.Parameter('spam', inspect.Parameter.POSITIONAL_ONLY) test.__signature__ = inspect.Signature(parameters=(spam_param,)) - self.assertFullArgSpecEquals(test, ['spam'], formatted='(spam)') + self.assertFullArgSpecEquals(test, ['spam']) def test_getfullargspec_signature_annos(self): def test(a:'spam') -> 'ham': pass @@ -984,18 +913,15 @@ def test(): pass @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_methods(self): - self.assertFullArgSpecEquals(_pickle.Pickler.dump, ['self', 'obj'], - formatted='(self, obj)') + self.assertFullArgSpecEquals(_pickle.Pickler.dump, ['self', 'obj']) - self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, ['self', 'obj'], - formatted='(self, obj)') + self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, ['self', 'obj']) self.assertFullArgSpecEquals( os.stat, args_e=['path'], kwonlyargs_e=['dir_fd', 'follow_symlinks'], - kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True}, - formatted='(path, *, dir_fd=None, follow_symlinks=True)') + kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True}) @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, @@ -1026,12 +952,6 @@ def test_getfullargspec_definition_order_preserved_on_kwonly(self): l = list(signature.kwonlyargs) self.assertEqual(l, unsorted_keyword_only_parameters) - def test_getargspec_method(self): - class A(object): - def m(self): - pass - self.assertArgSpecEquals(A.m, ['self']) - def test_classify_newstyle(self): class A(object): diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index fbfee1a582095b..682014cfd5b344 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -59,6 +59,15 @@ def bad_encoder2(*args): with self.assertRaises(ZeroDivisionError): enc('spam', 4) + def test_bad_markers_argument_to_encoder(self): + # https://bugs.python.org/issue45269 + with self.assertRaisesRegex( + TypeError, + r'make_encoder\(\) argument 1 must be dict or None, not int', + ): + self.json.encoder.c_make_encoder(1, None, None, None, ': ', ', ', + False, False, False) + def test_bad_bool_args(self): def test(name): self.json.encoder.JSONEncoder(**{name: BadBool()}).encode({'a': 1}) diff --git a/Lib/test/test_lltrace.py b/Lib/test/test_lltrace.py index 8f1a92e5c725c3..06e33f4c4c2f38 100644 --- a/Lib/test/test_lltrace.py +++ b/Lib/test/test_lltrace.py @@ -12,7 +12,7 @@ def test_lltrace_does_not_crash_on_subscript_operator(self): # If this test fails, it will reproduce a crash reported as # bpo-34113. The crash happened at the command line console of # debug Python builds with __ltrace__ enabled (only possible in console), - # when the interal Python stack was negatively adjusted + # when the internal Python stack was negatively adjusted with open(os_helper.TESTFN, 'w', encoding='utf-8') as fd: self.addCleanup(os_helper.unlink, os_helper.TESTFN) fd.write(textwrap.dedent("""\ diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 2c788f2dfa65e6..f844e62ca2e72b 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -496,7 +496,7 @@ def test_japanese(self): class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is - # valid. Futhermore LC_CTYPE=UTF is used by the UTF-8 locale coercing + # valid. Furthermore LC_CTYPE=UTF is used by the UTF-8 locale coercing # during interpreter startup (on macOS). import _locale import os diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 7a80244047d0f1..85b6e5f3921113 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5216,6 +5216,13 @@ def test_should_not_rollover(self): self.fn, encoding="utf-8", maxBytes=0) self.assertFalse(rh.shouldRollover(None)) rh.close() + # bpo-45401 - test with special file + # We set maxBytes to 1 so that rollover would normally happen, except + # for the check for regular files + rh = logging.handlers.RotatingFileHandler( + os.devnull, encoding="utf-8", maxBytes=1) + self.assertFalse(rh.shouldRollover(self.next_rec())) + rh.close() def test_should_rollover(self): rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=1) @@ -5310,6 +5317,15 @@ def rotator(source, dest): rh.close() class TimedRotatingFileHandlerTest(BaseFileTest): + def test_should_not_rollover(self): + # See bpo-45401. Should only ever rollover regular files + fh = logging.handlers.TimedRotatingFileHandler( + os.devnull, 'S', encoding="utf-8", backupCount=1) + time.sleep(1.1) # a little over a second ... + r = logging.makeLogRecord({'msg': 'testing - device file'}) + self.assertFalse(fh.shouldRollover(r)) + fh.close() + # other test methods added below def test_rollover(self): fh = logging.handlers.TimedRotatingFileHandler( diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index e15bf8b418676a..e5c4d7fc4220f8 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1260,7 +1260,7 @@ def equivalent_python(byte_array, byteorder, signed=False): expected) except Exception as err: raise AssertionError( - "failed to convert {} with default arugments" + "failed to convert {} with default arguments" .format(test)) from err try: diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 6d67d6293b2e7d..a9f1b1e11bcb39 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1803,16 +1803,22 @@ def test_prod(self): self.assertRaises(TypeError, prod) self.assertRaises(TypeError, prod, 42) self.assertRaises(TypeError, prod, ['a', 'b', 'c']) - self.assertRaises(TypeError, prod, ['a', 'b', 'c'], '') - self.assertRaises(TypeError, prod, [b'a', b'c'], b'') + self.assertRaises(TypeError, prod, ['a', 'b', 'c'], start='') + self.assertRaises(TypeError, prod, [b'a', b'c'], start=b'') values = [bytearray(b'a'), bytearray(b'b')] - self.assertRaises(TypeError, prod, values, bytearray(b'')) + self.assertRaises(TypeError, prod, values, start=bytearray(b'')) self.assertRaises(TypeError, prod, [[1], [2], [3]]) self.assertRaises(TypeError, prod, [{2:3}]) - self.assertRaises(TypeError, prod, [{2:3}]*2, {2:3}) - self.assertRaises(TypeError, prod, [[1], [2], [3]], []) + self.assertRaises(TypeError, prod, [{2:3}]*2, start={2:3}) + self.assertRaises(TypeError, prod, [[1], [2], [3]], start=[]) + + # Some odd cases + self.assertEqual(prod([2, 3], start='ab'), 'abababababab') + self.assertEqual(prod([2, 3], start=[1, 2]), [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2]) + self.assertEqual(prod([], start={2: 3}), {2:3}) + with self.assertRaises(TypeError): - prod([10, 20], [30, 40]) # start is a keyword-only argument + prod([10, 20], 1) # start is a keyword-only argument self.assertEqual(prod([0, 1, 2, 3]), 0) self.assertEqual(prod([1, 0, 2, 3]), 0) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index e716f4d3859326..5f6d9f47d13d75 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -324,7 +324,7 @@ def test_match_common(self): self.assertFalse(P('b/py').match('b.py')) self.assertFalse(P('/a.py').match('b.py')) self.assertFalse(P('b.py/c').match('b.py')) - # Wilcard relative pattern. + # Wildcard relative pattern. self.assertTrue(P('b.py').match('*.py')) self.assertTrue(P('a/b.py').match('*.py')) self.assertTrue(P('/a/b.py').match('*.py')) @@ -1284,7 +1284,7 @@ def test_is_reserved(self): self.assertIs(False, P('/foo/bar').is_reserved()) # UNC paths are never reserved. self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) - # Case-insenstive DOS-device names are reserved. + # Case-insensitive DOS-device names are reserved. self.assertIs(True, P('nul').is_reserved()) self.assertIs(True, P('aux').is_reserved()) self.assertIs(True, P('prn').is_reserved()) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 25ac1fb59d4c48..0a7d72c768424b 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -132,128 +132,70 @@ class C(builtins.object) expected_text_data_docstrings = tuple('\n | ' + s if s else '' for s in expected_data_docstrings) -expected_html_pattern = """ - - -
 
- 
test.pydoc_mod (version 1.2.3.4)		index
%s%s
-

This is a test module for test_pydoc

-

- - - -\x20\x20\x20\x20 - -
 
-Classes
       
-
builtins.object -
-
-
A -
B -
C -
-
-
-

- - - -\x20\x20\x20\x20 - - - -
 
-class A(builtins.object)
   Hello and goodbye
 
 Methods defined here:
-
__init__()
Wow, I have no function!
- -
-Data descriptors defined here:
-
__dict__
-
%s
-
-
__weakref__
-
%s
-
-

- - - -\x20\x20\x20\x20 - -
 
-class B(builtins.object)
    Data descriptors defined here:
-
__dict__
-
%s
-
-
__weakref__
-
%s
-
-
-Data and other attributes defined here:
-
NO_MEANING = 'eggs'
- -
__annotations__ = {'NO_MEANING': <class 'str'>}
- -

- - - -\x20\x20\x20\x20 - -
 
-class C(builtins.object)
    Methods defined here:
-
get_answer(self)
Return say_no()
- -
is_it_true(self)
Return self.get_answer()
- -
say_no(self)
- -
-Data descriptors defined here:
-
__dict__
-
dictionary for instance variables (if defined)
-
-
__weakref__
-
list of weak references to the object (if defined)
-
-

- - - -\x20\x20\x20\x20 - -
 
-Functions
       
doc_func()
This function solves all of the world's problems:
-hunger
-lack of Python
-war
-
nodoc_func()
-

- - - -\x20\x20\x20\x20 - -
 
-Data
       __xyz__ = 'X, Y and Z'

- - - -\x20\x20\x20\x20 - -
 
-Author
       Benjamin Peterson

- - - -\x20\x20\x20\x20 - -
 
-Credits
       Nobody
-""".strip() # ' <- emacs turd +html2text_of_expected = """ +test.pydoc_mod (version 1.2.3.4) +This is a test module for test_pydoc + +Classes + builtins.object + A + B + C + +class A(builtins.object) + Hello and goodbye + + Methods defined here: + __init__() + Wow, I have no function! + + Data descriptors defined here: + __dict__ + dictionary for instance variables (if defined) + __weakref__ + list of weak references to the object (if defined) + +class B(builtins.object) + Data descriptors defined here: + __dict__ + dictionary for instance variables (if defined) + __weakref__ + list of weak references to the object (if defined) + Data and other attributes defined here: + NO_MEANING = 'eggs' + __annotations__ = {'NO_MEANING': } + + +class C(builtins.object) + Methods defined here: + get_answer(self) + Return say_no() + is_it_true(self) + Return self.get_answer() + say_no(self) + Data descriptors defined here: + __dict__ + dictionary for instance variables (if defined) + __weakref__ + list of weak references to the object (if defined) + +Functions + doc_func() + This function solves all of the world's problems: + hunger + lack of Python + war + nodoc_func() + +Data + __xyz__ = 'X, Y and Z' + +Author + Benjamin Peterson + +Credits + Nobody +""" expected_html_data_docstrings = tuple(s.replace(' ', ' ') for s in expected_data_docstrings) @@ -394,6 +336,16 @@ def get_html_title(text): return title +def html2text(html): + """A quick and dirty implementation of html2text. + + Tailored for pydoc tests only. + """ + return pydoc.replace( + re.sub("<.*?>", "", html), + " ", " ", ">", ">", "<", "<") + + class PydocBaseTest(unittest.TestCase): def _restricted_walk_packages(self, walk_packages, path=None): @@ -434,12 +386,16 @@ class PydocDocTest(unittest.TestCase): @requires_docstrings def test_html_doc(self): result, doc_loc = get_pydoc_html(pydoc_mod) + text_result = html2text(result) + expected_lines = [line.strip() for line in html2text_of_expected if line] + for line in expected_lines: + self.assertIn(line, text_result) mod_file = inspect.getabsfile(pydoc_mod) mod_url = urllib.parse.quote(mod_file) - expected_html = expected_html_pattern % ( - (mod_url, mod_file, doc_loc) + - expected_html_data_docstrings) - self.assertEqual(result, expected_html) + self.assertIn(mod_url, result) + self.assertIn(mod_file, result) + self.assertIn(doc_loc, result) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") @@ -845,47 +801,39 @@ class B(A) ''' % __name__) doc = pydoc.render_doc(B, renderer=pydoc.HTMLDoc()) - self.assertEqual(doc, '''\ -Python Library Documentation: class B in module %s + expected_text = """ +Python Library Documentation -

- - - -\x20\x20\x20\x20 - -
 
-class B(A)
    
Method resolution order:
-
B
-
A
-
builtins.object
-
-
-Methods defined here:
-
b_size = a_size(self)
- -
itemconfig = itemconfigure(self, tagOrId, cnf=None, **kw)
- -
itemconfigure(self, tagOrId, cnf=None, **kw)
Configure resources of an item TAGORID.
- -
-Methods inherited from A:
-
a_size(self)
Return size
- -
lift = tkraise(self, aboveThis=None)
- -
tkraise(self, aboveThis=None)
Raise this widget in the stacking order.
- -
-Data descriptors inherited from A:
-
__dict__
-
dictionary for instance variables (if defined)
-
-
__weakref__
-
list of weak references to the object (if defined)
-
-
\ -''' % __name__) +class B in module test.test_pydoc +class B(A) + Method resolution order: + B + A + builtins.object + + Methods defined here: + b_size = a_size(self) + itemconfig = itemconfigure(self, tagOrId, cnf=None, **kw) + itemconfigure(self, tagOrId, cnf=None, **kw) + Configure resources of an item TAGORID. + + Methods inherited from A: + a_size(self) + Return size + lift = tkraise(self, aboveThis=None) + tkraise(self, aboveThis=None) + Raise this widget in the stacking order. + + Data descriptors inherited from A: + __dict__ + dictionary for instance variables (if defined) + __weakref__ + list of weak references to the object (if defined) +""" + as_text = html2text(doc) + expected_lines = [line.strip() for line in expected_text.split("\n") if line] + for expected_line in expected_lines: + self.assertIn(expected_line, as_text) class PydocImportTest(PydocBaseTest): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 448624b79be5db..cdae8897c2ae2f 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -481,50 +481,44 @@ def test_randrange_nonunit_step(self): self.assertEqual(rint, 0) def test_randrange_errors(self): - raises = partial(self.assertRaises, ValueError, self.gen.randrange) + raises_value_error = partial(self.assertRaises, ValueError, self.gen.randrange) + raises_type_error = partial(self.assertRaises, TypeError, self.gen.randrange) + # Empty range - raises(3, 3) - raises(-721) - raises(0, 100, -12) - # Non-integer start/stop - self.assertWarns(DeprecationWarning, raises, 3.14159) - self.assertWarns(DeprecationWarning, self.gen.randrange, 3.0) - self.assertWarns(DeprecationWarning, self.gen.randrange, Fraction(3, 1)) - self.assertWarns(DeprecationWarning, raises, '3') - self.assertWarns(DeprecationWarning, raises, 0, 2.71828) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 2.0) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, Fraction(2, 1)) - self.assertWarns(DeprecationWarning, raises, 0, '2') - # Zero and non-integer step - raises(0, 42, 0) - self.assertWarns(DeprecationWarning, raises, 0, 42, 0.0) - self.assertWarns(DeprecationWarning, raises, 0, 0, 0.0) - self.assertWarns(DeprecationWarning, raises, 0, 42, 3.14159) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, 3.0) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, Fraction(3, 1)) - self.assertWarns(DeprecationWarning, raises, 0, 42, '3') - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, 1.0) - self.assertWarns(DeprecationWarning, raises, 0, 0, 1.0) - - def test_randrange_argument_handling(self): - randrange = self.gen.randrange - with self.assertWarns(DeprecationWarning): - randrange(10.0, 20, 2) - with self.assertWarns(DeprecationWarning): - randrange(10, 20.0, 2) - with self.assertWarns(DeprecationWarning): - randrange(10, 20, 1.0) - with self.assertWarns(DeprecationWarning): - randrange(10, 20, 2.0) - with self.assertWarns(DeprecationWarning): - with self.assertRaises(ValueError): - randrange(10.5) - with self.assertWarns(DeprecationWarning): - with self.assertRaises(ValueError): - randrange(10, 20.5) - with self.assertWarns(DeprecationWarning): - with self.assertRaises(ValueError): - randrange(10, 20, 1.5) + raises_value_error(3, 3) + raises_value_error(-721) + raises_value_error(0, 100, -12) + + # Zero step + raises_value_error(0, 42, 0) + raises_type_error(0, 42, 0.0) + raises_type_error(0, 0, 0.0) + + # Non-integer stop + raises_type_error(3.14159) + raises_type_error(3.0) + raises_type_error(Fraction(3, 1)) + raises_type_error('3') + raises_type_error(0, 2.71827) + raises_type_error(0, 2.0) + raises_type_error(0, Fraction(2, 1)) + raises_type_error(0, '2') + raises_type_error(0, 2.71827, 2) + + # Non-integer start + raises_type_error(2.71827, 5) + raises_type_error(2.0, 5) + raises_type_error(Fraction(2, 1), 5) + raises_type_error('2', 5) + raises_type_error(2.71827, 5, 2) + + # Non-integer step + raises_type_error(0, 42, 3.14159) + raises_type_error(0, 42, 3.0) + raises_type_error(0, 42, Fraction(3, 1)) + raises_type_error(0, 42, '3') + raises_type_error(0, 42, 1.0) + raises_type_error(0, 0, 1.0) def test_randrange_step(self): # bpo-42772: When stop is None, the step argument was being ignored. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 3780feeda30e1b..dcc795de1a4b63 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1300,6 +1300,46 @@ def test_threading_excepthook(self): self.assertIn("Warning -- Uncaught thread exception", output) self.assertIn("Exception: bug in thread", output) + def test_print_warning(self): + # bpo-45410: The order of messages must be preserved when -W and + # support.print_warning() are used. + code = textwrap.dedent(r""" + import sys + import unittest + from test import support + + class MyObject: + pass + + def func_bug(): + raise Exception("bug in thread") + + class Tests(unittest.TestCase): + def test_print_warning(self): + print("msg1: stdout") + support.print_warning("msg2: print_warning") + # Fail with ENV CHANGED to see print_warning() log + support.environment_altered = True + """) + testname = self.create_test(code=code) + + # Expect an output like: + # + # test_threading_excepthook (test.test_x.Tests) ... msg1: stdout + # Warning -- msg2: print_warning + # ok + regex = (r"test_print_warning.*msg1: stdout\n" + r"Warning -- msg2: print_warning\n" + r"ok\n") + for option in ("-v", "-W"): + with self.subTest(option=option): + cmd = ["--fail-env-changed", option, testname] + output = self.run_tests(*cmd, exitcode=3) + self.check_executed_tests(output, [testname], + env_changed=[testname], + fail_env_changed=True) + self.assertRegex(output, regex) + def test_unicode_guard_env(self): guard = os.environ.get(setup.UNICODE_GUARD_ENV) self.assertIsNotNone(guard, f"{setup.UNICODE_GUARD_ENV} not set") diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index eeb8e8c98a1494..394d2942483fb2 100755 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1017,8 +1017,10 @@ def test_host_resolution(self): def test_host_resolution_bad_address(self): # These are all malformed IP addresses and expected not to resolve to - # any result. But some ISPs, e.g. AWS, may successfully resolve these - # IPs. + # any result. But some ISPs, e.g. AWS and AT&T, may successfully + # resolve these IPs. In particular, AT&T's DNS Error Assist service + # will break this test. See https://bugs.python.org/issue42092 for a + # workaround. explanation = ( "resolving an invalid IP address did not raise OSError; " "can be caused by a broken DNS server" diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index ec305e54ff24f0..be43c49e40aa50 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -114,7 +114,7 @@ def strftest1(self, now): ) for e in expectations: - # musn't raise a value error + # mustn't raise a value error try: result = time.strftime(e[0], now) except ValueError as error: diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 8b55352b6eecc6..d5a1d447f05634 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -469,12 +469,8 @@ def test_reap_children(self): if time.monotonic() > deadline: self.fail("timeout") - old_stderr = sys.__stderr__ - try: - sys.__stderr__ = stderr + with support.swap_attr(support.print_warning, 'orig_stderr', stderr): support.reap_children() - finally: - sys.__stderr__ = old_stderr # Use environment_altered to check if reap_children() found # the child process @@ -674,14 +670,8 @@ def test_fd_count(self): def check_print_warning(self, msg, expected): stderr = io.StringIO() - - old_stderr = sys.__stderr__ - try: - sys.__stderr__ = stderr + with support.swap_attr(support.print_warning, 'orig_stderr', stderr): support.print_warning(msg) - finally: - sys.__stderr__ = old_stderr - self.assertEqual(stderr.getvalue(), expected) def test_print_warning(self): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index aa86d0cd1d4801..3ce362788da6e0 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1274,7 +1274,8 @@ class SyntaxTestCase(unittest.TestCase): def _check_error(self, code, errtext, - filename="", mode="exec", subclass=None, lineno=None, offset=None): + filename="", mode="exec", subclass=None, + lineno=None, offset=None, end_lineno=None, end_offset=None): """Check that compiling code raises SyntaxError with errtext. errtest is a regular expression that must be present in the @@ -1294,6 +1295,11 @@ def _check_error(self, code, errtext, self.assertEqual(err.lineno, lineno) if offset is not None: self.assertEqual(err.offset, offset) + if end_lineno is not None: + self.assertEqual(err.end_lineno, end_lineno) + if end_offset is not None: + self.assertEqual(err.end_offset, end_offset) + else: self.fail("compile() did not raise SyntaxError") @@ -1434,6 +1440,11 @@ def test_kwargs_last3(self): "iterable argument unpacking follows " "keyword argument unpacking") + def test_generator_in_function_call(self): + self._check_error("foo(x, y for y in range(3) for z in range(2) if z , p)", + "Generator expression must be parenthesized", + lineno=1, end_lineno=1, offset=11, end_offset=53) + def test_empty_line_after_linecont(self): # See issue-40847 s = r"""\ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index e98803b48f6ac0..b0688e1e605fe5 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -13,6 +13,7 @@ from test.support import os_helper from test.support.script_helper import assert_python_ok, assert_python_failure from test.support import threading_helper +from test.support import import_helper import textwrap import unittest import warnings @@ -380,7 +381,7 @@ def g456(): self.assertTrue(frame is sys._getframe()) # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a litte tricky, since various bits of + # from f123. This is a little tricky, since various bits of # threading.py are also in the thread's call stack. frame = d.pop(thread_id) stack = traceback.extract_stack(frame) @@ -447,7 +448,7 @@ def g456(): self.assertEqual((None, None, None), d.pop(main_id)) # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a litte tricky, since various bits of + # from f123. This is a little tricky, since various bits of # threading.py are also in the thread's call stack. exc_type, exc_value, exc_tb = d.pop(thread_id) stack = traceback.extract_stack(exc_tb.tb_frame) @@ -507,7 +508,7 @@ def test_attributes(self): self.assertIsInstance(sys.hash_info.nan, int) self.assertIsInstance(sys.hash_info.imag, int) algo = sysconfig.get_config_var("Py_HASH_ALGORITHM") - if sys.hash_info.algorithm in {"fnv", "siphash24"}: + if sys.hash_info.algorithm in {"fnv", "siphash13", "siphash24"}: self.assertIn(sys.hash_info.hash_bits, {32, 64}) self.assertIn(sys.hash_info.seed_bits, {32, 64, 128}) @@ -515,8 +516,10 @@ def test_attributes(self): self.assertEqual(sys.hash_info.algorithm, "siphash24") elif algo == 2: self.assertEqual(sys.hash_info.algorithm, "fnv") + elif algo == 3: + self.assertEqual(sys.hash_info.algorithm, "siphash13") else: - self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash24"}) + self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash13", "siphash24"}) else: # PY_HASH_EXTERNAL self.assertEqual(algo, 0) @@ -822,7 +825,18 @@ def test_debugmallocstats(self): from test.support.script_helper import assert_python_ok args = ['-c', 'import sys; sys._debugmallocstats()'] ret, out, err = assert_python_ok(*args) - self.assertIn(b"free PyDictObjects", err) + + # Output of sys._debugmallocstats() depends on configure flags. + # The sysconfig vars are not available on Windows. + if sys.platform != "win32": + with_freelists = sysconfig.get_config_var("WITH_FREELISTS") + with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC") + if with_freelists: + self.assertIn(b"free PyDictObjects", err) + if with_pymalloc: + self.assertIn(b'Small block threshold', err) + if not with_freelists and not with_pymalloc: + self.assertFalse(err) # The function has no parameter self.assertRaises(TypeError, sys._debugmallocstats, True) @@ -994,6 +1008,15 @@ def test_module_names(self): for name in sys.stdlib_module_names: self.assertIsInstance(name, str) + def test_stdlib_dir(self): + os = import_helper.import_fresh_module('os') + marker = getattr(os, '__file__', None) + if marker and not os.path.exists(marker): + marker = None + expected = os.path.dirname(marker) if marker else None + self.assertEqual(os.path.normpath(sys._stdlib_dir), + os.path.normpath(expected)) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): @@ -1397,7 +1420,7 @@ def delx(self): del self.__x check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn11PIPP' + fmt = 'P2nPI13Pl4Pn9Pn12PIPP' s = vsize(fmt) check(int, s) # class @@ -1407,18 +1430,18 @@ def delx(self): del self.__x '3P' # PyMappingMethods '10P' # PySequenceMethods '2P' # PyBufferProcs - '5P') + '6P') class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries - check(newstyleclass, s + calcsize(DICT_KEY_STRUCT_FORMAT) + 8 + 5*calcsize("n2P")) + check(newstyleclass, s + calcsize(DICT_KEY_STRUCT_FORMAT) + 32 + 21*calcsize("n2P")) # dict with shared keys - check(newstyleclass().__dict__, size('nQ2P') + 5*self.P) + check(newstyleclass().__dict__, size('nQ2P') + 15*self.P) o = newstyleclass() o.a = o.b = o.c = o.d = o.e = o.f = o.g = o.h = 1 # Separate block for PyDictKeysObject with 16 keys and 10 entries - check(newstyleclass, s + calcsize(DICT_KEY_STRUCT_FORMAT) + 16 + 10*calcsize("n2P")) + check(newstyleclass, s + calcsize(DICT_KEY_STRUCT_FORMAT) + 32 + 21*calcsize("n2P")) # dict with shared keys - check(newstyleclass().__dict__, size('nQ2P') + 10*self.P) + check(newstyleclass().__dict__, size('nQ2P') + 13*self.P) # unicode # each tuple contains a string and its expected character size # don't put any static strings here, as they may contain diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index cfdda24a269f56..e4b5c52bf1eaf4 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -19,6 +19,10 @@ import gzip except ImportError: gzip = None +try: + import zlib +except ImportError: + zlib = None try: import bz2 except ImportError: @@ -687,6 +691,16 @@ def test_parallel_iteration(self): self.assertEqual(m1.offset, m2.offset) self.assertEqual(m1.get_info(), m2.get_info()) + @unittest.skipIf(zlib is None, "requires zlib") + def test_zlib_error_does_not_leak(self): + # bpo-39039: tarfile.open allowed zlib exceptions to bubble up when + # parsing certain types of invalid data + with unittest.mock.patch("tarfile.TarInfo.fromtarfile") as mock: + mock.side_effect = zlib.error + with self.assertRaises(tarfile.ReadError): + tarfile.open(self.tarname) + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index f18baa54e4a010..581c31ccb72fd1 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -729,7 +729,6 @@ def test_huge_string_builtins(self, size): self.assertRaises(OverflowError, tk.exprlong, value) self.assertRaises(OverflowError, tk.exprboolean, value) self.assertRaises(OverflowError, tk.splitlist, value) - self.assertRaises(OverflowError, tk.split, value) self.assertRaises(OverflowError, tk.createcommand, value, max) self.assertRaises(OverflowError, tk.deletecommand, value) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index f1d483733e2675..2b0ec46a103276 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -62,6 +62,25 @@ def test_infer_return_type_multiples_and_none(self): def test_infer_return_type_pathlib(self): self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + def test_infer_return_type_pathlike(self): + class Path: + def __init__(self, path): + self.path = path + + def __fspath__(self): + return self.path + + self.assertIs(str, tempfile._infer_return_type(Path('/'))) + self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) + self.assertIs(str, tempfile._infer_return_type('', Path(''))) + self.assertIs(bytes, tempfile._infer_return_type(b'', Path(b''))) + self.assertIs(bytes, tempfile._infer_return_type(None, Path(b''))) + self.assertIs(str, tempfile._infer_return_type(None, Path(''))) + + with self.assertRaises(TypeError): + tempfile._infer_return_type('', Path(b'')) + with self.assertRaises(TypeError): + tempfile._infer_return_type(b'', Path('')) # Common functionality. @@ -1435,7 +1454,7 @@ def test_explict_cleanup_ignore_errors(self): self.assertEqual( temp_path.exists(), sys.platform.startswith("win"), - f"TemporaryDirectory {temp_path!s} existance state unexpected") + f"TemporaryDirectory {temp_path!s} existence state unexpected") temp_dir.cleanup() self.assertFalse( temp_path.exists(), @@ -1494,7 +1513,7 @@ def test_del_on_collection_ignore_errors(self): self.assertEqual( temp_path.exists(), sys.platform.startswith("win"), - f"TemporaryDirectory {temp_path!s} existance state unexpected") + f"TemporaryDirectory {temp_path!s} existence state unexpected") def test_del_on_shutdown(self): # A TemporaryDirectory may be cleaned up during shutdown @@ -1559,7 +1578,7 @@ def test_del_on_shutdown_ignore_errors(self): self.assertEqual( temp_path.exists(), sys.platform.startswith("win"), - f"TemporaryDirectory {temp_path!s} existance state unexpected") + f"TemporaryDirectory {temp_path!s} existence state unexpected") err = err.decode('utf-8', 'backslashreplace') self.assertNotIn("Exception", err) self.assertNotIn("Error", err) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index f5ba16ea08ecec..a8f3c139b24be1 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -928,6 +928,39 @@ def test_debug_deprecation(self): b'is deprecated and will be removed in Python 3.12') self.assertIn(msg, err) + def test_import_from_another_thread(self): + # bpo-1596321: If the threading module is first import from a thread + # different than the main thread, threading._shutdown() must handle + # this case without logging an error at Python exit. + code = textwrap.dedent(''' + import _thread + import sys + + event = _thread.allocate_lock() + event.acquire() + + def import_threading(): + import threading + event.release() + + if 'threading' in sys.modules: + raise Exception('threading is already imported') + + _thread.start_new_thread(import_threading, ()) + + # wait until the threading module is imported + event.acquire() + event.release() + + if 'threading' not in sys.modules: + raise Exception('threading is not imported') + + # don't wait until the thread completes + ''') + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + class ThreadJoinOnShutdown(BaseTestCase): diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 325829864851c3..f98aec2651d9d7 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -38,6 +38,10 @@ class _PyTime(enum.IntEnum): # Round away from zero ROUND_UP = 3 +# _PyTime_t is int64_t +_PyTime_MIN = -2 ** 63 +_PyTime_MAX = 2 ** 63 - 1 + # Rounding modes supported by PyTime ROUNDING_MODES = ( # (PyTime rounding method, decimal rounding method) @@ -436,8 +440,8 @@ def test_mktime(self): @unittest.skipUnless(platform.libc_ver()[0] != 'glibc', "disabled because of a bug in glibc. Issue #13309") def test_mktime_error(self): - # It may not be possible to reliably make mktime return error - # on all platfom. This will make sure that no other exception + # It may not be possible to reliably make mktime return an error + # on all platforms. This will make sure that no other exception # than OverflowError is raised for an extreme value. tt = time.gmtime(self.t) tzname = time.strftime('%Z', tt) @@ -960,6 +964,49 @@ def timespec_converter(ns): NS_TO_SEC, value_filter=self.time_t_filter) + @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimeval_clamp'), + 'need _testcapi.PyTime_AsTimeval_clamp') + def test_AsTimeval_clamp(self): + from _testcapi import PyTime_AsTimeval_clamp + + if sys.platform == 'win32': + from _testcapi import LONG_MIN, LONG_MAX + tv_sec_max = LONG_MAX + tv_sec_min = LONG_MIN + else: + tv_sec_max = self.time_t_max + tv_sec_min = self.time_t_min + + for t in (_PyTime_MIN, _PyTime_MAX): + ts = PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) + with decimal.localcontext() as context: + context.rounding = decimal.ROUND_CEILING + us = self.decimal_round(decimal.Decimal(t) / US_TO_NS) + tv_sec, tv_usec = divmod(us, SEC_TO_US) + if tv_sec_max < tv_sec: + tv_sec = tv_sec_max + tv_usec = 0 + elif tv_sec < tv_sec_min: + tv_sec = tv_sec_min + tv_usec = 0 + self.assertEqual(ts, (tv_sec, tv_usec)) + + @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec_clamp'), + 'need _testcapi.PyTime_AsTimespec_clamp') + def test_AsTimespec_clamp(self): + from _testcapi import PyTime_AsTimespec_clamp + + for t in (_PyTime_MIN, _PyTime_MAX): + ts = PyTime_AsTimespec_clamp(t) + tv_sec, tv_nsec = divmod(t, NS_TO_SEC) + if self.time_t_max < tv_sec: + tv_sec = self.time_t_max + tv_nsec = 0 + elif tv_sec < self.time_t_min: + tv_sec = self.time_t_min + tv_nsec = 0 + self.assertEqual(ts, (tv_sec, tv_nsec)) + def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds @@ -1062,7 +1109,7 @@ def test_clock_functions(self): clock_names = [ "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", "clock_settime_ns", "clock_getres"] - + if mac_ver >= (10, 12): for name in clock_names: self.assertTrue(hasattr(time, name), f"time.{name} is not available") diff --git a/Lib/test/test_tk.py b/Lib/test/test_tk.py index 59842a5e25e366..69cc2322cd9aa8 100644 --- a/Lib/test/test_tk.py +++ b/Lib/test/test_tk.py @@ -6,11 +6,9 @@ # Skip test if tk cannot be initialized. support.requires('gui') -from tkinter.test import runtktests +def load_tests(loader, tests, pattern): + return loader.discover('tkinter.test.test_tkinter') -def test_main(): - support.run_unittest( - *runtktests.get_tests(text=False, packages=['test_tkinter'])) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index f8b16e52976451..ca2821de7c0816 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1933,7 +1933,7 @@ def test_string(self): c"""', """\ STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) """) - + self.check_tokenize('f"abc"', """\ STRING 'f"abc"' (1, 0) (1, 6) """) diff --git a/Lib/test/test_tools/test_fixcid.py b/Lib/test/test_tools/test_fixcid.py index 3df13680437194..a72f74b8b15b34 100644 --- a/Lib/test/test_tools/test_fixcid.py +++ b/Lib/test/test_tools/test_fixcid.py @@ -61,9 +61,10 @@ def test_directory(self): os.mkdir(os_helper.TESTFN) self.addCleanup(os_helper.rmtree, os_helper.TESTFN) c_filename = os.path.join(os_helper.TESTFN, "file.c") - with open(c_filename, "w") as file: + with open(c_filename, "w", encoding="utf-8") as file: file.write("int xx;\n") - with open(os.path.join(os_helper.TESTFN, "file.py"), "w") as file: + with open(os.path.join(os_helper.TESTFN, "file.py"), "w", + encoding="utf-8") as file: file.write("xx = 'unaltered'\n") script = os.path.join(scriptsdir, "fixcid.py") output = self.run_script(args=(os_helper.TESTFN,)) @@ -76,7 +77,7 @@ def test_directory(self): def run_script(self, input="", *, args=("-",), substfile="xx yy\n"): substfilename = os_helper.TESTFN + ".subst" - with open(substfilename, "w") as file: + with open(substfilename, "w", encoding="utf-8") as file: file.write(substfile) self.addCleanup(os_helper.unlink, substfilename) diff --git a/Lib/test/test_tools/test_gprof2html.py b/Lib/test/test_tools/test_gprof2html.py index 9489ed91f9c1cf..7cceb8faf8e5f4 100644 --- a/Lib/test/test_tools/test_gprof2html.py +++ b/Lib/test/test_tools/test_gprof2html.py @@ -25,7 +25,7 @@ def test_gprof(self): with mock.patch.object(self.gprof, 'webbrowser') as wmock, \ tempfile.TemporaryDirectory() as tmpdir: fn = os.path.join(tmpdir, 'abc') - open(fn, 'w').close() + open(fn, 'wb').close() sys.argv = ['gprof2html', fn] self.gprof.main() self.assertTrue(wmock.open.called) diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 12f778dbf8405d..7f18edaaa8ca60 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -57,10 +57,10 @@ def extract_docstrings_from_str(self, module_content): """ utility: return all msgids extracted from module_content """ filename = 'test_docstrings.py' with temp_cwd(None) as cwd: - with open(filename, 'w') as fp: + with open(filename, 'w', encoding='utf-8') as fp: fp.write(module_content) assert_python_ok(self.script, '-D', filename) - with open('messages.pot') as fp: + with open('messages.pot', encoding='utf-8') as fp: data = fp.read() return self.get_msgids(data) @@ -70,7 +70,7 @@ def test_header(self): """ with temp_cwd(None) as cwd: assert_python_ok(self.script) - with open('messages.pot') as fp: + with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) @@ -97,7 +97,7 @@ def test_POT_Creation_Date(self): from datetime import datetime with temp_cwd(None) as cwd: assert_python_ok(self.script) - with open('messages.pot') as fp: + with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) creationDate = header['POT-Creation-Date'] @@ -299,16 +299,19 @@ def test_files_list(self): text3 = 'Text to ignore' with temp_cwd(None), temp_dir(None) as sdir: os.mkdir(os.path.join(sdir, 'pypkg')) - with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w') as sfile: + with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w', + encoding='utf-8') as sfile: sfile.write(f'_({text1!r})') os.mkdir(os.path.join(sdir, 'pkg.py')) - with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w') as sfile: + with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w', + encoding='utf-8') as sfile: sfile.write(f'_({text2!r})') os.mkdir(os.path.join(sdir, 'CVS')) - with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w') as sfile: + with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w', + encoding='utf-8') as sfile: sfile.write(f'_({text3!r})') assert_python_ok(self.script, sdir) - with open('messages.pot') as fp: + with open('messages.pot', encoding='utf-8') as fp: data = fp.read() self.assertIn(f'msgid "{text1}"', data) self.assertIn(f'msgid "{text2}"', data) diff --git a/Lib/test/test_tools/test_lll.py b/Lib/test/test_tools/test_lll.py index ec0c97334fdebd..6eeb96ed9b67e4 100644 --- a/Lib/test/test_tools/test_lll.py +++ b/Lib/test/test_tools/test_lll.py @@ -22,7 +22,7 @@ def test_lll_multiple_dirs(self): fn1 = os.path.join(dir1, 'foo1') fn2 = os.path.join(dir2, 'foo2') for fn, dir in (fn1, dir1), (fn2, dir2): - open(fn, 'w').close() + open(fn, 'wb').close() os.symlink(fn, os.path.join(dir, 'symlink')) with support.captured_stdout() as output: diff --git a/Lib/test/test_tools/test_pdeps.py b/Lib/test/test_tools/test_pdeps.py index 27cbfe215d039d..a986d10e499d9b 100644 --- a/Lib/test/test_tools/test_pdeps.py +++ b/Lib/test/test_tools/test_pdeps.py @@ -19,7 +19,7 @@ def test_process_errors(self): # Issue #14492: m_import.match(line) can be None. with tempfile.TemporaryDirectory() as tmpdir: fn = os.path.join(tmpdir, 'foo') - with open(fn, 'w') as stream: + with open(fn, 'w', encoding='utf-8') as stream: stream.write("#!/this/will/fail") self.pdeps.process(fn, {}) diff --git a/Lib/test/test_tools/test_pindent.py b/Lib/test/test_tools/test_pindent.py index e7a547ad7d6127..01f13850eae733 100644 --- a/Lib/test/test_tools/test_pindent.py +++ b/Lib/test/test_tools/test_pindent.py @@ -37,9 +37,9 @@ def test_selftest(self): self.maxDiff = None with os_helper.temp_dir() as directory: data_path = os.path.join(directory, '_test.py') - with open(self.script) as f: + with open(self.script, encoding='utf-8') as f: closed = f.read() - with open(data_path, 'w') as f: + with open(data_path, 'w', encoding='utf-8') as f: f.write(closed) rc, out, err = assert_python_ok(self.script, '-d', data_path) @@ -47,9 +47,9 @@ def test_selftest(self): self.assertEqual(err, b'') backup = data_path + '~' self.assertTrue(os.path.exists(backup)) - with open(backup) as f: + with open(backup, encoding='utf-8') as f: self.assertEqual(f.read(), closed) - with open(data_path) as f: + with open(data_path, encoding='utf-8') as f: clean = f.read() compile(clean, '_test.py', 'exec') self.assertEqual(self.pindent(clean, '-c'), closed) @@ -58,20 +58,20 @@ def test_selftest(self): rc, out, err = assert_python_ok(self.script, '-c', data_path) self.assertEqual(out, b'') self.assertEqual(err, b'') - with open(backup) as f: + with open(backup, encoding='utf-8') as f: self.assertEqual(f.read(), clean) - with open(data_path) as f: + with open(data_path, encoding='utf-8') as f: self.assertEqual(f.read(), closed) broken = self.lstriplines(closed) - with open(data_path, 'w') as f: + with open(data_path, 'w', encoding='utf-8') as f: f.write(broken) rc, out, err = assert_python_ok(self.script, '-r', data_path) self.assertEqual(out, b'') self.assertEqual(err, b'') - with open(backup) as f: + with open(backup, encoding='utf-8') as f: self.assertEqual(f.read(), broken) - with open(data_path) as f: + with open(data_path, encoding='utf-8') as f: indented = f.read() compile(indented, '_test.py', 'exec') self.assertEqual(self.pindent(broken, '-r'), indented) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 363165d06ef834..1c7db9d3d47376 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -43,6 +43,9 @@ def syntax_error_with_caret(self): def syntax_error_with_caret_2(self): compile("1 +\n", "?", "exec") + def syntax_error_with_caret_range(self): + compile("f(x, y for y in range(30), z)", "?", "exec") + def syntax_error_bad_indentation(self): compile("def spam():\n print(1)\n print(2)", "?", "exec") @@ -52,6 +55,9 @@ def syntax_error_with_caret_non_ascii(self): def syntax_error_bad_indentation2(self): compile(" print(2)", "?", "exec") + def tokenizer_error_with_caret_range(self): + compile("blech ( ", "?", "exec") + def test_caret(self): err = self.get_exception_format(self.syntax_error_with_caret, SyntaxError) @@ -59,18 +65,35 @@ def test_caret(self): self.assertTrue(err[1].strip() == "return x!") self.assertIn("^", err[2]) # third line has caret self.assertEqual(err[1].find("!"), err[2].find("^")) # in the right place + self.assertEqual(err[2].count("^"), 1) err = self.get_exception_format(self.syntax_error_with_caret_2, SyntaxError) self.assertIn("^", err[2]) # third line has caret self.assertEqual(err[2].count('\n'), 1) # and no additional newline self.assertEqual(err[1].find("+") + 1, err[2].find("^")) # in the right place + self.assertEqual(err[2].count("^"), 1) err = self.get_exception_format(self.syntax_error_with_caret_non_ascii, SyntaxError) self.assertIn("^", err[2]) # third line has caret self.assertEqual(err[2].count('\n'), 1) # and no additional newline self.assertEqual(err[1].find("+") + 1, err[2].find("^")) # in the right place + self.assertEqual(err[2].count("^"), 1) + + err = self.get_exception_format(self.syntax_error_with_caret_range, + SyntaxError) + self.assertIn("^", err[2]) # third line has caret + self.assertEqual(err[2].count('\n'), 1) # and no additional newline + self.assertEqual(err[1].find("y"), err[2].find("^")) # in the right place + self.assertEqual(err[2].count("^"), len("y for y in range(30)")) + + err = self.get_exception_format(self.tokenizer_error_with_caret_range, + SyntaxError) + self.assertIn("^", err[2]) # third line has caret + self.assertEqual(err[2].count('\n'), 1) # and no additional newline + self.assertEqual(err[1].find("("), err[2].find("^")) # in the right place + self.assertEqual(err[2].count("^"), 1) def test_nocaret(self): exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) diff --git a/Lib/test/test_ttk_guionly.py b/Lib/test/test_ttk_guionly.py index abb26433652f17..8f59839d066e68 100644 --- a/Lib/test/test_ttk_guionly.py +++ b/Lib/test/test_ttk_guionly.py @@ -11,25 +11,26 @@ import tkinter from _tkinter import TclError from tkinter import ttk -from tkinter.test import runtktests - -root = None -try: - root = tkinter.Tk() - button = ttk.Button(root) - button.destroy() - del button -except TclError as msg: - # assuming ttk is not available - raise unittest.SkipTest("ttk not available: %s" % msg) -finally: - if root is not None: - root.destroy() - del root - -def test_main(): - support.run_unittest( - *runtktests.get_tests(text=False, packages=['test_ttk'])) + + +def setUpModule(): + root = None + try: + root = tkinter.Tk() + button = ttk.Button(root) + button.destroy() + del button + except TclError as msg: + # assuming ttk is not available + raise unittest.SkipTest("ttk not available: %s" % msg) + finally: + if root is not None: + root.destroy() + del root + +def load_tests(loader, tests, pattern): + return loader.discover('tkinter.test.test_ttk') + if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_ttk_textonly.py b/Lib/test/test_ttk_textonly.py index 013b0e9ee21da4..96dc179a69ecac 100644 --- a/Lib/test/test_ttk_textonly.py +++ b/Lib/test/test_ttk_textonly.py @@ -1,15 +1,463 @@ -from test import support from test.support import import_helper - # Skip this test if _tkinter does not exist. import_helper.import_module('_tkinter') -from tkinter.test import runtktests +import unittest +from tkinter import ttk + + +class MockTkApp: + + def splitlist(self, arg): + if isinstance(arg, tuple): + return arg + return arg.split(':') + + def wantobjects(self): + return True + + +class MockTclObj(object): + typename = 'test' + + def __init__(self, val): + self.val = val + + def __str__(self): + return str(self.val) + + +class MockStateSpec(object): + typename = 'StateSpec' + + def __init__(self, *args): + self.val = args + + def __str__(self): + return ' '.join(self.val) + + +class InternalFunctionsTest(unittest.TestCase): + + def test_format_optdict(self): + def check_against(fmt_opts, result): + for i in range(0, len(fmt_opts), 2): + self.assertEqual(result.pop(fmt_opts[i]), fmt_opts[i + 1]) + if result: + self.fail("result still got elements: %s" % result) + + # passing an empty dict should return an empty object (tuple here) + self.assertFalse(ttk._format_optdict({})) + + # check list formatting + check_against( + ttk._format_optdict({'fg': 'blue', 'padding': [1, 2, 3, 4]}), + {'-fg': 'blue', '-padding': '1 2 3 4'}) + + # check tuple formatting (same as list) + check_against( + ttk._format_optdict({'test': (1, 2, '', 0)}), + {'-test': '1 2 {} 0'}) + + # check untouched values + check_against( + ttk._format_optdict({'test': {'left': 'as is'}}), + {'-test': {'left': 'as is'}}) + + # check script formatting + check_against( + ttk._format_optdict( + {'test': [1, -1, '', '2m', 0], 'test2': 3, + 'test3': '', 'test4': 'abc def', + 'test5': '"abc"', 'test6': '{}', + 'test7': '} -spam {'}, script=True), + {'-test': '{1 -1 {} 2m 0}', '-test2': '3', + '-test3': '{}', '-test4': '{abc def}', + '-test5': '{"abc"}', '-test6': r'\{\}', + '-test7': r'\}\ -spam\ \{'}) + + opts = {'αβγ': True, 'á': False} + orig_opts = opts.copy() + # check if giving unicode keys is fine + check_against(ttk._format_optdict(opts), {'-αβγ': True, '-á': False}) + # opts should remain unchanged + self.assertEqual(opts, orig_opts) + + # passing values with spaces inside a tuple/list + check_against( + ttk._format_optdict( + {'option': ('one two', 'three')}), + {'-option': '{one two} three'}) + check_against( + ttk._format_optdict( + {'option': ('one\ttwo', 'three')}), + {'-option': '{one\ttwo} three'}) + + # passing empty strings inside a tuple/list + check_against( + ttk._format_optdict( + {'option': ('', 'one')}), + {'-option': '{} one'}) + + # passing values with braces inside a tuple/list + check_against( + ttk._format_optdict( + {'option': ('one} {two', 'three')}), + {'-option': r'one\}\ \{two three'}) + + # passing quoted strings inside a tuple/list + check_against( + ttk._format_optdict( + {'option': ('"one"', 'two')}), + {'-option': '{"one"} two'}) + check_against( + ttk._format_optdict( + {'option': ('{one}', 'two')}), + {'-option': r'\{one\} two'}) + + # ignore an option + amount_opts = len(ttk._format_optdict(opts, ignore=('á'))) / 2 + self.assertEqual(amount_opts, len(opts) - 1) + + # ignore non-existing options + amount_opts = len(ttk._format_optdict(opts, ignore=('á', 'b'))) / 2 + self.assertEqual(amount_opts, len(opts) - 1) + + # ignore every option + self.assertFalse(ttk._format_optdict(opts, ignore=list(opts.keys()))) + + + def test_format_mapdict(self): + opts = {'a': [('b', 'c', 'val'), ('d', 'otherval'), ('', 'single')]} + result = ttk._format_mapdict(opts) + self.assertEqual(len(result), len(list(opts.keys())) * 2) + self.assertEqual(result, ('-a', '{b c} val d otherval {} single')) + self.assertEqual(ttk._format_mapdict(opts, script=True), + ('-a', '{{b c} val d otherval {} single}')) + + self.assertEqual(ttk._format_mapdict({2: []}), ('-2', '')) + + opts = {'üñíćódè': [('á', 'vãl')]} + result = ttk._format_mapdict(opts) + self.assertEqual(result, ('-üñíćódè', 'á vãl')) + + self.assertEqual(ttk._format_mapdict({'opt': [('value',)]}), + ('-opt', '{} value')) + + # empty states + valid = {'opt': [('', '', 'hi')]} + self.assertEqual(ttk._format_mapdict(valid), ('-opt', '{ } hi')) + + # when passing multiple states, they all must be strings + invalid = {'opt': [(1, 2, 'valid val')]} + self.assertRaises(TypeError, ttk._format_mapdict, invalid) + invalid = {'opt': [([1], '2', 'valid val')]} + self.assertRaises(TypeError, ttk._format_mapdict, invalid) + # but when passing a single state, it can be anything + valid = {'opt': [[1, 'value']]} + self.assertEqual(ttk._format_mapdict(valid), ('-opt', '1 value')) + # special attention to single states which evaluate to False + for stateval in (None, 0, False, '', set()): # just some samples + valid = {'opt': [(stateval, 'value')]} + self.assertEqual(ttk._format_mapdict(valid), + ('-opt', '{} value')) + + # values must be iterable + opts = {'a': None} + self.assertRaises(TypeError, ttk._format_mapdict, opts) + + + def test_format_elemcreate(self): + self.assertTrue(ttk._format_elemcreate(None), (None, ())) + + ## Testing type = image + # image type expects at least an image name, so this should raise + # IndexError since it tries to access the index 0 of an empty tuple + self.assertRaises(IndexError, ttk._format_elemcreate, 'image') + + # don't format returned values as a tcl script + # minimum acceptable for image type + self.assertEqual(ttk._format_elemcreate('image', False, 'test'), + ("test ", ())) + # specifying a state spec + self.assertEqual(ttk._format_elemcreate('image', False, 'test', + ('', 'a')), ("test {} a", ())) + # state spec with multiple states + self.assertEqual(ttk._format_elemcreate('image', False, 'test', + ('a', 'b', 'c')), ("test {a b} c", ())) + # state spec and options + self.assertEqual(ttk._format_elemcreate('image', False, 'test', + ('a', 'b'), a='x'), ("test a b", ("-a", "x"))) + # format returned values as a tcl script + # state spec with multiple states and an option with a multivalue + self.assertEqual(ttk._format_elemcreate('image', True, 'test', + ('a', 'b', 'c', 'd'), x=[2, 3]), ("{test {a b c} d}", "-x {2 3}")) + + ## Testing type = vsapi + # vsapi type expects at least a class name and a part_id, so this + # should raise a ValueError since it tries to get two elements from + # an empty tuple + self.assertRaises(ValueError, ttk._format_elemcreate, 'vsapi') + + # don't format returned values as a tcl script + # minimum acceptable for vsapi + self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b'), + ("a b ", ())) + # now with a state spec with multiple states + self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', + ('a', 'b', 'c')), ("a b {a b} c", ())) + # state spec and option + self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', + ('a', 'b'), opt='x'), ("a b a b", ("-opt", "x"))) + # format returned values as a tcl script + # state spec with a multivalue and an option + self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', + ('a', 'b', [1, 2]), opt='x'), ("{a b {a b} {1 2}}", "-opt x")) + + # Testing type = from + # from type expects at least a type name + self.assertRaises(IndexError, ttk._format_elemcreate, 'from') + + self.assertEqual(ttk._format_elemcreate('from', False, 'a'), + ('a', ())) + self.assertEqual(ttk._format_elemcreate('from', False, 'a', 'b'), + ('a', ('b', ))) + self.assertEqual(ttk._format_elemcreate('from', True, 'a', 'b'), + ('{a}', 'b')) + + + def test_format_layoutlist(self): + def sample(indent=0, indent_size=2): + return ttk._format_layoutlist( + [('a', {'other': [1, 2, 3], 'children': + [('b', {'children': + [('c', {'children': + [('d', {'nice': 'opt'})], 'something': (1, 2) + })] + })] + })], indent=indent, indent_size=indent_size)[0] + + def sample_expected(indent=0, indent_size=2): + spaces = lambda amount=0: ' ' * (amount + indent) + return ( + "%sa -other {1 2 3} -children {\n" + "%sb -children {\n" + "%sc -something {1 2} -children {\n" + "%sd -nice opt\n" + "%s}\n" + "%s}\n" + "%s}" % (spaces(), spaces(indent_size), + spaces(2 * indent_size), spaces(3 * indent_size), + spaces(2 * indent_size), spaces(indent_size), spaces())) + + # empty layout + self.assertEqual(ttk._format_layoutlist([])[0], '') + + # _format_layoutlist always expects the second item (in every item) + # to act like a dict (except when the value evaluates to False). + self.assertRaises(AttributeError, + ttk._format_layoutlist, [('a', 'b')]) + + smallest = ttk._format_layoutlist([('a', None)], indent=0) + self.assertEqual(smallest, + ttk._format_layoutlist([('a', '')], indent=0)) + self.assertEqual(smallest[0], 'a') + + # testing indentation levels + self.assertEqual(sample(), sample_expected()) + for i in range(4): + self.assertEqual(sample(i), sample_expected(i)) + self.assertEqual(sample(i, i), sample_expected(i, i)) + + # invalid layout format, different kind of exceptions will be + # raised by internal functions + + # plain wrong format + self.assertRaises(ValueError, ttk._format_layoutlist, + ['bad', 'format']) + # will try to use iteritems in the 'bad' string + self.assertRaises(AttributeError, ttk._format_layoutlist, + [('name', 'bad')]) + # bad children formatting + self.assertRaises(ValueError, ttk._format_layoutlist, + [('name', {'children': {'a': None}})]) + + + def test_script_from_settings(self): + # empty options + self.assertFalse(ttk._script_from_settings({'name': + {'configure': None, 'map': None, 'element create': None}})) + + # empty layout + self.assertEqual( + ttk._script_from_settings({'name': {'layout': None}}), + "ttk::style layout name {\nnull\n}") + + configdict = {'αβγ': True, 'á': False} + self.assertTrue( + ttk._script_from_settings({'name': {'configure': configdict}})) + + mapdict = {'üñíćódè': [('á', 'vãl')]} + self.assertTrue( + ttk._script_from_settings({'name': {'map': mapdict}})) + + # invalid image element + self.assertRaises(IndexError, + ttk._script_from_settings, {'name': {'element create': ['image']}}) + + # minimal valid image + self.assertTrue(ttk._script_from_settings({'name': + {'element create': ['image', 'name']}})) + + image = {'thing': {'element create': + ['image', 'name', ('state1', 'state2', 'val')]}} + self.assertEqual(ttk._script_from_settings(image), + "ttk::style element create thing image {name {state1 state2} val} ") + + image['thing']['element create'].append({'opt': 30}) + self.assertEqual(ttk._script_from_settings(image), + "ttk::style element create thing image {name {state1 state2} val} " + "-opt 30") + + image['thing']['element create'][-1]['opt'] = [MockTclObj(3), + MockTclObj('2m')] + self.assertEqual(ttk._script_from_settings(image), + "ttk::style element create thing image {name {state1 state2} val} " + "-opt {3 2m}") + + + def test_tclobj_to_py(self): + self.assertEqual( + ttk._tclobj_to_py((MockStateSpec('a', 'b'), 'val')), + [('a', 'b', 'val')]) + self.assertEqual( + ttk._tclobj_to_py([MockTclObj('1'), 2, MockTclObj('3m')]), + [1, 2, '3m']) + + + def test_list_from_statespec(self): + def test_it(sspec, value, res_value, states): + self.assertEqual(ttk._list_from_statespec( + (sspec, value)), [states + (res_value, )]) + + states_even = tuple('state%d' % i for i in range(6)) + statespec = MockStateSpec(*states_even) + test_it(statespec, 'val', 'val', states_even) + test_it(statespec, MockTclObj('val'), 'val', states_even) + + states_odd = tuple('state%d' % i for i in range(5)) + statespec = MockStateSpec(*states_odd) + test_it(statespec, 'val', 'val', states_odd) + + test_it(('a', 'b', 'c'), MockTclObj('val'), 'val', ('a', 'b', 'c')) + + + def test_list_from_layouttuple(self): + tk = MockTkApp() + + # empty layout tuple + self.assertFalse(ttk._list_from_layouttuple(tk, ())) + + # shortest layout tuple + self.assertEqual(ttk._list_from_layouttuple(tk, ('name', )), + [('name', {})]) + + # not so interesting ltuple + sample_ltuple = ('name', '-option', 'value') + self.assertEqual(ttk._list_from_layouttuple(tk, sample_ltuple), + [('name', {'option': 'value'})]) + + # empty children + self.assertEqual(ttk._list_from_layouttuple(tk, + ('something', '-children', ())), + [('something', {'children': []})] + ) + + # more interesting ltuple + ltuple = ( + 'name', '-option', 'niceone', '-children', ( + ('otherone', '-children', ( + ('child', )), '-otheropt', 'othervalue' + ) + ) + ) + self.assertEqual(ttk._list_from_layouttuple(tk, ltuple), + [('name', {'option': 'niceone', 'children': + [('otherone', {'otheropt': 'othervalue', 'children': + [('child', {})] + })] + })] + ) + + # bad tuples + self.assertRaises(ValueError, ttk._list_from_layouttuple, tk, + ('name', 'no_minus')) + self.assertRaises(ValueError, ttk._list_from_layouttuple, tk, + ('name', 'no_minus', 'value')) + self.assertRaises(ValueError, ttk._list_from_layouttuple, tk, + ('something', '-children')) # no children + + + def test_val_or_dict(self): + def func(res, opt=None, val=None): + if opt is None: + return res + if val is None: + return "test val" + return (opt, val) + + tk = MockTkApp() + tk.call = func + + self.assertEqual(ttk._val_or_dict(tk, {}, '-test:3'), + {'test': '3'}) + self.assertEqual(ttk._val_or_dict(tk, {}, ('-test', 3)), + {'test': 3}) + + self.assertEqual(ttk._val_or_dict(tk, {'test': None}, 'x:y'), + 'test val') + + self.assertEqual(ttk._val_or_dict(tk, {'test': 3}, 'x:y'), + {'test': 3}) + + + def test_convert_stringval(self): + tests = ( + (0, 0), ('09', 9), ('a', 'a'), ('áÚ', 'áÚ'), ([], '[]'), + (None, 'None') + ) + for orig, expected in tests: + self.assertEqual(ttk._convert_stringval(orig), expected) + + +class TclObjsToPyTest(unittest.TestCase): + + def test_unicode(self): + adict = {'opt': 'välúè'} + self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': 'välúè'}) + + adict['opt'] = MockTclObj(adict['opt']) + self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': 'välúè'}) + + def test_multivalues(self): + adict = {'opt': [1, 2, 3, 4]} + self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': [1, 2, 3, 4]}) + + adict['opt'] = [1, 'xm', 3] + self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': [1, 'xm', 3]}) + + adict['opt'] = (MockStateSpec('a', 'b'), 'válũè') + self.assertEqual(ttk.tclobjs_to_py(adict), + {'opt': [('a', 'b', 'válũè')]}) + + self.assertEqual(ttk.tclobjs_to_py({'x': ['y z']}), + {'x': ['y z']}) + + def test_nosplit(self): + self.assertEqual(ttk.tclobjs_to_py({'text': 'some text'}), + {'text': 'some text'}) -def test_main(): - support.run_unittest( - *runtktests.get_tests(gui=False, packages=['test_ttk'])) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index d1887f7eee4437..032fe91c7a8408 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2903,6 +2903,12 @@ def test_final_forward_ref(self): self.assertNotEqual(gth(Loop, globals())['attr'], Final[int]) self.assertNotEqual(gth(Loop, globals())['attr'], Final) + def test_or(self): + X = ForwardRef('X') + # __or__/__ror__ itself + self.assertEqual(X | "x", Union[X, "x"]) + self.assertEqual("x" | X, Union["x", X]) + class OverloadTests(BaseTestCase): @@ -4115,6 +4121,19 @@ def test_namedtuple_special_keyword_names(self): self.assertEqual(a.typename, 'foo') self.assertEqual(a.fields, [('bar', tuple)]) + def test_empty_namedtuple(self): + NT = NamedTuple('NT') + + class CNT(NamedTuple): + pass # empty body + + for struct in [NT, CNT]: + with self.subTest(struct=struct): + self.assertEqual(struct._fields, ()) + self.assertEqual(struct._field_defaults, {}) + self.assertEqual(struct.__annotations__, {}) + self.assertIsInstance(struct(), struct) + def test_namedtuple_errors(self): with self.assertRaises(TypeError): NamedTuple.__new__() diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 63151ec89d8551..d8ba487328b393 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -427,7 +427,7 @@ def test_type_ignore(self): class CosmeticTestCase(ASTTestCase): - """Test if there are cosmetic issues caused by unnecesary additions""" + """Test if there are cosmetic issues caused by unnecessary additions""" def test_simple_expressions_parens(self): self.check_src_roundtrip("(a := b)") diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 9db23e6ce04bd5..a2b1340e0bf5ba 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1163,7 +1163,7 @@ def test_redirect(self): o = h.parent = MockOpener() # ordinary redirect behaviour - for code in 301, 302, 303, 307: + for code in 301, 302, 303, 307, 308: for data in None, "blah\nblah\n": method = getattr(h, "http_error_%s" % code) req = Request(from_url, data) @@ -1176,8 +1176,8 @@ def test_redirect(self): method(req, MockFile(), code, "Blah", MockHeaders({"location": to_url})) except urllib.error.HTTPError: - # 307 in response to POST requires user OK - self.assertEqual(code, 307) + # 307 and 308 in response to POST require user OK + self.assertIn(code, (307, 308)) self.assertIsNotNone(data) self.assertEqual(o.req.get_full_url(), to_url) try: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 098ba17af59758..94d626598bac38 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -150,14 +150,20 @@ def test_prompt(self): def test_upgrade_dependencies(self): builder = venv.EnvBuilder() bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' - python_exe = 'python.exe' if sys.platform == 'win32' else 'python' + python_exe = os.path.split(sys.executable)[1] with tempfile.TemporaryDirectory() as fake_env_dir: + expect_exe = os.path.normcase( + os.path.join(fake_env_dir, bin_path, python_exe) + ) + if sys.platform == 'win32': + expect_exe = os.path.normcase(os.path.realpath(expect_exe)) def pip_cmd_checker(cmd): + cmd[0] = os.path.normcase(cmd[0]) self.assertEqual( cmd, [ - os.path.join(fake_env_dir, bin_path, python_exe), + expect_exe, '-m', 'pip', 'install', diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 3bdc86d6923e02..46dac0228295d7 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1473,7 +1473,7 @@ def check_weak_del_and_len_while_iterating(self, dict, testcontext): o = Object(123456) with testcontext(): n = len(dict) - # Since underlaying dict is ordered, first item is popped + # Since underlying dict is ordered, first item is popped dict.pop(next(dict.keys())) self.assertEqual(len(dict), n - 1) dict[o] = o diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 1807e47c66c387..70a85552cc03b0 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -92,9 +92,11 @@ def test_open_name(self): f.close() f.close() - f = open('C:/con', 'rb', buffering=0) - self.assertIsInstance(f, ConIO) - f.close() + # bpo-45354: Windows 11 changed MS-DOS device name handling + if sys.getwindowsversion()[:3] < (10, 0, 22000): + f = open('C:/con', 'rb', buffering=0) + self.assertIsInstance(f, ConIO) + f.close() @unittest.skipIf(sys.getwindowsversion()[:2] <= (6, 1), "test does not work on Windows 7 and earlier") @@ -114,7 +116,8 @@ def test_conout_path(self): conout_path = os.path.join(temp_path, 'CONOUT$') with open(conout_path, 'wb', buffering=0) as f: - if sys.getwindowsversion()[:2] > (6, 1): + # bpo-45354: Windows 11 changed MS-DOS device name handling + if (6, 1) < sys.getwindowsversion()[:3] < (10, 0, 22000): self.assertIsInstance(f, ConIO) else: self.assertNotIsInstance(f, ConIO) diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index cf40e5a5c85d2e..9316d0ecbcf1ae 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -556,7 +556,7 @@ def testEnviron(self): # Test handler.environ as a dict expected = {} setup_testing_defaults(expected) - # Handler inherits os_environ variables which are not overriden + # Handler inherits os_environ variables which are not overridden # by SimpleHandler.add_cgi_vars() (SimpleHandler.base_env) for key, value in os_environ.items(): if key not in expected: diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c79b5462b931df..285559a872a65f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -26,7 +26,7 @@ from test import support from test.support import os_helper from test.support import warnings_helper -from test.support import findfile, gc_collect, swap_attr +from test.support import findfile, gc_collect, swap_attr, swap_item from test.support.import_helper import import_fresh_module from test.support.os_helper import TESTFN @@ -167,12 +167,11 @@ def setUpClass(cls): cls.modules = {pyET, ET} def pickleRoundTrip(self, obj, name, dumper, loader, proto): - save_m = sys.modules[name] try: - sys.modules[name] = dumper - temp = pickle.dumps(obj, proto) - sys.modules[name] = loader - result = pickle.loads(temp) + with swap_item(sys.modules, name, dumper): + temp = pickle.dumps(obj, proto) + with swap_item(sys.modules, name, loader): + result = pickle.loads(temp) except pickle.PicklingError as pe: # pyET must be second, because pyET may be (equal to) ET. human = dict([(ET, "cET"), (pyET, "pyET")]) @@ -180,8 +179,6 @@ def pickleRoundTrip(self, obj, name, dumper, loader, proto): % (obj, human.get(dumper, dumper), human.get(loader, loader))) from pe - finally: - sys.modules[name] = save_m return result def assertEqualElements(self, alice, bob): @@ -3334,7 +3331,7 @@ class MyElement(ET.Element): self._check_element_factory_class(MyElement) def test_element_factory_pure_python_subclass(self): - # Mimick SimpleTAL's behaviour (issue #16089): both versions of + # Mimic SimpleTAL's behaviour (issue #16089): both versions of # TreeBuilder should be able to cope with a subclass of the # pure Python Element class. base = ET._Element_Py diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 85e27ad4631d29..1f06f5fdf483e9 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -561,7 +561,7 @@ def test_comparison(self): class BinaryTestCase(unittest.TestCase): - # XXX What should str(Binary(b"\xff")) return? I'm chosing "\xff" + # XXX What should str(Binary(b"\xff")) return? I'm choosing "\xff" # for now (i.e. interpreting the binary data as Latin-1-encoded # text). But this feels very unsatisfactory. Perhaps we should # only define repr(), and return r"Binary(b'\xff')" instead? diff --git a/Lib/threading.py b/Lib/threading.py index c2b94a5045514f..6068d06ab6c906 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -418,6 +418,11 @@ def __init__(self, value=1): self._cond = Condition(Lock()) self._value = value + def __repr__(self): + cls = self.__class__ + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" value={self._value}>") + def acquire(self, blocking=True, timeout=None): """Acquire a semaphore, decrementing the internal counter by one. @@ -504,6 +509,11 @@ def __init__(self, value=1): Semaphore.__init__(self, value) self._initial_value = value + def __repr__(self): + cls = self.__class__ + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" value={self._value}/{self._initial_value}>") + def release(self, n=1): """Release a semaphore, incrementing the internal counter by one or more. @@ -539,6 +549,11 @@ def __init__(self): self._cond = Condition(Lock()) self._flag = False + def __repr__(self): + cls = self.__class__ + status = 'set' if self._flag else 'unset' + return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: {status}>" + def _at_fork_reinit(self): # Private method called by Thread._reset_internal_locks() self._cond._at_fork_reinit() @@ -634,9 +649,16 @@ def __init__(self, parties, action=None, timeout=None): self._action = action self._timeout = timeout self._parties = parties - self._state = 0 #0 filling, 1, draining, -1 resetting, -2 broken + self._state = 0 # 0 filling, 1 draining, -1 resetting, -2 broken self._count = 0 + def __repr__(self): + cls = self.__class__ + if self.broken: + return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: broken>" + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" waiters={self.n_waiting}/{self.parties}>") + def wait(self, timeout=None): """Wait for the barrier. @@ -1094,11 +1116,24 @@ def _wait_for_tstate_lock(self, block=True, timeout=-1): # If the lock is acquired, the C code is done, and self._stop() is # called. That sets ._is_stopped to True, and ._tstate_lock to None. lock = self._tstate_lock - if lock is None: # already determined that the C code is done + if lock is None: + # already determined that the C code is done assert self._is_stopped - elif lock.acquire(block, timeout): - lock.release() - self._stop() + return + + try: + if lock.acquire(block, timeout): + lock.release() + self._stop() + except: + if lock.locked(): + # bpo-45274: lock.acquire() acquired the lock, but the function + # was interrupted with an exception before reaching the + # lock.release(). It can happen if a signal handler raises an + # exception, like CTRL+C which raises KeyboardInterrupt. + lock.release() + self._stop() + raise @property def name(self): @@ -1504,20 +1539,29 @@ def _shutdown(): global _SHUTTING_DOWN _SHUTTING_DOWN = True - # Main thread - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't have - # been released. - assert tlock is not None - assert tlock.locked() - tlock.release() - _main_thread._stop() # Call registered threading atexit functions before threads are joined. # Order is reversed, similar to atexit. for atexit_call in reversed(_threading_atexits): atexit_call() + # Main thread + if _main_thread.ident == get_ident(): + tlock = _main_thread._tstate_lock + # The main thread isn't finished yet, so its thread state lock can't + # have been released. + assert tlock is not None + assert tlock.locked() + tlock.release() + _main_thread._stop() + else: + # bpo-1596321: _shutdown() must be called in the main thread. + # If the threading module was not imported by the main thread, + # _main_thread is the thread which imported the threading module. + # In this case, ignore _main_thread, similar behavior than for threads + # spawned by C libraries or using _thread.start_new_thread(). + pass + # Join all non-deamon threads while True: with _shutdown_locks_lock: @@ -1528,7 +1572,7 @@ def _shutdown(): break for lock in locks: - # mimick Thread.join() + # mimic Thread.join() lock.acquire() lock.release() diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 2513c972bc77f0..2e7e21c6648ea8 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2735,7 +2735,7 @@ def addtag_closest(self, newtag, x, y, halo=None, start=None): """Add tag NEWTAG to item which is closest to pixel at X, Y. If several match take the top-most. All items closer than HALO are considered overlapping (all are - closests). If START is specified the next below this tag is taken.""" + closest). If START is specified the next below this tag is taken.""" self.addtag(newtag, 'closest', x, y, halo, start) def addtag_enclosed(self, newtag, x1, y1, x2, y2): @@ -3330,7 +3330,7 @@ def add_command(self, cnf={}, **kw): self.add('command', cnf or kw) def add_radiobutton(self, cnf={}, **kw): - """Addd radio menu item.""" + """Add radio menu item.""" self.add('radiobutton', cnf or kw) def add_separator(self, cnf={}, **kw): @@ -3355,7 +3355,7 @@ def insert_command(self, index, cnf={}, **kw): self.insert(index, 'command', cnf or kw) def insert_radiobutton(self, index, cnf={}, **kw): - """Addd radio menu item at INDEX.""" + """Add radio menu item at INDEX.""" self.insert(index, 'radiobutton', cnf or kw) def insert_separator(self, index, cnf={}, **kw): diff --git a/Lib/tkinter/test/runtktests.py b/Lib/tkinter/test/runtktests.py deleted file mode 100644 index 33dc54a1375bf8..00000000000000 --- a/Lib/tkinter/test/runtktests.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Use this module to get and run all tk tests. - -tkinter tests should live in a package inside the directory where this file -lives, like test_tkinter. -Extensions also should live in packages following the same rule as above. -""" - -import os -import importlib -import test.support - -this_dir_path = os.path.abspath(os.path.dirname(__file__)) - -def is_package(path): - for name in os.listdir(path): - if name in ('__init__.py', '__init__.pyc'): - return True - return False - -def get_tests_modules(basepath=this_dir_path, gui=True, packages=None): - """This will import and yield modules whose names start with test_ - and are inside packages found in the path starting at basepath. - - If packages is specified it should contain package names that - want their tests collected. - """ - py_ext = '.py' - - for dirpath, dirnames, filenames in os.walk(basepath): - for dirname in list(dirnames): - if dirname[0] == '.': - dirnames.remove(dirname) - - if is_package(dirpath) and filenames: - pkg_name = dirpath[len(basepath) + len(os.sep):].replace('/', '.') - if packages and pkg_name not in packages: - continue - - filenames = filter( - lambda x: x.startswith('test_') and x.endswith(py_ext), - filenames) - - for name in filenames: - try: - yield importlib.import_module( - ".%s.%s" % (pkg_name, name[:-len(py_ext)]), - "tkinter.test") - except test.support.ResourceDenied: - if gui: - raise - -def get_tests(text=True, gui=True, packages=None): - """Yield all the tests in the modules found by get_tests_modules. - - If nogui is True, only tests that do not require a GUI will be - returned.""" - attrs = [] - if text: - attrs.append('tests_nogui') - if gui: - attrs.append('tests_gui') - for module in get_tests_modules(gui=gui, packages=packages): - for attr in attrs: - for test in getattr(module, attr, ()): - yield test - -if __name__ == "__main__": - test.support.run_unittest(*get_tests()) diff --git a/Lib/tkinter/test/test_tkinter/test_colorchooser.py b/Lib/tkinter/test/test_tkinter/test_colorchooser.py index 41da86c2adef49..488162ff0dd966 100644 --- a/Lib/tkinter/test/test_tkinter/test_colorchooser.py +++ b/Lib/tkinter/test/test_tkinter/test_colorchooser.py @@ -1,6 +1,6 @@ import unittest import tkinter -from test.support import requires, run_unittest, swap_attr +from test.support import requires, swap_attr from tkinter.test.support import AbstractDefaultRootTest, AbstractTkTest from tkinter import colorchooser from tkinter.colorchooser import askcolor @@ -64,7 +64,5 @@ def test_callback(dialog, master): self.assertRaises(RuntimeError, askcolor) -tests_gui = (ChooserTest, DefaultRootTest,) - if __name__ == "__main__": - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_font.py b/Lib/tkinter/test/test_tkinter/test_font.py index 91f99741171972..058c53a9023647 100644 --- a/Lib/tkinter/test/test_tkinter/test_font.py +++ b/Lib/tkinter/test/test_tkinter/test_font.py @@ -1,7 +1,7 @@ import unittest import tkinter from tkinter import font -from test.support import requires, run_unittest, gc_collect, ALWAYS_EQ +from test.support import requires, gc_collect, ALWAYS_EQ from tkinter.test.support import AbstractTkTest, AbstractDefaultRootTest requires('gui') @@ -159,7 +159,5 @@ def test_nametofont(self): self.assertRaises(RuntimeError, font.nametofont, fontname) -tests_gui = (FontTest, DefaultRootTest) - if __name__ == "__main__": - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_images.py b/Lib/tkinter/test/test_tkinter/test_images.py index c7b468044d55eb..cc69ccac62d742 100644 --- a/Lib/tkinter/test/test_tkinter/test_images.py +++ b/Lib/tkinter/test/test_tkinter/test_images.py @@ -376,7 +376,5 @@ def test_transparency(self): self.assertEqual(image.transparency_get(4, 6), False) -tests_gui = (MiscTest, DefaultRootTest, BitmapImageTest, PhotoImageTest,) - if __name__ == "__main__": - support.run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_loadtk.py b/Lib/tkinter/test/test_tkinter/test_loadtk.py index 760ba721340829..61b0eda2fc750a 100644 --- a/Lib/tkinter/test/test_tkinter/test_loadtk.py +++ b/Lib/tkinter/test/test_tkinter/test_loadtk.py @@ -41,7 +41,6 @@ def testLoadTkFailure(self): self.assertRaises(TclError, tcl.winfo_geometry) self.assertRaises(TclError, tcl.loadtk) -tests_gui = (TkLoadTest, ) if __name__ == "__main__": - test_support.run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_messagebox.py b/Lib/tkinter/test/test_tkinter/test_messagebox.py index 0dec08e9041a0e..d38541a5a45e76 100644 --- a/Lib/tkinter/test/test_tkinter/test_messagebox.py +++ b/Lib/tkinter/test/test_tkinter/test_messagebox.py @@ -1,6 +1,6 @@ import unittest import tkinter -from test.support import requires, run_unittest, swap_attr +from test.support import requires, swap_attr from tkinter.test.support import AbstractDefaultRootTest from tkinter.commondialog import Dialog from tkinter.messagebox import showinfo @@ -32,7 +32,5 @@ def test_callback(dialog, master): self.assertRaises(RuntimeError, showinfo, "Spam", "Egg Information") -tests_gui = (DefaultRootTest,) - if __name__ == "__main__": - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index ab8f64790dfc0e..2c78eaca2c85b4 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -390,7 +390,5 @@ def test_mainloop(self): self.assertRaises(RuntimeError, tkinter.mainloop) -tests_gui = (MiscTest, DefaultRootTest) - if __name__ == "__main__": - support.run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_simpledialog.py b/Lib/tkinter/test/test_tkinter/test_simpledialog.py index b64b854c4db7ef..18cd2712b0c5ed 100644 --- a/Lib/tkinter/test/test_tkinter/test_simpledialog.py +++ b/Lib/tkinter/test/test_tkinter/test_simpledialog.py @@ -1,6 +1,6 @@ import unittest import tkinter -from test.support import requires, run_unittest, swap_attr +from test.support import requires, swap_attr from tkinter.test.support import AbstractDefaultRootTest from tkinter.simpledialog import Dialog, askinteger @@ -31,7 +31,5 @@ def mock_wait_window(w): self.assertRaises(RuntimeError, askinteger, "Go To Line", "Line number") -tests_gui = (DefaultRootTest,) - if __name__ == "__main__": - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_text.py b/Lib/tkinter/test/test_tkinter/test_text.py index 13b7c56a3978d9..482f150df559fc 100644 --- a/Lib/tkinter/test/test_tkinter/test_text.py +++ b/Lib/tkinter/test/test_tkinter/test_text.py @@ -1,6 +1,6 @@ import unittest import tkinter -from test.support import requires, run_unittest +from test.support import requires from tkinter.test.support import AbstractTkTest requires('gui') @@ -41,7 +41,5 @@ def test_search(self): self.assertEqual(text.search('test', '1.0', 'end'), '1.3') -tests_gui = (TextTest, ) - if __name__ == "__main__": - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_variables.py b/Lib/tkinter/test/test_tkinter/test_variables.py index 0be5282a3a3b30..427e168454362c 100644 --- a/Lib/tkinter/test/test_tkinter/test_variables.py +++ b/Lib/tkinter/test/test_tkinter/test_variables.py @@ -338,10 +338,5 @@ def test_variable(self): self.assertRaises(RuntimeError, Variable) -tests_gui = (TestVariable, TestStringVar, TestIntVar, - TestDoubleVar, TestBooleanVar, DefaultRootTest) - - if __name__ == "__main__": - from test.support import run_unittest - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index 39334de8cf41c3..cc227e579679b7 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -1241,8 +1241,11 @@ def test_configure_title(self): def test_configure_type(self): widget = self.create() - self.checkEnumParam(widget, 'type', - 'normal', 'tearoff', 'menubar') + self.checkEnumParam( + widget, 'type', + 'normal', 'tearoff', 'menubar', + errmsg='bad type "{}": must be normal, tearoff, or menubar', + ) def test_entryconfigure(self): m1 = self.create() diff --git a/Lib/tkinter/test/test_ttk/test_extensions.py b/Lib/tkinter/test/test_ttk/test_extensions.py index e6b3eccf7afb8a..438d21d0b37334 100644 --- a/Lib/tkinter/test/test_ttk/test_extensions.py +++ b/Lib/tkinter/test/test_ttk/test_extensions.py @@ -2,7 +2,7 @@ import unittest import tkinter from tkinter import ttk -from test.support import requires, run_unittest, gc_collect +from test.support import requires, gc_collect from tkinter.test.support import AbstractTkTest, AbstractDefaultRootTest requires('gui') @@ -308,7 +308,5 @@ def test_labeledscale(self): self._test_widget(ttk.LabeledScale) -tests_gui = (LabeledScaleTest, OptionMenuTest, DefaultRootTest) - if __name__ == "__main__": - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_ttk/test_functions.py b/Lib/tkinter/test/test_ttk/test_functions.py deleted file mode 100644 index 5c23d6fecf8b42..00000000000000 --- a/Lib/tkinter/test/test_ttk/test_functions.py +++ /dev/null @@ -1,460 +0,0 @@ -# -*- encoding: utf-8 -*- -import unittest -from tkinter import ttk - -class MockTkApp: - - def splitlist(self, arg): - if isinstance(arg, tuple): - return arg - return arg.split(':') - - def wantobjects(self): - return True - - -class MockTclObj(object): - typename = 'test' - - def __init__(self, val): - self.val = val - - def __str__(self): - return str(self.val) - - -class MockStateSpec(object): - typename = 'StateSpec' - - def __init__(self, *args): - self.val = args - - def __str__(self): - return ' '.join(self.val) - - -class InternalFunctionsTest(unittest.TestCase): - - def test_format_optdict(self): - def check_against(fmt_opts, result): - for i in range(0, len(fmt_opts), 2): - self.assertEqual(result.pop(fmt_opts[i]), fmt_opts[i + 1]) - if result: - self.fail("result still got elements: %s" % result) - - # passing an empty dict should return an empty object (tuple here) - self.assertFalse(ttk._format_optdict({})) - - # check list formatting - check_against( - ttk._format_optdict({'fg': 'blue', 'padding': [1, 2, 3, 4]}), - {'-fg': 'blue', '-padding': '1 2 3 4'}) - - # check tuple formatting (same as list) - check_against( - ttk._format_optdict({'test': (1, 2, '', 0)}), - {'-test': '1 2 {} 0'}) - - # check untouched values - check_against( - ttk._format_optdict({'test': {'left': 'as is'}}), - {'-test': {'left': 'as is'}}) - - # check script formatting - check_against( - ttk._format_optdict( - {'test': [1, -1, '', '2m', 0], 'test2': 3, - 'test3': '', 'test4': 'abc def', - 'test5': '"abc"', 'test6': '{}', - 'test7': '} -spam {'}, script=True), - {'-test': '{1 -1 {} 2m 0}', '-test2': '3', - '-test3': '{}', '-test4': '{abc def}', - '-test5': '{"abc"}', '-test6': r'\{\}', - '-test7': r'\}\ -spam\ \{'}) - - opts = {'αβγ': True, 'á': False} - orig_opts = opts.copy() - # check if giving unicode keys is fine - check_against(ttk._format_optdict(opts), {'-αβγ': True, '-á': False}) - # opts should remain unchanged - self.assertEqual(opts, orig_opts) - - # passing values with spaces inside a tuple/list - check_against( - ttk._format_optdict( - {'option': ('one two', 'three')}), - {'-option': '{one two} three'}) - check_against( - ttk._format_optdict( - {'option': ('one\ttwo', 'three')}), - {'-option': '{one\ttwo} three'}) - - # passing empty strings inside a tuple/list - check_against( - ttk._format_optdict( - {'option': ('', 'one')}), - {'-option': '{} one'}) - - # passing values with braces inside a tuple/list - check_against( - ttk._format_optdict( - {'option': ('one} {two', 'three')}), - {'-option': r'one\}\ \{two three'}) - - # passing quoted strings inside a tuple/list - check_against( - ttk._format_optdict( - {'option': ('"one"', 'two')}), - {'-option': '{"one"} two'}) - check_against( - ttk._format_optdict( - {'option': ('{one}', 'two')}), - {'-option': r'\{one\} two'}) - - # ignore an option - amount_opts = len(ttk._format_optdict(opts, ignore=('á'))) / 2 - self.assertEqual(amount_opts, len(opts) - 1) - - # ignore non-existing options - amount_opts = len(ttk._format_optdict(opts, ignore=('á', 'b'))) / 2 - self.assertEqual(amount_opts, len(opts) - 1) - - # ignore every option - self.assertFalse(ttk._format_optdict(opts, ignore=list(opts.keys()))) - - - def test_format_mapdict(self): - opts = {'a': [('b', 'c', 'val'), ('d', 'otherval'), ('', 'single')]} - result = ttk._format_mapdict(opts) - self.assertEqual(len(result), len(list(opts.keys())) * 2) - self.assertEqual(result, ('-a', '{b c} val d otherval {} single')) - self.assertEqual(ttk._format_mapdict(opts, script=True), - ('-a', '{{b c} val d otherval {} single}')) - - self.assertEqual(ttk._format_mapdict({2: []}), ('-2', '')) - - opts = {'üñíćódè': [('á', 'vãl')]} - result = ttk._format_mapdict(opts) - self.assertEqual(result, ('-üñíćódè', 'á vãl')) - - self.assertEqual(ttk._format_mapdict({'opt': [('value',)]}), - ('-opt', '{} value')) - - # empty states - valid = {'opt': [('', '', 'hi')]} - self.assertEqual(ttk._format_mapdict(valid), ('-opt', '{ } hi')) - - # when passing multiple states, they all must be strings - invalid = {'opt': [(1, 2, 'valid val')]} - self.assertRaises(TypeError, ttk._format_mapdict, invalid) - invalid = {'opt': [([1], '2', 'valid val')]} - self.assertRaises(TypeError, ttk._format_mapdict, invalid) - # but when passing a single state, it can be anything - valid = {'opt': [[1, 'value']]} - self.assertEqual(ttk._format_mapdict(valid), ('-opt', '1 value')) - # special attention to single states which evaluate to False - for stateval in (None, 0, False, '', set()): # just some samples - valid = {'opt': [(stateval, 'value')]} - self.assertEqual(ttk._format_mapdict(valid), - ('-opt', '{} value')) - - # values must be iterable - opts = {'a': None} - self.assertRaises(TypeError, ttk._format_mapdict, opts) - - - def test_format_elemcreate(self): - self.assertTrue(ttk._format_elemcreate(None), (None, ())) - - ## Testing type = image - # image type expects at least an image name, so this should raise - # IndexError since it tries to access the index 0 of an empty tuple - self.assertRaises(IndexError, ttk._format_elemcreate, 'image') - - # don't format returned values as a tcl script - # minimum acceptable for image type - self.assertEqual(ttk._format_elemcreate('image', False, 'test'), - ("test ", ())) - # specifying a state spec - self.assertEqual(ttk._format_elemcreate('image', False, 'test', - ('', 'a')), ("test {} a", ())) - # state spec with multiple states - self.assertEqual(ttk._format_elemcreate('image', False, 'test', - ('a', 'b', 'c')), ("test {a b} c", ())) - # state spec and options - self.assertEqual(ttk._format_elemcreate('image', False, 'test', - ('a', 'b'), a='x'), ("test a b", ("-a", "x"))) - # format returned values as a tcl script - # state spec with multiple states and an option with a multivalue - self.assertEqual(ttk._format_elemcreate('image', True, 'test', - ('a', 'b', 'c', 'd'), x=[2, 3]), ("{test {a b c} d}", "-x {2 3}")) - - ## Testing type = vsapi - # vsapi type expects at least a class name and a part_id, so this - # should raise a ValueError since it tries to get two elements from - # an empty tuple - self.assertRaises(ValueError, ttk._format_elemcreate, 'vsapi') - - # don't format returned values as a tcl script - # minimum acceptable for vsapi - self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b'), - ("a b ", ())) - # now with a state spec with multiple states - self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b', 'c')), ("a b {a b} c", ())) - # state spec and option - self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b'), opt='x'), ("a b a b", ("-opt", "x"))) - # format returned values as a tcl script - # state spec with a multivalue and an option - self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', - ('a', 'b', [1, 2]), opt='x'), ("{a b {a b} {1 2}}", "-opt x")) - - # Testing type = from - # from type expects at least a type name - self.assertRaises(IndexError, ttk._format_elemcreate, 'from') - - self.assertEqual(ttk._format_elemcreate('from', False, 'a'), - ('a', ())) - self.assertEqual(ttk._format_elemcreate('from', False, 'a', 'b'), - ('a', ('b', ))) - self.assertEqual(ttk._format_elemcreate('from', True, 'a', 'b'), - ('{a}', 'b')) - - - def test_format_layoutlist(self): - def sample(indent=0, indent_size=2): - return ttk._format_layoutlist( - [('a', {'other': [1, 2, 3], 'children': - [('b', {'children': - [('c', {'children': - [('d', {'nice': 'opt'})], 'something': (1, 2) - })] - })] - })], indent=indent, indent_size=indent_size)[0] - - def sample_expected(indent=0, indent_size=2): - spaces = lambda amount=0: ' ' * (amount + indent) - return ( - "%sa -other {1 2 3} -children {\n" - "%sb -children {\n" - "%sc -something {1 2} -children {\n" - "%sd -nice opt\n" - "%s}\n" - "%s}\n" - "%s}" % (spaces(), spaces(indent_size), - spaces(2 * indent_size), spaces(3 * indent_size), - spaces(2 * indent_size), spaces(indent_size), spaces())) - - # empty layout - self.assertEqual(ttk._format_layoutlist([])[0], '') - - # _format_layoutlist always expects the second item (in every item) - # to act like a dict (except when the value evaluates to False). - self.assertRaises(AttributeError, - ttk._format_layoutlist, [('a', 'b')]) - - smallest = ttk._format_layoutlist([('a', None)], indent=0) - self.assertEqual(smallest, - ttk._format_layoutlist([('a', '')], indent=0)) - self.assertEqual(smallest[0], 'a') - - # testing indentation levels - self.assertEqual(sample(), sample_expected()) - for i in range(4): - self.assertEqual(sample(i), sample_expected(i)) - self.assertEqual(sample(i, i), sample_expected(i, i)) - - # invalid layout format, different kind of exceptions will be - # raised by internal functions - - # plain wrong format - self.assertRaises(ValueError, ttk._format_layoutlist, - ['bad', 'format']) - # will try to use iteritems in the 'bad' string - self.assertRaises(AttributeError, ttk._format_layoutlist, - [('name', 'bad')]) - # bad children formatting - self.assertRaises(ValueError, ttk._format_layoutlist, - [('name', {'children': {'a': None}})]) - - - def test_script_from_settings(self): - # empty options - self.assertFalse(ttk._script_from_settings({'name': - {'configure': None, 'map': None, 'element create': None}})) - - # empty layout - self.assertEqual( - ttk._script_from_settings({'name': {'layout': None}}), - "ttk::style layout name {\nnull\n}") - - configdict = {'αβγ': True, 'á': False} - self.assertTrue( - ttk._script_from_settings({'name': {'configure': configdict}})) - - mapdict = {'üñíćódè': [('á', 'vãl')]} - self.assertTrue( - ttk._script_from_settings({'name': {'map': mapdict}})) - - # invalid image element - self.assertRaises(IndexError, - ttk._script_from_settings, {'name': {'element create': ['image']}}) - - # minimal valid image - self.assertTrue(ttk._script_from_settings({'name': - {'element create': ['image', 'name']}})) - - image = {'thing': {'element create': - ['image', 'name', ('state1', 'state2', 'val')]}} - self.assertEqual(ttk._script_from_settings(image), - "ttk::style element create thing image {name {state1 state2} val} ") - - image['thing']['element create'].append({'opt': 30}) - self.assertEqual(ttk._script_from_settings(image), - "ttk::style element create thing image {name {state1 state2} val} " - "-opt 30") - - image['thing']['element create'][-1]['opt'] = [MockTclObj(3), - MockTclObj('2m')] - self.assertEqual(ttk._script_from_settings(image), - "ttk::style element create thing image {name {state1 state2} val} " - "-opt {3 2m}") - - - def test_tclobj_to_py(self): - self.assertEqual( - ttk._tclobj_to_py((MockStateSpec('a', 'b'), 'val')), - [('a', 'b', 'val')]) - self.assertEqual( - ttk._tclobj_to_py([MockTclObj('1'), 2, MockTclObj('3m')]), - [1, 2, '3m']) - - - def test_list_from_statespec(self): - def test_it(sspec, value, res_value, states): - self.assertEqual(ttk._list_from_statespec( - (sspec, value)), [states + (res_value, )]) - - states_even = tuple('state%d' % i for i in range(6)) - statespec = MockStateSpec(*states_even) - test_it(statespec, 'val', 'val', states_even) - test_it(statespec, MockTclObj('val'), 'val', states_even) - - states_odd = tuple('state%d' % i for i in range(5)) - statespec = MockStateSpec(*states_odd) - test_it(statespec, 'val', 'val', states_odd) - - test_it(('a', 'b', 'c'), MockTclObj('val'), 'val', ('a', 'b', 'c')) - - - def test_list_from_layouttuple(self): - tk = MockTkApp() - - # empty layout tuple - self.assertFalse(ttk._list_from_layouttuple(tk, ())) - - # shortest layout tuple - self.assertEqual(ttk._list_from_layouttuple(tk, ('name', )), - [('name', {})]) - - # not so interesting ltuple - sample_ltuple = ('name', '-option', 'value') - self.assertEqual(ttk._list_from_layouttuple(tk, sample_ltuple), - [('name', {'option': 'value'})]) - - # empty children - self.assertEqual(ttk._list_from_layouttuple(tk, - ('something', '-children', ())), - [('something', {'children': []})] - ) - - # more interesting ltuple - ltuple = ( - 'name', '-option', 'niceone', '-children', ( - ('otherone', '-children', ( - ('child', )), '-otheropt', 'othervalue' - ) - ) - ) - self.assertEqual(ttk._list_from_layouttuple(tk, ltuple), - [('name', {'option': 'niceone', 'children': - [('otherone', {'otheropt': 'othervalue', 'children': - [('child', {})] - })] - })] - ) - - # bad tuples - self.assertRaises(ValueError, ttk._list_from_layouttuple, tk, - ('name', 'no_minus')) - self.assertRaises(ValueError, ttk._list_from_layouttuple, tk, - ('name', 'no_minus', 'value')) - self.assertRaises(ValueError, ttk._list_from_layouttuple, tk, - ('something', '-children')) # no children - - - def test_val_or_dict(self): - def func(res, opt=None, val=None): - if opt is None: - return res - if val is None: - return "test val" - return (opt, val) - - tk = MockTkApp() - tk.call = func - - self.assertEqual(ttk._val_or_dict(tk, {}, '-test:3'), - {'test': '3'}) - self.assertEqual(ttk._val_or_dict(tk, {}, ('-test', 3)), - {'test': 3}) - - self.assertEqual(ttk._val_or_dict(tk, {'test': None}, 'x:y'), - 'test val') - - self.assertEqual(ttk._val_or_dict(tk, {'test': 3}, 'x:y'), - {'test': 3}) - - - def test_convert_stringval(self): - tests = ( - (0, 0), ('09', 9), ('a', 'a'), ('áÚ', 'áÚ'), ([], '[]'), - (None, 'None') - ) - for orig, expected in tests: - self.assertEqual(ttk._convert_stringval(orig), expected) - - -class TclObjsToPyTest(unittest.TestCase): - - def test_unicode(self): - adict = {'opt': 'välúè'} - self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': 'välúè'}) - - adict['opt'] = MockTclObj(adict['opt']) - self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': 'välúè'}) - - def test_multivalues(self): - adict = {'opt': [1, 2, 3, 4]} - self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': [1, 2, 3, 4]}) - - adict['opt'] = [1, 'xm', 3] - self.assertEqual(ttk.tclobjs_to_py(adict), {'opt': [1, 'xm', 3]}) - - adict['opt'] = (MockStateSpec('a', 'b'), 'válũè') - self.assertEqual(ttk.tclobjs_to_py(adict), - {'opt': [('a', 'b', 'válũè')]}) - - self.assertEqual(ttk.tclobjs_to_py({'x': ['y z']}), - {'x': ['y z']}) - - def test_nosplit(self): - self.assertEqual(ttk.tclobjs_to_py({'text': 'some text'}), - {'text': 'some text'}) - -tests_nogui = (InternalFunctionsTest, TclObjsToPyTest) - -if __name__ == "__main__": - from test.support import run_unittest - run_unittest(*tests_nogui) diff --git a/Lib/tkinter/test/test_ttk/test_style.py b/Lib/tkinter/test/test_ttk/test_style.py index 38d70d7a89077d..a33c24ac55beef 100644 --- a/Lib/tkinter/test/test_ttk/test_style.py +++ b/Lib/tkinter/test/test_ttk/test_style.py @@ -3,7 +3,7 @@ import tkinter from tkinter import ttk from test import support -from test.support import requires, run_unittest +from test.support import requires from tkinter.test.support import AbstractTkTest requires('gui') @@ -175,7 +175,5 @@ def test_map_custom_copy(self): self.assertEqual(style.map(newname, key), value) -tests_gui = (StyleTest, ) - if __name__ == "__main__": - run_unittest(*tests_gui) + unittest.main() diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index ee5af82fd1b448..935be3d7f1b5ae 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -4,7 +4,7 @@ from test.support import requires, gc_collect import sys -from tkinter.test.test_ttk.test_functions import MockTclObj +from test.test_ttk_textonly import MockTclObj from tkinter.test.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from tkinter.test.widget_tests import (add_standard_options, noconv, @@ -169,10 +169,13 @@ def checkImageParam(self, widget, name): errmsg='image "spam" doesn\'t exist') def test_configure_compound(self): + options = 'none text image center top bottom left right'.split() + errmsg = ( + 'bad compound "{}": must be' + f' {", ".join(options[:-1])}, or {options[-1]}' + ) widget = self.create() - self.checkEnumParam(widget, 'compound', - 'none', 'text', 'image', 'center', - 'top', 'bottom', 'left', 'right') + self.checkEnumParam(widget, 'compound', *options, errmsg=errmsg) def test_configure_state(self): widget = self.create() @@ -968,7 +971,7 @@ def test_add_and_hidden(self): tabs = self.nb.tabs() curr = self.nb.index('current') - # verify that the tab gets readded at its previous position + # verify that the tab gets re-added at its previous position child2_index = self.nb.index(self.child2) self.nb.hide(self.child2) self.nb.add(self.child2) diff --git a/Lib/traceback.py b/Lib/traceback.py index 1b537dc5a91142..568f3ff28c29b2 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -622,10 +622,14 @@ class TracebackException: occurred. - :attr:`lineno` For syntax errors - the linenumber where the error occurred. + - :attr:`end_lineno` For syntax errors - the end linenumber where the error + occurred. Can be `None` if not present. - :attr:`text` For syntax errors - the text where the error occurred. - :attr:`offset` For syntax errors - the offset into the text where the error occurred. + - :attr:`end_offset` For syntax errors - the offset into the text where the + error occurred. Can be `None` if not present. - :attr:`msg` For syntax errors - the compiler error message. """ @@ -655,8 +659,11 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.filename = exc_value.filename lno = exc_value.lineno self.lineno = str(lno) if lno is not None else None + end_lno = exc_value.end_lineno + self.end_lineno = str(end_lno) if end_lno is not None else None self.text = exc_value.text self.offset = exc_value.offset + self.end_offset = exc_value.end_offset self.msg = exc_value.msg if lookup_lines: self._load_lines() @@ -771,12 +778,20 @@ def _format_syntax_error(self, stype): ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) yield ' {}\n'.format(ltext) - # Convert 1-based column offset to 0-based index into stripped text - caret = (self.offset or 0) - 1 - spaces - if caret >= 0: - # non-space whitespace (likes tabs) must be kept for alignment - caretspace = ((c if c.isspace() else ' ') for c in ltext[:caret]) - yield ' {}^\n'.format(''.join(caretspace)) + + if self.offset is not None: + offset = self.offset + end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + if offset == end_offset or end_offset == -1: + end_offset = offset + 1 + + # Convert 1-based column offset to 0-based index into stripped text + colno = offset - 1 - spaces + end_colno = end_offset - 1 - spaces + if colno >= 0: + # non-space whitespace (likes tabs) must be kept for alignment + caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) + yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n")) msg = self.msg or "" yield "{}: {}{}\n".format(stype, msg, filename_suffix) diff --git a/Lib/typing.py b/Lib/typing.py index ada9adb0d32a58..78d973d2bba056 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -719,6 +719,12 @@ def __eq__(self, other): def __hash__(self): return hash(self.__forward_arg__) + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] + def __repr__(self): return f'ForwardRef({self.__forward_arg__!r})' diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index 3e864d14d112fa..d8bfaf6b67e004 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -53,7 +53,7 @@ def addAsyncCleanup(self, func, /, *args, **kwargs): # We intentionally don't add inspect.iscoroutinefunction() check # for func argument because there is no way # to check for async function reliably: - # 1. It can be "async def func()" iself + # 1. It can be "async def func()" itself # 2. Class can implement "async def __call__()" method # 3. Regular "def func()" that returns awaitable object self.addCleanup(*(func, *args), **kwargs) diff --git a/Lib/unittest/test/testmock/testsealable.py b/Lib/unittest/test/testmock/testsealable.py index 11784c3678918f..daba2b49b46f63 100644 --- a/Lib/unittest/test/testmock/testsealable.py +++ b/Lib/unittest/test/testmock/testsealable.py @@ -128,7 +128,7 @@ def test_integration_with_spec_att_definition(self): m.attr_sample2 def test_integration_with_spec_method_definition(self): - """You need to defin the methods, even if they are in the spec""" + """You need to define the methods, even if they are in the spec""" m = mock.Mock(SampleObject) m.method_sample1.return_value = 1 diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index eca6cc350161f2..fd6fc36aee04b3 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -11,8 +11,8 @@ Handlers needed to open the requested URL. For example, the HTTPHandler performs HTTP GET and POST requests and deals with non-error returns. The HTTPRedirectHandler automatically deals with -HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler -deals with digest authentication. +HTTP 301, 302, 303, 307, and 308 redirect errors, and the +HTTPDigestAuthHandler deals with digest authentication. urlopen(url, data=None) -- Basic usage is the same as original urllib. pass the url and optionally data to post to an HTTP URL, and @@ -661,7 +661,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): but another Handler might. """ m = req.get_method() - if (not (code in (301, 302, 303, 307) and m in ("GET", "HEAD") + if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD") or code in (301, 302, 303) and m == "POST")): raise HTTPError(req.full_url, code, msg, headers, fp) @@ -748,7 +748,7 @@ def http_error_302(self, req, fp, code, msg, headers): return self.parent.open(new, timeout=req.timeout) - http_error_301 = http_error_303 = http_error_307 = http_error_302 + http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302 inf_msg = "The HTTP server returned a redirect error that would " \ "lead to an infinite loop.\n" \ @@ -2211,6 +2211,13 @@ def http_error_307(self, url, fp, errcode, errmsg, headers, data=None): else: return self.http_error_default(url, fp, errcode, errmsg, headers) + def http_error_308(self, url, fp, errcode, errmsg, headers, data=None): + """Error 308 -- relocated, but turn POST into error.""" + if data is None: + return self.http_error_301(url, fp, errcode, errmsg, headers, data) + else: + return self.http_error_default(url, fp, errcode, errmsg, headers) + def http_error_401(self, url, fp, errcode, errmsg, headers, data=None, retry=False): """Error 401 -- authentication required. diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index b007e25f7ea2cc..6f1af294ae63e3 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -142,6 +142,20 @@ def create_if_needed(d): context.bin_name = binname context.env_exe = os.path.join(binpath, exename) create_if_needed(binpath) + # Assign and update the command to use when launching the newly created + # environment, in case it isn't simply the executable script (e.g. bpo-45337) + context.env_exec_cmd = context.env_exe + if sys.platform == 'win32': + # bpo-45337: Fix up env_exec_cmd to account for file system redirections. + # Some redirects only apply to CreateFile and not CreateProcess + real_env_exe = os.path.realpath(context.env_exe) + if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + logger.warning('Actual environment location may have moved due to ' + 'redirects, links or junctions.\n' + ' Requested location: "%s"\n' + ' Actual location: "%s"', + context.env_exe, real_env_exe) + context.env_exec_cmd = real_env_exe return context def create_configuration(self, context): @@ -294,8 +308,8 @@ def _setup_pip(self, context): # We run ensurepip in isolated mode to avoid side effects from # environment vars, the current directory and anything else # intended for the global Python environment - cmd = [context.env_exe, '-Im', 'ensurepip', '--upgrade', - '--default-pip'] + cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade', + '--default-pip'] subprocess.check_output(cmd, stderr=subprocess.STDOUT) def setup_scripts(self, context): @@ -395,11 +409,7 @@ def upgrade_dependencies(self, context): logger.debug( f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) - if sys.platform == 'win32': - python_exe = os.path.join(context.bin_path, 'python.exe') - else: - python_exe = os.path.join(context.bin_path, 'python') - cmd = [python_exe, '-m', 'pip', 'install', '--upgrade'] + cmd = [context.env_exec_cmd, '-m', 'pip', 'install', '--upgrade'] cmd.extend(CORE_VENV_DEPS) subprocess.check_call(cmd) diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1 index 51fc55c4ec7cb9..eeea3583fa130d 100644 --- a/Lib/venv/scripts/common/Activate.ps1 +++ b/Lib/venv/scripts/common/Activate.ps1 @@ -202,7 +202,7 @@ else { $Prompt = $pyvenvCfg['prompt']; } else { - Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virutal environment)" + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" $Prompt = Split-Path -Path $venvDir -Leaf } diff --git a/Lib/wsgiref/validate.py b/Lib/wsgiref/validate.py index 48ac0070549b3f..6e16578dbb648f 100644 --- a/Lib/wsgiref/validate.py +++ b/Lib/wsgiref/validate.py @@ -137,7 +137,7 @@ def validator(application): """ When applied between a WSGI server and a WSGI application, this - middleware will check for WSGI compliancy on a number of levels. + middleware will check for WSGI compliance on a number of levels. This middleware does not modify the request or response in any way, but will raise an AssertionError if anything seems off (except for a failure to close the application iterator, which diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index 9e7449c88dfc0d..a614cef6ab2f1a 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -264,16 +264,22 @@ def __repr__(self): # Issue #13305: different format codes across platforms _day0 = datetime(1, 1, 1) -if _day0.strftime('%Y') == '0001': # Mac OS X +def _try(fmt): + try: + return _day0.strftime(fmt) == '0001' + except ValueError: + return False +if _try('%Y'): # Mac OS X def _iso8601_format(value): return value.strftime("%Y%m%dT%H:%M:%S") -elif _day0.strftime('%4Y') == '0001': # Linux +elif _try('%4Y'): # Linux def _iso8601_format(value): return value.strftime("%4Y%m%dT%H:%M:%S") else: def _iso8601_format(value): return value.strftime("%Y%m%dT%H:%M:%S").zfill(17) del _day0 +del _try def _strftime(value): diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 69a260f5b12c07..e22e480a829ff1 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -440,7 +440,7 @@ class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): # Class attribute listing the accessible path components; # paths not on this list will result in a 404 error. - rpc_paths = ('/', '/RPC2') + rpc_paths = ('/', '/RPC2', '/pydoc.css') #if not None, encode responses larger than this, if possible encode_threshold = 1400 #a common MTU @@ -801,7 +801,7 @@ def docserver(self, server_name, package_documentation, methods): server_name = self.escape(server_name) head = '%s' % server_name - result = self.heading(head, '#ffffff', '#7799ee') + result = self.heading(head) doc = self.markup(package_documentation, self.preformat, fdict) doc = doc and '%s' % doc @@ -812,10 +812,25 @@ def docserver(self, server_name, package_documentation, methods): for key, value in method_items: contents.append(self.docroutine(value, key, funcs=fdict)) result = result + self.bigsection( - 'Methods', '#ffffff', '#eeaa77', ''.join(contents)) + 'Methods', 'functions', ''.join(contents)) return result + + def page(self, title, contents): + """Format an HTML page.""" + css_path = "/pydoc.css" + css_link = ( + '' % + css_path) + return '''\ + + + + +Python: %s +%s%s''' % (title, css_link, contents) + class XMLRPCDocGenerator: """Generates documentation for an XML-RPC server. @@ -907,6 +922,12 @@ class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): for documentation. """ + def _get_css(self, url): + path_here = os.path.dirname(os.path.realpath(__file__)) + css_path = os.path.join(path_here, "..", "pydoc_data", "_pydoc.css") + with open(css_path, mode="rb") as fp: + return fp.read() + def do_GET(self): """Handles the HTTP GET request. @@ -918,9 +939,15 @@ def do_GET(self): self.report_404() return - response = self.server.generate_html_documentation().encode('utf-8') + if self.path.endswith('.css'): + content_type = 'text/css' + response = self._get_css(self.path) + else: + content_type = 'text/html' + response = self.server.generate_html_documentation().encode('utf-8') + self.send_response(200) - self.send_header("Content-type", "text/html") + self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) self.send_header("Content-length", str(len(response))) self.end_headers() self.wfile.write(response) diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index 9810637d3ef65e..de68380792f17c 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -338,7 +338,7 @@ def _utcoff_to_dstoff(trans_idx, utcoffsets, isdsts): comp_idx = trans_idx[i + 1] # If the following transition is also DST and we couldn't - # find the DST offset by this point, we're going ot have to + # find the DST offset by this point, we're going to have to # skip it and hope this transition gets assigned later if isdsts[comp_idx]: continue diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 529060d8a1303b..a6d5c349c348b8 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -454,7 +454,7 @@ def pkg_recipes(): source="/pydocs", readme="""\ This package installs the python documentation at a location - that is useable for pydoc and IDLE. + that is usable for pydoc and IDLE. """, postflight="scripts/postflight.documentation", required=False, @@ -1602,7 +1602,7 @@ def buildDMG(): # instead of 11. We should not run into that situation here.) # Also we should use "macos" instead of "macosx" going forward. # - # To maintain compability for legacy variants, the file name for + # To maintain compatibility for legacy variants, the file name for # builds on macOS 10.15 and earlier remains: # python-3.x.y-macosx10.z.{dmg->pkg} # e.g. python-3.9.4-macosx10.9.{dmg->pkg} diff --git a/Mac/IDLE/IDLE.app/Contents/Resources/idlemain.py b/Mac/IDLE/IDLE.app/Contents/Resources/idlemain.py index 5994c18ff876b3..ccc18d4aee4386 100644 --- a/Mac/IDLE/IDLE.app/Contents/Resources/idlemain.py +++ b/Mac/IDLE/IDLE.app/Contents/Resources/idlemain.py @@ -35,7 +35,7 @@ # idlemain.py running under the symlinked python. # This is the magic step. # 4. During interpreter initialization, because PYTHONEXECUTABLE is defined, -# sys.executable may get set to an unuseful value. +# sys.executable may get set to an useless value. # # (Note that the IDLE script and the setting of PYTHONEXECUTABLE is # generated automatically by bundlebuilder in the Python 2.x build. diff --git a/Mac/PythonLauncher/MyAppDelegate.m b/Mac/PythonLauncher/MyAppDelegate.m index 25779a2540a37e..9cc2aa0ad90982 100644 --- a/Mac/PythonLauncher/MyAppDelegate.m +++ b/Mac/PythonLauncher/MyAppDelegate.m @@ -22,7 +22,7 @@ - (void)applicationDidFinishLaunching:(NSNotification *)notification { // Test that the file mappings are correct [self testFileTypeBinding]; - // If we were opened because of a file drag or doubleclick + // If we were opened because of a file drag or double-click // we've set initial_action_done in shouldShowUI // Otherwise we open a preferences dialog. if (!initial_action_done) { diff --git a/Mac/README.rst b/Mac/README.rst index 7dd41a3ee4725a..7476639d0ff541 100644 --- a/Mac/README.rst +++ b/Mac/README.rst @@ -296,7 +296,7 @@ How do I create a binary distribution? Download and unpack the source release from https://www.python.org/download/. Go to the directory ``Mac/BuildScript``. There you will find a script ``build-installer.py`` that does all the work. This will download and build -a number of 3rd-party libaries, configures and builds a framework Python, +a number of 3rd-party libraries, configures and builds a framework Python, installs it, creates the installer package files and then packs this in a DMG image. The script also builds an HTML copy of the current Python documentation set for this release for inclusion in the framework. The diff --git a/Makefile.pre.in b/Makefile.pre.in index 5564c1bae1ce2a..f03f535f6faa60 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -752,7 +752,12 @@ FROZEN_FILES_IN = \ Lib/os.py \ Lib/site.py \ Lib/stat.py \ - Lib/__hello__.py + Lib/__hello__.py \ + Lib/__phello__/__init__.py \ + Lib/__phello__/ham/__init__.py \ + Lib/__phello__/ham/eggs.py \ + Lib/__phello__/spam.py \ + Tools/freeze/flag.py # End FROZEN_FILES_IN FROZEN_FILES_OUT = \ Python/frozen_modules/importlib._bootstrap.h \ @@ -769,7 +774,12 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/os.h \ Python/frozen_modules/site.h \ Python/frozen_modules/stat.h \ - Python/frozen_modules/__hello__.h + Python/frozen_modules/__hello__.h \ + Python/frozen_modules/__phello__.h \ + Python/frozen_modules/__phello__.ham.h \ + Python/frozen_modules/__phello__.ham.eggs.h \ + Python/frozen_modules/__phello__.spam.h \ + Python/frozen_modules/frozen_only.h # End FROZEN_FILES_OUT Programs/_freeze_module.o: Programs/_freeze_module.c Makefile @@ -824,6 +834,21 @@ Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py Python/frozen_modules/__hello__.h: Programs/_freeze_module Lib/__hello__.py Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py $(srcdir)/Python/frozen_modules/__hello__.h +Python/frozen_modules/__phello__.h: Programs/_freeze_module Lib/__phello__/__init__.py + Programs/_freeze_module __phello__ $(srcdir)/Lib/__phello__/__init__.py $(srcdir)/Python/frozen_modules/__phello__.h + +Python/frozen_modules/__phello__.ham.h: Programs/_freeze_module Lib/__phello__/ham/__init__.py + Programs/_freeze_module __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py $(srcdir)/Python/frozen_modules/__phello__.ham.h + +Python/frozen_modules/__phello__.ham.eggs.h: Programs/_freeze_module Lib/__phello__/ham/eggs.py + Programs/_freeze_module __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h + +Python/frozen_modules/__phello__.spam.h: Programs/_freeze_module Lib/__phello__/spam.py + Programs/_freeze_module __phello__.spam $(srcdir)/Lib/__phello__/spam.py $(srcdir)/Python/frozen_modules/__phello__.spam.h + +Python/frozen_modules/frozen_only.h: Programs/_freeze_module Tools/freeze/flag.py + Programs/_freeze_module frozen_only $(srcdir)/Tools/freeze/flag.py $(srcdir)/Python/frozen_modules/frozen_only.h + # END: freezing modules Tools/scripts/freeze_modules.py: Programs/_freeze_module @@ -1102,39 +1127,30 @@ PYTHON_HEADERS= \ $(srcdir)/Include/boolobject.h \ $(srcdir)/Include/bytearrayobject.h \ $(srcdir)/Include/bytesobject.h \ - $(srcdir)/Include/cellobject.h \ $(srcdir)/Include/ceval.h \ - $(srcdir)/Include/classobject.h \ $(srcdir)/Include/code.h \ $(srcdir)/Include/codecs.h \ $(srcdir)/Include/compile.h \ $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/context.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ $(srcdir)/Include/dynamic_annotations.h \ $(srcdir)/Include/enumobject.h \ $(srcdir)/Include/errcode.h \ - $(srcdir)/Include/eval.h \ $(srcdir)/Include/fileobject.h \ $(srcdir)/Include/fileutils.h \ $(srcdir)/Include/floatobject.h \ $(srcdir)/Include/frameobject.h \ - $(srcdir)/Include/funcobject.h \ - $(srcdir)/Include/genobject.h \ $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/longintrepr.h \ $(srcdir)/Include/longobject.h \ $(srcdir)/Include/marshal.h \ $(srcdir)/Include/memoryobject.h \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/namespaceobject.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ @@ -1154,7 +1170,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pyport.h \ $(srcdir)/Include/pystate.h \ $(srcdir)/Include/pystrcmp.h \ - $(srcdir)/Include/pystrhex.h \ $(srcdir)/Include/pystrtod.h \ $(srcdir)/Include/pythonrun.h \ $(srcdir)/Include/pythread.h \ @@ -1178,17 +1193,24 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/abstract.h \ $(srcdir)/Include/cpython/bytearrayobject.h \ $(srcdir)/Include/cpython/bytesobject.h \ + $(srcdir)/Include/cpython/cellobject.h \ $(srcdir)/Include/cpython/ceval.h \ + $(srcdir)/Include/cpython/classobject.h \ $(srcdir)/Include/cpython/code.h \ $(srcdir)/Include/cpython/compile.h \ + $(srcdir)/Include/cpython/context.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ $(srcdir)/Include/cpython/fileutils.h \ + $(srcdir)/Include/cpython/floatobject.h \ $(srcdir)/Include/cpython/frameobject.h \ + $(srcdir)/Include/cpython/funcobject.h \ + $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ + $(srcdir)/Include/cpython/longintrepr.h \ + $(srcdir)/Include/cpython/longobject.h \ $(srcdir)/Include/cpython/methodobject.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ @@ -1207,6 +1229,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tupleobject.h \ $(srcdir)/Include/cpython/unicodeobject.h \ + $(srcdir)/Include/cpython/warnings.h \ + $(srcdir)/Include/cpython/weakrefobject.h \ \ $(srcdir)/Include/internal/pycore_abstract.h \ $(srcdir)/Include/internal/pycore_accu.h \ @@ -1226,6 +1250,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dtoa.h \ $(srcdir)/Include/internal/pycore_fileutils.h \ + $(srcdir)/Include/internal/pycore_floatobject.h \ $(srcdir)/Include/internal/pycore_format.h \ $(srcdir)/Include/internal/pycore_getopt.h \ $(srcdir)/Include/internal/pycore_gil.h \ @@ -1234,9 +1259,11 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_initconfig.h \ $(srcdir)/Include/internal/pycore_interp.h \ + $(srcdir)/Include/internal/pycore_interpreteridobject.h \ $(srcdir)/Include/internal/pycore_list.h \ $(srcdir)/Include/internal/pycore_long.h \ $(srcdir)/Include/internal/pycore_moduleobject.h \ + $(srcdir)/Include/internal/pycore_namespace.h \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ @@ -1246,6 +1273,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pymem.h \ $(srcdir)/Include/internal/pycore_pystate.h \ $(srcdir)/Include/internal/pycore_runtime.h \ + $(srcdir)/Include/internal/pycore_strhex.h \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ @@ -1320,7 +1348,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing_fork test_multiprocessing_spawn \ test_multiprocessing_forkserver \ - test_mailbox test_socket test_poll \ + test_mailbox test_nntplib test_socket test_poll \ test_select test_zipfile test_concurrent_futures quicktest: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(QUICKTESTOPTS) @@ -1418,13 +1446,13 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ fi; \ fi if test "x$(LIPO_32BIT_FLAGS)" != "x" ; then \ - rm -f $(DESTDIR)$(BINDIR)python$(VERSION)-32$(EXE); \ + rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-32$(EXE); \ lipo $(LIPO_32BIT_FLAGS) \ -output $(DESTDIR)$(BINDIR)/python$(VERSION)-32$(EXE) \ $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \ fi if test "x$(LIPO_INTEL64_FLAGS)" != "x" ; then \ - rm -f $(DESTDIR)$(BINDIR)python$(VERSION)-intel64$(EXE); \ + rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-intel64$(EXE); \ lipo $(LIPO_INTEL64_FLAGS) \ -output $(DESTDIR)$(BINDIR)/python$(VERSION)-intel64$(EXE) \ $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \ @@ -1519,7 +1547,8 @@ LIBSUBDIRS= asyncio \ wsgiref \ $(XMLLIBSUBDIRS) \ xmlrpc \ - zoneinfo + zoneinfo \ + __phello__ TESTSUBDIRS= ctypes/test \ distutils/tests \ idlelib/idle_test \ diff --git a/Misc/HISTORY b/Misc/HISTORY index 32d814c31bfd14..805acf4bebcad9 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -1225,7 +1225,7 @@ Library - Issue #22396: On 32-bit AIX platform, don't expose os.posix_fadvise() nor os.posix_fallocate() because their prototypes in system headers are wrong. -- Issue #22517: When a io.BufferedRWPair object is deallocated, clear its +- Issue #22517: When an io.BufferedRWPair object is deallocated, clear its weakrefs. - Issue #22448: Improve canceled timer handles cleanup to prevent diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index 044bd20594cc37..1c1c2d54e8c20a 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -178,7 +178,7 @@ Convert the :mod:`_sha256` extension module types to heap types. .. section: Core and Builtins Fix a possible stack overflow in the parser when parsing functions and -classes with a huge ammount of arguments. Patch by Pablo Galindo. +classes with a huge amount of arguments. Patch by Pablo Galindo. .. diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index 3cfef17160336b..61a291914f9333 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -258,7 +258,7 @@ The :func:`repr` of :mod:`typing` types containing :ref:`Generic Alias Types .. nonce: 6aDbty .. section: Library -``binhex.binhex()`` consisently writes macOS 9 line endings. +``binhex.binhex()`` consistently writes macOS 9 line endings. .. @@ -534,7 +534,7 @@ belong to. Patch by Batuhan Taskaya. .. nonce: LR4fnY .. section: Library -Handle exceptions caused by unparseable date headers when using email +Handle exceptions caused by unparsable date headers when using email "default" policy. Patch by Tim Bell, Georges Toth .. diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 0b76367f944451..4f182e8e3f1f0c 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -949,7 +949,7 @@ branches, are now handled by docsbuild-script. .. nonce: W_updK .. section: Tests -Reenable test_gdb on gdb 9.2 and newer: +Re-enable test_gdb on gdb 9.2 and newer: https://bugzilla.redhat.com/show_bug.cgi?id=1866884 bug is fixed in gdb 10.1. diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 57da9254587b44..beac530fcb28f6 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -767,7 +767,7 @@ results. Patch by Ammar Askar. .. nonce: -7-XGz .. section: Tests -Update test_nntplib to use offical group name of news.aioe.org for testing. +Update test_nntplib to use official group name of news.aioe.org for testing. Patch by Dong-hee Na. .. @@ -786,7 +786,7 @@ Skip some asyncio tests on VxWorks. .. nonce: uzwlF_ .. section: Tests -Enhance ``test_select.test_select()``: it now takes 500 ms rather than 10 +Enhance ``test_select.test_select()``: it now takes 500 milliseconds rather than 10 seconds. Use Python rather than a shell to make the test more portable. .. @@ -863,7 +863,7 @@ and Peixing Xin. Now all platforms use a value for the "EXT_SUFFIX" build variable derived from SOABI (for instance in freeBSD, "EXT_SUFFIX" is now ".cpython-310d.so" -instead of ".so"). Previosuly only Linux, Mac and VxWorks were using a value +instead of ".so"). Previously only Linux, Mac and VxWorks were using a value for "EXT_SUFFIX" that included "SOABI". .. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index 989edb0b47e97f..1c7c7447cae065 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -549,7 +549,7 @@ Pass ``--timeout=$(TESTTIMEOUT)`` option to the default profile task .. section: Build Removed the grep -q and -E flags in the tzpath validation section of the -configure script to better accomodate users of some platforms (specifically +configure script to better accommodate users of some platforms (specifically Solaris 10). .. diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index 46d06add9115f7..a4ee9ae098bd93 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -49,7 +49,7 @@ The :data:`types.FunctionType` constructor now inherits the current builtins if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` -in Python is not affected, globals cannot be overriden with this syntax: it +in Python is not affected, globals cannot be overridden with this syntax: it also inherits the current builtins. Patch by Victor Stinner. .. @@ -355,7 +355,7 @@ in 4.0" to "3.12". See :pep:`623` for detail. .. nonce: LfTvL- .. section: Tests -Fix test_importlib to correctly skip Unicode file tests if the fileystem +Fix test_importlib to correctly skip Unicode file tests if the filesystem does not support them. .. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index a20072b7ae11e0..aa332631292a7e 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -172,7 +172,7 @@ regression introduced in python3.7 .. nonce: Ns3a_F .. section: Core and Builtins -Tracing now has correct line numbers for attribute accesses when the the +Tracing now has correct line numbers for attribute accesses when the attribute is on a different line from the object. Improves debugging and profiling for multi-line method chains. @@ -195,7 +195,7 @@ decoded as Unicode characters outside the [U+0000; U+10ffff] range. .. nonce: lCzIg0 .. section: Core and Builtins -Fix a bug that was causing the parser to crash when emiting syntax errors +Fix a bug that was causing the parser to crash when emitting syntax errors when reading input from stdin. Patch by Pablo Galindo .. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 25c3943abe50e5..e4391a1ee38709 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -276,7 +276,7 @@ cause any runtime effects with ``from __future__ import annotations``. .. nonce: 0Ik1AM .. section: Core and Builtins -:exc:`SyntaxError` exceptions raised by the intepreter will highlight the +:exc:`SyntaxError` exceptions raised by the interpreter will highlight the full error range of the expression that consistutes the syntax error itself, instead of just where the problem is detected. Patch by Pablo Galindo. @@ -547,7 +547,7 @@ enum or one of its members' value. .. nonce: ejjsyR .. section: Library -For backwards compatbility with previous minor versions of Python, if +For backwards compatibility with previous minor versions of Python, if :func:`typing.get_type_hints` receives no namespace dictionary arguments, :func:`typing.get_type_hints` will search through the global then local namespaces during evaluation of stringized type annotations (string forward @@ -720,9 +720,9 @@ now raise ``TypeError`` during substitution. .. nonce: xT9QjF .. section: Library -The :mod:`multiprocessing` ``Server`` class now explicitly catchs +The :mod:`multiprocessing` ``Server`` class now explicitly catches :exc:`SystemExit` and closes the client connection in this case. It happens -when the ``Server.serve_client()`` method reachs the end of file (EOF). +when the ``Server.serve_client()`` method reaches the end of file (EOF). .. @@ -1132,7 +1132,7 @@ preferred "user", "home", and "prefix" (default) scheme names. .. section: Library Improve :meth:`sqlite3.Connection.backup` error handling. The error message -for non-existant target database names is now ``unknown database `` instead of ``SQL logic error``. Patch by Erlend E. Aasland. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst new file mode 100644 index 00000000000000..28b81b75266fc2 --- /dev/null +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -0,0 +1,5098 @@ +.. bpo: 42278 +.. date: 2021-08-29-12-39-44 +.. nonce: jvmQz_ +.. release date: 2021-10-05 +.. section: Security + +Replaced usage of :func:`tempfile.mktemp` with +:class:`~tempfile.TemporaryDirectory` to avoid a potential race condition. + +.. + +.. bpo: 44600 +.. date: 2021-07-25-20-04-54 +.. nonce: 0WMldg +.. section: Security + +Fix incorrect line numbers while tracing some failed patterns in :ref:`match +` statements. Patch by Charles Burkland. + +.. + +.. bpo: 41180 +.. date: 2021-06-29-23-40-22 +.. nonce: uTWHv_ +.. section: Security + +Add auditing events to the :mod:`marshal` module, and stop raising +``code.__init__`` events for every unmarshalled code object. Directly +instantiated code objects will continue to raise an event, and audit event +handlers should inspect or collect the raw marshal data. This reduces a +significant performance overhead when loading from ``.pyc`` files. + +.. + +.. bpo: 44394 +.. date: 2021-06-29-02-45-53 +.. nonce: A220N1 +.. section: Security + +Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix +for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used +on Windows and macOS. + +.. + +.. bpo: 43124 +.. date: 2021-05-08-11-50-46 +.. nonce: 2CTM6M +.. section: Security + +Made the internal ``putcmd`` function in :mod:`smtplib` sanitize input for +presence of ``\r`` and ``\n`` characters to avoid (unlikely) command +injection. + +.. + +.. bpo: 44022 +.. date: 2021-05-05-17-37-04 +.. nonce: bS3XJ9 +.. section: Security + +:mod:`http.client` now avoids infinitely reading potential HTTP headers +after a ``100 Continue`` status response from the server. + +.. + +.. bpo: 43760 +.. date: 2021-10-04-16-11-50 +.. nonce: R9QoUv +.. section: Core and Builtins + +The number of hardware branches per instruction dispatch is reduced from two +to one by adding a special instruction for tracing. Patch by Mark Shannon. + +.. + +.. bpo: 45061 +.. date: 2021-09-21-22-27-25 +.. nonce: 5IOUf0 +.. section: Core and Builtins + +Add a deallocator to the bool type to detect refcount bugs in C extensions +which call Py_DECREF(Py_True) or Py_DECREF(Py_False) by mistake. Detect also +refcount bugs when the empty tuple singleton or the Unicode empty string +singleton is destroyed by mistake. Patch by Victor Stinner. + +.. + +.. bpo: 24076 +.. date: 2021-09-20-10-02-12 +.. nonce: ZFgFSj +.. section: Core and Builtins + +sum() was further optimised for summing up single digit integers. + +.. + +.. bpo: 45190 +.. date: 2021-09-14-10-02-12 +.. nonce: ZFRgSj +.. section: Core and Builtins + +Update Unicode databases to Unicode 14.0.0. + +.. + +.. bpo: 45167 +.. date: 2021-09-14-09-23-59 +.. nonce: CPSSoV +.. section: Core and Builtins + +Fix deepcopying of :class:`types.GenericAlias` objects. + +.. + +.. bpo: 45155 +.. date: 2021-09-09-15-05-17 +.. nonce: JRw9TG +.. section: Core and Builtins + +:meth:`int.to_bytes` and :meth:`int.from_bytes` now take a default value of +``"big"`` for the ``byteorder`` argument. :meth:`int.to_bytes` also takes a +default value of ``1`` for the ``length`` argument. + +.. + +.. bpo: 44219 +.. date: 2021-09-09-10-32-33 +.. nonce: WiYyjz +.. section: Core and Builtins + +Release the GIL while performing ``isatty`` system calls on arbitrary file +descriptors. In particular, this affects :func:`os.isatty`, +:func:`os.device_encoding` and :class:`io.TextIOWrapper`. By extension, +:func:`io.open` in text mode is also affected. This change solves a deadlock +in :func:`os.isatty`. Patch by Vincent Michel in :issue:`44219`. + +.. + +.. bpo: 44959 +.. date: 2021-09-08-08-29-41 +.. nonce: OSwwPf +.. section: Core and Builtins + +Added fallback to extension modules with '.sl' suffix on HP-UX + +.. + +.. bpo: 45121 +.. date: 2021-09-07-17-10-16 +.. nonce: iG-Hsf +.. section: Core and Builtins + +Fix issue where ``Protocol.__init__`` raises ``RecursionError`` when it's +called directly or via ``super()``. Patch provided by Yurii Karabas. + +.. + +.. bpo: 44348 +.. date: 2021-09-07-00-21-04 +.. nonce: f8w_Td +.. section: Core and Builtins + +The deallocator function of the :exc:`BaseException` type now uses the +trashcan mechanism to prevent stack overflow. For example, when a +:exc:`RecursionError` instance is raised, it can be linked to another +RecursionError through the ``__context__`` attribute or the +``__traceback__`` attribute, and then a chain of exceptions is created. When +the chain is destroyed, nested deallocator function calls can crash with a +stack overflow if the chain is too long compared to the available stack +memory. Patch by Victor Stinner. + +.. + +.. bpo: 45123 +.. date: 2021-09-06-21-52-45 +.. nonce: 8Eh9iI +.. section: Core and Builtins + +Fix PyAiter_Check to only check for the __anext__ presence (not for +__aiter__). Rename PyAiter_Check to PyAIter_Check, PyObject_GetAiter -> +PyObject_GetAIter. + +.. + +.. bpo: 1514420 +.. date: 2021-09-03-16-18-10 +.. nonce: 2Lumpj +.. section: Core and Builtins + +Interpreter no longer attempts to open files with names in angle brackets +(like "" or "") when formatting an exception. + +.. + +.. bpo: 41031 +.. date: 2021-09-03-12-35-17 +.. nonce: yPSJEs +.. section: Core and Builtins + +Match C and Python code formatting of unprintable exceptions and exceptions +in the :mod:`__main__` module. + +.. + +.. bpo: 37330 +.. date: 2021-09-02-01-28-01 +.. nonce: QDjM_l +.. section: Core and Builtins + +:func:`open`, :func:`io.open`, :func:`codecs.open` and +:class:`fileinput.FileInput` no longer accept ``'U'`` ("universal newline") +in the file mode. This flag was deprecated since Python 3.3. Patch by Victor +Stinner. + +.. + +.. bpo: 45083 +.. date: 2021-09-01-23-55-49 +.. nonce: cLi9G3 +.. section: Core and Builtins + +When the interpreter renders an exception, its name now has a complete +qualname. Previously only the class name was concatenated to the module +name, which sometimes resulted in an incorrect full name being displayed. + +(This issue impacted only the C code exception rendering, the +:mod:`traceback` module was using qualname already). + +.. + +.. bpo: 34561 +.. date: 2021-09-01-19-21-48 +.. nonce: uMAVA- +.. section: Core and Builtins + +List sorting now uses the merge-ordering strategy from Munro and Wild's +``powersort()``. Unlike the former strategy, this is provably near-optimal +in the entropy of the distribution of run lengths. Most uses of +``list.sort()`` probably won't see a significant time difference, but may +see significant improvements in cases where the former strategy was +exceptionally poor. However, as these are all fast linear-time +approximations to a problem that's inherently at best quadratic-time to +solve truly optimally, it's also possible to contrive cases where the former +strategy did better. + +.. + +.. bpo: 45056 +.. date: 2021-09-01-16-55-43 +.. nonce: 7AK2d9 +.. section: Core and Builtins + +Compiler now removes trailing unused constants from co_consts. + +.. + +.. bpo: 45020 +.. date: 2021-08-31-17-44-51 +.. nonce: ZPI_3L +.. section: Core and Builtins + +Add a new command line option, "-X frozen_modules=[on|off]" to opt out of +(or into) using optional frozen modules. This defaults to "on" (or "off" if +it's running out of the source tree). + +.. + +.. bpo: 45012 +.. date: 2021-08-31-11-09-52 +.. nonce: ueeOcx +.. section: Core and Builtins + +In :mod:`posix`, release GIL during ``stat()``, ``lstat()``, and +``fstatat()`` syscalls made by :func:`os.DirEntry.stat`. Patch by Stanisław +Skonieczny. + +.. + +.. bpo: 45018 +.. date: 2021-08-26-18-44-03 +.. nonce: pu8H9L +.. section: Core and Builtins + +Fixed pickling of range iterators that iterated for over ``2**32`` times. + +.. + +.. bpo: 45000 +.. date: 2021-08-25-23-17-32 +.. nonce: XjmyLl +.. section: Core and Builtins + +A :exc:`SyntaxError` is now raised when trying to delete :const:`__debug__`. +Patch by Dong-hee Na. + +.. + +.. bpo: 44963 +.. date: 2021-08-25-23-07-10 +.. nonce: 5EET8y +.. section: Core and Builtins + +Implement ``send()`` and ``throw()`` methods for ``anext_awaitable`` +objects. Patch by Pablo Galindo. + +.. + +.. bpo: 44962 +.. date: 2021-08-23-19-55-08 +.. nonce: J00ftt +.. section: Core and Builtins + +Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two +threads attempt to commit the last pending removal. This fixes +asyncio.create_task and fixes a data loss in asyncio.run where +shutdown_asyncgens is not run + +.. + +.. bpo: 24234 +.. date: 2021-08-23-10-36-55 +.. nonce: MGVUQi +.. section: Core and Builtins + +Implement the :meth:`__bytes__` special method on the :class:`bytes` type, +so a bytes object ``b`` passes an ``isinstance(b, typing.SupportsBytes)`` +check. + +.. + +.. bpo: 24234 +.. date: 2021-08-22-12-28-50 +.. nonce: n3oTdx +.. section: Core and Builtins + +Implement the :meth:`__complex__` special method on the :class:`complex` +type, so a complex number ``z`` passes an ``isinstance(z, +typing.SupportsComplex)`` check. + +.. + +.. bpo: 44954 +.. date: 2021-08-19-14-43-24 +.. nonce: dLn3lg +.. section: Core and Builtins + +Fixed a corner case bug where the result of ``float.fromhex('0x.8p-1074')`` +was rounded the wrong way. + +.. + +.. bpo: 44947 +.. date: 2021-08-18-19-09-28 +.. nonce: mcvGdS +.. section: Core and Builtins + +Refine the syntax error for trailing commas in import statements. Patch by +Pablo Galindo. + +.. + +.. bpo: 44945 +.. date: 2021-08-18-11-14-38 +.. nonce: CO3s77 +.. section: Core and Builtins + +Specialize the BINARY_ADD instruction using the PEP 659 machinery. Adds five +new instructions: + +* BINARY_ADD_ADAPTIVE +* BINARY_ADD_FLOAT +* BINARY_ADD_INT +* BINARY_ADD_UNICODE +* BINARY_ADD_UNICODE_INPLACE_FAST + +.. + +.. bpo: 44929 +.. date: 2021-08-16-23-16-17 +.. nonce: qpMEky +.. section: Core and Builtins + +Fix some edge cases of ``enum.Flag`` string representation in the REPL. +Patch by Pablo Galindo. + +.. + +.. bpo: 44914 +.. date: 2021-08-16-11-36-02 +.. nonce: 6Lgrx3 +.. section: Core and Builtins + +Class version tags are no longer recycled. + +This means that a version tag serves as a unique identifier for the state of +a class. We rely on this for effective specialization of the LOAD_ATTR and +other instructions. + +.. + +.. bpo: 44698 +.. date: 2021-08-15-10-39-06 +.. nonce: lITKNc +.. section: Core and Builtins + +Restore behaviour of complex exponentiation with integer-valued exponent of +type :class:`float` or :class:`complex`. + +.. + +.. bpo: 44895 +.. date: 2021-08-14-20-13-21 +.. nonce: Ic9m90 +.. section: Core and Builtins + +A debug variable :envvar:`PYTHONDUMPREFSFILE` is added for creating a dump +file which is generated by :option:`--with-trace-refs`. Patch by Dong-hee +Na. + +.. + +.. bpo: 44900 +.. date: 2021-08-12-14-00-57 +.. nonce: w2gpwy +.. section: Core and Builtins + +Add five superinstructions for PEP 659 quickening: + +* LOAD_FAST LOAD_FAST +* STORE_FAST LOAD_FAST +* LOAD_FAST LOAD_CONST +* LOAD_CONST LOAD_FAST +* STORE_FAST STORE_FAST + +.. + +.. bpo: 44889 +.. date: 2021-08-11-20-45-02 +.. nonce: 2T3nTn +.. section: Core and Builtins + +Initial implementation of adaptive specialization of ``LOAD_METHOD``. The +following specialized forms were added: + +* ``LOAD_METHOD_CACHED`` + +* ``LOAD_METHOD_MODULE`` + +* ``LOAD_METHOD_CLASS`` + +.. + +.. bpo: 44890 +.. date: 2021-08-11-16-46-27 +.. nonce: PwNg8N +.. section: Core and Builtins + +Specialization stats are always collected in debug builds. + +.. + +.. bpo: 44885 +.. date: 2021-08-11-15-39-57 +.. nonce: i4noUO +.. section: Core and Builtins + +Correct the ast locations of f-strings with format specs and repeated +expressions. Patch by Pablo Galindo + +.. + +.. bpo: 44878 +.. date: 2021-08-11-14-12-41 +.. nonce: pAbBfc +.. section: Core and Builtins + +Remove the loop from the bytecode interpreter. All instructions end with a +DISPATCH macro, so the loop is now redundant. + +.. + +.. bpo: 44878 +.. date: 2021-08-11-12-03-52 +.. nonce: nEhjLi +.. section: Core and Builtins + +Remove switch statement for interpreter loop when using computed gotos. This +makes sure that we only have one dispatch table in the interpreter. + +.. + +.. bpo: 44874 +.. date: 2021-08-09-19-05-20 +.. nonce: oOcfU4 +.. section: Core and Builtins + +Deprecate the old trashcan macros +(``Py_TRASHCAN_SAFE_BEGIN``/``Py_TRASHCAN_SAFE_END``). They should be +replaced by the new macros ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. + +.. + +.. bpo: 44872 +.. date: 2021-08-09-16-16-03 +.. nonce: OKRlhK +.. section: Core and Builtins + +Use new trashcan macros (Py_TRASHCAN_BEGIN/END) in frameobject.c instead of +the old ones (Py_TRASHCAN_SAFE_BEGIN/END). + +.. + +.. bpo: 33930 +.. date: 2021-08-09-14-29-52 +.. nonce: --5LQ- +.. section: Core and Builtins + +Fix segmentation fault with deep recursion when cleaning method objects. +Patch by Augusto Goulart and Pablo Galindo. + +.. + +.. bpo: 25782 +.. date: 2021-08-07-21-39-19 +.. nonce: B22lMx +.. section: Core and Builtins + +Fix bug where ``PyErr_SetObject`` hangs when the current exception has a +cycle in its context chain. + +.. + +.. bpo: 44856 +.. date: 2021-08-07-01-26-12 +.. nonce: 9rk3li +.. section: Core and Builtins + +Fix reference leaks in the error paths of ``update_bases()`` and +``__build_class__``. Patch by Pablo Galindo. + +.. + +.. bpo: 44826 +.. date: 2021-08-05-17-49-55 +.. nonce: zQsyK5 +.. section: Core and Builtins + +Initial implementation of adaptive specialization of STORE_ATTR + +Three specialized forms of STORE_ATTR are added: + +* STORE_ATTR_SLOT + +* STORE_ATTR_SPLIT_KEYS + +* STORE_ATTR_WITH_HINT + +.. + +.. bpo: 44838 +.. date: 2021-08-05-17-42-03 +.. nonce: r_Lkj_ +.. section: Core and Builtins + +Fixed a bug that was causing the parser to raise an incorrect custom +:exc:`SyntaxError` for invalid 'if' expressions. Patch by Pablo Galindo. + +.. + +.. bpo: 44821 +.. date: 2021-08-04-11-37-38 +.. nonce: 67YHGI +.. section: Core and Builtins + +Create instance dictionaries (__dict__) eagerly, to improve regularity of +object layout and assist specialization. + +.. + +.. bpo: 44792 +.. date: 2021-07-31-12-12-57 +.. nonce: mOReTW +.. section: Core and Builtins + +Improve syntax errors for if expressions. Patch by Miguel Brito + +.. + +.. bpo: 34013 +.. date: 2021-07-27-11-14-22 +.. nonce: SjLFe- +.. section: Core and Builtins + +Generalize the invalid legacy statement custom error message (like the one +generated when "print" is called without parentheses) to include more +generic expressions. Patch by Pablo Galindo + +.. + +.. bpo: 44732 +.. date: 2021-07-26-15-27-03 +.. nonce: IxObt3 +.. section: Core and Builtins + +Rename ``types.Union`` to ``types.UnionType``. + +.. + +.. bpo: 44725 +.. date: 2021-07-23-15-17-01 +.. nonce: qcuKaa +.. section: Core and Builtins + +Expose specialization stats in python via +:func:`_opcode.get_specialization_stats`. + +.. + +.. bpo: 44717 +.. date: 2021-07-23-01-52-13 +.. nonce: -vVmAh +.. section: Core and Builtins + +Improve AttributeError on circular imports of submodules. + +.. + +.. bpo: 44698 +.. date: 2021-07-21-15-26-56 +.. nonce: DA4_0o +.. section: Core and Builtins + +Fix undefined behaviour in complex object exponentiation. + +.. + +.. bpo: 44653 +.. date: 2021-07-19-20-49-06 +.. nonce: WcqGyI +.. section: Core and Builtins + +Support :mod:`typing` types in parameter substitution in the union type. + +.. + +.. bpo: 44676 +.. date: 2021-07-19-19-53-46 +.. nonce: WgIMvh +.. section: Core and Builtins + +Add ability to serialise ``types.Union`` objects. Patch provided by Yurii +Karabas. + +.. + +.. bpo: 44633 +.. date: 2021-07-17-21-04-04 +.. nonce: 5-zKeI +.. section: Core and Builtins + +Parameter substitution of the union type with wrong types now raises +``TypeError`` instead of returning ``NotImplemented``. + +.. + +.. bpo: 44661 +.. date: 2021-07-17-14-20-59 +.. nonce: BQbXiH +.. section: Core and Builtins + +Update ``property_descr_set`` to use vectorcall if possible. Patch by +Dong-hee Na. + +.. + +.. bpo: 44662 +.. date: 2021-07-17-13-41-58 +.. nonce: q22kWR +.. section: Core and Builtins + +Add ``__module__`` to ``types.Union``. This also fixes ``types.Union`` +issues with ``typing.Annotated``. Patch provided by Yurii Karabas. + +.. + +.. bpo: 44655 +.. date: 2021-07-16-21-35-14 +.. nonce: 95I7M6 +.. section: Core and Builtins + +Include the name of the type in unset __slots__ attribute errors. Patch by +Pablo Galindo + +.. + +.. bpo: 44655 +.. date: 2021-07-16-20-25-37 +.. nonce: I3wRjL +.. section: Core and Builtins + +Don't include a missing attribute with the same name as the failing one when +offering suggestions for missing attributes. Patch by Pablo Galindo + +.. + +.. bpo: 44646 +.. date: 2021-07-16-09-59-13 +.. nonce: Yb6s05 +.. section: Core and Builtins + +Fix the hash of the union type: it no longer depends on the order of +arguments. + +.. + +.. bpo: 44636 +.. date: 2021-07-16-09-36-12 +.. nonce: ZWebi8 +.. section: Core and Builtins + +Collapse union of equal types. E.g. the result of ``int | int`` is now +``int``. Fix comparison of the union type with non-hashable objects. E.g. +``int | str == {}`` no longer raises a TypeError. + +.. + +.. bpo: 44611 +.. date: 2021-07-16-01-01-11 +.. nonce: LcfHN- +.. section: Core and Builtins + +On Windows, :func:`os.urandom`: uses BCryptGenRandom API instead of +CryptGenRandom API which is deprecated from Microsoft Windows API. Patch by +Dong-hee Na. + +.. + +.. bpo: 44635 +.. date: 2021-07-14-13-54-07 +.. nonce: 7ZMAdB +.. section: Core and Builtins + +Convert ``None`` to ``type(None)`` in the union type constructor. + +.. + +.. bpo: 26280 +.. date: 2021-07-14-10-31-10 +.. nonce: cgpM4B +.. section: Core and Builtins + +Implement adaptive specialization for BINARY_SUBSCR + +Three specialized forms of BINARY_SUBSCR are added: + +* BINARY_SUBSCR_LIST_INT + +* BINARY_SUBSCR_TUPLE_INT + +* BINARY_SUBSCR_DICT + +.. + +.. bpo: 44589 +.. date: 2021-07-13-23-19-41 +.. nonce: 59OH8T +.. section: Core and Builtins + +Mapping patterns in ``match`` statements with two or more equal literal keys +will now raise a :exc:`SyntaxError` at compile-time. + +.. + +.. bpo: 44606 +.. date: 2021-07-13-20-22-12 +.. nonce: S3Bv2w +.. section: Core and Builtins + +Fix ``__instancecheck__`` and ``__subclasscheck__`` for the union type. + +.. + +.. bpo: 42073 +.. date: 2021-07-13-17-47-32 +.. nonce: 9wopiC +.. section: Core and Builtins + +The ``@classmethod`` decorator can now wrap other classmethod-like +descriptors. + +.. + +.. bpo: 41972 +.. date: 2021-07-12-04-06-57 +.. nonce: nDX5k_ +.. section: Core and Builtins + +Tuned the string-searching algorithm of fastsearch.h to have a shorter inner +loop for most cases. + +.. + +.. bpo: 44590 +.. date: 2021-07-09-12-08-17 +.. nonce: a2ntVX +.. section: Core and Builtins + +All necessary data for executing a Python function (local variables, stack, +etc) is now kept in a per-thread stack. Frame objects are lazily allocated +on demand. This increases performance by about 7% on the standard benchmark +suite. Introspection and debugging are unaffected as frame objects are +always available when needed. Patch by Mark Shannon. + +.. + +.. bpo: 44584 +.. date: 2021-07-08-12-18-56 +.. nonce: qKnSqV +.. section: Core and Builtins + +The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is +deprecated in Python 3.10 and will be removed in Python 3.12. This feature +requires a debug build of Python. Patch by Victor Stinner. + +.. + +.. bpo: 43895 +.. date: 2021-07-07-16-05-35 +.. nonce: JFjR0- +.. section: Core and Builtins + +An obsolete internal cache of shared object file handles added in 1995 that +attempted, but did not guarantee, that a .so would not be dlopen'ed twice to +work around flaws in mid-1990s posix-ish operating systems has been removed +from dynload_shlib.c. + +.. + +.. bpo: 44490 +.. date: 2021-07-06-22-22-15 +.. nonce: BJxPbZ +.. section: Core and Builtins + +:mod:`typing` now searches for type parameters in ``types.Union`` objects. +``get_type_hints`` will also properly resolve annotations with nested +``types.Union`` objects. Patch provided by Yurii Karabas. + +.. + +.. bpo: 43950 +.. date: 2021-07-06-15-27-11 +.. nonce: LhL2-q +.. section: Core and Builtins + +Code objects can now provide the column information for instructions when +available. This is levaraged during traceback printing to show the +expressions responsible for errors. + +Contributed by Pablo Galindo, Batuhan Taskaya and Ammar Askar as part of +:pep:`657`. + +.. + +.. bpo: 44562 +.. date: 2021-07-04-23-38-51 +.. nonce: QdeRQo +.. section: Core and Builtins + +Remove uses of :c:func:`PyObject_GC_Del` in error path when initializing +:class:`types.GenericAlias`. + +.. + +.. bpo: 41486 +.. date: 2021-07-04-17-41-47 +.. nonce: DiM24a +.. section: Core and Builtins + +Fix a memory consumption and copying performance regression in earlier 3.10 +beta releases if someone used an output buffer larger than 4GiB with +zlib.decompress on input data that expands that large. + +.. + +.. bpo: 43908 +.. date: 2021-07-03-00-20-39 +.. nonce: YHuV_s +.. section: Core and Builtins + +Heap types with the :const:`Py_TPFLAGS_IMMUTABLETYPE` flag can now inherit +the :pep:`590` vectorcall protocol. Previously, this was only possible for +:ref:`static types `. Patch by Erlend E. Aasland. + +.. + +.. bpo: 44553 +.. date: 2021-07-02-22-54-41 +.. nonce: l9YqGg +.. section: Core and Builtins + +Implement GC methods for ``types.Union`` to break reference cycles and +prevent memory leaks. + +.. + +.. bpo: 44490 +.. date: 2021-07-01-11-59-34 +.. nonce: xY80VR +.. section: Core and Builtins + +Add ``__parameters__`` attribute and ``__getitem__`` operator to +``types.Union``. Patch provided by Yurii Karabas. + +.. + +.. bpo: 44523 +.. date: 2021-06-29-11-49-29 +.. nonce: 67-ZIP +.. section: Core and Builtins + +Remove the pass-through for :func:`hash` of :class:`weakref.proxy` objects +to prevent unintended consequences when the original referred object dies +while the proxy is part of a hashable object. Patch by Pablo Galindo. + +.. + +.. bpo: 44483 +.. date: 2021-06-22-19-08-19 +.. nonce: eq2f7T +.. section: Core and Builtins + +Fix a crash in ``types.Union`` objects when creating a union of an object +with bad ``__module__`` field. + +.. + +.. bpo: 44486 +.. date: 2021-06-22-10-55-23 +.. nonce: wct-9X +.. section: Core and Builtins + +Modules will always have a dictionary, even when created by +``types.ModuleType.__new__()`` + +.. + +.. bpo: 44472 +.. date: 2021-06-21-11-19-54 +.. nonce: Vvm1yn +.. section: Core and Builtins + +Fix ltrace functionality when exceptions are raised. Patch by Pablo Galindo + +.. + +.. bpo: 12022 +.. date: 2021-06-20-10-53-21 +.. nonce: SW240M +.. section: Core and Builtins + +A :exc:`TypeError` is now raised instead of an :exc:`AttributeError` in +:keyword:`with` and :keyword:`async with` statements for objects which do +not support the :term:`context manager` or :term:`asynchronous context +manager` protocols correspondingly. + +.. + +.. bpo: 44297 +.. date: 2021-06-19-12-41-13 +.. nonce: F53vHj +.. section: Core and Builtins + +Make sure that the line number is set when entering a comprehension scope. +Ensures that backtraces inclusing generator expressions show the correct +line number. + +.. + +.. bpo: 44456 +.. date: 2021-06-18-22-08-25 +.. nonce: L0Rhko +.. section: Core and Builtins + +Improve the syntax error when mixing positional and keyword patterns. Patch +by Pablo Galindo. + +.. + +.. bpo: 44409 +.. date: 2021-06-13-23-12-18 +.. nonce: eW4LS- +.. section: Core and Builtins + +Fix error location information for tokenizer errors raised on initialization +of the tokenizer. Patch by Pablo Galindo. + +.. + +.. bpo: 44396 +.. date: 2021-06-11-18-17-42 +.. nonce: Z9EKim +.. section: Core and Builtins + +Fix a possible crash in the tokenizer when raising syntax errors for +unclosed strings. Patch by Pablo Galindo. + +.. + +.. bpo: 44376 +.. date: 2021-06-11-17-37-15 +.. nonce: zhM1UW +.. section: Core and Builtins + +Exact integer exponentiation (like ``i**2`` or ``pow(i, 2)``) with a small +exponent is much faster, due to reducing overhead in such cases. + +.. + +.. bpo: 44313 +.. date: 2021-06-10-16-10-39 +.. nonce: 34RjI8 +.. section: Core and Builtins + +Directly imported objects and modules (through import and from import +statements) don't generate ``LOAD_METHOD``/``CALL_METHOD`` for directly +accessed objects on their namespace. They now use the regular +``LOAD_ATTR``/``CALL_FUNCTION``. + +.. + +.. bpo: 44338 +.. date: 2021-06-10-10-06-18 +.. nonce: c4Myr4 +.. section: Core and Builtins + +Implement adaptive specialization for LOAD_GLOBAL + +Two specialized forms of LOAD_GLOBAL are added: + +* LOAD_GLOBAL_MODULE + +* LOAD_GLOBAL_BUILTIN + +.. + +.. bpo: 44368 +.. date: 2021-06-09-22-56-59 +.. nonce: vgT0Cx +.. section: Core and Builtins + +Improve syntax errors for invalid "as" targets. Patch by Pablo Galindo + +.. + +.. bpo: 44349 +.. date: 2021-06-08-22-49-06 +.. nonce: xgEgeA +.. section: Core and Builtins + +Fix an edge case when displaying text from files with encoding in syntax +errors. Patch by Pablo Galindo. + +.. + +.. bpo: 44337 +.. date: 2021-06-08-10-22-46 +.. nonce: RTjmIt +.. section: Core and Builtins + +Initial implementation of adaptive specialization of LOAD_ATTR + +Four specialized forms of LOAD_ATTR are added: + +* LOAD_ATTR_SLOT + +* LOAD_ATTR_SPLIT_KEYS + +* LOAD_ATTR_WITH_HINT + +* LOAD_ATTR_MODULE + +.. + +.. bpo: 44335 +.. date: 2021-06-08-01-13-47 +.. nonce: GQTTkl +.. section: Core and Builtins + +Fix a regression when identifying incorrect characters in syntax errors. +Patch by Pablo Galindo + +.. + +.. bpo: 43693 +.. date: 2021-06-07-15-13-44 +.. nonce: c_zDeY +.. section: Core and Builtins + +Computation of the offsets of cell variables is done in the compiler instead +of at runtime. This reduces the overhead of handling cell and free +variables, especially in the case where a variable is both an argument and +cell variable. + +.. + +.. bpo: 44317 +.. date: 2021-06-06-00-29-14 +.. nonce: xPPhcZ +.. section: Core and Builtins + +Improve tokenizer error with improved locations. Patch by Pablo Galindo. + +.. + +.. bpo: 44304 +.. date: 2021-06-05-02-34-57 +.. nonce: _MAoPc +.. section: Core and Builtins + +Fix a crash in the :mod:`sqlite3` module that happened when the garbage +collector clears :class:`sqlite.Statement` objects. Patch by Pablo Galindo + +.. + +.. bpo: 44305 +.. date: 2021-06-03-22-51-50 +.. nonce: 66dVDG +.. section: Core and Builtins + +Improve error message for ``try`` blocks without ``except`` or ``finally`` +blocks. Patch by Pablo Galindo. + +.. + +.. bpo: 43413 +.. date: 2021-05-30-16-37-47 +.. nonce: vYFPPC +.. section: Core and Builtins + +Constructors of subclasses of some builtin classes (e.g. :class:`tuple`, +:class:`list`, :class:`frozenset`) no longer accept arbitrary keyword +arguments. Subclass of :class:`set` can now define a ``__new__()`` method +with additional keyword parameters without overriding also ``__init__()``. + +.. + +.. bpo: 43667 +.. date: 2021-05-27-17-34-29 +.. nonce: ND9jP3 +.. section: Core and Builtins + +Improve Unicode support in non-UTF locales on Oracle Solaris. This issue +does not affect other Solaris systems. + +.. + +.. bpo: 43693 +.. date: 2021-05-26-19-10-47 +.. nonce: 1KSG9u +.. section: Core and Builtins + +A new opcode MAKE_CELL has been added that effectively moves some of the +work done on function entry into the compiler and into the eval loop. In +addition to creating the required cell objects, the new opcode converts +relevant arguments (and other locals) to cell variables on function entry. + +.. + +.. bpo: 44232 +.. date: 2021-05-25-18-20-10 +.. nonce: DMcCCf +.. section: Core and Builtins + +Fix a regression in :func:`type` when a metaclass raises an exception. The C +function :c:func:`type_new` must properly report the exception when a +metaclass constructor raises an exception and the winner class is not the +metaclass. Patch by Victor Stinner. + +.. + +.. bpo: 44201 +.. date: 2021-05-21-21-16-03 +.. nonce: bGaSjt +.. section: Core and Builtins + +Avoid side effects of checking for specialized syntax errors in the REPL +that was causing it to ask for extra tokens after a syntax error had been +detected. Patch by Pablo Galindo + +.. + +.. bpo: 43693 +.. date: 2021-05-21-20-53-49 +.. nonce: -NN3J_ +.. section: Core and Builtins + +``PyCodeObject`` gained ``co_fastlocalnames`` and ``co_fastlocalkinds`` as +the authoritative source of fast locals info. Marshaled code objects +have changed accordingly. + +.. + +.. bpo: 44184 +.. date: 2021-05-21-01-42-45 +.. nonce: 9qOptC +.. section: Core and Builtins + +Fix a crash at Python exit when a deallocator function removes the last +strong reference to a heap type. Patch by Victor Stinner. + +.. + +.. bpo: 44187 +.. date: 2021-05-20-12-43-04 +.. nonce: 3lk0L1 +.. section: Core and Builtins + +Implement quickening in the interpreter. This offers no advantages as yet, +but is an enabler of future optimizations. See PEP 659 for full explanation. + +.. + +.. bpo: 44180 +.. date: 2021-05-19-20-33-36 +.. nonce: mQVaAs +.. section: Core and Builtins + +The parser doesn't report generic syntax errors that happen in a position +further away that the one it reached in the first pass. Patch by Pablo +Galindo + +.. + +.. bpo: 44168 +.. date: 2021-05-18-11-27-02 +.. nonce: mgB-rt +.. section: Core and Builtins + +Fix error message in the parser involving keyword arguments with invalid +expressions. Patch by Pablo Galindo + +.. + +.. bpo: 44156 +.. date: 2021-05-17-20-44-45 +.. nonce: 8KSp9l +.. section: Core and Builtins + +String caches in ``compile.c`` are now subinterpreter compatible. + +.. + +.. bpo: 44143 +.. date: 2021-05-15-17-30-57 +.. nonce: 7UTS6H +.. section: Core and Builtins + +Fixed a crash in the parser that manifest when raising tokenizer errors when +an existing exception was present. Patch by Pablo Galindo. + +.. + +.. bpo: 44032 +.. date: 2021-05-14-20-03-32 +.. nonce: OzT1ob +.. section: Core and Builtins + +Move 'fast' locals and other variables from the frame object to a per-thread +datastack. + +.. + +.. bpo: 44114 +.. date: 2021-05-12-14-26-16 +.. nonce: p-WfAE +.. section: Core and Builtins + +Fix incorrect dictkeys_reversed and dictitems_reversed function signatures +in C code, which broke webassembly builds. + +.. + +.. bpo: 44110 +.. date: 2021-05-11-21-52-44 +.. nonce: VqbAsB +.. section: Core and Builtins + +Improve :func:`str.__getitem__` error message + +.. + +.. bpo: 26110 +.. date: 2021-05-10-18-49-13 +.. nonce: EQzqqA +.. section: Core and Builtins + +Add ``CALL_METHOD_KW`` opcode to speed up method calls with keyword +arguments. Idea originated from PyPy. A side effect is executing +``CALL_METHOD`` is now branchless in the evaluation loop. + +.. + +.. bpo: 28307 +.. date: 2021-05-08-19-54-57 +.. nonce: 7ysaVW +.. section: Core and Builtins + +Compiler now optimizes simple C-style formatting with literal format +containing only format codes %s, %r and %a by converting them to f-string +expressions. + +.. + +.. bpo: 43149 +.. date: 2021-05-08-17-18-37 +.. nonce: Kp5FxD +.. section: Core and Builtins + +Correct the syntax error message regarding multiple exception types to not +refer to "exception groups". Patch by Pablo Galindo + +.. + +.. bpo: 43822 +.. date: 2021-05-04-01-01-04 +.. nonce: 9VeCg0 +.. section: Core and Builtins + +The parser will prioritize tokenizer errors over custom syntax errors when +raising exceptions. Patch by Pablo Galindo. + +.. + +.. bpo: 40222 +.. date: 2021-04-30-15-48-36 +.. nonce: j3VxeX +.. section: Core and Builtins + +"Zero cost" exception handling. + +* Uses a lookup table to determine how to handle exceptions. +* Removes SETUP_FINALLY and POP_TOP block instructions, eliminating the runtime overhead of try statements. +* Reduces the size of the frame object by about 60%. + +Patch by Mark Shannon + +.. + +.. bpo: 43918 +.. date: 2021-04-23-03-46-45 +.. nonce: nNDY3S +.. section: Core and Builtins + +Document the signature and ``default`` argument in the docstring of the new +``anext`` builtin. + +.. + +.. bpo: 43833 +.. date: 2021-04-18-18-07-33 +.. nonce: oChkCi +.. section: Core and Builtins + +Emit a deprecation warning if the numeric literal is immediately followed by +one of keywords: and, else, for, if, in, is, or. Raise a syntax error with +more informative message if it is immediately followed by other keyword or +identifier. + +.. + +.. bpo: 43879 +.. date: 2021-04-17-16-08-00 +.. nonce: zkyJgh +.. section: Core and Builtins + +Add native_thread_id to PyThreadState. Patch by Gabriele N. Tornetta. + +.. + +.. bpo: 43693 +.. date: 2021-04-02-15-02-16 +.. nonce: l3Ureu +.. section: Core and Builtins + +Compute cell offsets relative to locals in compiler. Allows the interpreter +to treats locals and cells a single array, which is slightly more efficient. +Also make the LOAD_CLOSURE opcode an alias for LOAD_FAST. Preserving +LOAD_CLOSURE helps keep bytecode a bit more readable. + +.. + +.. bpo: 17792 +.. date: 2021-03-22-17-50-30 +.. nonce: _zssjS +.. section: Core and Builtins + +More accurate error messages for access of unbound locals or free vars. + +.. + +.. bpo: 28146 +.. date: 2021-01-13-19-34-41 +.. nonce: AZBBkH +.. section: Core and Builtins + +Fix a confusing error message in :func:`str.format`. + +.. + +.. bpo: 11105 +.. date: 2020-06-02-13-21-14 +.. nonce: wceryW +.. section: Core and Builtins + +When compiling :class:`ast.AST` objects with recursive references through +:func:`compile`, the interpreter doesn't crash anymore instead it raises a +:exc:`RecursionError`. + +.. + +.. bpo: 39091 +.. date: 2019-12-21-14-18-32 +.. nonce: dOexgQ +.. section: Core and Builtins + +Fix crash when using passing a non-exception to a generator's ``throw()`` +method. Patch by Noah Oxer + +.. + +.. bpo: 33346 +.. date: 2018-05-11-12-44-03 +.. nonce: ZgBkvB +.. section: Core and Builtins + +Asynchronous comprehensions are now allowed inside comprehensions in +asynchronous functions. Outer comprehensions implicitly become +asynchronous. + +.. + +.. bpo: 45371 +.. date: 2021-10-05-11-03-48 +.. nonce: NOwcDJ +.. section: Library + +Fix clang rpath issue in :mod:`distutils`. The UnixCCompiler now uses +correct clang option to add a runtime library directory (rpath) to a shared +library. + +.. + +.. bpo: 45329 +.. date: 2021-10-01-13-09-53 +.. nonce: 9iMYaO +.. section: Library + +Fix freed memory access in :class:`pyexpat.xmlparser` when building it with +an installed expat library <= 2.2.0. + +.. + +.. bpo: 41710 +.. date: 2021-09-30-23-00-18 +.. nonce: svuloZ +.. section: Library + +On Unix, if the ``sem_clockwait()`` function is available in the C library +(glibc 2.30 and newer), the :meth:`threading.Lock.acquire` method now uses +the monotonic clock (:data:`time.CLOCK_MONOTONIC`) for the timeout, rather +than using the system clock (:data:`time.CLOCK_REALTIME`), to not be +affected by system clock changes. Patch by Victor Stinner. + +.. + +.. bpo: 1596321 +.. date: 2021-09-24-17-20-23 +.. nonce: 3nhPUk +.. section: Library + +Fix the :func:`threading._shutdown` function when the :mod:`threading` +module was imported first from a thread different than the main thread: no +longer log an error at Python exit. + +.. + +.. bpo: 45274 +.. date: 2021-09-23-22-17-26 +.. nonce: gPpa4E +.. section: Library + +Fix a race condition in the :meth:`Thread.join() ` +method of the :mod:`threading` module. If the function is interrupted by a +signal and the signal handler raises an exception, make sure that the thread +remains in a consistent state to prevent a deadlock. Patch by Victor +Stinner. + +.. + +.. bpo: 21302 +.. date: 2021-09-22-23-56-15 +.. nonce: vvQ3Su +.. section: Library + +In Unix operating systems, :func:`time.sleep` now uses the ``nanosleep()`` +function, if ``clock_nanosleep()`` is not available but ``nanosleep()`` is +available. ``nanosleep()`` allows to sleep with nanosecond precision. + +.. + +.. bpo: 21302 +.. date: 2021-09-20-22-46-40 +.. nonce: h56430 +.. section: Library + +On Windows, :func:`time.sleep` now uses a waitable timer which has a +resolution of 100 nanoseconds (10\ :sup:`-7` seconds). Previously, it had a +resolution of 1 millisecond (10\ :sup:`-3` seconds). Patch by Benjamin Szőke and +Victor Stinner. + +.. + +.. bpo: 45238 +.. date: 2021-09-18-16-56-33 +.. nonce: Hng_9V +.. section: Library + +Fix :meth:`unittest.IsolatedAsyncioTestCase.debug`: it runs now asynchronous +methods and callbacks. + +.. + +.. bpo: 36674 +.. date: 2021-09-18-13-14-57 +.. nonce: a2k5Zb +.. section: Library + +:meth:`unittest.TestCase.debug` raises now a :class:`unittest.SkipTest` if +the class or the test method are decorated with the skipping decorator. + +.. + +.. bpo: 45235 +.. date: 2021-09-17-16-55-37 +.. nonce: sXnmPA +.. section: Library + +Fix an issue where argparse would not preserve values in a provided +namespace when using a subparser with defaults. + +.. + +.. bpo: 45183 +.. date: 2021-09-17-15-58-53 +.. nonce: Vv_vch +.. section: Library + +Have zipimport.zipimporter.find_spec() not raise an exception when the +underlying zip file has been deleted and the internal cache has been reset +via invalidate_cache(). + +.. + +.. bpo: 45234 +.. date: 2021-09-17-11-20-55 +.. nonce: qUcTVt +.. section: Library + +Fixed a regression in :func:`~shutil.copyfile`, :func:`~shutil.copy`, +:func:`~shutil.copy2` raising :exc:`FileNotFoundError` when source is a +directory, which should raise :exc:`IsADirectoryError` + +.. + +.. bpo: 45228 +.. date: 2021-09-17-09-59-33 +.. nonce: WV1dcT +.. section: Library + +Fix stack buffer overflow in parsing J1939 network address. + +.. + +.. bpo: 45225 +.. date: 2021-09-16-19-02-14 +.. nonce: xmKV4i +.. section: Library + +use map function instead of genexpr in capwords. + +.. + +.. bpo: 42135 +.. date: 2021-09-13-19-32-58 +.. nonce: 1ZAHqR +.. section: Library + +Fix typo: ``importlib.find_loader`` is really slated for removal in Python +3.12 not 3.10, like the others in GH-25169. + +Patch by Hugo van Kemenade. + +.. + +.. bpo: 20524 +.. date: 2021-09-13-14-59-01 +.. nonce: PMQ1Fh +.. section: Library + +Improves error messages on ``.format()`` operation for ``str``, ``float``, +``int``, and ``complex``. New format now shows the problematic pattern and +the object type. + +.. + +.. bpo: 45168 +.. date: 2021-09-13-14-28-49 +.. nonce: Z1mfW4 +.. section: Library + +Change :func:`dis.dis` output to omit op arg values that cannot be resolved +due to ``co_consts``, ``co_names`` etc not being provided. Previously the +oparg itself was repeated in the value field, which is not useful and can be +confusing. + +.. + +.. bpo: 21302 +.. date: 2021-09-11-18-44-40 +.. nonce: QxHRpR +.. section: Library + +In Unix operating systems, :func:`time.sleep` now uses the +``clock_nanosleep()`` function, if available, which allows to sleep for an +interval specified with nanosecond precision. + +.. + +.. bpo: 45173 +.. date: 2021-09-11-17-46-20 +.. nonce: UptGAn +.. section: Library + +Remove from the :mod:`configparser` module: the :class:`SafeConfigParser` +class, the :attr:`filename` property of the :class:`ParsingError` class, the +:meth:`readfp` method of the :class:`ConfigParser` class, deprecated since +Python 3.2. + +Patch by Hugo van Kemenade. + +.. + +.. bpo: 44987 +.. date: 2021-09-11-14-41-02 +.. nonce: Mt8DiX +.. section: Library + +Pure ASCII strings are now normalized in constant time by +:func:`unicodedata.normalize`. Patch by Dong-hee Na. + +.. + +.. bpo: 35474 +.. date: 2021-09-11-10-45-12 +.. nonce: tEY3SD +.. section: Library + +Calling :func:`mimetypes.guess_all_extensions` with ``strict=False`` no +longer affects the result of the following call with ``strict=True``. Also, +mutating the returned list no longer affects the global state. + +.. + +.. bpo: 45166 +.. date: 2021-09-10-21-35-53 +.. nonce: UHipXF +.. section: Library + +:func:`typing.get_type_hints` now works with :data:`~typing.Final` wrapped +in :class:`~typing.ForwardRef`. + +.. + +.. bpo: 45162 +.. date: 2021-09-10-13-20-53 +.. nonce: 2Jh-lq +.. section: Library + +Remove many old deprecated :mod:`unittest` features: + +* "``fail*``" and "``assert*``" aliases of :class:`~unittest.TestCase` methods. +* Broken from start :class:`~unittest.TestCase` method ``assertDictContainsSubset()``. +* Ignored :meth:` TestLoader.loadTestsFromModule` parameter *use_load_tests*. +* Old alias ``_TextTestResult`` of :class:`~unittest.TextTestResult`. + +.. + +.. bpo: 38371 +.. date: 2021-09-08-13-19-29 +.. nonce: y1kEfP +.. section: Library + +Remove the deprecated ``split()`` method of :class:`_tkinter.TkappType`. +Patch by Erlend E. Aasland. + +.. + +.. bpo: 20499 +.. date: 2021-09-08-01-19-31 +.. nonce: tSxx8Y +.. section: Library + +Improve the speed and accuracy of statistics.pvariance(). + +.. + +.. bpo: 45132 +.. date: 2021-09-07-16-33-51 +.. nonce: WI9zQY +.. section: Library + +Remove :meth:`__getitem__` methods of +:class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` +and :class:`fileinput.FileInput`, deprecated since Python 3.9. + +Patch by Hugo van Kemenade. + +.. + +.. bpo: 45129 +.. date: 2021-09-07-14-27-39 +.. nonce: vXH0gw +.. section: Library + +Due to significant security concerns, the *reuse_address* parameter of +:meth:`asyncio.loop.create_datagram_endpoint`, disabled in Python 3.9, is +now entirely removed. This is because of the behavior of the socket option +``SO_REUSEADDR`` in UDP. + +Patch by Hugo van Kemenade. + +.. + +.. bpo: 45124 +.. date: 2021-09-07-09-13-27 +.. nonce: Kw5AUs +.. section: Library + +The ``bdist_msi`` command, deprecated in Python 3.9, is now removed. + +Use ``bdist_wheel`` (wheel packages) instead. + +Patch by Hugo van Kemenade. + +.. + +.. bpo: 30856 +.. date: 2021-09-05-21-37-28 +.. nonce: jj86y0 +.. section: Library + +:class:`unittest.TestResult` methods +:meth:`~unittest.TestResult.addFailure`, +:meth:`~unittest.TestResult.addError`, :meth:`~unittest.TestResult.addSkip` +and :meth:`~unittest.TestResult.addSubTest` are now called immediately after +raising an exception in test or finishing a subtest. Previously they were +called only after finishing the test clean up. + +.. + +.. bpo: 45034 +.. date: 2021-09-05-20-33-25 +.. nonce: 62NLD5 +.. section: Library + +Changes how error is formatted for ``struct.pack`` with ``'H'`` and ``'h'`` +modes and too large / small numbers. Now it shows the actual numeric limits, +while previously it was showing arithmetic expressions. + +.. + +.. bpo: 25894 +.. date: 2021-09-05-13-15-08 +.. nonce: zjbi2f +.. section: Library + +:mod:`unittest` now always reports skipped and failed subtests separately: +separate characters in default mode and separate lines in verbose mode. Also +the test description is now output for errors in test method, class and +module cleanups. + +.. + +.. bpo: 45081 +.. date: 2021-09-02-12-42-25 +.. nonce: tOjJ1k +.. section: Library + +Fix issue when dataclasses that inherit from ``typing.Protocol`` subclasses +have wrong ``__init__``. Patch provided by Yurii Karabas. + +.. + +.. bpo: 45085 +.. date: 2021-09-02-00-47-14 +.. nonce: mMnaDv +.. section: Library + +The ``binhex`` module, deprecated in Python 3.9, is now removed. The +following :mod:`binascii` functions, deprecated in Python 3.9, are now also +removed: + +* ``a2b_hqx()``, ``b2a_hqx()``; +* ``rlecode_hqx()``, ``rledecode_hqx()``. + +The :func:`binascii.crc_hqx` function remains available. + +Patch by Victor Stinner. + +.. + +.. bpo: 40360 +.. date: 2021-09-02-00-18-32 +.. nonce: 9nmMtB +.. section: Library + +The :mod:`lib2to3` package is now deprecated and may not be able to parse +Python 3.10 or newer. See the :pep:`617` (New PEG parser for CPython). Patch +by Victor Stinner. + +.. + +.. bpo: 45075 +.. date: 2021-09-01-15-27-00 +.. nonce: 9xUpvt +.. section: Library + +Rename :meth:`traceback.StackSummary.format_frame` to +:meth:`traceback.StackSummary.format_frame_summary`. This method was added +for 3.11 so it was not released yet. + +Updated code and docs to better distinguish frame and FrameSummary. + +.. + +.. bpo: 31299 +.. date: 2021-08-30-13-55-09 +.. nonce: 9QzjZs +.. section: Library + +Add option to completely drop frames from a traceback by returning ``None`` +from a :meth:`~traceback.StackSummary.format_frame` override. + +.. + +.. bpo: 41620 +.. date: 2021-08-29-14-49-22 +.. nonce: WJ6PFL +.. section: Library + +:meth:`~unittest.TestCase.run` now always return a +:class:`~unittest.TestResult` instance. Previously it returned ``None`` if +the test class or method was decorated with a skipping decorator. + +.. + +.. bpo: 45021 +.. date: 2021-08-28-13-00-12 +.. nonce: rReeaj +.. section: Library + +Fix a potential deadlock at shutdown of forked children when using +:mod:`concurrent.futures` module + +.. + +.. bpo: 43913 +.. date: 2021-08-27-23-40-51 +.. nonce: Uo1Gt5 +.. section: Library + +Fix bugs in cleaning up classes and modules in :mod:`unittest`: + +* Functions registered with :func:`~unittest.addModuleCleanup` were not called unless the user defines ``tearDownModule()`` in their test module. +* Functions registered with :meth:`~unittest.TestCase.addClassCleanup` were not called if ``tearDownClass`` is set to ``None``. +* Buffering in :class:`~unittest.TestResult` did not work with functions registered with ``addClassCleanup()`` and ``addModuleCleanup()``. +* Errors in functions registered with ``addClassCleanup()`` and ``addModuleCleanup()`` were not handled correctly in buffered and debug modes. +* Errors in ``setUpModule()`` and functions registered with ``addModuleCleanup()`` were reported in wrong order. +* And several lesser bugs. + +.. + +.. bpo: 45030 +.. date: 2021-08-27-19-01-23 +.. nonce: tAmBbY +.. section: Library + +Fix integer overflow in pickling and copying the range iterator. + +.. + +.. bpo: 45001 +.. date: 2021-08-26-16-25-48 +.. nonce: tn_dKp +.. section: Library + +Made email date parsing more robust against malformed input, namely a +whitespace-only ``Date:`` header. Patch by Wouter Bolsterlee. + +.. + +.. bpo: 45010 +.. date: 2021-08-26-09-54-14 +.. nonce: Cn23bQ +.. section: Library + +Remove support of special method ``__div__`` in :mod:`unittest.mock`. It is +not used in Python 3. + +.. + +.. bpo: 39218 +.. date: 2021-08-25-20-18-31 +.. nonce: BlO6jW +.. section: Library + +Improve accuracy of variance calculations by using ``x*x`` instead of +``x**2``. + +.. + +.. bpo: 43613 +.. date: 2021-08-25-10-28-49 +.. nonce: WkYmI0 +.. section: Library + +Improve the speed of :func:`gzip.compress` and :func:`gzip.decompress` by +compressing and decompressing at once in memory instead of in a streamed +fashion. + +.. + +.. bpo: 37596 +.. date: 2021-08-23-21-39-59 +.. nonce: ojRcwB +.. section: Library + +Ensure that :class:`set` and :class:`frozenset` objects are always +:mod:`marshalled ` reproducibly. + +.. + +.. bpo: 44019 +.. date: 2021-08-22-13-25-17 +.. nonce: BN8HDy +.. section: Library + +A new function ``operator.call`` has been added, such that +``operator.call(obj, *args, **kwargs) == obj(*args, **kwargs)``. + +.. + +.. bpo: 42255 +.. date: 2021-08-19-23-49-10 +.. nonce: ofe3ms +.. section: Library + +:class:`webbrowser.MacOSX` is deprecated and will be removed in Python 3.13. +It is untested and undocumented and also not used by webbrowser itself. +Patch by Dong-hee Na. + +.. + +.. bpo: 44955 +.. date: 2021-08-19-15-03-54 +.. nonce: 1mxFQS +.. section: Library + +Method :meth:`~unittest.TestResult.stopTestRun` is now always called in pair +with method :meth:`~unittest.TestResult.startTestRun` for +:class:`~unittest.TestResult` objects implicitly created in +:meth:`~unittest.TestCase.run`. Previously it was not called for test +methods and classes decorated with a skipping decorator. + +.. + +.. bpo: 39039 +.. date: 2021-08-18-10-36-14 +.. nonce: A63LYh +.. section: Library + +tarfile.open raises :exc:`~tarfile.ReadError` when a zlib error occurs +during file extraction. + +.. + +.. bpo: 44935 +.. date: 2021-08-17-16-01-44 +.. nonce: roUl0G +.. section: Library + +:mod:`subprocess` on Solaris now also uses :func:`os.posix_spawn()` for +better performance. + +.. + +.. bpo: 44911 +.. date: 2021-08-14-00-55-16 +.. nonce: uk3hYk +.. section: Library + +:class:`~unittest.IsolatedAsyncioTestCase` will no longer throw an exception +while cancelling leaked tasks. Patch by Bar Harel. + +.. + +.. bpo: 41322 +.. date: 2021-08-12-16-22-16 +.. nonce: utscTd +.. section: Library + +Added ``DeprecationWarning`` for tests and async tests that return a +value!=None (as this may indicate an improperly written test, for example a +test written as a generator function). + +.. + +.. bpo: 44524 +.. date: 2021-08-10-16-57-10 +.. nonce: dk9QX4 +.. section: Library + +Make exception message more useful when subclass from typing special form +alias. Patch provided by Yurii Karabas. + +.. + +.. bpo: 38956 +.. date: 2021-08-09-13-17-10 +.. nonce: owWLNv +.. section: Library + +:class:`argparse.BooleanOptionalAction`'s default value is no longer printed +twice when used with :class:`argparse.ArgumentDefaultsHelpFormatter`. + +.. + +.. bpo: 44860 +.. date: 2021-08-07-22-51-32 +.. nonce: PTvRrU +.. section: Library + +Fix the ``posix_user`` scheme in :mod:`sysconfig` to not depend on +:data:`sys.platlibdir`. + +.. + +.. bpo: 44859 +.. date: 2021-08-07-17-28-56 +.. nonce: CCopjk +.. section: Library + +Improve error handling in :mod:`sqlite3` and raise more accurate exceptions. + +* :exc:`MemoryError` is now raised instead of :exc:`sqlite3.Warning` when memory is not enough for encoding a statement to UTF-8 in ``Connection.__call__()`` and ``Cursor.execute()``. +* :exc:`UnicodEncodeError` is now raised instead of :exc:`sqlite3.Warning` when the statement contains surrogate characters in ``Connection.__call__()`` and ``Cursor.execute()``. +* :exc:`TypeError` is now raised instead of :exc:`ValueError` for non-string script argument in ``Cursor.executescript()``. +* :exc:`ValueError` is now raised for script containing the null character instead of truncating it in ``Cursor.executescript()``. +* Correctly handle exceptions raised when getting boolean value of the result of the progress handler. +* Add many tests covering different corner cases. + +.. + +.. bpo: 44581 +.. date: 2021-08-06-19-15-52 +.. nonce: oFDBTB +.. section: Library + +Upgrade bundled pip to 21.2.3 and setuptools to 57.4.0 + +.. + +.. bpo: 44849 +.. date: 2021-08-06-13-00-28 +.. nonce: O78F_f +.. section: Library + +Fix the :func:`os.set_inheritable` function on FreeBSD 14 for file +descriptor opened with the :data:`~os.O_PATH` flag: ignore the +:data:`~errno.EBADF` error on ``ioctl()``, fallback on the ``fcntl()`` +implementation. Patch by Victor Stinner. + +.. + +.. bpo: 44605 +.. date: 2021-08-06-09-43-50 +.. nonce: q4YSBZ +.. section: Library + +The @functools.total_ordering() decorator now works with metaclasses. + +.. + +.. bpo: 44524 +.. date: 2021-08-05-18-20-17 +.. nonce: 9T1tfe +.. section: Library + +Fixed an issue wherein the ``__name__`` and ``__qualname__`` attributes of +subscribed specialforms could be ``None``. + +.. + +.. bpo: 44839 +.. date: 2021-08-05-14-59-39 +.. nonce: MURNL9 +.. section: Library + +:class:`MemoryError` raised in user-defined functions will now produce a +``MemoryError`` in :mod:`sqlite3`. :class:`OverflowError` will now be +converted to :class:`~sqlite3.DataError`. Previously +:class:`~sqlite3.OperationalError` was produced in these cases. + +.. + +.. bpo: 44822 +.. date: 2021-08-04-12-29-00 +.. nonce: zePNXA +.. section: Library + +:mod:`sqlite3` user-defined functions and aggregators returning +:class:`strings ` with embedded NUL characters are no longer truncated. +Patch by Erlend E. Aasland. + +.. + +.. bpo: 44801 +.. date: 2021-08-03-20-37-45 +.. nonce: i49Aug +.. section: Library + +Ensure that the :class:`~typing.ParamSpec` variable in Callable can only be +substituted with a parameters expression (a list of types, an ellipsis, +ParamSpec or Concatenate). + +.. + +.. bpo: 44806 +.. date: 2021-08-02-14-37-32 +.. nonce: wOW_Qn +.. section: Library + +Non-protocol subclasses of :class:`typing.Protocol` ignore now the +``__init__`` method inherited from protocol base classes. + +.. + +.. bpo: 27275 +.. date: 2021-08-01-19-49-09 +.. nonce: QsvE0k +.. section: Library + +:meth:`collections.OrderedDict.popitem` and +:meth:`collections.OrderedDict.pop` no longer call ``__getitem__`` and +``__delitem__`` methods of the OrderedDict subclasses. + +.. + +.. bpo: 44793 +.. date: 2021-07-31-20-28-20 +.. nonce: woaQSg +.. section: Library + +Fix checking the number of arguments when subscribe a generic type with +``ParamSpec`` parameter. + +.. + +.. bpo: 44784 +.. date: 2021-07-31-08-45-31 +.. nonce: fIMIDS +.. section: Library + +In importlib.metadata tests, override warnings behavior under expected +DeprecationWarnings (importlib_metadata 4.6.3). + +.. + +.. bpo: 44667 +.. date: 2021-07-30-23-27-30 +.. nonce: tu0Xrv +.. section: Library + +The :func:`tokenize.tokenize` doesn't incorrectly generate a ``NEWLINE`` +token if the source doesn't end with a new line character but the last line +is a comment, as the function is already generating a ``NL`` token. Patch by +Pablo Galindo + +.. + +.. bpo: 44771 +.. date: 2021-07-28-22-53-18 +.. nonce: BvLdnU +.. section: Library + +Added ``importlib.simple`` module implementing adapters from a low-level +resources reader interface to a ``TraversableResources`` interface. Legacy +API (``path``, ``contents``, ...) is now supported entirely by the +``.files()`` API with a compatibility shim supplied for resource loaders +without that functionality. Feature parity with ``importlib_resources`` 5.2. + +.. + +.. bpo: 44752 +.. date: 2021-07-27-22-11-29 +.. nonce: _bvbrZ +.. section: Library + +:mod:`rcompleter` does not call :func:`getattr` on :class:`property` objects +to avoid the side-effect of evaluating the corresponding method. + +.. + +.. bpo: 44747 +.. date: 2021-07-27-12-06-19 +.. nonce: epUzZz +.. section: Library + +Refactor usage of ``sys._getframe`` in ``typing`` module. Patch provided by +Yurii Karabas. + +.. + +.. bpo: 42378 +.. date: 2021-07-25-08-17-55 +.. nonce: WIhUZK +.. section: Library + +Fixes the issue with log file being overwritten when +:class:`logging.FileHandler` is used in :mod:`atexit` with *filemode* set to +``'w'``. Note this will cause the message in *atexit* not being logged if +the log stream is already closed due to shutdown of logging. + +.. + +.. bpo: 44720 +.. date: 2021-07-24-02-17-59 +.. nonce: shU5Qm +.. section: Library + +``weakref.proxy`` objects referencing non-iterators now raise ``TypeError`` +rather than dereferencing the null ``tp_iternext`` slot and crashing. + +.. + +.. bpo: 44704 +.. date: 2021-07-21-23-16-30 +.. nonce: iqHLxQ +.. section: Library + +The implementation of ``collections.abc.Set._hash()`` now matches that of +``frozenset.__hash__()``. + +.. + +.. bpo: 44666 +.. date: 2021-07-21-10-43-22 +.. nonce: CEThkv +.. section: Library + +Fixed issue in :func:`compileall.compile_file` when ``sys.stdout`` is +redirected. Patch by Stefan Hölzl. + +.. + +.. bpo: 44688 +.. date: 2021-07-20-23-28-26 +.. nonce: buFgz3 +.. section: Library + +:meth:`sqlite3.Connection.create_collation` now accepts non-ASCII collation +names. Patch by Erlend E. Aasland. + +.. + +.. bpo: 44690 +.. date: 2021-07-20-22-03-24 +.. nonce: tV7Zjg +.. section: Library + +Adopt *binacii.a2b_base64*'s strict mode in *base64.b64decode*. + +.. + +.. bpo: 42854 +.. date: 2021-07-20-21-51-35 +.. nonce: ThuDMI +.. section: Library + +Fixed a bug in the :mod:`_ssl` module that was throwing :exc:`OverflowError` +when using :meth:`_ssl._SSLSocket.write` and :meth:`_ssl._SSLSocket.read` +for a big value of the ``len`` parameter. Patch by Pablo Galindo + +.. + +.. bpo: 44686 +.. date: 2021-07-20-19-35-49 +.. nonce: ucCGhu +.. section: Library + +Replace ``unittest.mock._importer`` with ``pkgutil.resolve_name``. + +.. + +.. bpo: 44353 +.. date: 2021-07-20-18-34-16 +.. nonce: ATuYq4 +.. section: Library + +Make ``NewType.__call__`` faster by implementing it in C. Patch provided by +Yurii Karabas. + +.. + +.. bpo: 44682 +.. date: 2021-07-20-00-11-47 +.. nonce: 3m2qVV +.. section: Library + +Change the :mod:`pdb` *commands* directive to disallow setting commands for +an invalid breakpoint and to display an appropriate error. + +.. + +.. bpo: 44353 +.. date: 2021-07-19-22-43-15 +.. nonce: HF81_Q +.. section: Library + +Refactor ``typing.NewType`` from function into callable class. Patch +provided by Yurii Karabas. + +.. + +.. bpo: 44678 +.. date: 2021-07-19-18-45-00 +.. nonce: YMEAu0 +.. section: Library + +Added a separate error message for discontinuous padding in +*binascii.a2b_base64* strict mode. + +.. + +.. bpo: 44524 +.. date: 2021-07-19-14-04-42 +.. nonce: Nbf2JC +.. section: Library + +Add missing ``__name__`` and ``__qualname__`` attributes to ``typing`` +module classes. Patch provided by Yurii Karabas. + +.. + +.. bpo: 40897 +.. date: 2021-07-16-13-40-31 +.. nonce: aveAre +.. section: Library + +Give priority to using the current class constructor in +:func:`inspect.signature`. Patch by Weipeng Hong. + +.. + +.. bpo: 44638 +.. date: 2021-07-16-08-57-27 +.. nonce: EwYKne +.. section: Library + +Add a reference to the zipp project and hint as to how to use it. + +.. + +.. bpo: 44648 +.. date: 2021-07-15-16-51-32 +.. nonce: 2o49TB +.. section: Library + +Fixed wrong error being thrown by :func:`inspect.getsource` when examining a +class in the interactive session. Instead of :exc:`TypeError`, it should be +:exc:`OSError` with appropriate error message. + +.. + +.. bpo: 44608 +.. date: 2021-07-13-09-01-33 +.. nonce: R3IcM1 +.. section: Library + +Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence +or set, but not list or tuple. + +.. + +.. bpo: 44594 +.. date: 2021-07-12-10-32-48 +.. nonce: eEa5zi +.. section: Library + +Fix an edge case of :class:`ExitStack` and :class:`AsyncExitStack` exception +chaining. They will now match ``with`` block behavior when ``__context__`` +is explicitly set to ``None`` when the exception is in flight. + +.. + +.. bpo: 42799 +.. date: 2021-07-10-19-55-13 +.. nonce: ad4tq8 +.. section: Library + +In :mod:`fnmatch`, the cache size for compiled regex patterns +(:func:`functools.lru_cache`) was bumped up from 256 to 32768, affecting +functions: :func:`fnmatch.fnmatch`, :func:`fnmatch.fnmatchcase`, +:func:`fnmatch.filter`. + +.. + +.. bpo: 41928 +.. date: 2021-07-09-07-14-37 +.. nonce: Q1jMrr +.. section: Library + +Update :func:`shutil.copyfile` to raise :exc:`FileNotFoundError` instead of +confusing :exc:`IsADirectoryError` when a path ending with a +:const:`os.path.sep` does not exist; :func:`shutil.copy` and +:func:`shutil.copy2` are also affected. + +.. + +.. bpo: 44569 +.. date: 2021-07-08-12-22-54 +.. nonce: KZ02v9 +.. section: Library + +Added the :func:`StackSummary.format_frame` function in :mod:`traceback`. +This allows users to customize the way individual lines are formatted in +tracebacks without re-implementing logic to handle recursive tracebacks. + +.. + +.. bpo: 44566 +.. date: 2021-07-05-18-13-25 +.. nonce: o51Bd1 +.. section: Library + +handle StopIteration subclass raised from @contextlib.contextmanager +generator + +.. + +.. bpo: 44558 +.. date: 2021-07-04-21-16-53 +.. nonce: cm7Slv +.. section: Library + +Make the implementation consistency of :func:`~operator.indexOf` between C +and Python versions. Patch by Dong-hee Na. + +.. + +.. bpo: 41249 +.. date: 2021-07-04-11-33-34 +.. nonce: sHdwBE +.. section: Library + +Fixes ``TypedDict`` to work with ``typing.get_type_hints()`` and postponed +evaluation of annotations across modules. + +.. + +.. bpo: 44554 +.. date: 2021-07-02-18-17-56 +.. nonce: aBUmJo +.. section: Library + +Refactor argument processing in :func:`pdb.main` to simplify detection of +errors in input loading and clarify behavior around module or script +invocation. + +.. + +.. bpo: 34798 +.. date: 2021-06-30-13-29-49 +.. nonce: t7FCa0 +.. section: Library + +Break up paragraph about :class:`pprint.PrettyPrinter` construction +parameters to make it easier to read. + +.. + +.. bpo: 44539 +.. date: 2021-06-30-11-34-35 +.. nonce: nP0Xi4 +.. section: Library + +Added support for recognizing JPEG files without JFIF or Exif markers. + +.. + +.. bpo: 44461 +.. date: 2021-06-29-21-17-17 +.. nonce: acqRnV +.. section: Library + +Fix bug with :mod:`pdb`'s handling of import error due to a package which +does not have a ``__main__`` module + +.. + +.. bpo: 43625 +.. date: 2021-06-29-07-27-08 +.. nonce: ZlAxhp +.. section: Library + +Fix a bug in the detection of CSV file headers by +:meth:`csv.Sniffer.has_header` and improve documentation of same. + +.. + +.. bpo: 44516 +.. date: 2021-06-26-12-27-14 +.. nonce: BVyX_y +.. section: Library + +Update vendored pip to 21.1.3 + +.. + +.. bpo: 42892 +.. date: 2021-06-24-19-16-20 +.. nonce: qvRNhI +.. section: Library + +Fixed an exception thrown while parsing a malformed multipart email by +:class:`email.message.EmailMessage`. + +.. + +.. bpo: 44468 +.. date: 2021-06-23-19-02-00 +.. nonce: -klV5- +.. section: Library + +:func:`typing.get_type_hints` now finds annotations in classes and base +classes with unexpected ``__module__``. Previously, it skipped those MRO +elements. + +.. + +.. bpo: 44491 +.. date: 2021-06-23-01-33-01 +.. nonce: tiOlr5 +.. section: Library + +Allow clearing the :mod:`sqlite3` authorizer callback by passing +:const:`None` to :meth:`~sqlite3.Connection.set_authorizer`. Patch by Erlend +E. Aasland. + +.. + +.. bpo: 43977 +.. date: 2021-06-22-16-45-48 +.. nonce: bamAGF +.. section: Library + +Set the proper :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` +flags for subclasses created before a parent has been registered as a +:class:`collections.abc.Mapping` or :class:`collections.abc.Sequence`. + +.. + +.. bpo: 44482 +.. date: 2021-06-22-08-43-04 +.. nonce: U9GznK +.. section: Library + +Fix very unlikely resource leak in :mod:`glob` in alternate Python +implementations. + +.. + +.. bpo: 44466 +.. date: 2021-06-21-12-43-04 +.. nonce: NSm6mv +.. section: Library + +The :mod:`faulthandler` module now detects if a fatal error occurs during a +garbage collector collection. Patch by Victor Stinner. + +.. + +.. bpo: 44471 +.. date: 2021-06-21-10-46-58 +.. nonce: 2QjXv_ +.. section: Library + +A :exc:`TypeError` is now raised instead of an :exc:`AttributeError` in +:meth:`contextlib.ExitStack.enter_context` and +:meth:`contextlib.AsyncExitStack.enter_async_context` for objects which do +not support the :term:`context manager` or :term:`asynchronous context +manager` protocols correspondingly. + +.. + +.. bpo: 44404 +.. date: 2021-06-20-19-01-11 +.. nonce: McfrYB +.. section: Library + +:mod:`tkinter`'s ``after()`` method now supports callables without the +``__name__`` attribute. + +.. + +.. bpo: 41546 +.. date: 2021-06-20-14-03-18 +.. nonce: lO1jXU +.. section: Library + +Make :mod:`pprint` (like the builtin ``print``) not attempt to write to +``stdout`` when it is ``None``. + +.. + +.. bpo: 44458 +.. date: 2021-06-20-07-14-46 +.. nonce: myqCQ0 +.. section: Library + +``BUFFER_BLOCK_SIZE`` is now declared static, to avoid linking collisions +when bz2, lmza or zlib are statically linked. + +.. + +.. bpo: 44464 +.. date: 2021-06-19-21-52-27 +.. nonce: U2oa-a +.. section: Library + +Remove exception for flake8 in deprecated importlib.metadata interfaces. +Sync with importlib_metadata 4.6. + +.. + +.. bpo: 44446 +.. date: 2021-06-17-22-39-34 +.. nonce: qwdRic +.. section: Library + +Take into account that ``lineno`` might be ``None`` in +:class:`traceback.FrameSummary`. + +.. + +.. bpo: 44439 +.. date: 2021-06-17-15-01-51 +.. nonce: 1S7QhT +.. section: Library + +Fix in :meth:`bz2.BZ2File.write` / :meth:`lzma.LZMAFile.write` methods, when +the input data is an object that supports the buffer protocol, the file +length may be wrong. + +.. + +.. bpo: 44434 +.. date: 2021-06-16-16-52-14 +.. nonce: SQS4Pg +.. section: Library + +_thread.start_new_thread() no longer calls PyThread_exit_thread() explicitly +at the thread exit, the call was redundant. On Linux with the glibc, +pthread_exit() aborts the whole process if dlopen() fails to open +libgcc_s.so file (ex: EMFILE error). Patch by Victor Stinner. + +.. + +.. bpo: 42972 +.. date: 2021-06-15-13-51-25 +.. nonce: UnyYo1 +.. section: Library + +The _thread.RLock type now fully implement the GC protocol: add a traverse +function and the :const:`Py_TPFLAGS_HAVE_GC` flag. Patch by Victor Stinner. + +.. + +.. bpo: 44422 +.. date: 2021-06-14-23-28-17 +.. nonce: BlWOgv +.. section: Library + +The :func:`threading.enumerate` function now uses a reentrant lock to +prevent a hang on reentrant call. Patch by Victor Stinner. + +.. + +.. bpo: 38291 +.. date: 2021-06-14-14-19-11 +.. nonce: ee4cSX +.. section: Library + +Importing typing.io or typing.re now prints a ``DeprecationWarning``. + +.. + +.. bpo: 37880 +.. date: 2021-06-13-00-16-56 +.. nonce: 5bTrkw +.. section: Library + +argparse actions store_const and append_const each receive a default value +of None when the ``const`` kwarg is not provided. Previously, this raised a +:exc:`TypeError`. + +.. + +.. bpo: 44389 +.. date: 2021-06-12-22-58-20 +.. nonce: WTRnoC +.. section: Library + +Fix deprecation of :data:`ssl.OP_NO_TLSv1_3` + +.. + +.. bpo: 27827 +.. date: 2021-06-12-21-25-35 +.. nonce: TMWh1i +.. section: Library + +:meth:`pathlib.PureWindowsPath.is_reserved` now identifies a greater range +of reserved filenames, including those with trailing spaces or colons. + +.. + +.. bpo: 44395 +.. date: 2021-06-12-10-08-14 +.. nonce: PcW6Sx +.. section: Library + +Fix :meth:`~email.message.MIMEPart.as_string` to pass unixfrom properly. +Patch by Dong-hee Na. + +.. + +.. bpo: 34266 +.. date: 2021-06-10-21-53-46 +.. nonce: k3fxnm +.. section: Library + +Handle exceptions from parsing the arg of :mod:`pdb`'s run/restart command. + +.. + +.. bpo: 44362 +.. date: 2021-06-10-20-07-32 +.. nonce: oVOMfd +.. section: Library + +Improve :mod:`ssl` module's deprecation messages, error reporting, and +documentation for deprecations. + +.. + +.. bpo: 44342 +.. date: 2021-06-10-15-06-47 +.. nonce: qqkGlj +.. section: Library + +[Enum] Change pickling from by-value to by-name. + +.. + +.. bpo: 44356 +.. date: 2021-06-10-08-35-38 +.. nonce: 6oDFhO +.. section: Library + +[Enum] Allow multiple data-type mixins if they are all the same. + +.. + +.. bpo: 44351 +.. date: 2021-06-10-07-26-12 +.. nonce: rvyf2v +.. section: Library + +Restore back :func:`parse_makefile` in :mod:`distutils.sysconfig` because it +behaves differently than the similar implementation in :mod:`sysconfig`. + +.. + +.. bpo: 35800 +.. date: 2021-06-09-10-08-32 +.. nonce: 3hmkWw +.. section: Library + +:class:`smtpd.MailmanProxy` is now removed as it is unusable without an +external module, ``mailman``. Patch by Dong-hee Na. + +.. + +.. bpo: 44357 +.. date: 2021-06-09-08-32-39 +.. nonce: 70Futb +.. section: Library + +Added a function that returns cube root of the given number +:func:`math.cbrt` + +.. + +.. bpo: 44339 +.. date: 2021-06-08-17-47-38 +.. nonce: 9JwMSc +.. section: Library + +Change ``math.pow(±0.0, -math.inf)`` to return ``inf`` instead of raising +``ValueError``. This brings the special-case handling of ``math.pow`` into +compliance with the IEEE 754 standard. + +.. + +.. bpo: 44242 +.. date: 2021-06-07-10-26-14 +.. nonce: MKeMCQ +.. section: Library + +Remove missing flag check from Enum creation and move into a ``verify`` +decorator. + +.. + +.. bpo: 44246 +.. date: 2021-05-31-11-34-56 +.. nonce: yHAkF0 +.. section: Library + +In ``importlib.metadata``, restore compatibility in the result from +``Distribution.entry_points`` (``EntryPoints``) to honor expectations in +older implementations and issuing deprecation warnings for these cases: A. +``EntryPoints`` objects are once again mutable, allowing for ``sort()`` +and other list-based mutation operations. Avoid deprecation warnings by +casting to a mutable sequence (e.g. ``list(dist.entry_points).sort()``). +B. ``EntryPoints`` results once again allow for access by index. To avoid +deprecation warnings, cast the result to a Sequence first (e.g. +``tuple(dist.entry_points)[0]``). + +.. + +.. bpo: 44246 +.. date: 2021-05-31-11-28-03 +.. nonce: nhmt-v +.. section: Library + +In importlib.metadata.entry_points, de-duplication of distributions no +longer requires loading the full metadata for PathDistribution objects, +improving entry point loading performance by ~10x. + +.. + +.. bpo: 43858 +.. date: 2021-05-31-04-51-02 +.. nonce: r7LOu6 +.. section: Library + +Added a function that returns a copy of a dict of logging levels: +:func:`logging.getLevelNamesMapping` + +.. + +.. bpo: 44260 +.. date: 2021-05-30-13-32-09 +.. nonce: ROEbVd +.. section: Library + +The :class:`random.Random` constructor no longer reads system entropy +without need. + +.. + +.. bpo: 44254 +.. date: 2021-05-29-01-05-43 +.. nonce: f06xDm +.. section: Library + +On Mac, give turtledemo button text a color that works on both light or dark +background. Programmers cannot control the latter. + +.. + +.. bpo: 44258 +.. date: 2021-05-28-09-43-33 +.. nonce: nh5F7R +.. section: Library + +Support PEP 515 for Fraction's initialization from string. + +.. + +.. bpo: 44235 +.. date: 2021-05-26-22-04-40 +.. nonce: qFBYpp +.. section: Library + +Remove deprecated functions in the :mod:`gettext`. Patch by Dong-hee Na. + +.. + +.. bpo: 38693 +.. date: 2021-05-26-14-50-06 +.. nonce: NkMacJ +.. section: Library + +Prefer f-strings to ``.format`` in importlib.resources. + +.. + +.. bpo: 33693 +.. date: 2021-05-26-13-34-37 +.. nonce: 3okzdo +.. section: Library + +Importlib.metadata now prefers f-strings to .format. + +.. + +.. bpo: 44241 +.. date: 2021-05-26-13-15-51 +.. nonce: TBqej8 +.. section: Library + +Incorporate minor tweaks from importlib_metadata 4.1: SimplePath protocol, +support for Metadata 2.2. + +.. + +.. bpo: 43216 +.. date: 2021-05-25-23-26-38 +.. nonce: xTUyyX +.. section: Library + +Remove the :func:`@asyncio.coroutine ` :term:`decorator` +enabling legacy generator-based coroutines to be compatible with async/await +code; remove :class:`asyncio.coroutines.CoroWrapper` used for wrapping +legacy coroutine objects in the debug mode. The decorator has been +deprecated since Python 3.8 and the removal was initially scheduled for +Python 3.10. Patch by Illia Volochii. + +.. + +.. bpo: 44210 +.. date: 2021-05-21-21-23-43 +.. nonce: 5afQ3K +.. section: Library + +Make importlib.metadata._meta.PackageMetadata public. + +.. + +.. bpo: 43643 +.. date: 2021-05-21-12-12-35 +.. nonce: GWnmcF +.. section: Library + +Declare readers.MultiplexedPath.name as a property per the spec. + +.. + +.. bpo: 27334 +.. date: 2021-05-18-00-17-21 +.. nonce: 32EJZi +.. section: Library + +The :mod:`sqlite3` context manager now performs a rollback (thus releasing +the database lock) if commit failed. Patch by Luca Citi and Erlend E. +Aasland. + +.. + +.. bpo: 4928 +.. date: 2021-05-17-21-05-06 +.. nonce: Ot2yjO +.. section: Library + +Documented existing behavior on POSIX: NamedTemporaryFiles are not deleted +when creating process is killed with SIGKILL + +.. + +.. bpo: 44154 +.. date: 2021-05-17-07-24-24 +.. nonce: GRI5bf +.. section: Library + +Optimize :class:`fractions.Fraction` pickling for large components. + +.. + +.. bpo: 33433 +.. date: 2021-05-16-17-48-24 +.. nonce: MyzO71 +.. section: Library + +For IPv4 mapped IPv6 addresses (:rfc:`4291` Section 2.5.5.2), the +:mod:`ipaddress.IPv6Address.is_private` check is deferred to the mapped IPv4 +address. This solves a bug where public mapped IPv4 addresses were +considered private by the IPv6 check. + +.. + +.. bpo: 44150 +.. date: 2021-05-16-11-57-38 +.. nonce: xAhhik +.. section: Library + +Add optional *weights* argument to statistics.fmean(). + +.. + +.. bpo: 44142 +.. date: 2021-05-16-02-24-23 +.. nonce: t-XU8k +.. section: Library + +:func:`ast.unparse` will now drop the redundant parentheses when tuples used +as assignment targets (e.g in for loops). + +.. + +.. bpo: 44145 +.. date: 2021-05-16-00-00-38 +.. nonce: ko5SJ7 +.. section: Library + +:mod:`hmac` computations were not releasing the GIL while calling the +OpenSSL ``HMAC_Update`` C API (a new feature in 3.9). This unintentionally +prevented parallel computation as other :mod:`hashlib` algorithms support. + +.. + +.. bpo: 44095 +.. date: 2021-05-14-16-06-02 +.. nonce: v_pLwY +.. section: Library + +:class:`zipfile.Path` now supports :attr:`zipfile.Path.stem`, +:attr:`zipfile.Path.suffixes`, and :attr:`zipfile.Path.suffix` attributes. + +.. + +.. bpo: 44077 +.. date: 2021-05-13-19-44-38 +.. nonce: 04b2a4 +.. section: Library + +It's now possible to receive the type of service (ToS), a.k.a. +differentiated services (DS), a.k.a. differentiated services code point +(DSCP) and explicit congestion notification (ECN) IP header fields with +``socket.IP_RECVTOS``. + +.. + +.. bpo: 37788 +.. date: 2021-05-13-19-07-28 +.. nonce: adeFcf +.. section: Library + +Fix a reference leak when a Thread object is never joined. + +.. + +.. bpo: 38908 +.. date: 2021-05-12-16-43-21 +.. nonce: nM2_rO +.. section: Library + +Subclasses of ``typing.Protocol`` which only have data variables declared +will now raise a ``TypeError`` when checked with ``isinstance`` unless they +are decorated with :func:`runtime_checkable`. Previously, these checks +passed silently. Patch provided by Yurii Karabas. + +.. + +.. bpo: 44098 +.. date: 2021-05-10-17-45-00 +.. nonce: _MoxuZ +.. section: Library + +``typing.ParamSpec`` will no longer be found in the ``__parameters__`` of +most :mod:`typing` generics except in valid use locations specified by +:pep:`612`. This prevents incorrect usage like ``typing.List[P][int]``. This +change means incorrect usage which may have passed silently in 3.10 beta 1 +and earlier will now error. + +.. + +.. bpo: 44089 +.. date: 2021-05-09-22-52-34 +.. nonce: IoANsN +.. section: Library + +Allow subclassing ``csv.Error`` in 3.10 (it was allowed in 3.9 and earlier +but was disallowed in early versions of 3.10). + +.. + +.. bpo: 44081 +.. date: 2021-05-09-03-26-31 +.. nonce: A-Mrto +.. section: Library + +:func:`ast.unparse` now doesn't use redundant spaces to separate ``lambda`` +and the ``:`` if there are no parameters. + +.. + +.. bpo: 44061 +.. date: 2021-05-07-08-39-23 +.. nonce: MvElG6 +.. section: Library + +Fix regression in previous release when calling :func:`pkgutil.iter_modules` +with a list of :class:`pathlib.Path` objects + +.. + +.. bpo: 44059 +.. date: 2021-05-06-16-01-55 +.. nonce: GF5r6O +.. section: Library + +Register the SerenityOS Browser in the :mod:`webbrowser` module. + +.. + +.. bpo: 36515 +.. date: 2021-05-05-11-44-49 +.. nonce: uOSa3q +.. section: Library + +The :mod:`hashlib` module no longer does unaligned memory accesses when +compiled for ARM platforms. + +.. + +.. bpo: 40465 +.. date: 2021-05-03-19-59-14 +.. nonce: 1tB4Y0 +.. section: Library + +Remove random module features deprecated in Python 3.9. + +.. + +.. bpo: 44018 +.. date: 2021-05-03-10-07-43 +.. nonce: VDyW8f +.. section: Library + +random.seed() no longer mutates bytearray inputs. + +.. + +.. bpo: 38352 +.. date: 2021-05-02-13-54-25 +.. nonce: N9MlhV +.. section: Library + +Add ``IO``, ``BinaryIO``, ``TextIO``, ``Match``, and ``Pattern`` to +``typing.__all__``. Patch by Jelle Zijlstra. + +.. + +.. bpo: 44002 +.. date: 2021-05-01-15-43-37 +.. nonce: KLT_wd +.. section: Library + +:mod:`urllib.parse` now uses :func:`functool.lru_cache` for its internal URL +splitting and quoting caches instead of rolling its own like its the '90s. + +The undocumented internal :mod:`urllib.parse` ``Quoted`` class API is now +deprecated, for removal in 3.14. + +.. + +.. bpo: 43972 +.. date: 2021-04-30-16-58-24 +.. nonce: Y2r9lg +.. section: Library + +When :class:`http.server.SimpleHTTPRequestHandler` sends a ``301 (Moved +Permanently)`` for a directory path not ending with `/`, add a +``Content-Length: 0`` header. This improves the behavior for certain +clients. + +.. + +.. bpo: 28528 +.. date: 2021-04-29-00-48-00 +.. nonce: JLAVWj +.. section: Library + +Fix a bug in :mod:`pdb` where :meth:`~pdb.Pdb.checkline` raises +:exc:`AttributeError` if it is called after :meth:`~pdb.Pdb.reset`. + +.. + +.. bpo: 43853 +.. date: 2021-04-15-12-02-17 +.. nonce: XXCVAp +.. section: Library + +Improved string handling for :mod:`sqlite3` user-defined functions and +aggregates: + +* It is now possible to pass strings with embedded null characters to UDFs +* Conversion failures now correctly raise :exc:`MemoryError` + +Patch by Erlend E. Aasland. + +.. + +.. bpo: 43666 +.. date: 2021-03-30-08-39-08 +.. nonce: m72tlH +.. section: Library + +AIX: `Lib/_aix_support.get_platform()` may fail in an AIX WPAR. The fileset +bos.rte appears to have a builddate in both LPAR and WPAR so this fileset is +queried rather than bos.mp64. To prevent a similar situation (no builddate +in ODM) a value (9988) sufficient for completing a build is provided. Patch +by M Felt. + +.. + +.. bpo: 43650 +.. date: 2021-03-29-00-23-30 +.. nonce: v01tic +.. section: Library + +Fix :exc:`MemoryError` in :func:`shutil.unpack_archive` which fails inside +:func:`shutil._unpack_zipfile` on large files. Patch by Igor Bolshakov. + +.. + +.. bpo: 43612 +.. date: 2021-03-24-09-40-02 +.. nonce: vMGZ4y +.. section: Library + +:func:`zlib.compress` now accepts a wbits parameter which allows users to +compress data as a raw deflate block without zlib headers and trailers in +one go. Previously this required instantiating a ``zlib.compressobj``. It +also provides a faster alternative to ``gzip.compress`` when wbits=31 is +used. + +.. + +.. bpo: 43392 +.. date: 2021-03-03-13-32-37 +.. nonce: QQumou +.. section: Library + +:func:`importlib._bootstrap._find_and_load` now implements a two-step check +to avoid locking when modules have been already imported and are ready. This +improves performance of repeated calls to :func:`importlib.import_module` +and :func:`importlib.__import__`. + +.. + +.. bpo: 43318 +.. date: 2021-02-25-08-32-06 +.. nonce: bZJw6V +.. section: Library + +Fix a bug where :mod:`pdb` does not always echo cleared breakpoints. + +.. + +.. bpo: 43234 +.. date: 2021-02-15-22-14-31 +.. nonce: F-vKAT +.. section: Library + +Prohibit passing non-:class:`concurrent.futures.ThreadPoolExecutor` +executors to :meth:`loop.set_default_executor` following a deprecation in +Python 3.8. Patch by Illia Volochii. + +.. + +.. bpo: 43232 +.. date: 2021-02-15-21-17-46 +.. nonce: awc4yZ +.. section: Library + +Prohibit previously deprecated potentially disruptive operations on +:class:`asyncio.trsock.TransportSocket`. Patch by Illia Volochii. + +.. + +.. bpo: 30077 +.. date: 2021-02-04-23-16-03 +.. nonce: v6TqAi +.. section: Library + +Added support for Apple's aifc/sowt pseudo-compression + +.. + +.. bpo: 42971 +.. date: 2021-02-02-20-11-14 +.. nonce: OpVoFu +.. section: Library + +Add definition of ``errno.EQFULL`` for platforms that define this constant +(such as macOS). + +.. + +.. bpo: 43086 +.. date: 2021-01-31-18-24-54 +.. nonce: 2_P-SH +.. section: Library + +Added a new optional :code:`strict_mode` parameter to *binascii.a2b_base64*. +When :code:`scrict_mode` is set to :code:`True`, the *a2b_base64* function +will accept only valid base64 content. More details about what "valid base64 +content" is, can be found in the function's documentation. + +.. + +.. bpo: 43024 +.. date: 2021-01-25-21-24-55 +.. nonce: vAUrIi +.. section: Library + +Improve the help signature of :func:`traceback.print_exception`, +:func:`traceback.format_exception` and +:func:`traceback.format_exception_only`. + +.. + +.. bpo: 33809 +.. date: 2021-01-16-18-36-00 +.. nonce: BiMK6V +.. section: Library + +Add the :meth:`traceback.TracebackException.print` method which prints the +formatted exception information. + +.. + +.. bpo: 42862 +.. date: 2021-01-13-00-02-44 +.. nonce: Z6ACLN +.. section: Library + +:mod:`sqlite3` now utilizes :meth:`functools.lru_cache` to implement the +connection statement cache. As a small optimisation, the default statement +cache size has been increased from 100 to 128. Patch by Erlend E. Aasland. + +.. + +.. bpo: 41818 +.. date: 2020-12-08-01-08-58 +.. nonce: zO8vV7 +.. section: Library + +Soumendra Ganguly: add termios.tcgetwinsize(), termios.tcsetwinsize(). + +.. + +.. bpo: 40497 +.. date: 2020-10-18-09-42-53 +.. nonce: CRz2sG +.. section: Library + +:meth:`subprocess.check_output` now raises :exc:`ValueError` when the +invalid keyword argument *check* is passed by user code. Previously such use +would fail later with a :exc:`TypeError`. Patch by Rémi Lapeyre. + +.. + +.. bpo: 37449 +.. date: 2020-10-11-20-23-48 +.. nonce: f-t3V6 +.. section: Library + +``ensurepip`` now uses ``importlib.resources.files()`` traversable APIs + +.. + +.. bpo: 40956 +.. date: 2020-10-01-21-46-34 +.. nonce: _tvsZ7 +.. section: Library + +Use Argument Clinic in :mod:`sqlite3`. Patches by Erlend E. Aasland. + +.. + +.. bpo: 41730 +.. date: 2020-09-10-07-23-24 +.. nonce: DyKFi9 +.. section: Library + +``DeprecationWarning`` is now raised when importing :mod:`tkinter.tix`, +which has been deprecated in documentation since Python 3.6. + +.. + +.. bpo: 20684 +.. date: 2020-07-30-14-37-15 +.. nonce: qV35GU +.. section: Library + +Remove unused ``_signature_get_bound_param`` function from :mod:`inspect` - +by Anthony Sottile. + +.. + +.. bpo: 41402 +.. date: 2020-07-26-18-17-30 +.. nonce: YRkVkp +.. section: Library + +Fix :meth:`email.message.EmailMessage.set_content` when called with binary +data and ``7bit`` content transfer encoding. + +.. + +.. bpo: 32695 +.. date: 2020-07-13-23-46-59 +.. nonce: tTqqXe +.. section: Library + +The *compresslevel* and *preset* keyword arguments of :func:`tarfile.open` +are now both documented and tested. + +.. + +.. bpo: 41137 +.. date: 2020-07-01-17-42-41 +.. nonce: AnqbP- +.. section: Library + +Use utf-8 encoding while reading .pdbrc files. Patch by Srinivas Reddy +Thatiparthy + +.. + +.. bpo: 24391 +.. date: 2020-05-30-10-48-04 +.. nonce: ZCTnhX +.. section: Library + +Improved reprs of :mod:`threading` synchronization objects: +:class:`~threading.Semaphore`, :class:`~threading.BoundedSemaphore`, +:class:`~threading.Event` and :class:`~threading.Barrier`. + +.. + +.. bpo: 5846 +.. date: 2020-05-25-23-58-29 +.. nonce: O9BIfm +.. section: Library + +Deprecated the following :mod:`unittest` functions, scheduled for removal in +Python 3.13: + +* :func:`~unittest.findTestCases` +* :func:`~unittest.makeSuite` +* :func:`~unittest.getTestCaseNames` + +Use :class:`~unittest.TestLoader` methods instead: + +* :meth:`unittest.TestLoader.loadTestsFromModule` +* :meth:`unittest.TestLoader.loadTestsFromTestCase` +* :meth:`unittest.TestLoader.getTestCaseNames` + +Patch by Erlend E. Aasland. + +.. + +.. bpo: 40563 +.. date: 2020-05-21-01-42-32 +.. nonce: fDn5bP +.. section: Library + +Support pathlike objects on dbm/shelve. Patch by Hakan Çelik and +Henry-Joseph Audéoud. + +.. + +.. bpo: 34990 +.. date: 2020-04-24-20-39-38 +.. nonce: 3SmL9M +.. section: Library + +Fixed a Y2k38 bug in the compileall module where it would fail to compile +files with a modification time after the year 2038. + +.. + +.. bpo: 39549 +.. date: 2020-02-03-21-18-31 +.. nonce: l4a8uH +.. section: Library + +Whereas the code for reprlib.Repr had previously used a hardcoded string +value of '...', this PR updates it to use of a “fillvalue” attribute, whose +value defaults to '...' and can be reset in either individual reprlib.Repr +instances or in subclasses thereof. + +.. + +.. bpo: 37022 +.. date: 2020-01-25-12-58-20 +.. nonce: FUZI25 +.. section: Library + +:mod:`pdb` now displays exceptions from ``repr()`` with its ``p`` and ``pp`` +commands. + +.. + +.. bpo: 38840 +.. date: 2020-01-16-23-41-16 +.. nonce: VzzYZz +.. section: Library + +Fix ``test___all__`` on platforms lacking a shared memory implementation. + +.. + +.. bpo: 39359 +.. date: 2020-01-16-13-54-28 +.. nonce: hzTu0h +.. section: Library + +Add one missing check that the password is a bytes object for an encrypted +zipfile. + +.. + +.. bpo: 38741 +.. date: 2019-11-12-18-59-33 +.. nonce: W7IYkq +.. section: Library + +:mod:`configparser`: using ']' inside a section header will no longer cut +the section name short at the ']' + +.. + +.. bpo: 38415 +.. date: 2019-10-08-14-08-59 +.. nonce: N1bUw6 +.. section: Library + +Added missing behavior to :func:`contextlib.asynccontextmanager` to match +:func:`contextlib.contextmanager` so decorated functions can themselves be +decorators. + +.. + +.. bpo: 30256 +.. date: 2019-09-25-13-54-41 +.. nonce: wBkzox +.. section: Library + +Pass multiprocessing BaseProxy argument ``manager_owned`` through AutoProxy. + +.. + +.. bpo: 27513 +.. date: 2019-06-03-23-53-25 +.. nonce: qITN7d +.. section: Library + +:func:`email.utils.getaddresses` now accepts :class:`email.header.Header` +objects along with string values. Patch by Zackery Spytz. + +.. + +.. bpo: 16379 +.. date: 2019-05-08-15-14-32 +.. nonce: rN5JVe +.. section: Library + +Add SQLite error code and name to :mod:`sqlite3` exceptions. Patch by Aviv +Palivoda, Daniel Shahaf, and Erlend E. Aasland. + +.. + +.. bpo: 26228 +.. date: 2019-02-26-09-31-59 +.. nonce: wyrHKc +.. section: Library + +pty.spawn no longer hangs on FreeBSD, macOS, and Solaris. + +.. + +.. bpo: 33349 +.. date: 2018-04-24-14-25-07 +.. nonce: Y_0LIr +.. section: Library + +lib2to3 now recognizes async generators everywhere. + +.. + +.. bpo: 29298 +.. date: 2017-09-20-14-43-03 +.. nonce: _78CSN +.. section: Library + +Fix ``TypeError`` when required subparsers without ``dest`` do not receive +arguments. Patch by Anthony Sottile. + +.. + +.. bpo: 45216 +.. date: 2021-09-18-13-45-19 +.. nonce: o56nyt +.. section: Documentation + +Remove extra documentation listing methods in ``difflib``. It was rendering +twice in pydoc and was outdated in some places. + +.. + +.. bpo: 45024 +.. date: 2021-09-08-17-20-19 +.. nonce: dkNPNi +.. section: Documentation + +:mod:`collections.abc` documentation has been expanded to explicitly cover +how instance and subclass checks work, with additional doctest examples and +an exhaustive list of ABCs which test membership purely by presence of the +right :term:`special method`\s. Patch by Raymond Hettinger. + +.. + +.. bpo: 44957 +.. date: 2021-08-19-15-53-08 +.. nonce: imqrh3 +.. section: Documentation + +Promote PEP 604 union syntax by using it where possible. Also, mention ``X | +Y`` more prominently in section about ``Union`` and mention ``X | None`` at +all in section about ``Optional``. + +.. + +.. bpo: 16580 +.. date: 2021-08-13-20-17-59 +.. nonce: MZ_iK9 +.. section: Documentation + +Added code equivalents for the :meth:`int.to_bytes` and +:meth:`int.from_bytes` methods, as well as tests ensuring that these code +equivalents are valid. + +.. + +.. bpo: 44903 +.. date: 2021-08-13-19-08-03 +.. nonce: aJuvQF +.. section: Documentation + +Removed the othergui.rst file, any references to it, and the list of GUI +frameworks in the FAQ. In their place I've added links to the Python Wiki +`page on GUI frameworks `. + +.. + +.. bpo: 33479 +.. date: 2021-08-11-18-02-06 +.. nonce: rCe4c5 +.. section: Documentation + +Tkinter documentation has been greatly expanded with new "Architecture" and +"Threading model" sections. + +.. + +.. bpo: 36700 +.. date: 2021-08-09-19-58-45 +.. nonce: WPNW5f +.. section: Documentation + +:mod:`base64` RFC references were updated to point to :rfc:`4648`; a section +was added to point users to the new "security considerations" section of the +RFC. + +.. + +.. bpo: 44740 +.. date: 2021-07-26-23-48-31 +.. nonce: zMFGMV +.. section: Documentation + +Replaced occurrences of uppercase "Web" and "Internet" with lowercase +versions per the 2016 revised Associated Press Style Book. + +.. + +.. bpo: 44693 +.. date: 2021-07-25-23-04-15 +.. nonce: JuCbNq +.. section: Documentation + +Update the definition of __future__ in the glossary by replacing the +confusing word "pseudo-module" with a more accurate description. + +.. + +.. bpo: 35183 +.. date: 2021-07-22-08-28-03 +.. nonce: p9BWTB +.. section: Documentation + +Add typical examples to os.path.splitext docs + +.. + +.. bpo: 30511 +.. date: 2021-07-20-21-03-18 +.. nonce: eMFkRi +.. section: Documentation + +Clarify that :func:`shutil.make_archive` is not thread-safe due to reliance +on changing the current working directory. + +.. + +.. bpo: 44561 +.. date: 2021-07-18-22-43-14 +.. nonce: T7HpWm +.. section: Documentation + +Update of three expired hyperlinks in Doc/distributing/index.rst: "Project +structure", "Building and packaging the project", and "Uploading the project +to the Python Packaging Index". + +.. + +.. bpo: 44651 +.. date: 2021-07-18-22-26-02 +.. nonce: SjT9iY +.. section: Documentation + +Delete entry "coercion" in Doc/glossary.rst for its outdated definition. + +.. + +.. bpo: 42958 +.. date: 2021-07-15-11-19-03 +.. nonce: gC5IHM +.. section: Documentation + +Updated the docstring and docs of :func:`filecmp.cmp` to be more accurate +and less confusing especially in respect to *shallow* arg. + +.. + +.. bpo: 44631 +.. date: 2021-07-13-22-25-13 +.. nonce: qkGwe4 +.. section: Documentation + +Refactored the ``repr()`` code of the ``_Environ`` (os module). + +.. + +.. bpo: 44613 +.. date: 2021-07-12-11-39-20 +.. nonce: DIXNzc +.. section: Documentation + +importlib.metadata is no longer provisional. + +.. + +.. bpo: 44558 +.. date: 2021-07-03-18-25-17 +.. nonce: 0pTknl +.. section: Documentation + +Match the docstring and python implementation of :func:`~operator.countOf` +to the behavior of its c implementation. + +.. + +.. bpo: 44544 +.. date: 2021-07-02-14-02-29 +.. nonce: _5_aCz +.. section: Documentation + +List all kwargs for :func:`textwrap.wrap`, :func:`textwrap.fill`, and +:func:`textwrap.shorten`. Now, there are nav links to attributes of +:class:`TextWrap`, which makes navigation much easier while minimizing +duplication in the documentation. + +.. + +.. bpo: 38062 +.. date: 2021-06-28-12-13-48 +.. nonce: 9Ehp9O +.. section: Documentation + +Clarify that atexit uses equality comparisons internally. + +.. + +.. bpo: 40620 +.. date: 2021-06-26-17-41-06 +.. nonce: PAYDrB +.. section: Documentation + +Convert examples in tutorial controlflow.rst section 4.3 to be +interpreter-demo style. + +.. + +.. bpo: 43066 +.. date: 2021-06-24-14-37-16 +.. nonce: Ti7ahX +.. section: Documentation + +Added a warning to :mod:`zipfile` docs: filename arg with a leading slash +may cause archive to be un-openable on Windows systems. + +.. + +.. bpo: 39452 +.. date: 2021-06-23-15-21-36 +.. nonce: o_I-6d +.. section: Documentation + +Rewrote ``Doc/library/__main__.rst``. Broadened scope of the document to +explicitly discuss and differentiate between ``__main__.py`` in packages +versus the ``__name__ == '__main__'`` expression (and the idioms that +surround it). + +.. + +.. bpo: 13814 +.. date: 2021-06-21-15-46-32 +.. nonce: LDcslu +.. section: Documentation + +In the Design FAQ, answer "Why don't generators support the with statement?" + +.. + +.. bpo: 27752 +.. date: 2021-06-18-18-04-53 +.. nonce: NEByNk +.. section: Documentation + +Documentation of csv.Dialect is more descriptive. + +.. + +.. bpo: 44453 +.. date: 2021-06-18-06-44-45 +.. nonce: 3PIkj2 +.. section: Documentation + +Fix documentation for the return type of :func:`sysconfig.get_path`. + +.. + +.. bpo: 44392 +.. date: 2021-06-16-18-09-49 +.. nonce: 6RF1Sc +.. section: Documentation + +Added a new section in the C API documentation for types used in type +hinting. Documented ``Py_GenericAlias`` and ``Py_GenericAliasType``. + +.. + +.. bpo: 38291 +.. date: 2021-06-14-09-20-37 +.. nonce: VMYa_Q +.. section: Documentation + +Mark ``typing.io`` and ``typing.re`` as deprecated since Python 3.8 in the +documentation. They were never properly supported by type checkers. + +.. + +.. bpo: 44322 +.. date: 2021-06-06-14-12-00 +.. nonce: K0PHfE +.. section: Documentation + +Document that SyntaxError args have a details tuple and that details are +adjusted for errors in f-string field replacement expressions. + +.. + +.. bpo: 42392 +.. date: 2021-05-26-11-16-33 +.. nonce: oxRx6E +.. section: Documentation + +Document the deprecation and removal of the ``loop`` parameter for many +functions and classes in :mod:`asyncio`. + +.. + +.. bpo: 44195 +.. date: 2021-05-23-09-11-28 +.. nonce: 1bqkOs +.. section: Documentation + +Corrected references to ``TraversableResources`` in docs. There is no +``TraversableReader``. + +.. + +.. bpo: 41963 +.. date: 2021-05-17-20-03-47 +.. nonce: eUz9_o +.. section: Documentation + +Document that ``ConfigParser`` strips off comments when reading +configuration files. + +.. + +.. bpo: 44072 +.. date: 2021-05-08-09-48-05 +.. nonce: fb2x5I +.. section: Documentation + +Correct where in the numeric ABC hierarchy ``**`` support is added, i.e., in +numbers.Complex, not numbers.Integral. + +.. + +.. bpo: 43558 +.. date: 2021-05-07-12-27-09 +.. nonce: UGhA8R +.. section: Documentation + +Add the remark to :mod:`dataclasses` documentation that the :meth:`__init__` +of any base class has to be called in :meth:`__post_init__`, along with a +code example. + +.. + +.. bpo: 44025 +.. date: 2021-05-03-22-08-08 +.. nonce: gcB7iP +.. section: Documentation + +Clarify when '_' in match statements is a keyword, and when not. + +.. + +.. bpo: 41706 +.. date: 2020-09-03-13-37-19 +.. nonce: _zXWOR +.. section: Documentation + +Fix docs about how methods like ``__add__`` are invoked when evaluating +operator expressions. + +.. + +.. bpo: 41621 +.. date: 2020-08-24-13-35-04 +.. nonce: nqaw9G +.. section: Documentation + +Document that :class:`collections.defaultdict` parameter ``default_factory`` +defaults to None and is positional-only. + +.. + +.. bpo: 41576 +.. date: 2020-08-21-22-59-37 +.. nonce: 7a6CQR +.. section: Documentation + +document BaseException in favor of bare except + +.. + +.. bpo: 21760 +.. date: 2020-03-21-01-19-28 +.. nonce: CqofIc +.. section: Documentation + +The description for __file__ fixed. Patch by Furkan Onder + +.. + +.. bpo: 39498 +.. date: 2020-01-30-05-18-48 +.. nonce: Nu3sFL +.. section: Documentation + +Add a "Security Considerations" index which links to standard library +modules that have explicitly documented security considerations. + +.. + +.. bpo: 33479 +.. date: 2018-05-19-15-59-29 +.. nonce: 4cLlxo +.. section: Documentation + +Remove the unqualified claim that tkinter is threadsafe. It has not been +true for several years and likely never was. An explanation of what is true +may be added later, after more discussion, and possibly after patching +_tkinter.c, + +.. + +.. bpo: 40173 +.. date: 2021-09-30-16-54-39 +.. nonce: J_slCw +.. section: Tests + +Fix :func:`test.support.import_helper.import_fresh_module`. + +.. + +.. bpo: 45280 +.. date: 2021-09-25-11-05-31 +.. nonce: 3MA6lC +.. section: Tests + +Add a test case for empty :class:`typing.NamedTuple`. + +.. + +.. bpo: 45269 +.. date: 2021-09-24-10-41-49 +.. nonce: 8jKEr8 +.. section: Tests + +Cover case when invalid ``markers`` type is supplied to ``c_make_encoder``. + +.. + +.. bpo: 45128 +.. date: 2021-09-16-17-22-35 +.. nonce: Jz6fl2 +.. section: Tests + +Fix ``test_multiprocessing_fork`` failure due to ``test_logging`` and +``sys.modules`` manipulation. + +.. + +.. bpo: 45209 +.. date: 2021-09-15-23-32-39 +.. nonce: 55ntL5 +.. section: Tests + +Fix ``UserWarning: resource_tracker`` warning in +``_test_multiprocessing._TestSharedMemory.test_shared_memory_cleaned_after_process_termination`` + +.. + +.. bpo: 45185 +.. date: 2021-09-14-14-54-04 +.. nonce: qFx5I6 +.. section: Tests + +Enables ``TestEnumerations`` test cases in ``test_ssl`` suite. + +.. + +.. bpo: 45195 +.. date: 2021-09-14-13-16-18 +.. nonce: EyQR1G +.. section: Tests + +Fix test_readline.test_nonascii(): sometimes, the newline character is not +written at the end, so don't expect it in the output. Patch by Victor +Stinner. + +.. + +.. bpo: 45156 +.. date: 2021-09-13-00-28-17 +.. nonce: 8oomV3 +.. section: Tests + +Fixes infinite loop on :func:`unittest.mock.seal` of mocks created by +:func:`~unittest.create_autospec`. + +.. + +.. bpo: 45125 +.. date: 2021-09-11-22-08-18 +.. nonce: FVSzs2 +.. section: Tests + +Improves pickling tests and docs of ``SharedMemory`` and ``SharableList`` +objects. + +.. + +.. bpo: 44860 +.. date: 2021-09-08-13-01-37 +.. nonce: qXd0kx +.. section: Tests + +Update ``test_sysconfig.test_user_similar()`` for the posix_user scheme: +``platlib`` doesn't use :data:`sys.platlibdir`. Patch by Victor Stinner. + +.. + +.. bpo: 45052 +.. date: 2021-09-06-19-00-29 +.. nonce: yrOK3J +.. section: Tests + +``WithProcessesTestSharedMemory.test_shared_memory_basics`` test was +ignored, because ``self.assertEqual(sms.size, sms2.size)`` line was failing. +It is now removed and test is unskipped. + +The main motivation for this line to be removed from the test is that the +``size`` of ``SharedMemory`` is not ever guaranteed to be the same. It is +decided by the platform. + +.. + +.. bpo: 44895 +.. date: 2021-09-01-17-17-44 +.. nonce: kV7H77 +.. section: Tests + +libregrtest now clears the type cache later to reduce the risk of false +alarm when checking for reference leaks. Previously, the type cache was +cleared too early and libregrtest raised a false alarm about reference leaks +under very specific conditions. Patch by Irit Katriel and Victor Stinner. + +.. + +.. bpo: 45042 +.. date: 2021-08-30-11-54-14 +.. nonce: QMz3X8 +.. section: Tests + +Fixes that test classes decorated with +``@hashlib_helper.requires_hashdigest`` were skipped all the time. + +.. + +.. bpo: 25130 +.. date: 2021-08-27-22-37-19 +.. nonce: ig4oJe +.. section: Tests + +Add calls of :func:`gc.collect` in tests to support PyPy. + +.. + +.. bpo: 45011 +.. date: 2021-08-26-14-20-44 +.. nonce: mQZdXU +.. section: Tests + +Made tests relying on the :mod:`_asyncio` C extension module optional to +allow running on alternative Python implementations. Patch by Serhiy +Storchaka. + +.. + +.. bpo: 44949 +.. date: 2021-08-18-18-30-12 +.. nonce: VE5ENv +.. section: Tests + +Fix auto history tests of test_readline: sometimes, the newline character is +not written at the end, so don't expect it in the output. + +.. + +.. bpo: 44891 +.. date: 2021-08-13-12-11-06 +.. nonce: T9_mBT +.. section: Tests + +Tests were added to clarify :func:`id` is preserved when ``obj * 1`` is used +on :class:`str` and :class:`bytes` objects. Patch by Nikita Sobolev. + +.. + +.. bpo: 44852 +.. date: 2021-08-06-18-36-04 +.. nonce: sUL8YX +.. section: Tests + +Add ability to wholesale silence DeprecationWarnings while running the +regression test suite. + +.. + +.. bpo: 40928 +.. date: 2021-08-06-00-07-15 +.. nonce: aIwb6G +.. section: Tests + +Notify users running test_decimal regression tests on macOS of potential +harmless "malloc can't allocate region" messages spewed by test_decimal. + +.. + +.. bpo: 44734 +.. date: 2021-07-24-20-09-15 +.. nonce: KKsNOV +.. section: Tests + +Fixed floating point precision issue in turtle tests. + +.. + +.. bpo: 44708 +.. date: 2021-07-22-16-38-39 +.. nonce: SYNaac +.. section: Tests + +Regression tests, when run with -w, are now re-running only the affected +test methods instead of re-running the entire test file. + +.. + +.. bpo: 42095 +.. date: 2021-07-17-11-41-20 +.. nonce: kCB7oj +.. section: Tests + +Added interop tests for Apple plists: generate plist files with Python +plistlib and parse with Apple plutil; and the other way round. + +.. + +.. bpo: 44647 +.. date: 2021-07-16-14-02-33 +.. nonce: 5LzqIy +.. section: Tests + +Added a permanent Unicode-valued environment variable to regression tests to +ensure they handle this use case in the future. If your test environment +breaks because of that, report a bug to us, and temporarily set +PYTHONREGRTEST_UNICODE_GUARD=0 in your test environment. + +.. + +.. bpo: 44515 +.. date: 2021-06-26-18-37-36 +.. nonce: e9fO6f +.. section: Tests + +Adjust recently added contextlib tests to avoid assuming the use of a +refcounted GC + +.. + +.. bpo: 44287 +.. date: 2021-06-21-17-53-41 +.. nonce: YON57s +.. section: Tests + +Fix asyncio test_popen() of test_windows_utils by using a longer timeout. +Use military grade battle-tested :data:`test.support.SHORT_TIMEOUT` timeout +rather than a hardcoded timeout of 10 seconds: it's 30 seconds by default, +but it is made longer on slow buildbots. Patch by Victor Stinner. + +.. + +.. bpo: 44451 +.. date: 2021-06-18-15-19-35 +.. nonce: aj5pqE +.. section: Tests + +Reset ``DeprecationWarning`` filters in +``test.test_importlib.test_metadata_api.APITests.test_entry_points_by_index`` +to avoid ``StopIteration`` error if ``DeprecationWarnings`` are ignored. + +.. + +.. bpo: 44363 +.. date: 2021-06-10-11-19-43 +.. nonce: -K9jD0 +.. section: Tests + +Account for address sanitizer in test_capi. test_capi now passes when run +GCC address sanitizer. + +.. + +.. bpo: 44364 +.. date: 2021-06-09-15-32-05 +.. nonce: zu9Zee +.. section: Tests + +Add non integral tests for :func:`math.sqrt` function. + +.. + +.. bpo: 43921 +.. date: 2021-06-03-03-29-34 +.. nonce: nwH1FS +.. section: Tests + +Fix test_ssl.test_wrong_cert_tls13(): use ``suppress_ragged_eofs=False``, +since ``read()`` can raise :exc:`ssl.SSLEOFError` on Windows. Patch by +Victor Stinner. + +.. + +.. bpo: 43921 +.. date: 2021-06-02-17-41-42 +.. nonce: xP7yZ4 +.. section: Tests + +Fix test_pha_required_nocert() of test_ssl: catch two more EOF cases (when +the ``recv()`` method returns an empty string). Patch by Victor Stinner. + +.. + +.. bpo: 44131 +.. date: 2021-05-14-14-13-44 +.. nonce: YPizoC +.. section: Tests + +Add test_frozenmain to test_embed to test the :c:func:`Py_FrozenMain` C +function. Patch by Victor Stinner. + +.. + +.. bpo: 31904 +.. date: 2021-05-07-15-46-04 +.. nonce: 8dk3la +.. section: Tests + +Ignore error string case in test_file_not_exists(). + +.. + +.. bpo: 42083 +.. date: 2021-05-04-18-10-57 +.. nonce: EMS2TK +.. section: Tests + +Add test to check that ``PyStructSequence_NewType`` accepts a +``PyStructSequence_Desc`` with ``doc`` field set to ``NULL``. + +.. + +.. bpo: 35753 +.. date: 2020-10-25-19-20-26 +.. nonce: 2LT-hO +.. section: Tests + +Fix crash in doctest when doctest parses modules that include unwrappable +functions by skipping those functions. + +.. + +.. bpo: 30256 +.. date: 2019-09-25-18-10-10 +.. nonce: A5i76Q +.. section: Tests + +Add test for nested queues when using ``multiprocessing`` shared objects +``AutoProxy[Queue]`` inside ``ListProxy`` and ``DictProxy`` + +.. + +.. bpo: 45220 +.. date: 2021-09-16-18-00-43 +.. nonce: TgbkvW +.. section: Build + +Avoid building with the Windows 11 SDK previews automatically. This may be +overridden by setting the ``DefaultWindowsSDKVersion`` environment variable +before building. + +.. + +.. bpo: 45020 +.. date: 2021-09-14-10-07-23 +.. nonce: _VGGPv +.. section: Build + +Freeze stdlib modules that are imported during startup. This provides +significant performance improvements to startup. If necessary, use the +previously added "-X frozen_modules=off" commandline option to force +importing the source modules. + +.. + +.. bpo: 45188 +.. date: 2021-09-14-00-47-57 +.. nonce: MNbo_T +.. section: Build + +Windows builds now regenerate frozen modules as the first part of the build. +Previously the regeneration was later in the build, which would require it +to be restarted if any modules had changed. + +.. + +.. bpo: 45163 +.. date: 2021-09-11-06-05-23 +.. nonce: q7xT93 +.. section: Build + +Fixes Haiku platform build. + +.. + +.. bpo: 45067 +.. date: 2021-09-09-16-45-26 +.. nonce: mFmY92 +.. section: Build + +The ncurses function extended_color_content was introduced in 2017 + +(https://invisible-island.net/ncurses/NEWS.html#index-t20170401). The + +ncurses-devel package in CentOS 7 had a older version ncurses resulted in +compilation error. For compiling ncurses with extended color support, we +verify the version of the ncurses library >= 20170401. + +.. + +.. bpo: 45019 +.. date: 2021-08-26-13-10-46 +.. nonce: e0mo49 +.. section: Build + +Generate lines in relevant files for frozen modules. Up until now each of +the files had to be edited manually. This change makes it easier to add to +and modify the frozen modules. + +.. + +.. bpo: 44340 +.. date: 2021-07-19-01-09-56 +.. nonce: JNeOf4 +.. section: Build + +Add support for building with clang thin lto via --with-lto=thin/full. Patch +by Dong-hee Na and Brett Holman. + +.. + +.. bpo: 44535 +.. date: 2021-06-30-02-32-46 +.. nonce: M9iN4- +.. section: Build + +Enable building using a Visual Studio 2022 install on Windows. + +.. + +.. bpo: 43298 +.. date: 2021-06-19-11-50-03 +.. nonce: 9ircMb +.. section: Build + +Improved error message when building without a Windows SDK installed. + +.. + +.. bpo: 44381 +.. date: 2021-06-10-18-08-44 +.. nonce: Xpc1iX +.. section: Build + +The Windows build now accepts :envvar:`EnableControlFlowGuard` set to +``guard`` to enable CFG. + +.. + +.. bpo: 41282 +.. date: 2021-05-24-03-31-17 +.. nonce: L8nP44 +.. section: Build + +Fix broken ``make install`` that caused standard library extension modules +to be unnecessarily and incorrectly rebuilt during the install phase of +cpython. + +.. + +.. bpo: 45375 +.. date: 2021-10-05-12-41-53 +.. nonce: CohPP- +.. section: Windows + +Fixes an assertion failure due to searching for the standard library in +unnormalised paths. + +.. + +.. bpo: 45022 +.. date: 2021-09-03-18-05-21 +.. nonce: bgpD_r +.. section: Windows + +Update Windows release to include libffi 3.4.2 + +.. + +.. bpo: 45007 +.. date: 2021-08-27-23-50-02 +.. nonce: NIBlVG +.. section: Windows + +Update to OpenSSL 1.1.1l in Windows build + +.. + +.. bpo: 44848 +.. date: 2021-08-06-10-11-07 +.. nonce: ib3Jcz +.. section: Windows + +Upgrade Windows installer to use SQLite 3.36.0. + +.. + +.. bpo: 44572 +.. date: 2021-07-13-15-32-49 +.. nonce: gXvhDc +.. section: Windows + +Avoid consuming standard input in the :mod:`platform` module + +.. + +.. bpo: 44582 +.. date: 2021-07-07-21-07-18 +.. nonce: 4Mm6Hh +.. section: Windows + +Accelerate speed of :mod:`mimetypes` initialization using a native +implementation of the registry scan. + +.. + +.. bpo: 41299 +.. date: 2021-06-06-16-36-13 +.. nonce: Rg-vb_ +.. section: Windows + +Fix 16 milliseconds jitter when using timeouts in :mod:`threading`, such as +with :meth:`threading.Lock.acquire` or :meth:`threading.Condition.wait`. + +.. + +.. bpo: 42686 +.. date: 2021-01-01-21-21-03 +.. nonce: G_f-TC +.. section: Windows + +Build :mod:`sqlite3` with math functions enabled. Patch by Erlend E. +Aasland. + +.. + +.. bpo: 40263 +.. date: 2020-04-13-15-20-28 +.. nonce: 1KKEbu +.. section: Windows + +This is a follow-on bug from https://bugs.python.org/issue26903. Once that +is applied we run into an off-by-one assertion problem. The assert was not +correct. + +.. + +.. bpo: 45007 +.. date: 2021-08-30-00-04-10 +.. nonce: pixqUB +.. section: macOS + +Update macOS installer builds to use OpenSSL 1.1.1l. + +.. + +.. bpo: 34602 +.. date: 2021-08-27-16-55-10 +.. nonce: ZjHsYJ +.. section: macOS + +When building CPython on macOS with ``./configure +--with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now +quadrupled to allow for the entire test suite to pass. + +.. + +.. bpo: 44848 +.. date: 2021-08-06-10-08-41 +.. nonce: 0uYXsE +.. section: macOS + +Update macOS installer to use SQLite 3.36.0. + +.. + +.. bpo: 44689 +.. date: 2021-07-20-22-27-01 +.. nonce: mmT_xH +.. section: macOS + +:meth:`ctypes.util.find_library` now works correctly on macOS 11 Big Sur +even if Python is built on an older version of macOS. Previously, when +built on older macOS systems, ``find_library`` was not able to find macOS +system libraries when running on Big Sur due to changes in how system +libraries are stored. + +.. + +.. bpo: 41972 +.. date: 2021-07-12-15-42-02 +.. nonce: yUjE8j +.. section: macOS + +The framework build's user header path in sysconfig is changed to add a +'pythonX.Y' component to match distutils's behavior. + +.. + +.. bpo: 43109 +.. date: 2021-05-24-21-15-41 +.. nonce: npKJ9c +.. section: macOS + +Allow --with-lto configure option to work with Apple-supplied Xcode or +Command Line Tools. + +.. + +.. bpo: 34932 +.. date: 2021-03-29-21-11-23 +.. nonce: f3Hdyd +.. section: macOS + +Add socket.TCP_KEEPALIVE support for macOS. Patch by Shane Harvey. + +.. + +.. bpo: 45296 +.. date: 2021-09-27-01-21-59 +.. nonce: 9H8rdY +.. section: IDLE + +On Windows, change exit/quit message to suggest Ctrl-D, which works, instead +of , which does not work in IDLE. + +.. + +.. bpo: 45193 +.. date: 2021-09-15-03-20-06 +.. nonce: G61_GV +.. section: IDLE + +Make completion boxes appear on Ubuntu again. + +.. + +.. bpo: 40128 +.. date: 2021-06-11-17-43-39 +.. nonce: 7vDN3U +.. section: IDLE + +Mostly fix completions on macOS when not using tcl/tk 8.6.11 (as with 3.9). +The added update_idletask call should be harmless and possibly helpful +otherwise. + +.. + +.. bpo: 33962 +.. date: 2021-06-10-00-50-02 +.. nonce: ikAUNg +.. section: IDLE + +Move the indent space setting from the Font tab to the new Windows tab. +Patch by Mark Roseman and Terry Jan Reedy. + +.. + +.. bpo: 40468 +.. date: 2021-06-08-03-04-51 +.. nonce: tUCGUb +.. section: IDLE + +Split the settings dialog General tab into Windows and Shell/ED tabs. Move +help sources, which extend the Help menu, to the Extensions tab. Make space +for new options and shorten the dialog. The latter makes the dialog better +fit small screens. + +.. + +.. bpo: 41611 +.. date: 2021-05-27-18-22-46 +.. nonce: jOKpfc +.. section: IDLE + +Avoid uncaught exceptions in ``AutoCompleteWindow.winconfig_event()``. + +.. + +.. bpo: 41611 +.. date: 2021-05-27-13-39-43 +.. nonce: liNQqj +.. section: IDLE + +Fix IDLE sometimes freezing upon tab-completion on macOS. + +.. + +.. bpo: 44010 +.. date: 2021-05-09-09-02-09 +.. nonce: TaLe9x +.. section: IDLE + +Highlight the new :ref:`match ` statement's :ref:`soft keywords +`: :keyword:`match`, :keyword:`case `, and :keyword:`_ +`. However, this highlighting is not perfect and will be +incorrect in some rare cases, including some ``_``-s in ``case`` patterns. + +.. + +.. bpo: 44026 +.. date: 2021-05-05-09-45-24 +.. nonce: m2Z0zR +.. section: IDLE + +Include interpreter's typo fix suggestions in message line for NameErrors +and AttributeErrors. Patch by E. Paine. + +.. + +.. bpo: 44786 +.. date: 2021-09-14-11-44-26 +.. nonce: DU0LC0 +.. section: Tools/Demos + +Fix a warning in regular expression in the c-analyzer script. + +.. + +.. bpo: 44967 +.. date: 2021-08-26-11-57-31 +.. nonce: UT1RMV +.. section: Tools/Demos + +pydoc now returns a non-zero status code when a module cannot be found. + +.. + +.. bpo: 44978 +.. date: 2021-08-22-11-45-31 +.. nonce: QupKV3 +.. section: Tools/Demos + +Allow the Argument Clinic tool to handle ``__complex__`` special methods. + +.. + +.. bpo: 43425 +.. date: 2021-07-01-22-21-25 +.. nonce: t65len +.. section: Tools/Demos + +Removed the 'test2to3' demo project that demonstrated using lib2to3 to +support Python 2.x and Python 3.x from a single source in a distutils +package. Patch by Dong-hee Na + +.. + +.. bpo: 44074 +.. date: 2021-05-08-13-57-00 +.. nonce: F09kCK +.. section: Tools/Demos + +Make patchcheck automatically detect the correct base branch name +(previously it was hardcoded to 'master') + +.. + +.. bpo: 20291 +.. date: 2020-02-25-18-22-09 +.. nonce: AyrDiZ +.. section: Tools/Demos + +Added support for variadic positional parameters in Argument Clinic. + +.. + +.. bpo: 41710 +.. date: 2021-09-30-03-14-35 +.. nonce: DDWJKx +.. section: C API + +The PyThread_acquire_lock_timed() function now clamps the timeout if it is +too large, rather than aborting the process. Patch by Victor Stinner. + +.. + +.. bpo: 44687 +.. date: 2021-09-19-17-18-25 +.. nonce: 3fqDRC +.. section: C API + +:meth:`BufferedReader.peek` no longer raises :exc:`ValueError` when the +entire file has already been buffered. + +.. + +.. bpo: 45116 +.. date: 2021-09-16-18-05-20 +.. nonce: WxXewl +.. section: C API + +Add the :c:macro:`Py_ALWAYS_INLINE` macro to ask the compiler to always +inline a static inline function. The compiler can ignore it and decides to +not inline the function. Patch by Victor Stinner. + +.. + +.. bpo: 45094 +.. date: 2021-09-03-15-53-43 +.. nonce: tinXwL +.. section: C API + +Add the :c:macro:`Py_NO_INLINE` macro to disable inlining on a function. +Patch by Victor Stinner. + +.. + +.. bpo: 45061 +.. date: 2021-08-31-15-21-36 +.. nonce: ZH0HVe +.. section: C API + +Add a deallocator to the :class:`bool` type to detect refcount bugs in C +extensions which call ``Py_DECREF(Py_True);`` or ``Py_DECREF(Py_False);`` by +mistake. Patch by Victor Stinner. + +.. + +.. bpo: 42035 +.. date: 2021-08-02-20-49-36 +.. nonce: HTBcZt +.. section: C API + +Add a new :c:func:`PyType_GetQualName` function to get type's qualified +name. + +.. + +.. bpo: 41103 +.. date: 2021-07-29-16-04-28 +.. nonce: hiKKcF +.. section: C API + +Reverts removal of the old buffer protocol because they are part of stable +ABI. + +.. + +.. bpo: 44751 +.. date: 2021-07-27-17-29-12 +.. nonce: 4qmbDG +.. section: C API + +Remove ``crypt.h`` include from the public ``Python.h`` header. + +.. + +.. bpo: 42747 +.. date: 2021-07-20-16-21-06 +.. nonce: rRxjUY +.. section: C API + +The ``Py_TPFLAGS_HAVE_VERSION_TAG`` type flag now does nothing. The +``Py_TPFLAGS_HAVE_AM_SEND`` flag (which was added in 3.10) is removed. Both +were unnecessary because it is not possible to have type objects with the +relevant fields missing. + +.. + +.. bpo: 44530 +.. date: 2021-06-28-23-44-47 +.. nonce: qij7YC +.. section: C API + +Added the ``co_qualname`` to the ``PyCodeObject`` structure to propagate the +qualified name from the compiler to code objects. + +Patch by Gabriele N. Tornetta + +.. + +.. bpo: 44441 +.. date: 2021-06-23-12-12-04 +.. nonce: 3p14JB +.. section: C API + +:c:func:`Py_RunMain` now resets :c:data:`PyImport_Inittab` to its initial +value at exit. It must be possible to call :c:func:`PyImport_AppendInittab` +or :c:func:`PyImport_ExtendInittab` at each Python initialization. Patch by +Victor Stinner. + +.. + +.. bpo: 39947 +.. date: 2021-06-23-10-31-45 +.. nonce: je_HMo +.. section: C API + +Remove 4 private trashcan C API functions which were only kept for the +backward compatibility of the stable ABI with Python 3.8 and older, since +the trashcan API was not usable with the limited C API on Python 3.8 and +older. The trashcan API was excluded from the limited C API in Python 3.9. + +Removed functions: + +* _PyTrash_deposit_object() +* _PyTrash_destroy_chain() +* _PyTrash_thread_deposit_object() +* _PyTrash_thread_destroy_chain() + +The trashcan C API was never usable with the limited C API, since old +trashcan macros accessed directly :c:type:`PyThreadState` members like +``_tstate->trash_delete_nesting``, whereas the :c:type:`PyThreadState` +structure is opaque in the limited C API. + +Exclude also the ``PyTrash_UNWIND_LEVEL`` constant from the C API. + +Patch by Victor Stinner. + +.. + +.. bpo: 40939 +.. date: 2021-06-22-17-00-06 +.. nonce: CGB0I5 +.. section: C API + +Removed documentation for the removed ``PyParser_*`` C API. + +.. + +.. bpo: 43795 +.. date: 2021-06-15-16-28-18 +.. nonce: fy0AXK +.. section: C API + +The list in :ref:`stable-abi-list` now shows the public name +:c:struct:`PyFrameObject` rather than ``_frame``. The non-existing entry +``_node`` no longer appears in the list. + +.. + +.. bpo: 44378 +.. date: 2021-06-10-15-22-31 +.. nonce: jGYakF +.. section: C API + +:c:func:`Py_IS_TYPE` no longer uses :c:func:`Py_TYPE` to avoid a compiler +warning: no longer cast ``const PyObject*`` to ``PyObject*``. Patch by +Victor Stinner. + +.. + +.. bpo: 39573 +.. date: 2021-06-03-00-59-48 +.. nonce: -elHTJ +.. section: C API + +Convert the :c:func:`Py_TYPE` and :c:func:`Py_SIZE` macros to static inline +functions. The :c:func:`Py_SET_TYPE` and :c:func:`Py_SET_SIZE` functions +must now be used to set an object type and size. Patch by Victor Stinner. + +.. + +.. bpo: 44263 +.. date: 2021-05-31-11-31-13 +.. nonce: 8mIOfV +.. section: C API + +The :c:func:`PyType_Ready` function now raises an error if a type is defined +with the :const:`Py_TPFLAGS_HAVE_GC` flag set but has no traverse function +(:c:member:`PyTypeObject.tp_traverse`). Patch by Victor Stinner. + +.. + +.. bpo: 43795 +.. date: 2021-05-19-15-09-47 +.. nonce: WAHRxt +.. section: C API + +The undocumented function :c:func:`Py_FrozenMain` is removed from the +Limited API. + +.. + +.. bpo: 44113 +.. date: 2021-05-12-12-24-45 +.. nonce: DcgOqE +.. section: C API + +Deprecate the following functions to configure the Python initialization: + +* :c:func:`PySys_AddWarnOptionUnicode` +* :c:func:`PySys_AddWarnOption` +* :c:func:`PySys_AddXOption` +* :c:func:`PySys_HasWarnOptions` +* :c:func:`Py_SetPath` +* :c:func:`Py_SetProgramName` +* :c:func:`Py_SetPythonHome` +* :c:func:`Py_SetStandardStreamEncoding` +* :c:func:`_Py_SetProgramFullPath` + +Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization +Configuration ` instead (:pep:`587`). + +.. + +.. bpo: 44094 +.. date: 2021-05-10-14-34-22 +.. nonce: HayXZO +.. section: C API + +Remove ``PyErr_SetFromErrnoWithUnicodeFilename()``, +``PyErr_SetFromWindowsErrWithUnicodeFilename()``, and +``PyErr_SetExcFromWindowsErrWithUnicodeFilename()``. They are not documented +and have been deprecated since Python 3.3. + +.. + +.. bpo: 43795 +.. date: 2021-05-05-19-04-50 +.. nonce: 9Ojj73 +.. section: C API + +:c:func:`PyCodec_Unregister` is now properly exported as a function in the +Windows Stable ABI DLL. + +.. + +.. bpo: 44029 +.. date: 2021-05-04-17-43-39 +.. nonce: ayX4PR +.. section: C API + +Remove deprecated ``Py_UNICODE`` APIs: ``PyUnicode_Encode``, +``PyUnicode_EncodeUTF7``, ``PyUnicode_EncodeUTF8``, +``PyUnicode_EncodeUTF16``, ``PyUnicode_EncodeUTF32``, +``PyUnicode_EncodeLatin1``, ``PyUnicode_EncodeMBCS``, +``PyUnicode_EncodeDecimal``, ``PyUnicode_EncodeRawUnicodeEscape``, +``PyUnicode_EncodeCharmap``, ``PyUnicode_EncodeUnicodeEscape``, +``PyUnicode_TransformDecimalToASCII``, ``PyUnicode_TranslateCharmap``, +``PyUnicodeEncodeError_Create``, ``PyUnicodeTranslateError_Create``. See +:pep:`393` and :pep:`624` for reference. + +.. + +.. bpo: 42035 +.. date: 2020-12-23-01-28-50 +.. nonce: S9eUm0 +.. section: C API + +Add a new :c:func:`PyType_GetName` function to get type's short name. diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst index a9eba80f5f9670..97bdef6c93213f 100644 --- a/Misc/NEWS.d/3.5.0a1.rst +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -2648,7 +2648,7 @@ module. .. nonce: THJSYB .. section: Library -Changed FeedParser feed() to avoid O(N**2) behavior when parsing long line. +Changed FeedParser feed() to avoid O(N\ :sup:`2`) behavior when parsing long line. Original patch by Raymond Hettinger. .. diff --git a/Misc/NEWS.d/3.6.4rc1.rst b/Misc/NEWS.d/3.6.4rc1.rst index 36dfadda0fe1b3..dc9ab7ad56de89 100644 --- a/Misc/NEWS.d/3.6.4rc1.rst +++ b/Misc/NEWS.d/3.6.4rc1.rst @@ -22,7 +22,7 @@ Setting sys.tracebacklimit to 0 or less now suppresses printing tracebacks. Setting sys.tracebacklimit to None now causes using the default limit. Setting sys.tracebacklimit to an integer larger than LONG_MAX now means using the limit LONG_MAX rather than the default limit. -Fixed integer overflows in the case of more than 2**31 traceback items on +Fixed integer overflows in the case of more than ``2**31`` traceback items on Windows. Fixed output errors handling. diff --git a/Misc/NEWS.d/3.7.0a3.rst b/Misc/NEWS.d/3.7.0a3.rst index 8ef7a5118a1f55..067720efa516e0 100644 --- a/Misc/NEWS.d/3.7.0a3.rst +++ b/Misc/NEWS.d/3.7.0a3.rst @@ -100,7 +100,7 @@ Setting sys.tracebacklimit to 0 or less now suppresses printing tracebacks. Setting sys.tracebacklimit to None now causes using the default limit. Setting sys.tracebacklimit to an integer larger than LONG_MAX now means using the limit LONG_MAX rather than the default limit. -Fixed integer overflows in the case of more than 2**31 traceback items on +Fixed integer overflows in the case of more than ``2**31`` traceback items on Windows. Fixed output errors handling. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index af9cf4d29f902f..f19d1a1823584b 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -473,7 +473,7 @@ match. .. nonce: uxVOpk .. section: Library -Abort asyncio SSLProtocol connection if handshake not complete within 10s +Abort asyncio SSLProtocol connection if handshake not complete within 10 seconds. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 89811c12a7948e..5cd3fa32105c27 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -3355,7 +3355,7 @@ if the ``PATH`` environment variable is not set. On Windows, fix multiprocessing.Connection for very large read: fix _winapi.PeekNamedPipe() and _winapi.ReadFile() for read larger than INT_MAX -(usually 2^31-1). +(usually ``2**31-1``). .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index fe9fc58e3979ae..a9b6694c133f19 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -213,7 +213,7 @@ objects. Patch by Dong-hee Na and Inada Naoki. .. section: Core and Builtins :class:`bytearray`, :class:`~array.array` and :class:`~mmap.mmap` objects -allow now to export more than 2**31 buffers at a time. +allow now to export more than ``2**31`` buffers at a time. .. @@ -279,7 +279,7 @@ visited by ``tp_traverse()`` are valid. .. nonce: Xgc6F_ .. section: Core and Builtins -Remove unecessary intersection and update set operation in dictview with +Remove unnecessary intersection and update set operation in dictview with empty set. (Contributed by Dong-hee Na in :issue:`38210`.) .. @@ -868,7 +868,7 @@ Fix the :c:func:`PySys_Audit` call in :class:`mmap.mmap`. .. nonce: WJkgKV .. section: Core and Builtins -Remove an unnecssary Py_XINCREF in classobject.c. +Remove an unnecessary Py_XINCREF in classobject.c. .. @@ -1224,7 +1224,7 @@ Anthony Sottile. .. nonce: cyq5nr .. section: Library -Now :func:`~logging.config.fileConfig` correcty sets the .name of handlers +Now :func:`~logging.config.fileConfig` correctly sets the .name of handlers loaded. .. @@ -1637,7 +1637,7 @@ Preserve subclassing in inspect.Signature.from_callable. .. nonce: nHAbuJ .. section: Library -Names of hashing algorithms frome OpenSSL are now normalized to follow +Names of hashing algorithms from OpenSSL are now normalized to follow Python's naming conventions. For example OpenSSL uses sha3-512 instead of sha3_512 or blake2b512 instead of blake2b. @@ -2403,7 +2403,7 @@ Fixed comparisons of :class:`datetime.timedelta` and .. nonce: 7UV5d0 .. section: Library -Syncronize ``importlib.metadata`` with `importlib_metadata 0.19 +Synchronize ``importlib.metadata`` with `importlib_metadata 0.19 `_, improving handling of EGG-INFO files and fixing a crash when entry point names contained colons. @@ -3002,7 +3002,7 @@ on platforms with 16-bit :c:type:`wchar_t` (for example, Windows and AIX). In a subinterpreter, spawning a daemon thread now raises an exception. Daemon threads were never supported in subinterpreters. Previously, the -subinterpreter finalization crashed with a Pyton fatal error if a daemon +subinterpreter finalization crashed with a Python fatal error if a daemon thread was still running. .. @@ -3065,7 +3065,7 @@ internal tasks weak set is changed by another thread during iteration. .. section: Library :class:`_pyio.IOBase` destructor now does nothing if getting the ``closed`` -attribute fails to better mimick :class:`_io.IOBase` finalizer. +attribute fails to better mimic :class:`_io.IOBase` finalizer. .. @@ -4038,7 +4038,7 @@ crypto policies. Use PKCS#8 format with AES256 encryption instead. .. nonce: _x-9uH .. section: Tests -test.support now has a helper function to check for availibility of a hash +test.support now has a helper function to check for availability of a hash digest function. Several tests are refactored avoid MD5 and use SHA256 instead. Other tests are marked to use MD5 and skipped when MD5 is disabled. diff --git a/Misc/NEWS.d/3.9.0a3.rst b/Misc/NEWS.d/3.9.0a3.rst index 6c71d7e839d05f..77ccc7453c2157 100644 --- a/Misc/NEWS.d/3.9.0a3.rst +++ b/Misc/NEWS.d/3.9.0a3.rst @@ -178,7 +178,7 @@ last iteration of asynchronous for loops. Patch by Pablo Galindo. .. nonce: WG9alt .. section: Core and Builtins -Fix incorrent line execution reporting in trace functions when tracing +Fix incorrect line execution reporting in trace functions when tracing exception handlers with name binding. Patch by Pablo Galindo. .. @@ -685,7 +685,7 @@ but not required. Patch by Juergen Gmach. .. section: Library Fixed __subclasshook__ of :class:`os.PathLike` to return a correct result -upon inheritence. Patch by Bar Harel. +upon inheritance. Patch by Bar Harel. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index 355a3fc22350cb..49a118ad7e4308 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -175,7 +175,7 @@ convention. Patch by Dong-hee Na. .. nonce: FE9S21 .. section: Core and Builtins -Chaged list overallocation strategy. It no longer overallocates if the new +Changed list overallocation strategy. It no longer overallocates if the new size is closer to overallocated size than to the old size and adds padding. .. diff --git a/Misc/NEWS.d/3.9.0b1.rst b/Misc/NEWS.d/3.9.0b1.rst index 51dc9ce0ec0377..529be0eba586ac 100644 --- a/Misc/NEWS.d/3.9.0b1.rst +++ b/Misc/NEWS.d/3.9.0b1.rst @@ -641,7 +641,7 @@ than alphabetical. On AIX, :func:`~time.thread_time` is now implemented with ``thread_cputime()`` which has nanosecond resolution, rather than -``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` which has a resolution of 10 ms. +``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` which has a resolution of 10 milliseconds. Patch by Batuhan Taskaya. .. diff --git a/Misc/NEWS.d/next/Build/2021-05-24-03-31-17.bpo-41282.L8nP44.rst b/Misc/NEWS.d/next/Build/2021-05-24-03-31-17.bpo-41282.L8nP44.rst deleted file mode 100644 index cc6eadefc6cba7..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-05-24-03-31-17.bpo-41282.L8nP44.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix broken ``make install`` that caused standard library extension modules -to be unnecessarily and incorrectly rebuilt during the install phase of -cpython. diff --git a/Misc/NEWS.d/next/Build/2021-06-10-18-08-44.bpo-44381.Xpc1iX.rst b/Misc/NEWS.d/next/Build/2021-06-10-18-08-44.bpo-44381.Xpc1iX.rst deleted file mode 100644 index 002112c4b55674..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-06-10-18-08-44.bpo-44381.Xpc1iX.rst +++ /dev/null @@ -1,2 +0,0 @@ -The Windows build now accepts :envvar:`EnableControlFlowGuard` set to -``guard`` to enable CFG. diff --git a/Misc/NEWS.d/next/Build/2021-06-19-11-50-03.bpo-43298.9ircMb.rst b/Misc/NEWS.d/next/Build/2021-06-19-11-50-03.bpo-43298.9ircMb.rst deleted file mode 100644 index 3bdc24b147a3ef..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-06-19-11-50-03.bpo-43298.9ircMb.rst +++ /dev/null @@ -1 +0,0 @@ -Improved error message when building without a Windows SDK installed. diff --git a/Misc/NEWS.d/next/Build/2021-06-30-02-32-46.bpo-44535.M9iN4-.rst b/Misc/NEWS.d/next/Build/2021-06-30-02-32-46.bpo-44535.M9iN4-.rst deleted file mode 100644 index e06d0d304852f3..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-06-30-02-32-46.bpo-44535.M9iN4-.rst +++ /dev/null @@ -1 +0,0 @@ -Enable building using a Visual Studio 2022 install on Windows. diff --git a/Misc/NEWS.d/next/Build/2021-07-19-01-09-56.bpo-44340.JNeOf4.rst b/Misc/NEWS.d/next/Build/2021-07-19-01-09-56.bpo-44340.JNeOf4.rst deleted file mode 100644 index cf19eb6052e8db..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-07-19-01-09-56.bpo-44340.JNeOf4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support for building with clang thin lto via --with-lto=thin/full. Patch -by Dong-hee Na and Brett Holman. diff --git a/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst b/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst deleted file mode 100644 index d11c6451462bd5..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst +++ /dev/null @@ -1,3 +0,0 @@ -Generate lines in relevant files for frozen modules. Up until now each of -the files had to be edited manually. This change makes it easier to add to -and modify the frozen modules. diff --git a/Misc/NEWS.d/next/Build/2021-09-09-16-45-26.bpo-45067.mFmY92.rst b/Misc/NEWS.d/next/Build/2021-09-09-16-45-26.bpo-45067.mFmY92.rst deleted file mode 100644 index a89736eb33e82b..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-09-09-16-45-26.bpo-45067.mFmY92.rst +++ /dev/null @@ -1,7 +0,0 @@ -The ncurses function extended_color_content was introduced in 2017 - -(https://invisible-island.net/ncurses/NEWS.html#index-t20170401). The - -ncurses-devel package in CentOS 7 had a older version ncurses resulted in -compilation error. For compiling ncurses with extended color support, we -verify the version of the ncurses library >= 20170401. diff --git a/Misc/NEWS.d/next/Build/2021-09-11-06-05-23.bpo-45163.q7xT93.rst b/Misc/NEWS.d/next/Build/2021-09-11-06-05-23.bpo-45163.q7xT93.rst deleted file mode 100644 index 2b656bcbc230bc..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-09-11-06-05-23.bpo-45163.q7xT93.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes Haiku platform build. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst b/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst deleted file mode 100644 index df470e8eeb30c0..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst +++ /dev/null @@ -1,3 +0,0 @@ -Windows builds now regenerate frozen modules as the first part of the build. -Previously the regeneration was later in the build, which would require it -to be restarted if any modules had changed. diff --git a/Misc/NEWS.d/next/Build/2021-09-14-10-07-23.bpo-45020._VGGPv.rst b/Misc/NEWS.d/next/Build/2021-09-14-10-07-23.bpo-45020._VGGPv.rst deleted file mode 100644 index 67d61c3f880403..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-09-14-10-07-23.bpo-45020._VGGPv.rst +++ /dev/null @@ -1,4 +0,0 @@ -Freeze stdlib modules that are imported during startup. This provides -significant performance improvements to startup. If necessary, use the -previously added "-X frozen_modules=off" commandline option to force -importing the source modules. diff --git a/Misc/NEWS.d/next/Build/2021-09-16-18-00-43.bpo-45220.TgbkvW.rst b/Misc/NEWS.d/next/Build/2021-09-16-18-00-43.bpo-45220.TgbkvW.rst deleted file mode 100644 index 8bbd634fa61a37..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-09-16-18-00-43.bpo-45220.TgbkvW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Avoid building with the Windows 11 SDK previews automatically. This may be -overridden by setting the ``DefaultWindowsSDKVersion`` environment variable -before building. diff --git a/Misc/NEWS.d/next/Build/2021-10-11-16-08-37.bpo-45433.pVDkMV.rst b/Misc/NEWS.d/next/Build/2021-10-11-16-08-37.bpo-45433.pVDkMV.rst new file mode 100644 index 00000000000000..27869f08fbb9b7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-11-16-08-37.bpo-45433.pVDkMV.rst @@ -0,0 +1 @@ +Avoid linking libpython with libcrypt. diff --git a/Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst b/Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst new file mode 100644 index 00000000000000..a2dc5bcc322175 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst @@ -0,0 +1,2 @@ +Prevent ``internal configure error`` when running ``configure`` +with recent versions of non-Apple clang. Patch by David Bohman. diff --git a/Misc/NEWS.d/next/Build/2021-10-12-02-13-08.bpo-45440.-zYgDb.rst b/Misc/NEWS.d/next/Build/2021-10-12-02-13-08.bpo-45440.-zYgDb.rst new file mode 100644 index 00000000000000..93f7f3139065fb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-12-02-13-08.bpo-45440.-zYgDb.rst @@ -0,0 +1,4 @@ +Building Python now requires a C99 ```` header file providing +the following functions: ``copysign()``, ``hypot()``, ``isfinite()``, +``isinf()``, ``isnan()``, ``round()``. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst b/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst new file mode 100644 index 00000000000000..cb981d96f3047a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst @@ -0,0 +1,3 @@ +Fixed regression in handling of ``LDFLAGS`` and ``CPPFLAGS`` options +where :meth:`argparse.parse_known_args` could interpret an option as +one of the built-in command line argument, for example ``-h`` for help. diff --git a/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst b/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst new file mode 100644 index 00000000000000..e560b71ede2d29 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst @@ -0,0 +1,3 @@ +The ``configure`` script now checks whether OpenSSL headers and libraries +provide required APIs. Most common APIs are verified. The check detects +outdated or missing OpenSSL. Failures do not stop configure. diff --git a/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst b/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst new file mode 100644 index 00000000000000..575e2fb9ae93b8 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst @@ -0,0 +1,2 @@ +Update :data:`sys.version` to use ``main`` as fallback information. +Patch by Jeong YunWon. diff --git a/Misc/NEWS.d/next/Build/2021-10-20-17-02-56.bpo-45548.BoggEf.rst b/Misc/NEWS.d/next/Build/2021-10-20-17-02-56.bpo-45548.BoggEf.rst new file mode 100644 index 00000000000000..a52afa21698f96 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-20-17-02-56.bpo-45548.BoggEf.rst @@ -0,0 +1 @@ +Fill in missing entries in Modules/Setup. diff --git a/Misc/NEWS.d/next/C API/2020-12-23-01-28-50.bpo-42035.S9eUm0.rst b/Misc/NEWS.d/next/C API/2020-12-23-01-28-50.bpo-42035.S9eUm0.rst deleted file mode 100644 index 8adb20e62d1c46..00000000000000 --- a/Misc/NEWS.d/next/C API/2020-12-23-01-28-50.bpo-42035.S9eUm0.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new :c:func:`PyType_GetName` function to get type's short name. diff --git a/Misc/NEWS.d/next/C API/2021-05-04-17-43-39.bpo-44029.ayX4PR.rst b/Misc/NEWS.d/next/C API/2021-05-04-17-43-39.bpo-44029.ayX4PR.rst deleted file mode 100644 index cf55e41bf332c7..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-05-04-17-43-39.bpo-44029.ayX4PR.rst +++ /dev/null @@ -1,9 +0,0 @@ -Remove deprecated ``Py_UNICODE`` APIs: ``PyUnicode_Encode``, -``PyUnicode_EncodeUTF7``, ``PyUnicode_EncodeUTF8``, -``PyUnicode_EncodeUTF16``, ``PyUnicode_EncodeUTF32``, -``PyUnicode_EncodeLatin1``, ``PyUnicode_EncodeMBCS``, -``PyUnicode_EncodeDecimal``, ``PyUnicode_EncodeRawUnicodeEscape``, -``PyUnicode_EncodeCharmap``, ``PyUnicode_EncodeUnicodeEscape``, -``PyUnicode_TransformDecimalToASCII``, ``PyUnicode_TranslateCharmap``, -``PyUnicodeEncodeError_Create``, ``PyUnicodeTranslateError_Create``. See -:pep:`393` and :pep:`624` for reference. diff --git a/Misc/NEWS.d/next/C API/2021-05-05-19-04-50.bpo-43795.9Ojj73.rst b/Misc/NEWS.d/next/C API/2021-05-05-19-04-50.bpo-43795.9Ojj73.rst deleted file mode 100644 index 20a3823f1f01c0..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-05-05-19-04-50.bpo-43795.9Ojj73.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`PyCodec_Unregister` is now properly exported as a function in the -Windows Stable ABI DLL. diff --git a/Misc/NEWS.d/next/C API/2021-05-10-14-34-22.bpo-44094.HayXZO.rst b/Misc/NEWS.d/next/C API/2021-05-10-14-34-22.bpo-44094.HayXZO.rst deleted file mode 100644 index eea9e0bf282081..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-05-10-14-34-22.bpo-44094.HayXZO.rst +++ /dev/null @@ -1,4 +0,0 @@ -Remove ``PyErr_SetFromErrnoWithUnicodeFilename()``, -``PyErr_SetFromWindowsErrWithUnicodeFilename()``, and -``PyErr_SetExcFromWindowsErrWithUnicodeFilename()``. They are not documented -and have been deprecated since Python 3.3. diff --git a/Misc/NEWS.d/next/C API/2021-05-12-12-24-45.bpo-44113.DcgOqE.rst b/Misc/NEWS.d/next/C API/2021-05-12-12-24-45.bpo-44113.DcgOqE.rst deleted file mode 100644 index 45f67efa10573b..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-05-12-12-24-45.bpo-44113.DcgOqE.rst +++ /dev/null @@ -1,14 +0,0 @@ -Deprecate the following functions to configure the Python initialization: - -* :c:func:`PySys_AddWarnOptionUnicode` -* :c:func:`PySys_AddWarnOption` -* :c:func:`PySys_AddXOption` -* :c:func:`PySys_HasWarnOptions` -* :c:func:`Py_SetPath` -* :c:func:`Py_SetProgramName` -* :c:func:`Py_SetPythonHome` -* :c:func:`Py_SetStandardStreamEncoding` -* :c:func:`_Py_SetProgramFullPath` - -Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization -Configuration ` instead (:pep:`587`). diff --git a/Misc/NEWS.d/next/C API/2021-05-19-15-09-47.bpo-43795.WAHRxt.rst b/Misc/NEWS.d/next/C API/2021-05-19-15-09-47.bpo-43795.WAHRxt.rst deleted file mode 100644 index 23db2330ac3960..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-05-19-15-09-47.bpo-43795.WAHRxt.rst +++ /dev/null @@ -1 +0,0 @@ -The undocumented function :c:func:`Py_FrozenMain` is removed from the Limited API. diff --git a/Misc/NEWS.d/next/C API/2021-05-24-22-12-40.bpo-44220.H9CUGl.rst b/Misc/NEWS.d/next/C API/2021-05-24-22-12-40.bpo-44220.H9CUGl.rst new file mode 100644 index 00000000000000..79ba8aa07cde68 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-05-24-22-12-40.bpo-44220.H9CUGl.rst @@ -0,0 +1 @@ +:c:var:`PyStructSequence_UnnamedField` is added to the Stable ABI. diff --git a/Misc/NEWS.d/next/C API/2021-05-31-11-31-13.bpo-44263.8mIOfV.rst b/Misc/NEWS.d/next/C API/2021-05-31-11-31-13.bpo-44263.8mIOfV.rst deleted file mode 100644 index aa831a2083c48f..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-05-31-11-31-13.bpo-44263.8mIOfV.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :c:func:`PyType_Ready` function now raises an error if a type is defined -with the :const:`Py_TPFLAGS_HAVE_GC` flag set but has no traverse function -(:c:member:`PyTypeObject.tp_traverse`). -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-06-03-00-59-48.bpo-39573.-elHTJ.rst b/Misc/NEWS.d/next/C API/2021-06-03-00-59-48.bpo-39573.-elHTJ.rst deleted file mode 100644 index d9641ed97e170d..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-06-03-00-59-48.bpo-39573.-elHTJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Convert the :c:func:`Py_TYPE` and :c:func:`Py_SIZE` macros to static inline -functions. The :c:func:`Py_SET_TYPE` and :c:func:`Py_SET_SIZE` functions -must now be used to set an object type and size. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-06-10-15-22-31.bpo-44378.jGYakF.rst b/Misc/NEWS.d/next/C API/2021-06-10-15-22-31.bpo-44378.jGYakF.rst deleted file mode 100644 index b620b499f23512..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-06-10-15-22-31.bpo-44378.jGYakF.rst +++ /dev/null @@ -1,3 +0,0 @@ -:c:func:`Py_IS_TYPE` no longer uses :c:func:`Py_TYPE` to avoid a compiler -warning: no longer cast ``const PyObject*`` to ``PyObject*``. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-06-15-16-28-18.bpo-43795.fy0AXK.rst b/Misc/NEWS.d/next/C API/2021-06-15-16-28-18.bpo-43795.fy0AXK.rst deleted file mode 100644 index 8d029a04579086..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-06-15-16-28-18.bpo-43795.fy0AXK.rst +++ /dev/null @@ -1,3 +0,0 @@ -The list in :ref:`stable-abi-list` now shows the public name -:c:struct:`PyFrameObject` rather than ``_frame``. The non-existing -entry ``_node`` no longer appears in the list. diff --git a/Misc/NEWS.d/next/C API/2021-06-22-17-00-06.bpo-40939.CGB0I5.rst b/Misc/NEWS.d/next/C API/2021-06-22-17-00-06.bpo-40939.CGB0I5.rst deleted file mode 100644 index 2531ac1ba3c12c..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-06-22-17-00-06.bpo-40939.CGB0I5.rst +++ /dev/null @@ -1 +0,0 @@ -Removed documentation for the removed ``PyParser_*`` C API. diff --git a/Misc/NEWS.d/next/C API/2021-06-23-10-31-45.bpo-39947.je_HMo.rst b/Misc/NEWS.d/next/C API/2021-06-23-10-31-45.bpo-39947.je_HMo.rst deleted file mode 100644 index 43adbffc7cce24..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-06-23-10-31-45.bpo-39947.je_HMo.rst +++ /dev/null @@ -1,20 +0,0 @@ -Remove 4 private trashcan C API functions which were only kept for the backward -compatibility of the stable ABI with Python 3.8 and older, since the trashcan -API was not usable with the limited C API on Python 3.8 and older. The -trashcan API was excluded from the limited C API in Python 3.9. - -Removed functions: - -* _PyTrash_deposit_object() -* _PyTrash_destroy_chain() -* _PyTrash_thread_deposit_object() -* _PyTrash_thread_destroy_chain() - -The trashcan C API was never usable with the limited C API, since old trashcan -macros accessed directly :c:type:`PyThreadState` members like -``_tstate->trash_delete_nesting``, whereas the :c:type:`PyThreadState` -structure is opaque in the limited C API. - -Exclude also the the ``PyTrash_UNWIND_LEVEL`` constant from the C API. - -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst b/Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst deleted file mode 100644 index 80c4282c18ea64..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst +++ /dev/null @@ -1,4 +0,0 @@ -:c:func:`Py_RunMain` now resets :c:data:`PyImport_Inittab` to its initial value -at exit. It must be possible to call :c:func:`PyImport_AppendInittab` or -:c:func:`PyImport_ExtendInittab` at each Python initialization. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-06-28-23-44-47.bpo-44530.qij7YC.rst b/Misc/NEWS.d/next/C API/2021-06-28-23-44-47.bpo-44530.qij7YC.rst deleted file mode 100644 index 6200f9b97d7fcd..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-06-28-23-44-47.bpo-44530.qij7YC.rst +++ /dev/null @@ -1,4 +0,0 @@ -Added the ``co_qualname`` to the ``PyCodeObject`` structure to propagate the -qualified name from the compiler to code objects. - -Patch by Gabriele N. Tornetta diff --git a/Misc/NEWS.d/next/C API/2021-07-20-16-21-06.bpo-42747.rRxjUY.rst b/Misc/NEWS.d/next/C API/2021-07-20-16-21-06.bpo-42747.rRxjUY.rst deleted file mode 100644 index c7ac5a776e2ed9..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-07-20-16-21-06.bpo-42747.rRxjUY.rst +++ /dev/null @@ -1,4 +0,0 @@ -The ``Py_TPFLAGS_HAVE_VERSION_TAG`` type flag now does nothing. The -``Py_TPFLAGS_HAVE_AM_SEND`` flag (which was added in 3.10) is removed. Both -were unnecessary because it is not possible to have type objects with the -relevant fields missing. diff --git a/Misc/NEWS.d/next/C API/2021-07-27-17-29-12.bpo-44751.4qmbDG.rst b/Misc/NEWS.d/next/C API/2021-07-27-17-29-12.bpo-44751.4qmbDG.rst deleted file mode 100644 index d7b9f098196694..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-07-27-17-29-12.bpo-44751.4qmbDG.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``crypt.h`` include from the public ``Python.h`` header. diff --git a/Misc/NEWS.d/next/C API/2021-07-29-16-04-28.bpo-41103.hiKKcF.rst b/Misc/NEWS.d/next/C API/2021-07-29-16-04-28.bpo-41103.hiKKcF.rst deleted file mode 100644 index af06654ba20409..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-07-29-16-04-28.bpo-41103.hiKKcF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Reverts removal of the old buffer protocol because they are part of stable -ABI. diff --git a/Misc/NEWS.d/next/C API/2021-08-02-20-49-36.bpo-42035.HTBcZt.rst b/Misc/NEWS.d/next/C API/2021-08-02-20-49-36.bpo-42035.HTBcZt.rst deleted file mode 100644 index 4631c43fdd5312..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-08-02-20-49-36.bpo-42035.HTBcZt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a new :c:func:`PyType_GetQualName` function to get type's qualified -name. diff --git a/Misc/NEWS.d/next/C API/2021-08-31-15-21-36.bpo-45061.ZH0HVe.rst b/Misc/NEWS.d/next/C API/2021-08-31-15-21-36.bpo-45061.ZH0HVe.rst deleted file mode 100644 index 58bd534601fb9d..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-08-31-15-21-36.bpo-45061.ZH0HVe.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add a deallocator to the :class:`bool` type to detect refcount bugs in C -extensions which call ``Py_DECREF(Py_True);`` or ``Py_DECREF(Py_False);`` by -mistake. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-09-03-15-53-43.bpo-45094.tinXwL.rst b/Misc/NEWS.d/next/C API/2021-09-03-15-53-43.bpo-45094.tinXwL.rst deleted file mode 100644 index 84b01b23b435fb..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-09-03-15-53-43.bpo-45094.tinXwL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add the :c:macro:`Py_NO_INLINE` macro to disable inlining on a function. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-09-16-18-05-20.bpo-45116.WxXewl.rst b/Misc/NEWS.d/next/C API/2021-09-16-18-05-20.bpo-45116.WxXewl.rst deleted file mode 100644 index cf3db5ca2a0236..00000000000000 --- a/Misc/NEWS.d/next/C API/2021-09-16-18-05-20.bpo-45116.WxXewl.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add the :c:macro:`Py_ALWAYS_INLINE` macro to ask the compiler to always -inline a static inline function. The compiler can ignore it and decides to -not inline the function. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-09-24-11-12-21.bpo-43760.Bfxd1-.rst b/Misc/NEWS.d/next/C API/2021-09-24-11-12-21.bpo-43760.Bfxd1-.rst new file mode 100644 index 00000000000000..16bea5ff8f57c2 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-09-24-11-12-21.bpo-43760.Bfxd1-.rst @@ -0,0 +1,4 @@ +Add new :c:func:`PyThreadState_EnterTracing`, and +:c:func:`PyThreadState_LeaveTracing` functions to the limited C API to suspend +and resume tracing and profiling. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-08-15-54-07.bpo-45412.KHyJCT.rst b/Misc/NEWS.d/next/C API/2021-10-08-15-54-07.bpo-45412.KHyJCT.rst new file mode 100644 index 00000000000000..49746810ee5b9f --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-08-15-54-07.bpo-45412.KHyJCT.rst @@ -0,0 +1,9 @@ +Remove the following math macros using the ``errno`` variable: + +* ``Py_ADJUST_ERANGE1()`` +* ``Py_ADJUST_ERANGE2()`` +* ``Py_OVERFLOWED()`` +* ``Py_SET_ERANGE_IF_OVERFLOW()`` +* ``Py_SET_ERRNO_ON_MATH_ERROR()`` + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-11-22-58-33.bpo-41123.myrlIp.rst b/Misc/NEWS.d/next/C API/2021-10-11-22-58-33.bpo-41123.myrlIp.rst new file mode 100644 index 00000000000000..3ae302bcea245f --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-11-22-58-33.bpo-41123.myrlIp.rst @@ -0,0 +1,4 @@ +Remove ``Py_UNICODE_COPY()`` and ``Py_UNICODE_FILL()`` macros, deprecated +since Python 3.3. Use ``PyUnicode_CopyCharacters()`` or ``memcpy()`` +(``wchar_t*`` string), and ``PyUnicode_Fill()`` functions instead. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-11-23-03-49.bpo-45434.tsS8I_.rst b/Misc/NEWS.d/next/C API/2021-10-11-23-03-49.bpo-45434.tsS8I_.rst new file mode 100644 index 00000000000000..627f687272f5a9 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-11-23-03-49.bpo-45434.tsS8I_.rst @@ -0,0 +1,5 @@ +```` no longer includes the header files ````, +````, ```` and ```` when the ``Py_LIMITED_API`` +macro is set to ``0x030b0000`` (Python 3.11) or higher. C extensions should +explicitly include the header files after ``#include ``. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-12-02-13-41.bpo-45440.Gf94rE.rst b/Misc/NEWS.d/next/C API/2021-10-12-02-13-41.bpo-45440.Gf94rE.rst new file mode 100644 index 00000000000000..d9d695fd2ace4b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-12-02-13-41.bpo-45440.Gf94rE.rst @@ -0,0 +1,2 @@ +Remove the ``Py_FORCE_DOUBLE()`` macro. It was used by the ``Py_IS_INFINITY()`` +macro. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-13-14-42-46.bpo-45434.INNEEt.rst b/Misc/NEWS.d/next/C API/2021-10-13-14-42-46.bpo-45434.INNEEt.rst new file mode 100644 index 00000000000000..933f80670f1aeb --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-13-14-42-46.bpo-45434.INNEEt.rst @@ -0,0 +1,3 @@ +Remove the ``pystrhex.h`` header file. It only contains private functions. C +extensions should only include the main ```` header file. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-14-22-16-56.bpo-45474.1OkJQh.rst b/Misc/NEWS.d/next/C API/2021-10-14-22-16-56.bpo-45474.1OkJQh.rst new file mode 100644 index 00000000000000..d41f1b72b8966f --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-14-22-16-56.bpo-45474.1OkJQh.rst @@ -0,0 +1,11 @@ +The following items are no longer available when ``Py_LIMITED_API`` is defined: + +* :c:func:`PyMarshal_WriteLongToFile` +* :c:func:`PyMarshal_WriteObjectToFile` +* :c:func:`PyMarshal_ReadObjectFromString` +* :c:func:`PyMarshal_WriteObjectToString` +* the ``Py_MARSHAL_VERSION`` macro + +These are not part of the :ref:`limited API `. + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-15-00-11-51.bpo-35134.eX4zqy.rst b/Misc/NEWS.d/next/C API/2021-10-15-00-11-51.bpo-35134.eX4zqy.rst new file mode 100644 index 00000000000000..d0d3ce6b34f6f1 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-15-00-11-51.bpo-35134.eX4zqy.rst @@ -0,0 +1,7 @@ +The non-limited API files ``cellobject.h``, ``classobject.h``, ``context.h``, +``funcobject.h``, ``genobject.h`` and ``longintrepr.h`` have been moved to +the ``Include/cpython`` directory. Moreover, the ``eval.h`` header file was +removed. These files must not be included directly, as they are already +included in ``Python.h``: :ref:`Include Files `. If they have +been included directly, consider including ``Python.h`` instead. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-15-09-29-59.bpo-35081.2teFD3.rst b/Misc/NEWS.d/next/C API/2021-10-15-09-29-59.bpo-35081.2teFD3.rst new file mode 100644 index 00000000000000..79d4a9b9180d95 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-15-09-29-59.bpo-35081.2teFD3.rst @@ -0,0 +1,3 @@ +Move the ``interpreteridobject.h`` header file from ``Include/`` to +``Include/internal/``. It only provides private functions. Patch by Victor +Stinner. diff --git a/Misc/NEWS.d/next/C API/2021-10-19-00-20-40.bpo-35134.Z0Zk_m.rst b/Misc/NEWS.d/next/C API/2021-10-19-00-20-40.bpo-35134.Z0Zk_m.rst new file mode 100644 index 00000000000000..57c4fa53bee668 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-19-00-20-40.bpo-35134.Z0Zk_m.rst @@ -0,0 +1,3 @@ +Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never +worked since the :c:type:`PyWeakReference` structure is opaque in the +limited C API. diff --git a/Misc/NEWS.d/next/C API/2021-10-19-13-07-46.bpo-45522.kGAwmZ.rst b/Misc/NEWS.d/next/C API/2021-10-19-13-07-46.bpo-45522.kGAwmZ.rst new file mode 100644 index 00000000000000..658261f8ce693b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-19-13-07-46.bpo-45522.kGAwmZ.rst @@ -0,0 +1,2 @@ +The internal freelists for frame, float, list, dict, async generators, and +context objects can now be disabled. diff --git a/Misc/NEWS.d/next/C API/2021-10-20-18-41-17.bpo-29103.CMRLyq.rst b/Misc/NEWS.d/next/C API/2021-10-20-18-41-17.bpo-29103.CMRLyq.rst new file mode 100644 index 00000000000000..e923bfd3ae2935 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-20-18-41-17.bpo-29103.CMRLyq.rst @@ -0,0 +1,3 @@ +:c:func:`PyType_FromSpec* ` now copies the class name +from the spec to a buffer owned by the class, so the original can be safely +deallocated. Patch by Petr Viktorin. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst deleted file mode 100644 index 9c91a8c0bf9ee4..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst +++ /dev/null @@ -1,3 +0,0 @@ -Asynchronous comprehensions are now allowed inside comprehensions in -asynchronous functions. Outer comprehensions implicitly become -asynchronous. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-12-21-14-18-32.bpo-39091.dOexgQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-12-21-14-18-32.bpo-39091.dOexgQ.rst deleted file mode 100644 index c3b4e810d658b2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-12-21-14-18-32.bpo-39091.dOexgQ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash when using passing a non-exception to a generator's ``throw()`` method. Patch by Noah Oxer diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-02-13-21-14.bpo-11105.wceryW.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-02-13-21-14.bpo-11105.wceryW.rst deleted file mode 100644 index 8891936cd88716..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-06-02-13-21-14.bpo-11105.wceryW.rst +++ /dev/null @@ -1,3 +0,0 @@ -When compiling :class:`ast.AST` objects with recursive references -through :func:`compile`, the interpreter doesn't crash anymore instead -it raises a :exc:`RecursionError`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-01-13-19-34-41.bpo-28146.AZBBkH.rst b/Misc/NEWS.d/next/Core and Builtins/2021-01-13-19-34-41.bpo-28146.AZBBkH.rst deleted file mode 100644 index e6198819389532..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-01-13-19-34-41.bpo-28146.AZBBkH.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a confusing error message in :func:`str.format`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-03-22-17-50-30.bpo-17792._zssjS.rst b/Misc/NEWS.d/next/Core and Builtins/2021-03-22-17-50-30.bpo-17792._zssjS.rst deleted file mode 100644 index 768cbf14efad9c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-03-22-17-50-30.bpo-17792._zssjS.rst +++ /dev/null @@ -1 +0,0 @@ -More accurate error messages for access of unbound locals or free vars. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-02-15-02-16.bpo-43693.l3Ureu.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-02-15-02-16.bpo-43693.l3Ureu.rst deleted file mode 100644 index 948c4d52482dc0..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-04-02-15-02-16.bpo-43693.l3Ureu.rst +++ /dev/null @@ -1,4 +0,0 @@ -Compute cell offsets relative to locals in compiler. Allows the interpreter -to treats locals and cells a single array, which is slightly more efficient. -Also make the LOAD_CLOSURE opcode an alias for LOAD_FAST. Preserving -LOAD_CLOSURE helps keep bytecode a bit more readable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-17-16-08-00.bpo-43879.zkyJgh.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-17-16-08-00.bpo-43879.zkyJgh.rst deleted file mode 100644 index 98b51736904a00..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-04-17-16-08-00.bpo-43879.zkyJgh.rst +++ /dev/null @@ -1 +0,0 @@ -Add native_thread_id to PyThreadState. Patch by Gabriele N. Tornetta. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-18-18-07-33.bpo-43833.oChkCi.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-18-18-07-33.bpo-43833.oChkCi.rst deleted file mode 100644 index 2adbdba651b831..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-04-18-18-07-33.bpo-43833.oChkCi.rst +++ /dev/null @@ -1,4 +0,0 @@ -Emit a deprecation warning if the numeric literal is immediately followed by -one of keywords: and, else, for, if, in, is, or. Raise a syntax error with -more informative message if it is immediately followed by other keyword or -identifier. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-23-03-46-45.bpo-43918.nNDY3S.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-23-03-46-45.bpo-43918.nNDY3S.rst deleted file mode 100644 index f2f33f02abbd99..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-04-23-03-46-45.bpo-43918.nNDY3S.rst +++ /dev/null @@ -1 +0,0 @@ -Document the signature and ``default`` argument in the docstring of the new ``anext`` builtin. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-30-15-48-36.bpo-40222.j3VxeX.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-30-15-48-36.bpo-40222.j3VxeX.rst deleted file mode 100644 index b7744755122783..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-04-30-15-48-36.bpo-40222.j3VxeX.rst +++ /dev/null @@ -1,7 +0,0 @@ -"Zero cost" exception handling. - -* Uses a lookup table to determine how to handle exceptions. -* Removes SETUP_FINALLY and POP_TOP block instructions, eliminating the runtime overhead of try statements. -* Reduces the size of the frame object by about 60%. - -Patch by Mark Shannon diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-04-01-01-04.bpo-43822.9VeCg0.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-04-01-01-04.bpo-43822.9VeCg0.rst deleted file mode 100644 index b8815cddf4e2c1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-04-01-01-04.bpo-43822.9VeCg0.rst +++ /dev/null @@ -1,2 +0,0 @@ -The parser will prioritize tokenizer errors over custom syntax errors when -raising exceptions. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-08-17-18-37.bpo-43149.Kp5FxD.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-08-17-18-37.bpo-43149.Kp5FxD.rst deleted file mode 100644 index cc1983ec3d4648..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-08-17-18-37.bpo-43149.Kp5FxD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Corrent the syntax error message regarding multiple exception types to not -refer to "exception groups". Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-08-19-54-57.bpo-28307.7ysaVW.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-08-19-54-57.bpo-28307.7ysaVW.rst deleted file mode 100644 index 86ac3254eaea87..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-08-19-54-57.bpo-28307.7ysaVW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Compiler now optimizes simple C-style formatting with literal format -containing only format codes %s, %r and %a by converting them to f-string -expressions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-10-18-49-13.bpo-26110.EQzqqA.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-10-18-49-13.bpo-26110.EQzqqA.rst deleted file mode 100644 index b5d9159000bad6..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-10-18-49-13.bpo-26110.EQzqqA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add ``CALL_METHOD_KW`` opcode to speed up method calls with keyword -arguments. Idea originated from PyPy. A side effect is executing -``CALL_METHOD`` is now branchless in the evaluation loop. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-11-21-52-44.bpo-44110.VqbAsB.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-11-21-52-44.bpo-44110.VqbAsB.rst deleted file mode 100644 index 3b82e425ab4a74..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-11-21-52-44.bpo-44110.VqbAsB.rst +++ /dev/null @@ -1 +0,0 @@ -Improve :func:`str.__getitem__` error message diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-12-14-26-16.bpo-44114.p-WfAE.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-12-14-26-16.bpo-44114.p-WfAE.rst deleted file mode 100644 index c50b1594cae356..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-12-14-26-16.bpo-44114.p-WfAE.rst +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect dictkeys_reversed and dictitems_reversed function signatures in C code, which broke webassembly builds. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-14-20-03-32.bpo-44032.OzT1ob.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-14-20-03-32.bpo-44032.OzT1ob.rst deleted file mode 100644 index fd2dec80cddf10..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-14-20-03-32.bpo-44032.OzT1ob.rst +++ /dev/null @@ -1,2 +0,0 @@ -Move 'fast' locals and other variables from the frame object to a per-thread -datastack. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-15-17-30-57.bpo-44143.7UTS6H.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-15-17-30-57.bpo-44143.7UTS6H.rst deleted file mode 100644 index a4e88e55723e4d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-15-17-30-57.bpo-44143.7UTS6H.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a crash in the parser that manifest when raising tokenizer errors when -an existing exception was present. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-17-20-44-45.bpo-44156.8KSp9l.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-17-20-44-45.bpo-44156.8KSp9l.rst deleted file mode 100644 index 31e63c2b613161..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-17-20-44-45.bpo-44156.8KSp9l.rst +++ /dev/null @@ -1 +0,0 @@ -String caches in ``compile.c`` are now subinterpreter compatible. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-18-11-27-02.bpo-44168.mgB-rt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-18-11-27-02.bpo-44168.mgB-rt.rst deleted file mode 100644 index ace01e49b508a2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-18-11-27-02.bpo-44168.mgB-rt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix error message in the parser involving keyword arguments with invalid -expressions. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-19-20-33-36.bpo-44180.mQVaAs.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-19-20-33-36.bpo-44180.mQVaAs.rst deleted file mode 100644 index c67eb70b5823a2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-19-20-33-36.bpo-44180.mQVaAs.rst +++ /dev/null @@ -1,3 +0,0 @@ -The parser doesn't report generic syntax errors that happen in a position -further away that the one it reached in the first pass. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-20-12-43-04.bpo-44187.3lk0L1.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-20-12-43-04.bpo-44187.3lk0L1.rst deleted file mode 100644 index 067dedd0f7dda3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-20-12-43-04.bpo-44187.3lk0L1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Implement quickening in the interpreter. This offers no advantages as -yet, but is an enabler of future optimizations. See PEP 659 for full -explanation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-01-42-45.bpo-44184.9qOptC.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-01-42-45.bpo-44184.9qOptC.rst deleted file mode 100644 index 3aba9a58475c61..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-01-42-45.bpo-44184.9qOptC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash at Python exit when a deallocator function removes the last strong -reference to a heap type. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-20-53-49.bpo-43693.-NN3J_.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-20-53-49.bpo-43693.-NN3J_.rst deleted file mode 100644 index 83b7ba260e6a22..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-20-53-49.bpo-43693.-NN3J_.rst +++ /dev/null @@ -1,3 +0,0 @@ -``PyCodeObject`` gained ``co_fastlocalnames`` and ``co_fastlocalkinds`` as -the the authoritative source of fast locals info. Marshaled code objects -have changed accordingly. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst deleted file mode 100644 index 6f61aac7817b2b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Avoid side effects of checking for specialized syntax errors in the REPL -that was causing it to ask for extra tokens after a syntax error had been -detected. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-25-18-20-10.bpo-44232.DMcCCf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-25-18-20-10.bpo-44232.DMcCCf.rst deleted file mode 100644 index fcd124d51442a5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-25-18-20-10.bpo-44232.DMcCCf.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a regression in :func:`type` when a metaclass raises an exception. The C -function :c:func:`type_new` must properly report the exception when a metaclass -constructor raises an exception and the winner class is not the metaclass. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-26-19-10-47.bpo-43693.1KSG9u.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-26-19-10-47.bpo-43693.1KSG9u.rst deleted file mode 100644 index c5fb29ba462e7d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-26-19-10-47.bpo-43693.1KSG9u.rst +++ /dev/null @@ -1,5 +0,0 @@ -A new opcode MAKE_CELL has been added that effectively moves some of -the work done on function entry into the compiler and into the eval -loop. In addition to creating the required cell objects, the new -opcode converts relevant arguments (and other locals) to cell -variables on function entry. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-27-17-34-29.bpo-43667.ND9jP3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-27-17-34-29.bpo-43667.ND9jP3.rst deleted file mode 100644 index 53130cce7180a7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-27-17-34-29.bpo-43667.ND9jP3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve Unicode support in non-UTF locales on Oracle Solaris. This issue -does not affect other Solaris systems. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-30-16-37-47.bpo-43413.vYFPPC.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-30-16-37-47.bpo-43413.vYFPPC.rst deleted file mode 100644 index 579b57ee2d4333..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-30-16-37-47.bpo-43413.vYFPPC.rst +++ /dev/null @@ -1,4 +0,0 @@ -Constructors of subclasses of some buitin classes (e.g. :class:`tuple`, -:class:`list`, :class:`frozenset`) no longer accept arbitrary keyword -arguments. Subclass of :class:`set` can now define a ``__new__()`` method -with additional keyword parameters without overriding also ``__init__()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-03-22-51-50.bpo-44305.66dVDG.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-03-22-51-50.bpo-44305.66dVDG.rst deleted file mode 100644 index eebc26f1cc777a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-03-22-51-50.bpo-44305.66dVDG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve error message for ``try`` blocks without ``except`` or ``finally`` -blocks. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-05-02-34-57.bpo-44304._MAoPc.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-05-02-34-57.bpo-44304._MAoPc.rst deleted file mode 100644 index 89104e8e387ed1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-05-02-34-57.bpo-44304._MAoPc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in the :mod:`sqlite3` module that happened when the garbage -collector clears :class:`sqlite.Statement` objects. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst deleted file mode 100644 index 8ac32adf8b5535..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst +++ /dev/null @@ -1 +0,0 @@ -Improve tokenizer error with improved locations. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-07-15-13-44.bpo-43693.c_zDeY.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-07-15-13-44.bpo-43693.c_zDeY.rst deleted file mode 100644 index c3db81072254b8..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-07-15-13-44.bpo-43693.c_zDeY.rst +++ /dev/null @@ -1,4 +0,0 @@ -Computation of the offsets of cell variables is done in the compiler instead -of at runtime. This reduces the overhead of handling cell and free -variables, especially in the case where a variable is both an argument and -cell variable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-08-01-13-47.bpo-44335.GQTTkl.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-08-01-13-47.bpo-44335.GQTTkl.rst deleted file mode 100644 index b57904e5da607f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-08-01-13-47.bpo-44335.GQTTkl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a regression when identifying incorrect characters in syntax errors. -Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-08-10-22-46.bpo-44337.RTjmIt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-08-10-22-46.bpo-44337.RTjmIt.rst deleted file mode 100644 index 2df082a078e309..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-08-10-22-46.bpo-44337.RTjmIt.rst +++ /dev/null @@ -1,11 +0,0 @@ -Initial implementation of adaptive specialization of LOAD_ATTR - -Four specialized forms of LOAD_ATTR are added: - -* LOAD_ATTR_SLOT - -* LOAD_ATTR_SPLIT_KEYS - -* LOAD_ATTR_WITH_HINT - -* LOAD_ATTR_MODULE diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-08-22-49-06.bpo-44349.xgEgeA.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-08-22-49-06.bpo-44349.xgEgeA.rst deleted file mode 100644 index b386a8ed2c846d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-08-22-49-06.bpo-44349.xgEgeA.rst +++ /dev/null @@ -1 +0,0 @@ -Fix an edge case when displaying text from files with encoding in syntax errors. Patch by Pablo Galindo. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-09-22-56-59.bpo-44368.vgT0Cx.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-09-22-56-59.bpo-44368.vgT0Cx.rst deleted file mode 100644 index e0d19134510eeb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-09-22-56-59.bpo-44368.vgT0Cx.rst +++ /dev/null @@ -1 +0,0 @@ -Improve syntax errors for invalid "as" targets. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-10-10-06-18.bpo-44338.c4Myr4.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-10-10-06-18.bpo-44338.c4Myr4.rst deleted file mode 100644 index beaa3e56334ba9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-10-10-06-18.bpo-44338.c4Myr4.rst +++ /dev/null @@ -1,7 +0,0 @@ -Implement adaptive specialization for LOAD_GLOBAL - -Two specialized forms of LOAD_GLOBAL are added: - -* LOAD_GLOBAL_MODULE - -* LOAD_GLOBAL_BUILTIN diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-10-16-10-39.bpo-44313.34RjI8.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-10-16-10-39.bpo-44313.34RjI8.rst deleted file mode 100644 index e48d4a471f802f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-10-16-10-39.bpo-44313.34RjI8.rst +++ /dev/null @@ -1,4 +0,0 @@ -Directly imported objects and modules (through import and from import -statements) don't generate ``LOAD_METHOD``/``CALL_METHOD`` for directly -accessed objects on their namespace. They now use the regular -``LOAD_ATTR``/``CALL_FUNCTION``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-11-17-37-15.bpo-44376.zhM1UW.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-11-17-37-15.bpo-44376.zhM1UW.rst deleted file mode 100644 index f854d56b3c8419..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-11-17-37-15.bpo-44376.zhM1UW.rst +++ /dev/null @@ -1 +0,0 @@ -Exact integer exponentiation (like ``i**2`` or ``pow(i, 2)``) with a small exponent is much faster, due to reducing overhead in such cases. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst deleted file mode 100644 index be72a7111dc8ae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible crash in the tokenizer when raising syntax errors for -unclosed strings. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst deleted file mode 100644 index 0f204ed812b27a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-13-23-12-18.bpo-44409.eW4LS-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix error location information for tokenizer errors raised on initialization -of the tokenizer. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-18-22-08-25.bpo-44456.L0Rhko.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-18-22-08-25.bpo-44456.L0Rhko.rst deleted file mode 100644 index 86a8c03ce561e7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-18-22-08-25.bpo-44456.L0Rhko.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve the syntax error when mixing positional and keyword patterns. Patch -by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-19-12-41-13.bpo-44297.F53vHj.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-19-12-41-13.bpo-44297.F53vHj.rst deleted file mode 100644 index bdcb5b2db39e09..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-19-12-41-13.bpo-44297.F53vHj.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make sure that the line number is set when entering a comprehension scope. -Ensures that backtraces inclusing generator expressions show the correct -line number. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-20-10-53-21.bpo-12022.SW240M.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-20-10-53-21.bpo-12022.SW240M.rst deleted file mode 100644 index 98c42283169d8e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-20-10-53-21.bpo-12022.SW240M.rst +++ /dev/null @@ -1,4 +0,0 @@ -A :exc:`TypeError` is now raised instead of an :exc:`AttributeError` in -:keyword:`with` and :keyword:`async with` statements for objects which do -not support the :term:`context manager` or :term:`asynchronous context -manager` protocols correspondingly. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-21-11-19-54.bpo-44472.Vvm1yn.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-21-11-19-54.bpo-44472.Vvm1yn.rst deleted file mode 100644 index 34fa2a9e8615fa..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-21-11-19-54.bpo-44472.Vvm1yn.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ltrace functionality when exceptions are raised. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-22-10-55-23.bpo-44486.wct-9X.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-22-10-55-23.bpo-44486.wct-9X.rst deleted file mode 100644 index cc419602541b74..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-22-10-55-23.bpo-44486.wct-9X.rst +++ /dev/null @@ -1,2 +0,0 @@ -Modules will always have a dictionary, even when created by -``types.ModuleType.__new__()`` diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-22-19-08-19.bpo-44483.eq2f7T.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-22-19-08-19.bpo-44483.eq2f7T.rst deleted file mode 100644 index ea54e79acfd9d8..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-22-19-08-19.bpo-44483.eq2f7T.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in ``types.Union`` objects when creating a union of an object -with bad ``__module__`` field. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-28-22-23-59.bpo-44525.sSvUKG.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-28-22-23-59.bpo-44525.sSvUKG.rst new file mode 100644 index 00000000000000..8963d028cf8a9b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-06-28-22-23-59.bpo-44525.sSvUKG.rst @@ -0,0 +1,10 @@ +Setup initial specialization infrastructure for the ``CALL_FUNCTION`` opcode. +Implemented initial specializations for C function calls: + +* ``CALL_FUNCTION_BUILTIN_O`` for ``METH_O`` flag. + +* ``CALL_FUNCTION_BUILTIN_FAST`` for ``METH_FASTCALL`` flag without keywords. + +* ``CALL_FUNCTION_LEN`` for ``len(o)``. + +* ``CALL_FUNCTION_ISINSTANCE`` for ``isinstance(o, t)``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-29-11-49-29.bpo-44523.67-ZIP.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-29-11-49-29.bpo-44523.67-ZIP.rst deleted file mode 100644 index aa51a7478583b1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-29-11-49-29.bpo-44523.67-ZIP.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove the pass-through for :func:`hash` of :class:`weakref.proxy` objects -to prevent unintended consequences when the original referred object -dies while the proxy is part of a hashable object. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst deleted file mode 100644 index 4912bca837bb10..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``__parameters__`` attribute and ``__getitem__`` -operator to ``types.Union``. Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-02-22-54-41.bpo-44553.l9YqGg.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-02-22-54-41.bpo-44553.l9YqGg.rst deleted file mode 100644 index e97df02994806a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-02-22-54-41.bpo-44553.l9YqGg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement GC methods for ``types.Union`` to break reference cycles and -prevent memory leaks. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-03-00-20-39.bpo-43908.YHuV_s.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-03-00-20-39.bpo-43908.YHuV_s.rst deleted file mode 100644 index 6113d0f912a5ac..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-03-00-20-39.bpo-43908.YHuV_s.rst +++ /dev/null @@ -1,3 +0,0 @@ -Heap types with the :const:`Py_TPFLAGS_IMMUTABLETYPE` flag can now inherit the -:pep:`590` vectorcall protocol. Previously, this was only possible for -:ref:`static types `. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-04-17-41-47.bpo-41486.DiM24a.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-04-17-41-47.bpo-41486.DiM24a.rst deleted file mode 100644 index 6a373f67f62f92..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-04-17-41-47.bpo-41486.DiM24a.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a memory consumption and copying performance regression in earlier 3.10 -beta releases if someone used an output buffer larger than 4GiB with -zlib.decompress on input data that expands that large. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-04-23-38-51.bpo-44562.QdeRQo.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-04-23-38-51.bpo-44562.QdeRQo.rst deleted file mode 100644 index 2fc65bcfdeef43..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-04-23-38-51.bpo-44562.QdeRQo.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove uses of :c:func:`PyObject_GC_Del` in error path when initializing -:class:`types.GenericAlias`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-06-15-27-11.bpo-43950.LhL2-q.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-06-15-27-11.bpo-43950.LhL2-q.rst deleted file mode 100644 index dde5399626b7ef..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-06-15-27-11.bpo-43950.LhL2-q.rst +++ /dev/null @@ -1,6 +0,0 @@ -Code objects can now provide the column information for instructions when -available. This is levaraged during traceback printing to show the -expressions responsible for errors. - -Contributed by Pablo Galindo, Batuhan Taskaya and Ammar Askar as part of -:pep:`657`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-06-22-22-15.bpo-44490.BJxPbZ.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-06-22-22-15.bpo-44490.BJxPbZ.rst deleted file mode 100644 index d49181cd82c818..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-06-22-22-15.bpo-44490.BJxPbZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`typing` now searches for type parameters in ``types.Union`` objects. -``get_type_hints`` will also properly resolve annotations with nested -``types.Union`` objects. Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-07-16-05-35.bpo-43895.JFjR0-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-07-16-05-35.bpo-43895.JFjR0-.rst deleted file mode 100644 index 49deb48fa43582..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-07-16-05-35.bpo-43895.JFjR0-.rst +++ /dev/null @@ -1,4 +0,0 @@ -An obsolete internal cache of shared object file handles added in 1995 that -attempted, but did not guarantee, that a .so would not be dlopen'ed twice to -work around flaws in mid-1990s posix-ish operating systems has been removed -from dynload_shlib.c. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-08-12-18-56.bpo-44584.qKnSqV.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-08-12-18-56.bpo-44584.qKnSqV.rst deleted file mode 100644 index 4afb33b047d4be..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-08-12-18-56.bpo-44584.qKnSqV.rst +++ /dev/null @@ -1,3 +0,0 @@ -The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is -deprecated in Python 3.10 and will be removed in Python 3.12. This feature -requires a debug build of Python. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-09-12-08-17.bpo-44590.a2ntVX.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-09-12-08-17.bpo-44590.a2ntVX.rst deleted file mode 100644 index ed4d969a11ab43..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-09-12-08-17.bpo-44590.a2ntVX.rst +++ /dev/null @@ -1,5 +0,0 @@ -All necessary data for executing a Python function (local variables, stack, -etc) is now kept in a per-thread stack. Frame objects are lazily allocated -on demand. This increases performance by about 7% on the standard benchmark -suite. Introspection and debugging are unaffected as frame objects are -always available when needed. Patch by Mark Shannon. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-12-04-06-57.bpo-41972.nDX5k_.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-12-04-06-57.bpo-41972.nDX5k_.rst deleted file mode 100644 index 3daffb9c0e1dff..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-12-04-06-57.bpo-41972.nDX5k_.rst +++ /dev/null @@ -1 +0,0 @@ -Tuned the string-searching algorithm of fastsearch.h to have a shorter inner loop for most cases. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-13-17-47-32.bpo-42073.9wopiC.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-13-17-47-32.bpo-42073.9wopiC.rst deleted file mode 100644 index 988fe67b99dc96..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-13-17-47-32.bpo-42073.9wopiC.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``@classmethod`` decorator can now wrap other classmethod-like -descriptors. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-13-20-22-12.bpo-44606.S3Bv2w.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-13-20-22-12.bpo-44606.S3Bv2w.rst deleted file mode 100644 index 3c7ef3b21fe092..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-13-20-22-12.bpo-44606.S3Bv2w.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``__instancecheck__`` and ``__subclasscheck__`` for the union type. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-13-23-19-41.bpo-44589.59OH8T.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-13-23-19-41.bpo-44589.59OH8T.rst deleted file mode 100644 index 23b2472b00e40f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-13-23-19-41.bpo-44589.59OH8T.rst +++ /dev/null @@ -1,2 +0,0 @@ -Mapping patterns in ``match`` statements with two or more equal literal -keys will now raise a :exc:`SyntaxError` at compile-time. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-14-10-31-10.bpo-26280.cgpM4B.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-14-10-31-10.bpo-26280.cgpM4B.rst deleted file mode 100644 index cb561e79c78cee..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-14-10-31-10.bpo-26280.cgpM4B.rst +++ /dev/null @@ -1,9 +0,0 @@ -Implement adaptive specialization for BINARY_SUBSCR - - Three specialized forms of BINARY_SUBSCR are added: - - * BINARY_SUBSCR_LIST_INT - - * BINARY_SUBSCR_TUPLE_INT - - * BINARY_SUBSCR_DICT \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-14-13-54-07.bpo-44635.7ZMAdB.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-14-13-54-07.bpo-44635.7ZMAdB.rst deleted file mode 100644 index ea00554aeeda66..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-14-13-54-07.bpo-44635.7ZMAdB.rst +++ /dev/null @@ -1 +0,0 @@ -Convert ``None`` to ``type(None)`` in the union type constructor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-01-01-11.bpo-44611.LcfHN-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-16-01-01-11.bpo-44611.LcfHN-.rst deleted file mode 100644 index 1cc8b127382390..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-01-01-11.bpo-44611.LcfHN-.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Windows, :func:`os.urandom`: uses BCryptGenRandom API instead of CryptGenRandom API -which is deprecated from Microsoft Windows API. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-09-36-12.bpo-44636.ZWebi8.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-16-09-36-12.bpo-44636.ZWebi8.rst deleted file mode 100644 index 1a053ae69e97e0..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-09-36-12.bpo-44636.ZWebi8.rst +++ /dev/null @@ -1 +0,0 @@ -Collapse union of equal types. E.g. the result of ``int | int`` is now ``int``. Fix comparison of the union type with non-hashable objects. E.g. ``int | str == {}`` no longer raises a TypeError. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-09-59-13.bpo-44646.Yb6s05.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-16-09-59-13.bpo-44646.Yb6s05.rst deleted file mode 100644 index 0e28eac54fe987..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-09-59-13.bpo-44646.Yb6s05.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the hash of the union type: it no longer depends on the order of -arguments. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-20-25-37.bpo-44655.I3wRjL.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-16-20-25-37.bpo-44655.I3wRjL.rst deleted file mode 100644 index 4ea4a6d0859799..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-20-25-37.bpo-44655.I3wRjL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Don't include a missing attribute with the same name as the failing one when -offering suggestions for missing attributes. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-21-35-14.bpo-44655.95I7M6.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-16-21-35-14.bpo-44655.95I7M6.rst deleted file mode 100644 index 17733b3619a8f3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-16-21-35-14.bpo-44655.95I7M6.rst +++ /dev/null @@ -1,2 +0,0 @@ -Include the name of the type in unset __slots__ attribute errors. Patch by -Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-17-13-41-58.bpo-44662.q22kWR.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-17-13-41-58.bpo-44662.q22kWR.rst deleted file mode 100644 index c165774a4cacfb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-17-13-41-58.bpo-44662.q22kWR.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add ``__module__`` to ``types.Union``. This also fixes -``types.Union`` issues with ``typing.Annotated``. Patch provided by -Yurii Karabas. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-17-14-20-59.bpo-44661.BQbXiH.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-17-14-20-59.bpo-44661.BQbXiH.rst deleted file mode 100644 index bafa98e5826cd4..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-17-14-20-59.bpo-44661.BQbXiH.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update ``property_descr_set`` to use vectorcall if possible. Patch by Dong-hee -Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-17-21-04-04.bpo-44633.5-zKeI.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-17-21-04-04.bpo-44633.5-zKeI.rst deleted file mode 100644 index 507a68b65d4c37..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-17-21-04-04.bpo-44633.5-zKeI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Parameter substitution of the union type with wrong types now raises -``TypeError`` instead of returning ``NotImplemented``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-19-19-53-46.bpo-44676.WgIMvh.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-19-19-53-46.bpo-44676.WgIMvh.rst deleted file mode 100644 index 4a1815e041dcc2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-19-19-53-46.bpo-44676.WgIMvh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ability to serialise ``types.Union`` objects. Patch provided by Yurii -Karabas. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-19-20-49-06.bpo-44653.WcqGyI.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-19-20-49-06.bpo-44653.WcqGyI.rst deleted file mode 100644 index 8254d9bbad4a6e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-19-20-49-06.bpo-44653.WcqGyI.rst +++ /dev/null @@ -1 +0,0 @@ -Support :mod:`typing` types in parameter substitution in the union type. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-21-15-26-56.bpo-44698.DA4_0o.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-21-15-26-56.bpo-44698.DA4_0o.rst deleted file mode 100644 index ed389630c8ba11..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-21-15-26-56.bpo-44698.DA4_0o.rst +++ /dev/null @@ -1 +0,0 @@ -Fix undefined behaviour in complex object exponentiation. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-23-01-52-13.bpo-44717.-vVmAh.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-23-01-52-13.bpo-44717.-vVmAh.rst deleted file mode 100644 index 7a4d77f7d6520f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-23-01-52-13.bpo-44717.-vVmAh.rst +++ /dev/null @@ -1 +0,0 @@ -Improve AttributeError on circular imports of submodules. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-23-15-17-01.bpo-44725.qcuKaa.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-23-15-17-01.bpo-44725.qcuKaa.rst deleted file mode 100644 index 995cf148001430..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-23-15-17-01.bpo-44725.qcuKaa.rst +++ /dev/null @@ -1 +0,0 @@ -Expose specialization stats in python via :func:`_opcode.get_specialization_stats`. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-26-15-27-03.bpo-44732.IxObt3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-26-15-27-03.bpo-44732.IxObt3.rst deleted file mode 100644 index 5094688a294314..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-26-15-27-03.bpo-44732.IxObt3.rst +++ /dev/null @@ -1 +0,0 @@ -Rename ``types.Union`` to ``types.UnionType``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-27-11-14-22.bpo-34013.SjLFe-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-27-11-14-22.bpo-34013.SjLFe-.rst deleted file mode 100644 index c0d3dd99f98272..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-27-11-14-22.bpo-34013.SjLFe-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Generalize the invalid legacy statement custom error message (like the one -generated when "print" is called without parentheses) to include more -generic expressions. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-31-12-12-57.bpo-44792.mOReTW.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-31-12-12-57.bpo-44792.mOReTW.rst deleted file mode 100644 index 2e9000d09ee44c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-31-12-12-57.bpo-44792.mOReTW.rst +++ /dev/null @@ -1 +0,0 @@ -Improve syntax errors for if expressions. Patch by Miguel Brito diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-04-11-37-38.bpo-44821.67YHGI.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-04-11-37-38.bpo-44821.67YHGI.rst deleted file mode 100644 index 1e254a62773bbd..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-04-11-37-38.bpo-44821.67YHGI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Create instance dictionaries (__dict__) eagerly, to improve regularity of -object layout and assist specialization. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-05-17-42-03.bpo-44838.r_Lkj_.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-05-17-42-03.bpo-44838.r_Lkj_.rst deleted file mode 100644 index a542a5d70933aa..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-05-17-42-03.bpo-44838.r_Lkj_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a bug that was causing the parser to raise an incorrect custom -:exc:`SyntaxError` for invalid 'if' expressions. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-05-17-49-55.bpo-44826.zQsyK5.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-05-17-49-55.bpo-44826.zQsyK5.rst deleted file mode 100644 index ccdb5e0c606c0f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-05-17-49-55.bpo-44826.zQsyK5.rst +++ /dev/null @@ -1,9 +0,0 @@ -Initial implementation of adaptive specialization of STORE_ATTR - -Three specialized forms of STORE_ATTR are added: - -* STORE_ATTR_SLOT - -* STORE_ATTR_SPLIT_KEYS - -* STORE_ATTR_WITH_HINT diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst deleted file mode 100644 index 1111d01b726fa2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst +++ /dev/null @@ -1 +0,0 @@ -Fix reference leaks in the error paths of ``update_bases()`` and ``__build_class__``. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-21-39-19.bpo-25782.B22lMx.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-07-21-39-19.bpo-25782.B22lMx.rst deleted file mode 100644 index 1c52059f76c8f3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-21-39-19.bpo-25782.B22lMx.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug where ``PyErr_SetObject`` hangs when the current exception has a cycle in its context chain. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-14-29-52.bpo-33930.--5LQ-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-09-14-29-52.bpo-33930.--5LQ-.rst deleted file mode 100644 index 827dd3f8b65131..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-14-29-52.bpo-33930.--5LQ-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix segmentation fault with deep recursion when cleaning method objects. -Patch by Augusto Goulart and Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-16-16-03.bpo-44872.OKRlhK.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-09-16-16-03.bpo-44872.OKRlhK.rst deleted file mode 100644 index 9a0d00018b2a7c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-16-16-03.bpo-44872.OKRlhK.rst +++ /dev/null @@ -1 +0,0 @@ -Use new trashcan macros (Py_TRASHCAN_BEGIN/END) in frameobject.c instead of the old ones (Py_TRASHCAN_SAFE_BEGIN/END). \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-19-05-20.bpo-44874.oOcfU4.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-09-19-05-20.bpo-44874.oOcfU4.rst deleted file mode 100644 index 1aed5351d74173..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-19-05-20.bpo-44874.oOcfU4.rst +++ /dev/null @@ -1 +0,0 @@ -Deprecate the old trashcan macros (``Py_TRASHCAN_SAFE_BEGIN``/``Py_TRASHCAN_SAFE_END``). They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-12-03-52.bpo-44878.nEhjLi.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-11-12-03-52.bpo-44878.nEhjLi.rst deleted file mode 100644 index 7998140d4aab0b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-12-03-52.bpo-44878.nEhjLi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove switch statement for interpreter loop when using computed gotos. This -makes sure that we only have one dispatch table in the interpreter. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-14-12-41.bpo-44878.pAbBfc.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-11-14-12-41.bpo-44878.pAbBfc.rst deleted file mode 100644 index 9e07fa8a25df0b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-14-12-41.bpo-44878.pAbBfc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove the loop from the bytecode interpreter. All instructions end with a -DISPATCH macro, so the loop is now redundant. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-15-39-57.bpo-44885.i4noUO.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-11-15-39-57.bpo-44885.i4noUO.rst deleted file mode 100644 index c6abd7363af711..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-15-39-57.bpo-44885.i4noUO.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correct the ast locations of f-strings with format specs and repeated -expressions. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-16-46-27.bpo-44890.PwNg8N.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-11-16-46-27.bpo-44890.PwNg8N.rst deleted file mode 100644 index 48a1c8b690e65d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-16-46-27.bpo-44890.PwNg8N.rst +++ /dev/null @@ -1 +0,0 @@ -Specialization stats are always collected in debug builds. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-20-45-02.bpo-44889.2T3nTn.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-11-20-45-02.bpo-44889.2T3nTn.rst deleted file mode 100644 index a50b6851c14b7a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-20-45-02.bpo-44889.2T3nTn.rst +++ /dev/null @@ -1,8 +0,0 @@ -Initial implementation of adaptive specialization of ``LOAD_METHOD``. The -following specialized forms were added: - -* ``LOAD_METHOD_CACHED`` - -* ``LOAD_METHOD_MODULE`` - -* ``LOAD_METHOD_CLASS`` diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-12-14-00-57.bpo-44900.w2gpwy.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-12-14-00-57.bpo-44900.w2gpwy.rst deleted file mode 100644 index 8d94d6aed80ead..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-12-14-00-57.bpo-44900.w2gpwy.rst +++ /dev/null @@ -1,7 +0,0 @@ -Add five superinstructions for PEP 659 quickening: - -* LOAD_FAST LOAD_FAST -* STORE_FAST LOAD_FAST -* LOAD_FAST LOAD_CONST -* LOAD_CONST LOAD_FAST -* STORE_FAST STORE_FAST diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-14-20-13-21.bpo-44895.Ic9m90.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-14-20-13-21.bpo-44895.Ic9m90.rst deleted file mode 100644 index d369ac76505974..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-14-20-13-21.bpo-44895.Ic9m90.rst +++ /dev/null @@ -1,2 +0,0 @@ -A debug variable :envvar:`PYTHONDUMPREFSFILE` is added for creating a dump file -which is generated by :option:`--with-trace-refs`. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-15-10-39-06.bpo-44698.lITKNc.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-15-10-39-06.bpo-44698.lITKNc.rst deleted file mode 100644 index f197253e10ec69..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-15-10-39-06.bpo-44698.lITKNc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restore behaviour of complex exponentiation with integer-valued exponent of -type :class:`float` or :class:`complex`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-16-11-36-02.bpo-44914.6Lgrx3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-16-11-36-02.bpo-44914.6Lgrx3.rst deleted file mode 100644 index 5e306ffaf5de84..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-16-11-36-02.bpo-44914.6Lgrx3.rst +++ /dev/null @@ -1,5 +0,0 @@ -Class version tags are no longer recycled. - -This means that a version tag serves as a unique identifier for the state of -a class. We rely on this for effective specialization of the LOAD_ATTR and -other instructions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-16-23-16-17.bpo-44929.qpMEky.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-16-23-16-17.bpo-44929.qpMEky.rst deleted file mode 100644 index e883e031a4afd8..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-16-23-16-17.bpo-44929.qpMEky.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix some edge cases of ``enum.Flag`` string representation in the REPL. -Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-18-11-14-38.bpo-44945.CO3s77.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-18-11-14-38.bpo-44945.CO3s77.rst deleted file mode 100644 index 66d53ec523de3c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-18-11-14-38.bpo-44945.CO3s77.rst +++ /dev/null @@ -1,7 +0,0 @@ -Specialize the BINARY_ADD instruction using the PEP 659 machinery. Adds five new instructions: - -* BINARY_ADD_ADAPTIVE -* BINARY_ADD_FLOAT -* BINARY_ADD_INT -* BINARY_ADD_UNICODE -* BINARY_ADD_UNICODE_INPLACE_FAST diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-18-19-09-28.bpo-44947.mcvGdS.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-18-19-09-28.bpo-44947.mcvGdS.rst deleted file mode 100644 index d531ba9faf3de5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-18-19-09-28.bpo-44947.mcvGdS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Refine the syntax error for trailing commas in import statements. Patch by -Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-19-14-43-24.bpo-44954.dLn3lg.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-19-14-43-24.bpo-44954.dLn3lg.rst deleted file mode 100644 index 4cdeb34b8b6116..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-19-14-43-24.bpo-44954.dLn3lg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a corner case bug where the result of ``float.fromhex('0x.8p-1074')`` -was rounded the wrong way. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-22-12-28-50.bpo-24234.n3oTdx.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-22-12-28-50.bpo-24234.n3oTdx.rst deleted file mode 100644 index 52397e90fbdfa1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-22-12-28-50.bpo-24234.n3oTdx.rst +++ /dev/null @@ -1,3 +0,0 @@ -Implement the :meth:`__complex__` special method on the :class:`complex` type, -so a complex number ``z`` passes an ``isinstance(z, typing.SupportsComplex)`` -check. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-10-36-55.bpo-24234.MGVUQi.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-10-36-55.bpo-24234.MGVUQi.rst deleted file mode 100644 index 3f724da08898a1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-10-36-55.bpo-24234.MGVUQi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Implement the :meth:`__bytes__` special method on the :class:`bytes` type, -so a bytes object ``b`` passes an ``isinstance(b, typing.SupportsBytes)`` -check. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst deleted file mode 100644 index 6b4b9dfd8bc3c0..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-25-23-07-10.bpo-44963.5EET8y.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-25-23-07-10.bpo-44963.5EET8y.rst deleted file mode 100644 index 9a54bda118e002..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-25-23-07-10.bpo-44963.5EET8y.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement ``send()`` and ``throw()`` methods for ``anext_awaitable`` -objects. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-25-23-17-32.bpo-45000.XjmyLl.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-25-23-17-32.bpo-45000.XjmyLl.rst deleted file mode 100644 index 96c95cc6e0296f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-25-23-17-32.bpo-45000.XjmyLl.rst +++ /dev/null @@ -1,2 +0,0 @@ -A :exc:`SyntaxError` is now raised when trying to delete :const:`__debug__`. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst deleted file mode 100644 index 5bf13ef06f34c3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed pickling of range iterators that iterated for over 2**32 times. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-11-09-52.bpo-45012.ueeOcx.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-11-09-52.bpo-45012.ueeOcx.rst deleted file mode 100644 index 91cb3a9e69cc52..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-11-09-52.bpo-45012.ueeOcx.rst +++ /dev/null @@ -1,2 +0,0 @@ -In :mod:`posix`, release GIL during ``stat()``, ``lstat()``, and -``fstatat()`` syscalls made by :func:`os.DirEntry.stat`. Patch by Stanisław Skonieczny. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst deleted file mode 100644 index f6dffa0831c54a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add a new command line option, "-X frozen_modules=[on|off]" to opt out -of (or into) using optional frozen modules. This defaults to "on" (or -"off" if it's a debug build). diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-16-55-43.bpo-45056.7AK2d9.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-01-16-55-43.bpo-45056.7AK2d9.rst deleted file mode 100644 index 6c790f5c50c48e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-16-55-43.bpo-45056.7AK2d9.rst +++ /dev/null @@ -1 +0,0 @@ -Compiler now removes trailing unused constants from co_consts. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-19-21-48.bpo-34561.uMAVA-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-01-19-21-48.bpo-34561.uMAVA-.rst deleted file mode 100644 index 7c48cb39df1c5a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-19-21-48.bpo-34561.uMAVA-.rst +++ /dev/null @@ -1 +0,0 @@ -List sorting now uses the merge-ordering strategy from Munro and Wild's ``powersort()``. Unlike the former strategy, this is provably near-optimal in the entropy of the distribution of run lengths. Most uses of ``list.sort()`` probably won't see a significant time difference, but may see significant improvements in cases where the former strategy was exceptionally poor. However, as these are all fast linear-time approximations to a problem that's inherently at best quadratic-time to solve truly optimally, it's also possible to contrive cases where the former strategy did better. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-23-55-49.bpo-45083.cLi9G3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-01-23-55-49.bpo-45083.cLi9G3.rst deleted file mode 100644 index 7bfd87b9420593..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-23-55-49.bpo-45083.cLi9G3.rst +++ /dev/null @@ -1,3 +0,0 @@ -When the interpreter renders an exception, its name now has a complete qualname. Previously only the class name was concatenated to the module name, which sometimes resulted in an incorrect full name being displayed. - -(This issue impacted only the C code exception rendering, the :mod:`traceback` module was using qualname already). \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-02-01-28-01.bpo-37330.QDjM_l.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-02-01-28-01.bpo-37330.QDjM_l.rst deleted file mode 100644 index 3f09449de70d09..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-02-01-28-01.bpo-37330.QDjM_l.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`open`, :func:`io.open`, :func:`codecs.open` and -:class:`fileinput.FileInput` no longer accept ``'U'`` ("universal newline") -in the file mode. This flag was deprecated since Python 3.3. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-03-12-35-17.bpo-41031.yPSJEs.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-03-12-35-17.bpo-41031.yPSJEs.rst deleted file mode 100644 index 5dcfaa0046c65c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-03-12-35-17.bpo-41031.yPSJEs.rst +++ /dev/null @@ -1 +0,0 @@ -Match C and Python code formatting of unprintable exceptions and exceptions in the :mod:`__main__` module. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-03-16-18-10.bpo-1514420.2Lumpj.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-03-16-18-10.bpo-1514420.2Lumpj.rst deleted file mode 100644 index fdd5cd70c5c2fb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-03-16-18-10.bpo-1514420.2Lumpj.rst +++ /dev/null @@ -1 +0,0 @@ -Interpreter no longer attempts to open files with names in angle brackets (like "" or "") when formatting an exception. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-06-21-52-45.bpo-45123.8Eh9iI.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-06-21-52-45.bpo-45123.8Eh9iI.rst deleted file mode 100644 index 6cc7303766f25d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-06-21-52-45.bpo-45123.8Eh9iI.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix PyAiter_Check to only check for the __anext__ presence (not for -__aiter__). Rename PyAiter_Check to PyAIter_Check, PyObject_GetAiter -> -PyObject_GetAIter. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-07-00-21-04.bpo-44348.f8w_Td.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-07-00-21-04.bpo-44348.f8w_Td.rst deleted file mode 100644 index c222a07725b8a3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-07-00-21-04.bpo-44348.f8w_Td.rst +++ /dev/null @@ -1,8 +0,0 @@ -The deallocator function of the :exc:`BaseException` type now uses the -trashcan mecanism to prevent stack overflow. For example, when a -:exc:`RecursionError` instance is raised, it can be linked to another -RecursionError through the ``__context__`` attribute or the -``__traceback__`` attribute, and then a chain of exceptions is created. When -the chain is destroyed, nested deallocator function calls can crash with a -stack overflow if the chain is too long compared to the available stack -memory. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-07-17-10-16.bpo-45121.iG-Hsf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-07-17-10-16.bpo-45121.iG-Hsf.rst deleted file mode 100644 index 19eb3314125167..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-07-17-10-16.bpo-45121.iG-Hsf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where ``Protocol.__init__`` raises ``RecursionError`` when it's -called directly or via ``super()``. Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-08-00-30-09.bpo-44050.mFI15u.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-08-00-30-09.bpo-44050.mFI15u.rst new file mode 100644 index 00000000000000..d6eed9f1bcfe9d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-09-08-00-30-09.bpo-44050.mFI15u.rst @@ -0,0 +1,3 @@ +Extensions that indicate they use global state (by setting ``m_size`` to -1) +can again be used in multiple interpreters. This reverts to behavior of +Python 3.8. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-08-08-29-41.bpo-44959.OSwwPf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-08-08-29-41.bpo-44959.OSwwPf.rst deleted file mode 100644 index 02e11ae94e430b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-08-08-29-41.bpo-44959.OSwwPf.rst +++ /dev/null @@ -1 +0,0 @@ -Added fallback to extension modules with '.sl' suffix on HP-UX \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-09-10-32-33.bpo-44219.WiYyjz.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-09-10-32-33.bpo-44219.WiYyjz.rst deleted file mode 100644 index 2abd81673663b8..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-09-10-32-33.bpo-44219.WiYyjz.rst +++ /dev/null @@ -1,5 +0,0 @@ -Release the GIL while performing ``isatty`` system calls on arbitrary file -descriptors. In particular, this affects :func:`os.isatty`, -:func:`os.device_encoding` and :class:`io.TextIOWrapper`. By extension, -:func:`io.open` in text mode is also affected. This change solves -a deadlock in :func:`os.isatty`. Patch by Vincent Michel in :issue:`44219`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-09-15-05-17.bpo-45155.JRw9TG.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-09-15-05-17.bpo-45155.JRw9TG.rst deleted file mode 100644 index eab023bf89471e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-09-15-05-17.bpo-45155.JRw9TG.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`int.to_bytes` and :meth:`int.from_bytes` now take a default value of -``"big"`` for the ``byteorder`` argument. :meth:`int.to_bytes` also takes a -default value of ``1`` for the ``length`` argument. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-14-09-23-59.bpo-45167.CPSSoV.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-14-09-23-59.bpo-45167.CPSSoV.rst deleted file mode 100644 index 47755ae59be2bd..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-14-09-23-59.bpo-45167.CPSSoV.rst +++ /dev/null @@ -1 +0,0 @@ -Fix deepcopying of :class:`types.GenericAlias` objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-14-10-02-12.bpo-45190.ZFRgSj.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-14-10-02-12.bpo-45190.ZFRgSj.rst deleted file mode 100644 index c6a4c554aff5a5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-14-10-02-12.bpo-45190.ZFRgSj.rst +++ /dev/null @@ -1 +0,0 @@ -Update Unicode databases to Unicode 14.0.0. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-20-10-02-12.bpo-24076.ZFgFSj.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-20-10-02-12.bpo-24076.ZFgFSj.rst deleted file mode 100644 index b680884ff8b1e0..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-20-10-02-12.bpo-24076.ZFgFSj.rst +++ /dev/null @@ -1 +0,0 @@ -sum() was further optimised for summing up single digit integers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-21-22-27-25.bpo-45061.5IOUf0.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-21-22-27-25.bpo-45061.5IOUf0.rst deleted file mode 100644 index caeb36ba52646a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-21-22-27-25.bpo-45061.5IOUf0.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add a deallocator to the bool type to detect refcount bugs in C extensions -which call Py_DECREF(Py_True) or Py_DECREF(Py_False) by mistake. Detect also -refcount bugs when the empty tuple singleton or the Unicode empty string -singleton is destroyed by mistake. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-23-14-00-05.bpo-40116.KaoeFs.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-23-14-00-05.bpo-40116.KaoeFs.rst new file mode 100644 index 00000000000000..24ce96376d87e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-09-23-14-00-05.bpo-40116.KaoeFs.rst @@ -0,0 +1,4 @@ +Change to the implementation of split dictionaries. Classes where the +instances differ either in the exact set of attributes, or in the order in +which those attributes are set, can still share keys. This should have no +observable effect on users of Python or the C-API. Patch by Mark Shannon. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-29-12-02-39.bpo-45324.BTQElX.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-29-12-02-39.bpo-45324.BTQElX.rst new file mode 100644 index 00000000000000..7b16847bb73397 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-09-29-12-02-39.bpo-45324.BTQElX.rst @@ -0,0 +1,3 @@ +In FrozenImporter.find_spec(), we now preserve the information needed in +exec_module() to load the module. This change mostly impacts internal +details, rather than changing the importer's behavior. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst new file mode 100644 index 00000000000000..839604357d1214 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst @@ -0,0 +1,5 @@ +For frozen stdlib modules, record the original module name as +``module.__spec__.loader_state.origname``. If the value is different than +``module.__spec__.name`` then the module was defined as an alias in +Tools/scripts/freeze_modules.py. If it is ``None`` then the module comes +from a source file outside the stdlib. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-21-02.bpo-21736.RI47BU.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-21-02.bpo-21736.RI47BU.rst new file mode 100644 index 00000000000000..8396a49c3cf736 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-21-02.bpo-21736.RI47BU.rst @@ -0,0 +1,9 @@ +Frozen stdlib modules now have ``__file__`` to the .py file they would +otherwise be loaded from, if possible. For packages, ``__path__`` now has +the correct entry instead of being an empty list, which allows unfrozen +submodules to be imported. These are set only if the stdlib directory is +known when the runtime is initialized. Note that the file at ``__file__`` +is not guaranteed to exist. None of this affects non-stdlib frozen modules +nor, for now, frozen modules imported using +``PyImport_ImportFrozenModule()``. Also, at the moment ``co_filename`` is +not updated for the module. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst new file mode 100644 index 00000000000000..aae518da1e86f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst @@ -0,0 +1 @@ +Specialized the ``BINARY_MULTIPLY`` opcode to ``BINARY_MULTIPLY_INT`` and ``BINARY_MULTIPLY_FLOAT`` using the PEP 659 machinery. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-06-21-20-11.bpo-45385.CTUT8s.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-06-21-20-11.bpo-45385.CTUT8s.rst new file mode 100644 index 00000000000000..8047c102634f04 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-06-21-20-11.bpo-45385.CTUT8s.rst @@ -0,0 +1 @@ +Fix reference leak from descr_check. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-07-19-09-12.bpo-29410.bg5SYp.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-07-19-09-12.bpo-29410.bg5SYp.rst new file mode 100644 index 00000000000000..b08999e78d84ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-07-19-09-12.bpo-29410.bg5SYp.rst @@ -0,0 +1 @@ +Add SipHash13 for string hash algorithm and use it by default. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-07-21-26-44.bpo-45408.qUqzcd.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-07-21-26-44.bpo-45408.qUqzcd.rst new file mode 100644 index 00000000000000..e4d4db9cb95362 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-07-21-26-44.bpo-45408.qUqzcd.rst @@ -0,0 +1,2 @@ +Fix a crash in the parser when reporting tokenizer errors that occur at the +same time unclosed parentheses are detected. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-08-09-47-38.bpo-45340.ukHgDb.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-08-09-47-38.bpo-45340.ukHgDb.rst new file mode 100644 index 00000000000000..7760773d4fb205 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-08-09-47-38.bpo-45340.ukHgDb.rst @@ -0,0 +1,3 @@ +Object attributes are held in an array instead of a dictionary. An object's +dictionary are created lazily, only when needed. Reduces the memory +consumption of a typical Python object by about 30%. Patch by Mark Shannon. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst new file mode 100644 index 00000000000000..d497ae26bd5775 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst @@ -0,0 +1,2 @@ +Python now fails to initialize if it finds an invalid :option:`-X` option in the +command line. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst new file mode 100644 index 00000000000000..97e29813ab2663 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst @@ -0,0 +1,2 @@ +Fix parser crash when reporting errors involving invalid continuation +characters. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-18-22-40-33.bpo-45521.GdMiuW.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-18-22-40-33.bpo-45521.GdMiuW.rst new file mode 100644 index 00000000000000..3a082a4ffdbcb1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-18-22-40-33.bpo-45521.GdMiuW.rst @@ -0,0 +1,3 @@ +Fix a bug in the obmalloc radix tree code. On 64-bit machines, the bug +causes the tree to hold 46-bits of virtual addresses, rather than the +intended 48-bits. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-20-11-57-31.bpo-44525.veL4lJ.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-20-11-57-31.bpo-44525.veL4lJ.rst new file mode 100644 index 00000000000000..6ab1d05603db87 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-20-11-57-31.bpo-44525.veL4lJ.rst @@ -0,0 +1 @@ +Specialize simple calls to Python functions (no starargs, keyowrd dict, or closure) diff --git a/Misc/NEWS.d/next/Documentation/2018-05-19-15-59-29.bpo-33479.4cLlxo.rst b/Misc/NEWS.d/next/Documentation/2018-05-19-15-59-29.bpo-33479.4cLlxo.rst deleted file mode 100644 index db4973d3923957..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2018-05-19-15-59-29.bpo-33479.4cLlxo.rst +++ /dev/null @@ -1,4 +0,0 @@ -Remove the unqualified claim that tkinter is threadsafe. It has not been -true for several years and likely never was. An explanation of what is true -may be added later, after more discussion, and possibly after patching -_tkinter.c, diff --git a/Misc/NEWS.d/next/Documentation/2020-01-30-05-18-48.bpo-39498.Nu3sFL.rst b/Misc/NEWS.d/next/Documentation/2020-01-30-05-18-48.bpo-39498.Nu3sFL.rst deleted file mode 100644 index a3e899a80a0fc2..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-01-30-05-18-48.bpo-39498.Nu3sFL.rst +++ /dev/null @@ -1 +0,0 @@ -Add a "Security Considerations" index which links to standard library modules that have explicitly documented security considerations. diff --git a/Misc/NEWS.d/next/Documentation/2020-03-21-01-19-28.bpo-21760.CqofIc.rst b/Misc/NEWS.d/next/Documentation/2020-03-21-01-19-28.bpo-21760.CqofIc.rst deleted file mode 100644 index 119ef3d4c4378e..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-03-21-01-19-28.bpo-21760.CqofIc.rst +++ /dev/null @@ -1,2 +0,0 @@ -The description for __file__ fixed. -Patch by Furkan Onder \ No newline at end of file diff --git a/Misc/NEWS.d/next/Documentation/2020-08-21-22-59-37.bpo-41576.7a6CQR.rst b/Misc/NEWS.d/next/Documentation/2020-08-21-22-59-37.bpo-41576.7a6CQR.rst deleted file mode 100644 index f74ef62ca47ab2..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-08-21-22-59-37.bpo-41576.7a6CQR.rst +++ /dev/null @@ -1 +0,0 @@ -document BaseException in favor of bare except \ No newline at end of file diff --git a/Misc/NEWS.d/next/Documentation/2020-08-24-13-35-04.bpo-41621.nqaw9G.rst b/Misc/NEWS.d/next/Documentation/2020-08-24-13-35-04.bpo-41621.nqaw9G.rst deleted file mode 100644 index bd193d9163073a..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-08-24-13-35-04.bpo-41621.nqaw9G.rst +++ /dev/null @@ -1 +0,0 @@ -Document that :class:`collections.defaultdict` parameter ``default_factory`` defaults to None and is positional-only. diff --git a/Misc/NEWS.d/next/Documentation/2020-09-03-13-37-19.bpo-41706._zXWOR.rst b/Misc/NEWS.d/next/Documentation/2020-09-03-13-37-19.bpo-41706._zXWOR.rst deleted file mode 100644 index 6406494ec03c7b..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-09-03-13-37-19.bpo-41706._zXWOR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix docs about how methods like ``__add__`` are invoked when evaluating -operator expressions. diff --git a/Misc/NEWS.d/next/Documentation/2021-05-03-22-08-08.bpo-44025.gcB7iP.rst b/Misc/NEWS.d/next/Documentation/2021-05-03-22-08-08.bpo-44025.gcB7iP.rst deleted file mode 100644 index 1432236a32f63f..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-05-03-22-08-08.bpo-44025.gcB7iP.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify when '_' in match statements is a keyword, and when not. diff --git a/Misc/NEWS.d/next/Documentation/2021-05-07-12-27-09.bpo-43558.UGhA8R.rst b/Misc/NEWS.d/next/Documentation/2021-05-07-12-27-09.bpo-43558.UGhA8R.rst deleted file mode 100644 index b0ecb171ef7314..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-05-07-12-27-09.bpo-43558.UGhA8R.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add the remark to :mod:`dataclasses` documentation that the :meth:`__init__` of any base class -has to be called in :meth:`__post_init__`, along with a code example. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Documentation/2021-05-08-09-48-05.bpo-44072.fb2x5I.rst b/Misc/NEWS.d/next/Documentation/2021-05-08-09-48-05.bpo-44072.fb2x5I.rst deleted file mode 100644 index a5b0c95d85e66e..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-05-08-09-48-05.bpo-44072.fb2x5I.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correct where in the numeric ABC hierarchy ``**`` support is added, i.e., in -numbers.Complex, not numbers.Integral. diff --git a/Misc/NEWS.d/next/Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst b/Misc/NEWS.d/next/Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst deleted file mode 100644 index b9fe722fa3a795..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst +++ /dev/null @@ -1 +0,0 @@ -Document that ``ConfigParser`` strips off comments when reading configuration files. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Documentation/2021-05-23-09-11-28.bpo-44195.1bqkOs.rst b/Misc/NEWS.d/next/Documentation/2021-05-23-09-11-28.bpo-44195.1bqkOs.rst deleted file mode 100644 index 5f165f166a377f..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-05-23-09-11-28.bpo-44195.1bqkOs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Corrected references to ``TraversableResources`` in docs. There is no -``TraversableReader``. diff --git a/Misc/NEWS.d/next/Documentation/2021-05-26-11-16-33.bpo-42392.oxRx6E.rst b/Misc/NEWS.d/next/Documentation/2021-05-26-11-16-33.bpo-42392.oxRx6E.rst deleted file mode 100644 index 5c840de6f68ef6..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-05-26-11-16-33.bpo-42392.oxRx6E.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document the deprecation and removal of the ``loop`` parameter for many -functions and classes in :mod:`asyncio`. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-06-14-12-00.bpo-44322.K0PHfE.rst b/Misc/NEWS.d/next/Documentation/2021-06-06-14-12-00.bpo-44322.K0PHfE.rst deleted file mode 100644 index 48dd7e6d97662d..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-06-14-12-00.bpo-44322.K0PHfE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that SyntaxError args have a details tuple and that details are -adjusted for errors in f-string field replacement expressions. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-14-09-20-37.bpo-38291.VMYa_Q.rst b/Misc/NEWS.d/next/Documentation/2021-06-14-09-20-37.bpo-38291.VMYa_Q.rst deleted file mode 100644 index 23ce35eb176d9d..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-14-09-20-37.bpo-38291.VMYa_Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Mark ``typing.io`` and ``typing.re`` as deprecated since Python 3.8 in the -documentation. They were never properly supported by type checkers. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-16-18-09-49.bpo-44392.6RF1Sc.rst b/Misc/NEWS.d/next/Documentation/2021-06-16-18-09-49.bpo-44392.6RF1Sc.rst deleted file mode 100644 index ac197f22929d14..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-16-18-09-49.bpo-44392.6RF1Sc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added a new section in the C API documentation for types used in type -hinting. Documented ``Py_GenericAlias`` and ``Py_GenericAliasType``. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-18-06-44-45.bpo-44453.3PIkj2.rst b/Misc/NEWS.d/next/Documentation/2021-06-18-06-44-45.bpo-44453.3PIkj2.rst deleted file mode 100644 index fd72cf525c32fb..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-18-06-44-45.bpo-44453.3PIkj2.rst +++ /dev/null @@ -1 +0,0 @@ -Fix documentation for the return type of :func:`sysconfig.get_path`. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst b/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst deleted file mode 100644 index ccb7767a6b6936..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst +++ /dev/null @@ -1 +0,0 @@ -Documentation of csv.Dialect is more descriptive. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-21-15-46-32.bpo-13814.LDcslu.rst b/Misc/NEWS.d/next/Documentation/2021-06-21-15-46-32.bpo-13814.LDcslu.rst deleted file mode 100644 index db0c6d6524beeb..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-21-15-46-32.bpo-13814.LDcslu.rst +++ /dev/null @@ -1 +0,0 @@ -In the Design FAQ, answer "Why don't generators support the with statement?" diff --git a/Misc/NEWS.d/next/Documentation/2021-06-23-15-21-36.bpo-39452.o_I-6d.rst b/Misc/NEWS.d/next/Documentation/2021-06-23-15-21-36.bpo-39452.o_I-6d.rst deleted file mode 100644 index 5c8cbd8e652232..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-23-15-21-36.bpo-39452.o_I-6d.rst +++ /dev/null @@ -1,4 +0,0 @@ -Rewrote ``Doc/library/__main__.rst``. Broadened scope of the document to -explicitly discuss and differentiate between ``__main__.py`` in packages -versus the ``__name__ == '__main__'`` expression (and the idioms that -surround it). diff --git a/Misc/NEWS.d/next/Documentation/2021-06-24-14-37-16.bpo-43066.Ti7ahX.rst b/Misc/NEWS.d/next/Documentation/2021-06-24-14-37-16.bpo-43066.Ti7ahX.rst deleted file mode 100644 index 3e38522839e8be..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-24-14-37-16.bpo-43066.Ti7ahX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added a warning to :mod:`zipfile` docs: filename arg with a leading slash may cause archive to -be un-openable on Windows systems. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-26-17-41-06.bpo-40620.PAYDrB.rst b/Misc/NEWS.d/next/Documentation/2021-06-26-17-41-06.bpo-40620.PAYDrB.rst deleted file mode 100644 index 52b451b492640e..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-26-17-41-06.bpo-40620.PAYDrB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Convert examples in tutorial controlflow.rst section 4.3 to be interpreter-demo -style. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-28-12-13-48.bpo-38062.9Ehp9O.rst b/Misc/NEWS.d/next/Documentation/2021-06-28-12-13-48.bpo-38062.9Ehp9O.rst deleted file mode 100644 index 1d90096e20bfa4..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-28-12-13-48.bpo-38062.9Ehp9O.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify that atexit uses equality comparisons internally. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-02-14-02-29.bpo-44544._5_aCz.rst b/Misc/NEWS.d/next/Documentation/2021-07-02-14-02-29.bpo-44544._5_aCz.rst deleted file mode 100644 index 4bb69977e0c1a4..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-02-14-02-29.bpo-44544._5_aCz.rst +++ /dev/null @@ -1,4 +0,0 @@ -List all kwargs for :func:`textwrap.wrap`, :func:`textwrap.fill`, and -:func:`textwrap.shorten`. Now, there are nav links to attributes of -:class:`TextWrap`, which makes navigation much easier while minimizing -duplication in the documentation. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-03-18-25-17.bpo-44558.0pTknl.rst b/Misc/NEWS.d/next/Documentation/2021-07-03-18-25-17.bpo-44558.0pTknl.rst deleted file mode 100644 index a12a49ccd80cc2..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-03-18-25-17.bpo-44558.0pTknl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Match the docstring and python implementation of :func:`~operator.countOf` to the behavior -of its c implementation. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-12-11-39-20.bpo-44613.DIXNzc.rst b/Misc/NEWS.d/next/Documentation/2021-07-12-11-39-20.bpo-44613.DIXNzc.rst deleted file mode 100644 index baf591073620c8..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-12-11-39-20.bpo-44613.DIXNzc.rst +++ /dev/null @@ -1 +0,0 @@ -importlib.metadata is no longer provisional. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-13-22-25-13.bpo-44631.qkGwe4.rst b/Misc/NEWS.d/next/Documentation/2021-07-13-22-25-13.bpo-44631.qkGwe4.rst deleted file mode 100644 index b0898fe1ad9995..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-13-22-25-13.bpo-44631.qkGwe4.rst +++ /dev/null @@ -1 +0,0 @@ -Refactored the ``repr()`` code of the ``_Environ`` (os module). \ No newline at end of file diff --git a/Misc/NEWS.d/next/Documentation/2021-07-15-11-19-03.bpo-42958.gC5IHM.rst b/Misc/NEWS.d/next/Documentation/2021-07-15-11-19-03.bpo-42958.gC5IHM.rst deleted file mode 100644 index c93b84d095526e..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-15-11-19-03.bpo-42958.gC5IHM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Updated the docstring and docs of :func:`filecmp.cmp` to be more accurate -and less confusing especially in respect to *shallow* arg. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-18-22-26-02.bpo-44651.SjT9iY.rst b/Misc/NEWS.d/next/Documentation/2021-07-18-22-26-02.bpo-44651.SjT9iY.rst deleted file mode 100644 index 20796e2a9bb693..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-18-22-26-02.bpo-44651.SjT9iY.rst +++ /dev/null @@ -1 +0,0 @@ -Delete entry "coercion" in Doc/glossary.rst for its outdated definition. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst b/Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst deleted file mode 100644 index 53238533edabbf..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update of three expired hyperlinks in Doc/distributing/index.rst: -"Project structure", "Building and packaging the project", and "Uploading the -project to the Python Packaging Index". diff --git a/Misc/NEWS.d/next/Documentation/2021-07-20-21-03-18.bpo-30511.eMFkRi.rst b/Misc/NEWS.d/next/Documentation/2021-07-20-21-03-18.bpo-30511.eMFkRi.rst deleted file mode 100644 index a358fb9cc2860b..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-20-21-03-18.bpo-30511.eMFkRi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Clarify that :func:`shutil.make_archive` is not thread-safe due to -reliance on changing the current working directory. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-22-08-28-03.bpo-35183.p9BWTB.rst b/Misc/NEWS.d/next/Documentation/2021-07-22-08-28-03.bpo-35183.p9BWTB.rst deleted file mode 100644 index 02c5fe82611fbf..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-22-08-28-03.bpo-35183.p9BWTB.rst +++ /dev/null @@ -1 +0,0 @@ -Add typical examples to os.path.splitext docs \ No newline at end of file diff --git a/Misc/NEWS.d/next/Documentation/2021-07-25-23-04-15.bpo-44693.JuCbNq.rst b/Misc/NEWS.d/next/Documentation/2021-07-25-23-04-15.bpo-44693.JuCbNq.rst deleted file mode 100644 index 614abb412df8ea..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-25-23-04-15.bpo-44693.JuCbNq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update the definition of __future__ in the glossary by replacing the confusing -word "pseudo-module" with a more accurate description. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-26-23-48-31.bpo-44740.zMFGMV.rst b/Misc/NEWS.d/next/Documentation/2021-07-26-23-48-31.bpo-44740.zMFGMV.rst deleted file mode 100644 index c01273f5ddc269..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-26-23-48-31.bpo-44740.zMFGMV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Replaced occurences of uppercase "Web" and "Internet" with lowercase -versions per the 2016 revised Associated Press Style Book. diff --git a/Misc/NEWS.d/next/Documentation/2021-08-09-19-58-45.bpo-36700.WPNW5f.rst b/Misc/NEWS.d/next/Documentation/2021-08-09-19-58-45.bpo-36700.WPNW5f.rst deleted file mode 100644 index 5bc1e23b285970..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-09-19-58-45.bpo-36700.WPNW5f.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`base64` RFC references were updated to point to :rfc:`4648`; a section -was added to point users to the new "security considerations" section of the -RFC. diff --git a/Misc/NEWS.d/next/Documentation/2021-08-11-18-02-06.bpo-33479.rCe4c5.rst b/Misc/NEWS.d/next/Documentation/2021-08-11-18-02-06.bpo-33479.rCe4c5.rst deleted file mode 100644 index c4a8a981939dec..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-11-18-02-06.bpo-33479.rCe4c5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Tkinter documentation has been greatly expanded with new "Architecture" and -"Threading model" sections. diff --git a/Misc/NEWS.d/next/Documentation/2021-08-13-19-08-03.bpo-44903.aJuvQF.rst b/Misc/NEWS.d/next/Documentation/2021-08-13-19-08-03.bpo-44903.aJuvQF.rst deleted file mode 100644 index e357405085ca06..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-13-19-08-03.bpo-44903.aJuvQF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Removed the othergui.rst file, any references to it, and the list of GUI -frameworks in the FAQ. In their place I've added links to the Python Wiki -`page on GUI frameworks `. diff --git a/Misc/NEWS.d/next/Documentation/2021-08-13-20-17-59.bpo-16580.MZ_iK9.rst b/Misc/NEWS.d/next/Documentation/2021-08-13-20-17-59.bpo-16580.MZ_iK9.rst deleted file mode 100644 index edeca6f66e5394..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-13-20-17-59.bpo-16580.MZ_iK9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added code equivalents for the :meth:`int.to_bytes` and :meth:`int.from_bytes` -methods, as well as tests ensuring that these code equivalents are valid. diff --git a/Misc/NEWS.d/next/Documentation/2021-08-19-15-53-08.bpo-44957.imqrh3.rst b/Misc/NEWS.d/next/Documentation/2021-08-19-15-53-08.bpo-44957.imqrh3.rst deleted file mode 100644 index 20a2aecc94ee19..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-19-15-53-08.bpo-44957.imqrh3.rst +++ /dev/null @@ -1,3 +0,0 @@ -Promote PEP 604 union syntax by using it where possible. Also, mention ``X | -Y`` more prominently in section about ``Union`` and mention ``X | None`` at -all in section about ``Optional``. diff --git a/Misc/NEWS.d/next/Documentation/2021-09-08-17-20-19.bpo-45024.dkNPNi.rst b/Misc/NEWS.d/next/Documentation/2021-09-08-17-20-19.bpo-45024.dkNPNi.rst deleted file mode 100644 index e73d52b8cc514e..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-09-08-17-20-19.bpo-45024.dkNPNi.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`collections.abc` documentation has been expanded to explicitly cover -how instance and subclass checks work, with additional doctest examples and -an exhaustive list of ABCs which test membership purely by presence of the -right :term:`special method`\s. Patch by Raymond Hettinger. diff --git a/Misc/NEWS.d/next/Documentation/2021-09-18-13-45-19.bpo-45216.o56nyt.rst b/Misc/NEWS.d/next/Documentation/2021-09-18-13-45-19.bpo-45216.o56nyt.rst deleted file mode 100644 index d10b18ecdb8fd9..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-09-18-13-45-19.bpo-45216.o56nyt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove extra documentation listing methods in ``difflib``. It was rendering -twice in pydoc and was outdated in some places. diff --git a/Misc/NEWS.d/next/Documentation/2021-10-13-00-42-54.bpo-20692.K5rGtP.rst b/Misc/NEWS.d/next/Documentation/2021-10-13-00-42-54.bpo-20692.K5rGtP.rst new file mode 100644 index 00000000000000..44ae468d1bccf3 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-10-13-00-42-54.bpo-20692.K5rGtP.rst @@ -0,0 +1,2 @@ +Add Programming FAQ entry explaining that int literal attribute access +requires either a space after or parentheses around the literal. diff --git a/Misc/NEWS.d/next/Documentation/2021-10-18-20-12-18.bpo-45516.EJh4K8.rst b/Misc/NEWS.d/next/Documentation/2021-10-18-20-12-18.bpo-45516.EJh4K8.rst new file mode 100644 index 00000000000000..98f5d3432db05c --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-10-18-20-12-18.bpo-45516.EJh4K8.rst @@ -0,0 +1,2 @@ +Add protocol description to the :class:`importlib.abc.Traversable` +documentation. diff --git a/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst b/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst new file mode 100644 index 00000000000000..fb817757a14870 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst @@ -0,0 +1 @@ +Add note about :pep:`585` in :mod:`collections.abc`. diff --git a/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst b/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst new file mode 100644 index 00000000000000..1721aa2c2dfc4e --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst @@ -0,0 +1,4 @@ +Mention in the documentation of :ref:`Built-in Exceptions +` that inheriting from multiple exception types in a +single subclass is not recommended due to possible memory layout +incompatibility. diff --git a/Misc/NEWS.d/next/IDLE/2021-05-05-09-45-24.bpo-44026.m2Z0zR.rst b/Misc/NEWS.d/next/IDLE/2021-05-05-09-45-24.bpo-44026.m2Z0zR.rst deleted file mode 100644 index bc4b680983a59a..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-05-05-09-45-24.bpo-44026.m2Z0zR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Include interpreter's typo fix suggestions in message line for -NameErrors and AttributeErrors. Patch by E. Paine. diff --git a/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst b/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst deleted file mode 100644 index becd331f6d789c..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst +++ /dev/null @@ -1,5 +0,0 @@ -Highlight the new :ref:`match ` statement's -:ref:`soft keywords `: :keyword:`match`, -:keyword:`case `, and :keyword:`_ `. -However, this highlighting is not perfect and will be incorrect in some -rare cases, including some ``_``-s in ``case`` patterns. diff --git a/Misc/NEWS.d/next/IDLE/2021-05-27-13-39-43.bpo-41611.liNQqj.rst b/Misc/NEWS.d/next/IDLE/2021-05-27-13-39-43.bpo-41611.liNQqj.rst deleted file mode 100644 index 27d778bbe41009..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-05-27-13-39-43.bpo-41611.liNQqj.rst +++ /dev/null @@ -1 +0,0 @@ -Fix IDLE sometimes freezing upon tab-completion on macOS. diff --git a/Misc/NEWS.d/next/IDLE/2021-05-27-18-22-46.bpo-41611.jOKpfc.rst b/Misc/NEWS.d/next/IDLE/2021-05-27-18-22-46.bpo-41611.jOKpfc.rst deleted file mode 100644 index a80c9f7c5a73b6..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-05-27-18-22-46.bpo-41611.jOKpfc.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid uncaught exceptions in ``AutoCompleteWindow.winconfig_event()``. diff --git a/Misc/NEWS.d/next/IDLE/2021-06-08-03-04-51.bpo-40468.tUCGUb.rst b/Misc/NEWS.d/next/IDLE/2021-06-08-03-04-51.bpo-40468.tUCGUb.rst deleted file mode 100644 index 526036ccf841ee..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-06-08-03-04-51.bpo-40468.tUCGUb.rst +++ /dev/null @@ -1,4 +0,0 @@ -Split the settings dialog General tab into Windows and Shell/ED tabs. -Move help sources, which extend the Help menu, to the Extensions tab. -Make space for new options and shorten the dialog. -The latter makes the dialog better fit small screens. diff --git a/Misc/NEWS.d/next/IDLE/2021-06-10-00-50-02.bpo-33962.ikAUNg.rst b/Misc/NEWS.d/next/IDLE/2021-06-10-00-50-02.bpo-33962.ikAUNg.rst deleted file mode 100644 index b15fa8f184792a..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-06-10-00-50-02.bpo-33962.ikAUNg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Move the indent space setting from the Font tab to the new Windows tab. -Patch by Mark Roseman and Terry Jan Reedy. diff --git a/Misc/NEWS.d/next/IDLE/2021-06-11-17-43-39.bpo-40128.7vDN3U.rst b/Misc/NEWS.d/next/IDLE/2021-06-11-17-43-39.bpo-40128.7vDN3U.rst deleted file mode 100644 index dafbe2cd5c3a8f..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-06-11-17-43-39.bpo-40128.7vDN3U.rst +++ /dev/null @@ -1,3 +0,0 @@ -Mostly fix completions on macOS when not using tcl/tk 8.6.11 (as with 3.9). -The added update_idletask call should be harmless and possibly helpful -otherwise. diff --git a/Misc/NEWS.d/next/IDLE/2021-09-15-03-20-06.bpo-45193.G61_GV.rst b/Misc/NEWS.d/next/IDLE/2021-09-15-03-20-06.bpo-45193.G61_GV.rst deleted file mode 100644 index 94729640c71eb4..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-09-15-03-20-06.bpo-45193.G61_GV.rst +++ /dev/null @@ -1 +0,0 @@ -Make completion boxes appear on Ubuntu again. diff --git a/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst b/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst new file mode 100644 index 00000000000000..3868f8d136a046 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst @@ -0,0 +1 @@ +Add context keywords 'case' and 'match' to completions list. diff --git a/Misc/NEWS.d/next/Library/2017-09-20-14-43-03.bpo-29298._78CSN.rst b/Misc/NEWS.d/next/Library/2017-09-20-14-43-03.bpo-29298._78CSN.rst deleted file mode 100644 index e84c6de02cd27e..00000000000000 --- a/Misc/NEWS.d/next/Library/2017-09-20-14-43-03.bpo-29298._78CSN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``TypeError`` when required subparsers without ``dest`` do not receive -arguments. Patch by Anthony Sottile. diff --git a/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst b/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst deleted file mode 100644 index be68b3ea7c4a2a..00000000000000 --- a/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst +++ /dev/null @@ -1 +0,0 @@ -lib2to3 now recognizes async generators everywhere. diff --git a/Misc/NEWS.d/next/Library/2019-02-11-19-06-10.bpo-35970.ZRvh51.rst b/Misc/NEWS.d/next/Library/2019-02-11-19-06-10.bpo-35970.ZRvh51.rst new file mode 100644 index 00000000000000..bdae153a500f49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-02-11-19-06-10.bpo-35970.ZRvh51.rst @@ -0,0 +1,2 @@ +Add help flag to the base64 module's command line interface. Patch contributed +by Robert Kuska. diff --git a/Misc/NEWS.d/next/Library/2019-02-26-09-31-59.bpo-26228.wyrHKc.rst b/Misc/NEWS.d/next/Library/2019-02-26-09-31-59.bpo-26228.wyrHKc.rst deleted file mode 100644 index c6ca84ae3b6399..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-02-26-09-31-59.bpo-26228.wyrHKc.rst +++ /dev/null @@ -1 +0,0 @@ -pty.spawn no longer hangs on FreeBSD, macOS, and Solaris. diff --git a/Misc/NEWS.d/next/Library/2019-05-08-15-14-32.bpo-16379.rN5JVe.rst b/Misc/NEWS.d/next/Library/2019-05-08-15-14-32.bpo-16379.rN5JVe.rst deleted file mode 100644 index 874a9cf77d8c01..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-05-08-15-14-32.bpo-16379.rN5JVe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add SQLite error code and name to :mod:`sqlite3` exceptions. -Patch by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2019-06-03-23-53-25.bpo-27513.qITN7d.rst b/Misc/NEWS.d/next/Library/2019-06-03-23-53-25.bpo-27513.qITN7d.rst deleted file mode 100644 index 90d49bb2a993f1..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-06-03-23-53-25.bpo-27513.qITN7d.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`email.utils.getaddresses` now accepts -:class:`email.header.Header` objects along with string values. -Patch by Zackery Spytz. diff --git a/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst b/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst deleted file mode 100644 index 698b0e8a61c0d8..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst +++ /dev/null @@ -1 +0,0 @@ -Pass multiprocessing BaseProxy argument ``manager_owned`` through AutoProxy. diff --git a/Misc/NEWS.d/next/Library/2019-10-08-14-08-59.bpo-38415.N1bUw6.rst b/Misc/NEWS.d/next/Library/2019-10-08-14-08-59.bpo-38415.N1bUw6.rst deleted file mode 100644 index f99bf0d19b1f8e..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-10-08-14-08-59.bpo-38415.N1bUw6.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added missing behavior to :func:`contextlib.asynccontextmanager` to match -:func:`contextlib.contextmanager` so decorated functions can themselves be -decorators. diff --git a/Misc/NEWS.d/next/Library/2019-11-12-18-59-33.bpo-38741.W7IYkq.rst b/Misc/NEWS.d/next/Library/2019-11-12-18-59-33.bpo-38741.W7IYkq.rst deleted file mode 100644 index 39d84ccea55b8c..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-11-12-18-59-33.bpo-38741.W7IYkq.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`configparser`: using ']' inside a section header will no longer cut the section name short at the ']' \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-01-16-13-54-28.bpo-39359.hzTu0h.rst b/Misc/NEWS.d/next/Library/2020-01-16-13-54-28.bpo-39359.hzTu0h.rst deleted file mode 100644 index ed4eb0c873d86a..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-01-16-13-54-28.bpo-39359.hzTu0h.rst +++ /dev/null @@ -1 +0,0 @@ -Add one missing check that the password is a bytes object for an encrypted zipfile. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-01-16-23-41-16.bpo-38840.VzzYZz.rst b/Misc/NEWS.d/next/Library/2020-01-16-23-41-16.bpo-38840.VzzYZz.rst deleted file mode 100644 index 727f62b52a710b..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-01-16-23-41-16.bpo-38840.VzzYZz.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``test___all__`` on platforms lacking a shared memory implementation. diff --git a/Misc/NEWS.d/next/Library/2020-01-25-12-58-20.bpo-37022.FUZI25.rst b/Misc/NEWS.d/next/Library/2020-01-25-12-58-20.bpo-37022.FUZI25.rst deleted file mode 100644 index 7b923b3aa6e444..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-01-25-12-58-20.bpo-37022.FUZI25.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`pdb` now displays exceptions from ``repr()`` with its ``p`` and ``pp`` commands. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-02-03-21-18-31.bpo-39549.l4a8uH.rst b/Misc/NEWS.d/next/Library/2020-02-03-21-18-31.bpo-39549.l4a8uH.rst deleted file mode 100644 index 91d63a96763cee..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-02-03-21-18-31.bpo-39549.l4a8uH.rst +++ /dev/null @@ -1,4 +0,0 @@ -Whereas the code for reprlib.Repr had previously used a hardcoded string -value of '...', this PR updates it to use of a “fillvalue” attribute, whose -value defaults to '...' and can be reset in either individual reprlib.Repr -instances or in subclasses thereof. diff --git a/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst b/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst deleted file mode 100644 index d420b5dce1e3b5..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a Y2k38 bug in the compileall module where it would fail to compile -files with a modification time after the year 2038. diff --git a/Misc/NEWS.d/next/Library/2020-05-21-01-42-32.bpo-40563.fDn5bP.rst b/Misc/NEWS.d/next/Library/2020-05-21-01-42-32.bpo-40563.fDn5bP.rst deleted file mode 100644 index f20664637669af..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-21-01-42-32.bpo-40563.fDn5bP.rst +++ /dev/null @@ -1 +0,0 @@ -Support pathlike objects on dbm/shelve. Patch by Hakan Çelik and Henry-Joseph Audéoud. diff --git a/Misc/NEWS.d/next/Library/2020-05-25-23-58-29.bpo-5846.O9BIfm.rst b/Misc/NEWS.d/next/Library/2020-05-25-23-58-29.bpo-5846.O9BIfm.rst deleted file mode 100644 index 556c54d0d1718b..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-25-23-58-29.bpo-5846.O9BIfm.rst +++ /dev/null @@ -1,14 +0,0 @@ -Deprecated the following :mod:`unittest` functions, scheduled for removal in -Python 3.13: - -* :func:`~unittest.findTestCases` -* :func:`~unittest.makeSuite` -* :func:`~unittest.getTestCaseNames` - -Use :class:`~unittest.TestLoader` methods instead: - -* :meth:`unittest.TestLoader.loadTestsFromModule` -* :meth:`unittest.TestLoader.loadTestsFromTestCase` -* :meth:`unittest.TestLoader.getTestCaseNames` - -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2020-07-01-17-42-41.bpo-41137.AnqbP-.rst b/Misc/NEWS.d/next/Library/2020-07-01-17-42-41.bpo-41137.AnqbP-.rst deleted file mode 100644 index f91b47dd724619..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-01-17-42-41.bpo-41137.AnqbP-.rst +++ /dev/null @@ -1 +0,0 @@ -Use utf-8 encoding while reading .pdbrc files. Patch by Srinivas Reddy Thatiparthy \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-07-13-23-46-59.bpo-32695.tTqqXe.rst b/Misc/NEWS.d/next/Library/2020-07-13-23-46-59.bpo-32695.tTqqXe.rst deleted file mode 100644 index c71316ed656a2b..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-13-23-46-59.bpo-32695.tTqqXe.rst +++ /dev/null @@ -1,2 +0,0 @@ -The *compresslevel* and *preset* keyword arguments of :func:`tarfile.open` -are now both documented and tested. diff --git a/Misc/NEWS.d/next/Library/2020-07-26-18-17-30.bpo-41402.YRkVkp.rst b/Misc/NEWS.d/next/Library/2020-07-26-18-17-30.bpo-41402.YRkVkp.rst deleted file mode 100644 index 45585a469e7247..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-26-18-17-30.bpo-41402.YRkVkp.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :meth:`email.message.EmailMessage.set_content` when called with binary data and ``7bit`` content transfer encoding. diff --git a/Misc/NEWS.d/next/Library/2020-07-27-19-21-05.bpo-41374.cd-kFL.rst b/Misc/NEWS.d/next/Library/2020-07-27-19-21-05.bpo-41374.cd-kFL.rst new file mode 100644 index 00000000000000..a5b2e042a3faec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-27-19-21-05.bpo-41374.cd-kFL.rst @@ -0,0 +1,2 @@ +Ensure that ``socket.TCP_*`` constants are exposed on Cygwin 3.1.6 and +greater. diff --git a/Misc/NEWS.d/next/Library/2020-07-30-14-37-15.bpo-20684.qV35GU.rst b/Misc/NEWS.d/next/Library/2020-07-30-14-37-15.bpo-20684.qV35GU.rst deleted file mode 100644 index 56bc1e4cb4ef06..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-30-14-37-15.bpo-20684.qV35GU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove unused ``_signature_get_bound_param`` function from :mod:`inspect` - -by Anthony Sottile. diff --git a/Misc/NEWS.d/next/Library/2020-09-10-07-23-24.bpo-41730.DyKFi9.rst b/Misc/NEWS.d/next/Library/2020-09-10-07-23-24.bpo-41730.DyKFi9.rst deleted file mode 100644 index 63d8353a7aab2e..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-09-10-07-23-24.bpo-41730.DyKFi9.rst +++ /dev/null @@ -1 +0,0 @@ -``DeprecationWarning`` is now raised when importing :mod:`tkinter.tix`, which has been deprecated in documentation since Python 3.6. diff --git a/Misc/NEWS.d/next/Library/2020-10-01-21-46-34.bpo-40956._tvsZ7.rst b/Misc/NEWS.d/next/Library/2020-10-01-21-46-34.bpo-40956._tvsZ7.rst deleted file mode 100644 index adec299d2316a8..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-01-21-46-34.bpo-40956._tvsZ7.rst +++ /dev/null @@ -1 +0,0 @@ -Use Argument Clinic in :mod:`sqlite3`. Patches by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2020-10-11-20-23-48.bpo-37449.f-t3V6.rst b/Misc/NEWS.d/next/Library/2020-10-11-20-23-48.bpo-37449.f-t3V6.rst deleted file mode 100644 index 2202ae0a9ac969..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-11-20-23-48.bpo-37449.f-t3V6.rst +++ /dev/null @@ -1 +0,0 @@ -``ensurepip`` now uses ``importlib.resources.files()`` traversable APIs diff --git a/Misc/NEWS.d/next/Library/2020-10-18-09-42-53.bpo-40497.CRz2sG.rst b/Misc/NEWS.d/next/Library/2020-10-18-09-42-53.bpo-40497.CRz2sG.rst deleted file mode 100644 index 067c48626c3083..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-18-09-42-53.bpo-40497.CRz2sG.rst +++ /dev/null @@ -1,4 +0,0 @@ -:meth:`subprocess.check_output` now raises :exc:`ValueError` when the -invalid keyword argument *check* is passed by user code. Previously -such use would fail later with a :exc:`TypeError`. -Patch by Rémi Lapeyre. diff --git a/Misc/NEWS.d/next/Library/2020-12-08-01-08-58.bpo-41818.zO8vV7.rst b/Misc/NEWS.d/next/Library/2020-12-08-01-08-58.bpo-41818.zO8vV7.rst deleted file mode 100644 index e8d6063d4b5e79..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-12-08-01-08-58.bpo-41818.zO8vV7.rst +++ /dev/null @@ -1 +0,0 @@ -Soumendra Ganguly: add termios.tcgetwinsize(), termios.tcsetwinsize(). \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-01-13-00-02-44.bpo-42862.Z6ACLN.rst b/Misc/NEWS.d/next/Library/2021-01-13-00-02-44.bpo-42862.Z6ACLN.rst deleted file mode 100644 index beda25fd335b08..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-01-13-00-02-44.bpo-42862.Z6ACLN.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`sqlite3` now utilizes :meth:`functools.lru_cache` to implement the -connection statement cache. As a small optimisation, the default -statement cache size has been increased from 100 to 128. -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-01-16-18-36-00.bpo-33809.BiMK6V.rst b/Misc/NEWS.d/next/Library/2021-01-16-18-36-00.bpo-33809.BiMK6V.rst deleted file mode 100644 index a8a550dc0d4183..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-01-16-18-36-00.bpo-33809.BiMK6V.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add the :meth:`traceback.TracebackException.print` method which prints -the formatted exception information. diff --git a/Misc/NEWS.d/next/Library/2021-01-25-21-24-55.bpo-43024.vAUrIi.rst b/Misc/NEWS.d/next/Library/2021-01-25-21-24-55.bpo-43024.vAUrIi.rst deleted file mode 100644 index 56596ce3c2a978..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-01-25-21-24-55.bpo-43024.vAUrIi.rst +++ /dev/null @@ -1 +0,0 @@ -Improve the help signature of :func:`traceback.print_exception`, :func:`traceback.format_exception` and :func:`traceback.format_exception_only`. diff --git a/Misc/NEWS.d/next/Library/2021-01-31-18-24-54.bpo-43086.2_P-SH.rst b/Misc/NEWS.d/next/Library/2021-01-31-18-24-54.bpo-43086.2_P-SH.rst deleted file mode 100644 index f49e7a84cc5375..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-01-31-18-24-54.bpo-43086.2_P-SH.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added a new optional :code:`strict_mode` parameter to *binascii.a2b_base64*. -When :code:`scrict_mode` is set to :code:`True`, the *a2b_base64* function will accept only valid base64 content. -More details about what "valid base64 content" is, can be found in the function's documentation. diff --git a/Misc/NEWS.d/next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst b/Misc/NEWS.d/next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst deleted file mode 100644 index 97c8d2d79aa404..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add definition of ``errno.EQFULL`` for platforms that define this constant -(such as macOS). diff --git a/Misc/NEWS.d/next/Library/2021-02-04-23-16-03.bpo-30077.v6TqAi.rst b/Misc/NEWS.d/next/Library/2021-02-04-23-16-03.bpo-30077.v6TqAi.rst deleted file mode 100644 index 4af17eed8f7336..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-02-04-23-16-03.bpo-30077.v6TqAi.rst +++ /dev/null @@ -1 +0,0 @@ -Added support for Apple's aifc/sowt pseudo-compression \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-02-15-21-17-46.bpo-43232.awc4yZ.rst b/Misc/NEWS.d/next/Library/2021-02-15-21-17-46.bpo-43232.awc4yZ.rst deleted file mode 100644 index a527a7ba95657f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-02-15-21-17-46.bpo-43232.awc4yZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prohibit previously deprecated potentially disruptive operations on -:class:`asyncio.trsock.TransportSocket`. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2021-02-15-22-14-31.bpo-43234.F-vKAT.rst b/Misc/NEWS.d/next/Library/2021-02-15-22-14-31.bpo-43234.F-vKAT.rst deleted file mode 100644 index 7f195cc752fc13..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-02-15-22-14-31.bpo-43234.F-vKAT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Prohibit passing non-:class:`concurrent.futures.ThreadPoolExecutor` -executors to :meth:`loop.set_default_executor` following a deprecation in -Python 3.8. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2021-02-25-08-32-06.bpo-43318.bZJw6V.rst b/Misc/NEWS.d/next/Library/2021-02-25-08-32-06.bpo-43318.bZJw6V.rst deleted file mode 100644 index c2c9c8776fd86d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-02-25-08-32-06.bpo-43318.bZJw6V.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where :mod:`pdb` does not always echo cleared breakpoints. diff --git a/Misc/NEWS.d/next/Library/2021-03-03-13-32-37.bpo-43392.QQumou.rst b/Misc/NEWS.d/next/Library/2021-03-03-13-32-37.bpo-43392.QQumou.rst deleted file mode 100644 index 175836b89170aa..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-03-03-13-32-37.bpo-43392.QQumou.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`importlib._bootstrap._find_and_load` now implements a two-step -check to avoid locking when modules have been already imported and are -ready. This improves performance of repeated calls to -:func:`importlib.import_module` and :func:`importlib.__import__`. diff --git a/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst b/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst deleted file mode 100644 index e6fc88f45eea5e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst +++ /dev/null @@ -1,5 +0,0 @@ -:func:`zlib.compress` now accepts a wbits parameter which allows users to -compress data as a raw deflate block without zlib headers and trailers in -one go. Previously this required instantiating a ``zlib.compressobj``. It -also provides a faster alternative to ``gzip.compress`` when wbits=31 is -used. diff --git a/Misc/NEWS.d/next/Library/2021-03-29-00-23-30.bpo-43650.v01tic.rst b/Misc/NEWS.d/next/Library/2021-03-29-00-23-30.bpo-43650.v01tic.rst deleted file mode 100644 index a2ea4a4800a738..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-03-29-00-23-30.bpo-43650.v01tic.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :exc:`MemoryError` in :func:`shutil.unpack_archive` which fails inside -:func:`shutil._unpack_zipfile` on large files. Patch by Igor Bolshakov. diff --git a/Misc/NEWS.d/next/Library/2021-03-30-08-39-08.bpo-43666.m72tlH.rst b/Misc/NEWS.d/next/Library/2021-03-30-08-39-08.bpo-43666.m72tlH.rst deleted file mode 100644 index 6a3432191d61ba..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-03-30-08-39-08.bpo-43666.m72tlH.rst +++ /dev/null @@ -1,6 +0,0 @@ -AIX: `Lib/_aix_support.get_platform()` may fail in an AIX WPAR. -The fileset bos.rte appears to have a builddate in both LPAR and WPAR -so this fileset is queried rather than bos.mp64. -To prevent a similiar situation (no builddate in ODM) a value (9988) -sufficient for completing a build is provided. -Patch by M Felt. diff --git a/Misc/NEWS.d/next/Library/2021-04-15-12-02-17.bpo-43853.XXCVAp.rst b/Misc/NEWS.d/next/Library/2021-04-15-12-02-17.bpo-43853.XXCVAp.rst deleted file mode 100644 index a0164c4665a056..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-04-15-12-02-17.bpo-43853.XXCVAp.rst +++ /dev/null @@ -1,7 +0,0 @@ -Improved string handling for :mod:`sqlite3` user-defined functions and -aggregates: - -* It is now possible to pass strings with embedded null characters to UDFs -* Conversion failures now correctly raise :exc:`MemoryError` - -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-04-29-00-48-00.bpo-28528.JLAVWj.rst b/Misc/NEWS.d/next/Library/2021-04-29-00-48-00.bpo-28528.JLAVWj.rst deleted file mode 100644 index 97731ad882e032..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-04-29-00-48-00.bpo-28528.JLAVWj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in :mod:`pdb` where :meth:`~pdb.Pdb.checkline` raises -:exc:`AttributeError` if it is called after :meth:`~pdb.Pdb.reset`. diff --git a/Misc/NEWS.d/next/Library/2021-04-30-16-58-24.bpo-43972.Y2r9lg.rst b/Misc/NEWS.d/next/Library/2021-04-30-16-58-24.bpo-43972.Y2r9lg.rst deleted file mode 100644 index 3d67b885bab105..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-04-30-16-58-24.bpo-43972.Y2r9lg.rst +++ /dev/null @@ -1,3 +0,0 @@ -When :class:`http.server.SimpleHTTPRequestHandler` sends a -``301 (Moved Permanently)`` for a directory path not ending with `/`, add a -``Content-Length: 0`` header. This improves the behavior for certain clients. diff --git a/Misc/NEWS.d/next/Library/2021-05-01-15-43-37.bpo-44002.KLT_wd.rst b/Misc/NEWS.d/next/Library/2021-05-01-15-43-37.bpo-44002.KLT_wd.rst deleted file mode 100644 index 9d662d9827a91d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-01-15-43-37.bpo-44002.KLT_wd.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`urllib.parse` now uses :func:`functool.lru_cache` for its internal URL -splitting and quoting caches instead of rolling its own like its the '90s. - -The undocumented internal :mod:`urllib.parse` ``Quoted`` class API is now -deprecated, for removal in 3.14. diff --git a/Misc/NEWS.d/next/Library/2021-05-02-13-54-25.bpo-38352.N9MlhV.rst b/Misc/NEWS.d/next/Library/2021-05-02-13-54-25.bpo-38352.N9MlhV.rst deleted file mode 100644 index bf8fe758f35702..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-02-13-54-25.bpo-38352.N9MlhV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``IO``, ``BinaryIO``, ``TextIO``, ``Match``, and ``Pattern`` to -``typing.__all__``. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2021-05-03-10-07-43.bpo-44018.VDyW8f.rst b/Misc/NEWS.d/next/Library/2021-05-03-10-07-43.bpo-44018.VDyW8f.rst deleted file mode 100644 index 87c7d83a7f35c5..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-03-10-07-43.bpo-44018.VDyW8f.rst +++ /dev/null @@ -1 +0,0 @@ -random.seed() no longer mutates bytearray inputs. diff --git a/Misc/NEWS.d/next/Library/2021-05-03-19-59-14.bpo-40465.1tB4Y0.rst b/Misc/NEWS.d/next/Library/2021-05-03-19-59-14.bpo-40465.1tB4Y0.rst deleted file mode 100644 index b8b63debdbc195..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-03-19-59-14.bpo-40465.1tB4Y0.rst +++ /dev/null @@ -1 +0,0 @@ -Remove random module features deprecated in Python 3.9. diff --git a/Misc/NEWS.d/next/Library/2021-05-05-11-44-49.bpo-36515.uOSa3q.rst b/Misc/NEWS.d/next/Library/2021-05-05-11-44-49.bpo-36515.uOSa3q.rst deleted file mode 100644 index dd24474c2fde7e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-05-11-44-49.bpo-36515.uOSa3q.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :mod:`hashlib` module no longer does unaligned memory accesses when -compiled for ARM platforms. diff --git a/Misc/NEWS.d/next/Library/2021-05-06-16-01-55.bpo-44059.GF5r6O.rst b/Misc/NEWS.d/next/Library/2021-05-06-16-01-55.bpo-44059.GF5r6O.rst deleted file mode 100644 index f734bdb0bce09f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-06-16-01-55.bpo-44059.GF5r6O.rst +++ /dev/null @@ -1 +0,0 @@ -Register the SerenityOS Browser in the :mod:`webbrowser` module. diff --git a/Misc/NEWS.d/next/Library/2021-05-07-08-39-23.bpo-44061.MvElG6.rst b/Misc/NEWS.d/next/Library/2021-05-07-08-39-23.bpo-44061.MvElG6.rst deleted file mode 100644 index e41f285fae9491..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-07-08-39-23.bpo-44061.MvElG6.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix regression in previous release when calling :func:`pkgutil.iter_modules` -with a list of :class:`pathlib.Path` objects diff --git a/Misc/NEWS.d/next/Library/2021-05-09-03-26-31.bpo-44081.A-Mrto.rst b/Misc/NEWS.d/next/Library/2021-05-09-03-26-31.bpo-44081.A-Mrto.rst deleted file mode 100644 index e4a09e366bd807..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-09-03-26-31.bpo-44081.A-Mrto.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`ast.unparse` now doesn't use redundant spaces to separate ``lambda`` -and the ``:`` if there are no parameters. diff --git a/Misc/NEWS.d/next/Library/2021-05-09-22-52-34.bpo-44089.IoANsN.rst b/Misc/NEWS.d/next/Library/2021-05-09-22-52-34.bpo-44089.IoANsN.rst deleted file mode 100644 index b9bd963582fdc2..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-09-22-52-34.bpo-44089.IoANsN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow subclassing ``csv.Error`` in 3.10 (it was allowed in 3.9 and earlier but -was disallowed in early versions of 3.10). diff --git a/Misc/NEWS.d/next/Library/2021-05-10-17-45-00.bpo-44098._MoxuZ.rst b/Misc/NEWS.d/next/Library/2021-05-10-17-45-00.bpo-44098._MoxuZ.rst deleted file mode 100644 index 2aaa8b695af465..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-10-17-45-00.bpo-44098._MoxuZ.rst +++ /dev/null @@ -1,5 +0,0 @@ -``typing.ParamSpec`` will no longer be found in the ``__parameters__`` of -most :mod:`typing` generics except in valid use locations specified by -:pep:`612`. This prevents incorrect usage like ``typing.List[P][int]``. This -change means incorrect usage which may have passed silently in 3.10 beta 1 -and earlier will now error. diff --git a/Misc/NEWS.d/next/Library/2021-05-12-16-43-21.bpo-38908.nM2_rO.rst b/Misc/NEWS.d/next/Library/2021-05-12-16-43-21.bpo-38908.nM2_rO.rst deleted file mode 100644 index 18e3dd4066c4af..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-12-16-43-21.bpo-38908.nM2_rO.rst +++ /dev/null @@ -1,5 +0,0 @@ -Subclasses of ``typing.Protocol`` which only have data variables declared -will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`runtime_checkable`. Previously, these checks -passed silently. -Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Library/2021-05-13-19-07-28.bpo-37788.adeFcf.rst b/Misc/NEWS.d/next/Library/2021-05-13-19-07-28.bpo-37788.adeFcf.rst deleted file mode 100644 index 0c33923e992452..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-13-19-07-28.bpo-37788.adeFcf.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a reference leak when a Thread object is never joined. diff --git a/Misc/NEWS.d/next/Library/2021-05-13-19-44-38.bpo-44077.04b2a4.rst b/Misc/NEWS.d/next/Library/2021-05-13-19-44-38.bpo-44077.04b2a4.rst deleted file mode 100644 index 7bb4379f571b68..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-13-19-44-38.bpo-44077.04b2a4.rst +++ /dev/null @@ -1,3 +0,0 @@ -It's now possible to receive the type of service (ToS), a.k.a. differentiated -services (DS), a.k.a. differenciated services code point (DSCP) and excplicit -congestion notification (ECN) IP header fields with ``socket.IP_RECVTOS``. diff --git a/Misc/NEWS.d/next/Library/2021-05-14-16-06-02.bpo-44095.v_pLwY.rst b/Misc/NEWS.d/next/Library/2021-05-14-16-06-02.bpo-44095.v_pLwY.rst deleted file mode 100644 index ee03e933f35d63..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-14-16-06-02.bpo-44095.v_pLwY.rst +++ /dev/null @@ -1,2 +0,0 @@ -:class:`zipfile.Path` now supports :attr:`zipfile.Path.stem`, -:attr:`zipfile.Path.suffixes`, and :attr:`zipfile.Path.suffix` attributes. diff --git a/Misc/NEWS.d/next/Library/2021-05-16-00-00-38.bpo-44145.ko5SJ7.rst b/Misc/NEWS.d/next/Library/2021-05-16-00-00-38.bpo-44145.ko5SJ7.rst deleted file mode 100644 index 40222185d50678..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-16-00-00-38.bpo-44145.ko5SJ7.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`hmac` computations were not releasing the GIL while calling the -OpenSSL ``HMAC_Update`` C API (a new feature in 3.9). This unintentionally -prevented parallel computation as other :mod:`hashlib` algorithms support. diff --git a/Misc/NEWS.d/next/Library/2021-05-16-02-24-23.bpo-44142.t-XU8k.rst b/Misc/NEWS.d/next/Library/2021-05-16-02-24-23.bpo-44142.t-XU8k.rst deleted file mode 100644 index 96fdd7c6566b20..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-16-02-24-23.bpo-44142.t-XU8k.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`ast.unparse` will now drop the redundant parentheses when tuples used -as assignment targets (e.g in for loops). diff --git a/Misc/NEWS.d/next/Library/2021-05-16-11-57-38.bpo-44150.xAhhik.rst b/Misc/NEWS.d/next/Library/2021-05-16-11-57-38.bpo-44150.xAhhik.rst deleted file mode 100644 index f4c2786d13b05e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-16-11-57-38.bpo-44150.xAhhik.rst +++ /dev/null @@ -1 +0,0 @@ -Add optional *weights* argument to statistics.fmean(). diff --git a/Misc/NEWS.d/next/Library/2021-05-16-17-48-24.bpo-33433.MyzO71.rst b/Misc/NEWS.d/next/Library/2021-05-16-17-48-24.bpo-33433.MyzO71.rst deleted file mode 100644 index 703e038fac9856..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-16-17-48-24.bpo-33433.MyzO71.rst +++ /dev/null @@ -1 +0,0 @@ -For IPv4 mapped IPv6 addresses (:rfc:`4291` Section 2.5.5.2), the :mod:`ipaddress.IPv6Address.is_private` check is deferred to the mapped IPv4 address. This solves a bug where public mapped IPv4 addresses were considered private by the IPv6 check. diff --git a/Misc/NEWS.d/next/Library/2021-05-17-07-24-24.bpo-44154.GRI5bf.rst b/Misc/NEWS.d/next/Library/2021-05-17-07-24-24.bpo-44154.GRI5bf.rst deleted file mode 100644 index 3ec326e875eccd..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-17-07-24-24.bpo-44154.GRI5bf.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize :class:`fractions.Fraction` pickling for large components. diff --git a/Misc/NEWS.d/next/Library/2021-05-17-21-05-06.bpo-4928.Ot2yjO.rst b/Misc/NEWS.d/next/Library/2021-05-17-21-05-06.bpo-4928.Ot2yjO.rst deleted file mode 100644 index 359f8015821544..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-17-21-05-06.bpo-4928.Ot2yjO.rst +++ /dev/null @@ -1 +0,0 @@ -Documented existing behavior on POSIX: NamedTemporaryFiles are not deleted when creating process is killed with SIGKILL \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst b/Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst deleted file mode 100644 index dc0cdf33ec5acf..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :mod:`sqlite3` context manager now performs a rollback (thus releasing the -database lock) if commit failed. Patch by Luca Citi and Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-05-21-12-12-35.bpo-43643.GWnmcF.rst b/Misc/NEWS.d/next/Library/2021-05-21-12-12-35.bpo-43643.GWnmcF.rst deleted file mode 100644 index 57157dfe217ee5..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-21-12-12-35.bpo-43643.GWnmcF.rst +++ /dev/null @@ -1 +0,0 @@ -Declare readers.MultiplexedPath.name as a property per the spec. diff --git a/Misc/NEWS.d/next/Library/2021-05-21-21-23-43.bpo-44210.5afQ3K.rst b/Misc/NEWS.d/next/Library/2021-05-21-21-23-43.bpo-44210.5afQ3K.rst deleted file mode 100644 index e5a14a1a265f1d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-21-21-23-43.bpo-44210.5afQ3K.rst +++ /dev/null @@ -1 +0,0 @@ -Make importlib.metadata._meta.PackageMetadata public. diff --git a/Misc/NEWS.d/next/Library/2021-05-25-23-26-38.bpo-43216.xTUyyX.rst b/Misc/NEWS.d/next/Library/2021-05-25-23-26-38.bpo-43216.xTUyyX.rst deleted file mode 100644 index 845ef95d1ad296..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-25-23-26-38.bpo-43216.xTUyyX.rst +++ /dev/null @@ -1,6 +0,0 @@ -Remove the :func:`@asyncio.coroutine ` :term:`decorator` -enabling legacy generator-based coroutines to be compatible with async/await -code; remove :class:`asyncio.coroutines.CoroWrapper` used for wrapping -legacy coroutine objects in the debug mode. The decorator has been deprecated -since Python 3.8 and the removal was initially scheduled for Python 3.10. -Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2021-05-26-13-15-51.bpo-44241.TBqej8.rst b/Misc/NEWS.d/next/Library/2021-05-26-13-15-51.bpo-44241.TBqej8.rst deleted file mode 100644 index c160cf70abb528..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-26-13-15-51.bpo-44241.TBqej8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Incorporate minor tweaks from importlib_metadata 4.1: SimplePath protocol, -support for Metadata 2.2. diff --git a/Misc/NEWS.d/next/Library/2021-05-26-13-34-37.bpo-33693.3okzdo.rst b/Misc/NEWS.d/next/Library/2021-05-26-13-34-37.bpo-33693.3okzdo.rst deleted file mode 100644 index 2a568a4f469f8d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-26-13-34-37.bpo-33693.3okzdo.rst +++ /dev/null @@ -1 +0,0 @@ -Importlib.metadata now prefers f-strings to .format. diff --git a/Misc/NEWS.d/next/Library/2021-05-26-14-50-06.bpo-38693.NkMacJ.rst b/Misc/NEWS.d/next/Library/2021-05-26-14-50-06.bpo-38693.NkMacJ.rst deleted file mode 100644 index 10d014b5da562b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-26-14-50-06.bpo-38693.NkMacJ.rst +++ /dev/null @@ -1 +0,0 @@ -Prefer f-strings to ``.format`` in importlib.resources. diff --git a/Misc/NEWS.d/next/Library/2021-05-26-22-04-40.bpo-44235.qFBYpp.rst b/Misc/NEWS.d/next/Library/2021-05-26-22-04-40.bpo-44235.qFBYpp.rst deleted file mode 100644 index 41af18175d95bb..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-26-22-04-40.bpo-44235.qFBYpp.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated functions in the :mod:`gettext`. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-05-28-09-43-33.bpo-44258.nh5F7R.rst b/Misc/NEWS.d/next/Library/2021-05-28-09-43-33.bpo-44258.nh5F7R.rst deleted file mode 100644 index b9636899700f6e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-28-09-43-33.bpo-44258.nh5F7R.rst +++ /dev/null @@ -1 +0,0 @@ -Support PEP 515 for Fraction's initialization from string. diff --git a/Misc/NEWS.d/next/Library/2021-05-29-01-05-43.bpo-44254.f06xDm.rst b/Misc/NEWS.d/next/Library/2021-05-29-01-05-43.bpo-44254.f06xDm.rst deleted file mode 100644 index 7438d5ce044b87..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-29-01-05-43.bpo-44254.f06xDm.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Mac, give turtledemo button text a color that works on both light -or dark background. Programmers cannot control the latter. diff --git a/Misc/NEWS.d/next/Library/2021-05-30-13-32-09.bpo-44260.ROEbVd.rst b/Misc/NEWS.d/next/Library/2021-05-30-13-32-09.bpo-44260.ROEbVd.rst deleted file mode 100644 index a63af627e6fbf9..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-30-13-32-09.bpo-44260.ROEbVd.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :class:`random.Random` constructor no longer reads system entropy -without need. diff --git a/Misc/NEWS.d/next/Library/2021-05-31-04-51-02.bpo-43858.r7LOu6.rst b/Misc/NEWS.d/next/Library/2021-05-31-04-51-02.bpo-43858.r7LOu6.rst deleted file mode 100644 index d864e1b4e51e3d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-31-04-51-02.bpo-43858.r7LOu6.rst +++ /dev/null @@ -1 +0,0 @@ -Added a function that returns a copy of a dict of logging levels: :func:`logging.getLevelNamesMapping` diff --git a/Misc/NEWS.d/next/Library/2021-05-31-11-28-03.bpo-44246.nhmt-v.rst b/Misc/NEWS.d/next/Library/2021-05-31-11-28-03.bpo-44246.nhmt-v.rst deleted file mode 100644 index 727d9fd0a19d8a..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-31-11-28-03.bpo-44246.nhmt-v.rst +++ /dev/null @@ -1,3 +0,0 @@ -In importlib.metadata.entry_points, de-duplication of distributions no -longer requires loading the full metadata for PathDistribution objects, -improving entry point loading performance by ~10x. diff --git a/Misc/NEWS.d/next/Library/2021-05-31-11-34-56.bpo-44246.yHAkF0.rst b/Misc/NEWS.d/next/Library/2021-05-31-11-34-56.bpo-44246.yHAkF0.rst deleted file mode 100644 index b93f8b02dc4760..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-31-11-34-56.bpo-44246.yHAkF0.rst +++ /dev/null @@ -1,7 +0,0 @@ -In ``importlib.metadata``, restore compatibility in the result from -``Distribution.entry_points`` (``EntryPoints``) to honor expectations in -older implementations and issuing deprecation warnings for these cases: A. ``EntryPoints`` objects are once again mutable, allowing for ``sort()`` -and other list-based mutation operations. Avoid deprecation warnings by -casting to a mutable sequence (e.g. ``list(dist.entry_points).sort()``). B. ``EntryPoints`` results once again allow for access by index. To avoid -deprecation warnings, cast the result to a Sequence first (e.g. -``tuple(dist.entry_points)[0]``). diff --git a/Misc/NEWS.d/next/Library/2021-06-07-10-26-14.bpo-44242.MKeMCQ.rst b/Misc/NEWS.d/next/Library/2021-06-07-10-26-14.bpo-44242.MKeMCQ.rst deleted file mode 100644 index 39740b67365918..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-07-10-26-14.bpo-44242.MKeMCQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove missing flag check from Enum creation and move into a ``verify`` -decorator. diff --git a/Misc/NEWS.d/next/Library/2021-06-08-17-47-38.bpo-44339.9JwMSc.rst b/Misc/NEWS.d/next/Library/2021-06-08-17-47-38.bpo-44339.9JwMSc.rst deleted file mode 100644 index 10499eb02bb3c9..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-08-17-47-38.bpo-44339.9JwMSc.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change ``math.pow(±0.0, -math.inf)`` to return ``inf`` instead of raising -``ValueError``. This brings the special-case handling of ``math.pow`` into -compliance with the IEEE 754 standard. diff --git a/Misc/NEWS.d/next/Library/2021-06-09-08-32-39.bpo-44357.70Futb.rst b/Misc/NEWS.d/next/Library/2021-06-09-08-32-39.bpo-44357.70Futb.rst deleted file mode 100644 index f169a464f9fe7c..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-09-08-32-39.bpo-44357.70Futb.rst +++ /dev/null @@ -1 +0,0 @@ -Added a function that returns cube root of the given number :func:`math.cbrt` \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-06-09-10-08-32.bpo-35800.3hmkWw.rst b/Misc/NEWS.d/next/Library/2021-06-09-10-08-32.bpo-35800.3hmkWw.rst deleted file mode 100644 index d3bf596b75028f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-09-10-08-32.bpo-35800.3hmkWw.rst +++ /dev/null @@ -1,2 +0,0 @@ -:class:`smtpd.MailmanProxy` is now removed as it is unusable without an -external module, ``mailman``. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-06-10-07-26-12.bpo-44351.rvyf2v.rst b/Misc/NEWS.d/next/Library/2021-06-10-07-26-12.bpo-44351.rvyf2v.rst deleted file mode 100644 index d731a549632b5f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-10-07-26-12.bpo-44351.rvyf2v.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restore back :func:`parse_makefile` in :mod:`distutils.sysconfig` because it -behaves differently than the similar implementation in :mod:`sysconfig`. diff --git a/Misc/NEWS.d/next/Library/2021-06-10-08-35-38.bpo-44356.6oDFhO.rst b/Misc/NEWS.d/next/Library/2021-06-10-08-35-38.bpo-44356.6oDFhO.rst deleted file mode 100644 index 954a803fe25c18..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-10-08-35-38.bpo-44356.6oDFhO.rst +++ /dev/null @@ -1 +0,0 @@ -[Enum] Allow multiple data-type mixins if they are all the same. diff --git a/Misc/NEWS.d/next/Library/2021-06-10-15-06-47.bpo-44342.qqkGlj.rst b/Misc/NEWS.d/next/Library/2021-06-10-15-06-47.bpo-44342.qqkGlj.rst deleted file mode 100644 index 6db75e3e9bcf11..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-10-15-06-47.bpo-44342.qqkGlj.rst +++ /dev/null @@ -1 +0,0 @@ -[Enum] Change pickling from by-value to by-name. diff --git a/Misc/NEWS.d/next/Library/2021-06-10-20-07-32.bpo-44362.oVOMfd.rst b/Misc/NEWS.d/next/Library/2021-06-10-20-07-32.bpo-44362.oVOMfd.rst deleted file mode 100644 index 0e6aef3c90e6fc..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-10-20-07-32.bpo-44362.oVOMfd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :mod:`ssl` module's deprecation messages, error reporting, and -documentation for deprecations. diff --git a/Misc/NEWS.d/next/Library/2021-06-10-21-53-46.bpo-34266.k3fxnm.rst b/Misc/NEWS.d/next/Library/2021-06-10-21-53-46.bpo-34266.k3fxnm.rst deleted file mode 100644 index 22ef84e9626ad6..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-10-21-53-46.bpo-34266.k3fxnm.rst +++ /dev/null @@ -1 +0,0 @@ -Handle exceptions from parsing the arg of :mod:`pdb`'s run/restart command. diff --git a/Misc/NEWS.d/next/Library/2021-06-12-10-08-14.bpo-44395.PcW6Sx.rst b/Misc/NEWS.d/next/Library/2021-06-12-10-08-14.bpo-44395.PcW6Sx.rst deleted file mode 100644 index 6172eec0a9bd3d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-12-10-08-14.bpo-44395.PcW6Sx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`~email.message.MIMEPart.as_string` to pass unixfrom properly. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-06-12-21-25-35.bpo-27827.TMWh1i.rst b/Misc/NEWS.d/next/Library/2021-06-12-21-25-35.bpo-27827.TMWh1i.rst deleted file mode 100644 index 1b8cc04533ed85..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-12-21-25-35.bpo-27827.TMWh1i.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`pathlib.PureWindowsPath.is_reserved` now identifies a greater range of -reserved filenames, including those with trailing spaces or colons. diff --git a/Misc/NEWS.d/next/Library/2021-06-12-22-58-20.bpo-44389.WTRnoC.rst b/Misc/NEWS.d/next/Library/2021-06-12-22-58-20.bpo-44389.WTRnoC.rst deleted file mode 100644 index e7e3b874899005..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-12-22-58-20.bpo-44389.WTRnoC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix deprecation of :data:`ssl.OP_NO_TLSv1_3` diff --git a/Misc/NEWS.d/next/Library/2021-06-13-00-16-56.bpo-37880.5bTrkw.rst b/Misc/NEWS.d/next/Library/2021-06-13-00-16-56.bpo-37880.5bTrkw.rst deleted file mode 100644 index 42821572aa67da..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-13-00-16-56.bpo-37880.5bTrkw.rst +++ /dev/null @@ -1,3 +0,0 @@ -argparse actions store_const and append_const each receive a default value -of None when the ``const`` kwarg is not provided. Previously, this raised a -:exc:`TypeError`. diff --git a/Misc/NEWS.d/next/Library/2021-06-14-14-19-11.bpo-38291.ee4cSX.rst b/Misc/NEWS.d/next/Library/2021-06-14-14-19-11.bpo-38291.ee4cSX.rst deleted file mode 100644 index 078d78d1778b1f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-14-14-19-11.bpo-38291.ee4cSX.rst +++ /dev/null @@ -1 +0,0 @@ -Importing typing.io or typing.re now prints a ``DeprecationWarning``. diff --git a/Misc/NEWS.d/next/Library/2021-06-14-23-28-17.bpo-44422.BlWOgv.rst b/Misc/NEWS.d/next/Library/2021-06-14-23-28-17.bpo-44422.BlWOgv.rst deleted file mode 100644 index 09bace01fc7794..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-14-23-28-17.bpo-44422.BlWOgv.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :func:`threading.enumerate` function now uses a reentrant lock to -prevent a hang on reentrant call. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-06-15-13-51-25.bpo-42972.UnyYo1.rst b/Misc/NEWS.d/next/Library/2021-06-15-13-51-25.bpo-42972.UnyYo1.rst deleted file mode 100644 index fbcc12c9f90a20..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-15-13-51-25.bpo-42972.UnyYo1.rst +++ /dev/null @@ -1,2 +0,0 @@ -The _thread.RLock type now fully implement the GC protocol: add a traverse -function and the :const:`Py_TPFLAGS_HAVE_GC` flag. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-06-16-16-52-14.bpo-44434.SQS4Pg.rst b/Misc/NEWS.d/next/Library/2021-06-16-16-52-14.bpo-44434.SQS4Pg.rst deleted file mode 100644 index 37b5b57ce65693..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-16-16-52-14.bpo-44434.SQS4Pg.rst +++ /dev/null @@ -1,4 +0,0 @@ -_thread.start_new_thread() no longer calls PyThread_exit_thread() explicitly -at the thread exit, the call was redundant. On Linux with the glibc, -pthread_exit() aborts the whole process if dlopen() fails to open -libgcc_s.so file (ex: EMFILE error). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-06-17-15-01-51.bpo-44439.1S7QhT.rst b/Misc/NEWS.d/next/Library/2021-06-17-15-01-51.bpo-44439.1S7QhT.rst deleted file mode 100644 index 27396683700a83..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-17-15-01-51.bpo-44439.1S7QhT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix in :meth:`bz2.BZ2File.write` / :meth:`lzma.LZMAFile.write` methods, when -the input data is an object that supports the buffer protocol, the file length -may be wrong. diff --git a/Misc/NEWS.d/next/Library/2021-06-17-22-39-34.bpo-44446.qwdRic.rst b/Misc/NEWS.d/next/Library/2021-06-17-22-39-34.bpo-44446.qwdRic.rst deleted file mode 100644 index 6d9758f42dd04f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-17-22-39-34.bpo-44446.qwdRic.rst +++ /dev/null @@ -1 +0,0 @@ -Take into account that ``lineno`` might be ``None`` in :class:`traceback.FrameSummary`. diff --git a/Misc/NEWS.d/next/Library/2021-06-19-21-52-27.bpo-44464.U2oa-a.rst b/Misc/NEWS.d/next/Library/2021-06-19-21-52-27.bpo-44464.U2oa-a.rst deleted file mode 100644 index 6b1c10783d569e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-19-21-52-27.bpo-44464.U2oa-a.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove exception for flake8 in deprecated importlib.metadata interfaces. -Sync with importlib_metadata 4.6. diff --git a/Misc/NEWS.d/next/Library/2021-06-20-07-14-46.bpo-44458.myqCQ0.rst b/Misc/NEWS.d/next/Library/2021-06-20-07-14-46.bpo-44458.myqCQ0.rst deleted file mode 100644 index f15104b75e31c9..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-20-07-14-46.bpo-44458.myqCQ0.rst +++ /dev/null @@ -1 +0,0 @@ -``BUFFER_BLOCK_SIZE`` is now declared static, to avoid linking collisions when bz2, lmza or zlib are statically linked. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-06-20-14-03-18.bpo-41546.lO1jXU.rst b/Misc/NEWS.d/next/Library/2021-06-20-14-03-18.bpo-41546.lO1jXU.rst deleted file mode 100644 index 050da761570d42..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-20-14-03-18.bpo-41546.lO1jXU.rst +++ /dev/null @@ -1 +0,0 @@ -Make :mod:`pprint` (like the builtin ``print``) not attempt to write to ``stdout`` when it is ``None``. diff --git a/Misc/NEWS.d/next/Library/2021-06-20-19-01-11.bpo-44404.McfrYB.rst b/Misc/NEWS.d/next/Library/2021-06-20-19-01-11.bpo-44404.McfrYB.rst deleted file mode 100644 index ff6ca1bfa7242d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-20-19-01-11.bpo-44404.McfrYB.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`tkinter`'s ``after()`` method now supports callables without the ``__name__`` attribute. diff --git a/Misc/NEWS.d/next/Library/2021-06-21-10-46-58.bpo-44471.2QjXv_.rst b/Misc/NEWS.d/next/Library/2021-06-21-10-46-58.bpo-44471.2QjXv_.rst deleted file mode 100644 index 0675ef3262a090..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-21-10-46-58.bpo-44471.2QjXv_.rst +++ /dev/null @@ -1,5 +0,0 @@ -A :exc:`TypeError` is now raised instead of an :exc:`AttributeError` in -:meth:`contextlib.ExitStack.enter_context` and -:meth:`contextlib.AsyncExitStack.enter_async_context` for objects which do -not support the :term:`context manager` or :term:`asynchronous context -manager` protocols correspondingly. diff --git a/Misc/NEWS.d/next/Library/2021-06-21-12-43-04.bpo-44466.NSm6mv.rst b/Misc/NEWS.d/next/Library/2021-06-21-12-43-04.bpo-44466.NSm6mv.rst deleted file mode 100644 index 69de3edb5a7f95..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-21-12-43-04.bpo-44466.NSm6mv.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :mod:`faulthandler` module now detects if a fatal error occurs during a -garbage collector collection. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-06-22-08-43-04.bpo-44482.U9GznK.rst b/Misc/NEWS.d/next/Library/2021-06-22-08-43-04.bpo-44482.U9GznK.rst deleted file mode 100644 index d05fe908e3eba3..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-22-08-43-04.bpo-44482.U9GznK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix very unlikely resource leak in :mod:`glob` in alternate Python -implementations. diff --git a/Misc/NEWS.d/next/Library/2021-06-22-16-45-48.bpo-43977.bamAGF.rst b/Misc/NEWS.d/next/Library/2021-06-22-16-45-48.bpo-43977.bamAGF.rst deleted file mode 100644 index 5f8cb7b7ea7294..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-22-16-45-48.bpo-43977.bamAGF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Set the proper :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` -flags for subclasses created before a parent has been registered as a -:class:`collections.abc.Mapping` or :class:`collections.abc.Sequence`. diff --git a/Misc/NEWS.d/next/Library/2021-06-23-01-33-01.bpo-44491.tiOlr5.rst b/Misc/NEWS.d/next/Library/2021-06-23-01-33-01.bpo-44491.tiOlr5.rst deleted file mode 100644 index ebe54484187ab8..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-23-01-33-01.bpo-44491.tiOlr5.rst +++ /dev/null @@ -1,3 +0,0 @@ -Allow clearing the :mod:`sqlite3` authorizer callback by passing -:const:`None` to :meth:`~sqlite3.Connection.set_authorizer`. Patch by -Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-06-23-19-02-00.bpo-44468.-klV5-.rst b/Misc/NEWS.d/next/Library/2021-06-23-19-02-00.bpo-44468.-klV5-.rst deleted file mode 100644 index 78251c7d55068d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-23-19-02-00.bpo-44468.-klV5-.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`typing.get_type_hints` now finds annotations in classes and base classes -with unexpected ``__module__``. Previously, it skipped those MRO elements. diff --git a/Misc/NEWS.d/next/Library/2021-06-24-19-16-20.bpo-42892.qvRNhI.rst b/Misc/NEWS.d/next/Library/2021-06-24-19-16-20.bpo-42892.qvRNhI.rst deleted file mode 100644 index 3c70b0534ecabf..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-24-19-16-20.bpo-42892.qvRNhI.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an exception thrown while parsing a malformed multipart email by :class:`email.message.EmailMessage`. diff --git a/Misc/NEWS.d/next/Library/2021-06-26-12-27-14.bpo-44516.BVyX_y.rst b/Misc/NEWS.d/next/Library/2021-06-26-12-27-14.bpo-44516.BVyX_y.rst deleted file mode 100644 index a9822881135ea7..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-26-12-27-14.bpo-44516.BVyX_y.rst +++ /dev/null @@ -1 +0,0 @@ -Update vendored pip to 21.1.3 diff --git a/Misc/NEWS.d/next/Library/2021-06-29-07-27-08.bpo-43625.ZlAxhp.rst b/Misc/NEWS.d/next/Library/2021-06-29-07-27-08.bpo-43625.ZlAxhp.rst deleted file mode 100644 index a21975b948ef9e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-29-07-27-08.bpo-43625.ZlAxhp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in the detection of CSV file headers by -:meth:`csv.Sniffer.has_header` and improve documentation of same. diff --git a/Misc/NEWS.d/next/Library/2021-06-29-21-17-17.bpo-44461.acqRnV.rst b/Misc/NEWS.d/next/Library/2021-06-29-21-17-17.bpo-44461.acqRnV.rst deleted file mode 100644 index 02e25e928b9cff..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-29-21-17-17.bpo-44461.acqRnV.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug with :mod:`pdb`'s handling of import error due to a package which does not have a ``__main__`` module \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-06-30-11-34-35.bpo-44539.nP0Xi4.rst b/Misc/NEWS.d/next/Library/2021-06-30-11-34-35.bpo-44539.nP0Xi4.rst deleted file mode 100644 index f5e831afce8358..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-30-11-34-35.bpo-44539.nP0Xi4.rst +++ /dev/null @@ -1 +0,0 @@ -Added support for recognizing JPEG files without JFIF or Exif markers. diff --git a/Misc/NEWS.d/next/Library/2021-06-30-13-29-49.bpo-34798.t7FCa0.rst b/Misc/NEWS.d/next/Library/2021-06-30-13-29-49.bpo-34798.t7FCa0.rst deleted file mode 100644 index ab9fd8e33799df..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-30-13-29-49.bpo-34798.t7FCa0.rst +++ /dev/null @@ -1 +0,0 @@ -Break up paragraph about :class:`pprint.PrettyPrinter` construction parameters to make it easier to read. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-02-18-17-56.bpo-44554.aBUmJo.rst b/Misc/NEWS.d/next/Library/2021-07-02-18-17-56.bpo-44554.aBUmJo.rst deleted file mode 100644 index 2c225b80839516..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-02-18-17-56.bpo-44554.aBUmJo.rst +++ /dev/null @@ -1 +0,0 @@ -Refactor argument processing in :func:`pdb.main` to simplify detection of errors in input loading and clarify behavior around module or script invocation. diff --git a/Misc/NEWS.d/next/Library/2021-07-04-11-33-34.bpo-41249.sHdwBE.rst b/Misc/NEWS.d/next/Library/2021-07-04-11-33-34.bpo-41249.sHdwBE.rst deleted file mode 100644 index 06dae4a6e93565..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-04-11-33-34.bpo-41249.sHdwBE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes ``TypedDict`` to work with ``typing.get_type_hints()`` and postponed evaluation of -annotations across modules. diff --git a/Misc/NEWS.d/next/Library/2021-07-04-21-16-53.bpo-44558.cm7Slv.rst b/Misc/NEWS.d/next/Library/2021-07-04-21-16-53.bpo-44558.cm7Slv.rst deleted file mode 100644 index 647a70490d1bac..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-04-21-16-53.bpo-44558.cm7Slv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make the implementation consistency of :func:`~operator.indexOf` between -C and Python versions. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-07-05-18-13-25.bpo-44566.o51Bd1.rst b/Misc/NEWS.d/next/Library/2021-07-05-18-13-25.bpo-44566.o51Bd1.rst deleted file mode 100644 index 3b00a1b715feef..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-05-18-13-25.bpo-44566.o51Bd1.rst +++ /dev/null @@ -1 +0,0 @@ -handle StopIteration subclass raised from @contextlib.contextmanager generator \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-08-12-22-54.bpo-44569.KZ02v9.rst b/Misc/NEWS.d/next/Library/2021-07-08-12-22-54.bpo-44569.KZ02v9.rst deleted file mode 100644 index 5f693b290dfb83..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-08-12-22-54.bpo-44569.KZ02v9.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added the :func:`StackSummary.format_frame` function in :mod:`traceback`. -This allows users to customize the way individual lines are formatted in -tracebacks without re-implementing logic to handle recursive tracebacks. diff --git a/Misc/NEWS.d/next/Library/2021-07-09-07-14-37.bpo-41928.Q1jMrr.rst b/Misc/NEWS.d/next/Library/2021-07-09-07-14-37.bpo-41928.Q1jMrr.rst deleted file mode 100644 index e6bd758980b003..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-09-07-14-37.bpo-41928.Q1jMrr.rst +++ /dev/null @@ -1,4 +0,0 @@ -Update :func:`shutil.copyfile` to raise :exc:`FileNotFoundError` instead of -confusing :exc:`IsADirectoryError` when a path ending with a -:const:`os.path.sep` does not exist; :func:`shutil.copy` and -:func:`shutil.copy2` are also affected. diff --git a/Misc/NEWS.d/next/Library/2021-07-10-19-55-13.bpo-42799.ad4tq8.rst b/Misc/NEWS.d/next/Library/2021-07-10-19-55-13.bpo-42799.ad4tq8.rst deleted file mode 100644 index 8a25800611a5a6..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-10-19-55-13.bpo-42799.ad4tq8.rst +++ /dev/null @@ -1,4 +0,0 @@ -In :mod:`fnmatch`, the cache size for compiled regex patterns -(:func:`functools.lru_cache`) was bumped up from 256 to 32768, affecting -functions: :func:`fnmatch.fnmatch`, :func:`fnmatch.fnmatchcase`, -:func:`fnmatch.filter`. diff --git a/Misc/NEWS.d/next/Library/2021-07-13-09-01-33.bpo-44608.R3IcM1.rst b/Misc/NEWS.d/next/Library/2021-07-13-09-01-33.bpo-44608.R3IcM1.rst deleted file mode 100644 index e0cf948f3cba68..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-13-09-01-33.bpo-44608.R3IcM1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence -or set, but not list or tuple. diff --git a/Misc/NEWS.d/next/Library/2021-07-15-16-51-32.bpo-44648.2o49TB.rst b/Misc/NEWS.d/next/Library/2021-07-15-16-51-32.bpo-44648.2o49TB.rst deleted file mode 100644 index f7171c3c84c5ea..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-15-16-51-32.bpo-44648.2o49TB.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed wrong error being thrown by :func:`inspect.getsource` when examining a -class in the interactive session. Instead of :exc:`TypeError`, it should be -:exc:`OSError` with appropriate error message. diff --git a/Misc/NEWS.d/next/Library/2021-07-16-08-57-27.bpo-44638.EwYKne.rst b/Misc/NEWS.d/next/Library/2021-07-16-08-57-27.bpo-44638.EwYKne.rst deleted file mode 100644 index eeaa91c16c4cfc..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-16-08-57-27.bpo-44638.EwYKne.rst +++ /dev/null @@ -1 +0,0 @@ -Add a reference to the zipp project and hint as to how to use it. diff --git a/Misc/NEWS.d/next/Library/2021-07-16-13-40-31.bpo-40897.aveAre.rst b/Misc/NEWS.d/next/Library/2021-07-16-13-40-31.bpo-40897.aveAre.rst deleted file mode 100644 index 04f1465f0ac67f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-16-13-40-31.bpo-40897.aveAre.rst +++ /dev/null @@ -1,2 +0,0 @@ -Give priority to using the current class constructor in -:func:`inspect.signature`. Patch by Weipeng Hong. diff --git a/Misc/NEWS.d/next/Library/2021-07-19-14-04-42.bpo-44524.Nbf2JC.rst b/Misc/NEWS.d/next/Library/2021-07-19-14-04-42.bpo-44524.Nbf2JC.rst deleted file mode 100644 index 0acdc7dff029f0..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-19-14-04-42.bpo-44524.Nbf2JC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add missing ``__name__`` and ``__qualname__`` attributes to ``typing`` module -classes. Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Library/2021-07-19-18-45-00.bpo-44678.YMEAu0.rst b/Misc/NEWS.d/next/Library/2021-07-19-18-45-00.bpo-44678.YMEAu0.rst deleted file mode 100644 index 991b1579b2e2b6..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-19-18-45-00.bpo-44678.YMEAu0.rst +++ /dev/null @@ -1 +0,0 @@ -Added a separate error message for discontinuous padding in *binascii.a2b_base64* strict mode. diff --git a/Misc/NEWS.d/next/Library/2021-07-19-22-43-15.bpo-44353.HF81_Q.rst b/Misc/NEWS.d/next/Library/2021-07-19-22-43-15.bpo-44353.HF81_Q.rst deleted file mode 100644 index 8a1e0f77b31772..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-19-22-43-15.bpo-44353.HF81_Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Refactor ``typing.NewType`` from function into callable class. Patch -provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Library/2021-07-20-00-11-47.bpo-44682.3m2qVV.rst b/Misc/NEWS.d/next/Library/2021-07-20-00-11-47.bpo-44682.3m2qVV.rst deleted file mode 100644 index 308053a62c389f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-20-00-11-47.bpo-44682.3m2qVV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change the :mod:`pdb` *commands* directive to disallow setting commands -for an invalid breakpoint and to display an appropriate error. diff --git a/Misc/NEWS.d/next/Library/2021-07-20-18-34-16.bpo-44353.ATuYq4.rst b/Misc/NEWS.d/next/Library/2021-07-20-18-34-16.bpo-44353.ATuYq4.rst deleted file mode 100644 index 7332770ac487e7..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-20-18-34-16.bpo-44353.ATuYq4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make ``NewType.__call__`` faster by implementing it in C. -Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Library/2021-07-20-19-35-49.bpo-44686.ucCGhu.rst b/Misc/NEWS.d/next/Library/2021-07-20-19-35-49.bpo-44686.ucCGhu.rst deleted file mode 100644 index d9c78020e4a9ba..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-20-19-35-49.bpo-44686.ucCGhu.rst +++ /dev/null @@ -1 +0,0 @@ -Replace ``unittest.mock._importer`` with ``pkgutil.resolve_name``. diff --git a/Misc/NEWS.d/next/Library/2021-07-20-21-51-35.bpo-42854.ThuDMI.rst b/Misc/NEWS.d/next/Library/2021-07-20-21-51-35.bpo-42854.ThuDMI.rst deleted file mode 100644 index 33cbb63a5e14b8..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-20-21-51-35.bpo-42854.ThuDMI.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug in the :mod:`_ssl` module that was throwing :exc:`OverflowError` -when using :meth:`_ssl._SSLSocket.write` and :meth:`_ssl._SSLSocket.read` -for a big value of the ``len`` parameter. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2021-07-20-22-03-24.bpo-44690.tV7Zjg.rst b/Misc/NEWS.d/next/Library/2021-07-20-22-03-24.bpo-44690.tV7Zjg.rst deleted file mode 100644 index 1d1184805471d4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-20-22-03-24.bpo-44690.tV7Zjg.rst +++ /dev/null @@ -1 +0,0 @@ -Adopt *binacii.a2b_base64*'s strict mode in *base64.b64decode*. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-20-23-28-26.bpo-44688.buFgz3.rst b/Misc/NEWS.d/next/Library/2021-07-20-23-28-26.bpo-44688.buFgz3.rst deleted file mode 100644 index 15f6a521f2d4f9..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-20-23-28-26.bpo-44688.buFgz3.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`sqlite3.Connection.create_collation` now accepts non-ASCII collation -names. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-07-21-10-43-22.bpo-44666.CEThkv.rst b/Misc/NEWS.d/next/Library/2021-07-21-10-43-22.bpo-44666.CEThkv.rst deleted file mode 100644 index ab2ef22d0c4558..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-21-10-43-22.bpo-44666.CEThkv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed issue in :func:`compileall.compile_file` when ``sys.stdout`` is redirected. -Patch by Stefan Hölzl. diff --git a/Misc/NEWS.d/next/Library/2021-07-21-23-16-30.bpo-44704.iqHLxQ.rst b/Misc/NEWS.d/next/Library/2021-07-21-23-16-30.bpo-44704.iqHLxQ.rst deleted file mode 100644 index 586661876dedba..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-21-23-16-30.bpo-44704.iqHLxQ.rst +++ /dev/null @@ -1 +0,0 @@ -The implementation of ``collections.abc.Set._hash()`` now matches that of ``frozenset.__hash__()``. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-22-21-25-56.bpo-40321.gBlFmw.rst b/Misc/NEWS.d/next/Library/2021-07-22-21-25-56.bpo-40321.gBlFmw.rst new file mode 100644 index 00000000000000..fede2a0e5e35fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-07-22-21-25-56.bpo-40321.gBlFmw.rst @@ -0,0 +1,2 @@ +Adds support for HTTP 308 redirects to :mod:`urllib`. See :rfc:`7538` for +details. Patch by Jochem Schulenklopper. diff --git a/Misc/NEWS.d/next/Library/2021-07-24-02-17-59.bpo-44720.shU5Qm.rst b/Misc/NEWS.d/next/Library/2021-07-24-02-17-59.bpo-44720.shU5Qm.rst deleted file mode 100644 index 83694f3988ae9a..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-24-02-17-59.bpo-44720.shU5Qm.rst +++ /dev/null @@ -1 +0,0 @@ -``weakref.proxy`` objects referencing non-iterators now raise ``TypeError`` rather than dereferencing the null ``tp_iternext`` slot and crashing. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-25-08-17-55.bpo-42378.WIhUZK.rst b/Misc/NEWS.d/next/Library/2021-07-25-08-17-55.bpo-42378.WIhUZK.rst deleted file mode 100644 index 90c3961dc87d8b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-25-08-17-55.bpo-42378.WIhUZK.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixes the issue with log file being overwritten when -:class:`logging.FileHandler` is used in :mod:`atexit` with *filemode* set to -``'w'``. Note this will cause the message in *atexit* not being logged if -the log stream is already closed due to shutdown of logging. diff --git a/Misc/NEWS.d/next/Library/2021-07-27-12-06-19.bpo-44747.epUzZz.rst b/Misc/NEWS.d/next/Library/2021-07-27-12-06-19.bpo-44747.epUzZz.rst deleted file mode 100644 index e63d77f76de92d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-27-12-06-19.bpo-44747.epUzZz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Refactor usage of ``sys._getframe`` in ``typing`` module. Patch provided by -Yurii Karabas. diff --git a/Misc/NEWS.d/next/Library/2021-07-27-22-11-29.bpo-44752._bvbrZ.rst b/Misc/NEWS.d/next/Library/2021-07-27-22-11-29.bpo-44752._bvbrZ.rst deleted file mode 100644 index 0d8a2cd6a5e0db..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-27-22-11-29.bpo-44752._bvbrZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`rcompleter` does not call :func:`getattr` on :class:`property` objects -to avoid the side-effect of evaluating the corresponding method. diff --git a/Misc/NEWS.d/next/Library/2021-07-28-22-53-18.bpo-44771.BvLdnU.rst b/Misc/NEWS.d/next/Library/2021-07-28-22-53-18.bpo-44771.BvLdnU.rst deleted file mode 100644 index 0d47a55a7d74f9..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-28-22-53-18.bpo-44771.BvLdnU.rst +++ /dev/null @@ -1,5 +0,0 @@ -Added ``importlib.simple`` module implementing adapters from a low-level -resources reader interface to a ``TraversableResources`` interface. Legacy -API (``path``, ``contents``, ...) is now supported entirely by the -``.files()`` API with a compatibility shim supplied for resource loaders -without that functionality. Feature parity with ``importlib_resources`` 5.2. diff --git a/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst b/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst deleted file mode 100644 index 5b7e20e0afdf5e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :func:`tokenize.tokenize` doesn't incorrectly generate a ``NEWLINE`` -token if the source doesn't end with a new line character but the last line -is a comment, as the function is already generating a ``NL`` token. Patch by -Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2021-07-31-08-45-31.bpo-44784.fIMIDS.rst b/Misc/NEWS.d/next/Library/2021-07-31-08-45-31.bpo-44784.fIMIDS.rst deleted file mode 100644 index 6ad10ef3f59805..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-31-08-45-31.bpo-44784.fIMIDS.rst +++ /dev/null @@ -1,2 +0,0 @@ -In importlib.metadata tests, override warnings behavior under expected -DeprecationWarnings (importlib_metadata 4.6.3). diff --git a/Misc/NEWS.d/next/Library/2021-07-31-20-28-20.bpo-44793.woaQSg.rst b/Misc/NEWS.d/next/Library/2021-07-31-20-28-20.bpo-44793.woaQSg.rst deleted file mode 100644 index 1d94d67615a479..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-31-20-28-20.bpo-44793.woaQSg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix checking the number of arguments when subscribe a generic type with -``ParamSpec`` parameter. diff --git a/Misc/NEWS.d/next/Library/2021-08-01-19-49-09.bpo-27275.QsvE0k.rst b/Misc/NEWS.d/next/Library/2021-08-01-19-49-09.bpo-27275.QsvE0k.rst deleted file mode 100644 index 1f5afaf4d108e4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-01-19-49-09.bpo-27275.QsvE0k.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`collections.OrderedDict.popitem` and :meth:`collections.OrderedDict.pop` -no longer call ``__getitem__`` and ``__delitem__`` methods of the OrderedDict -subclasses. diff --git a/Misc/NEWS.d/next/Library/2021-08-02-14-37-32.bpo-44806.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-02-14-37-32.bpo-44806.wOW_Qn.rst deleted file mode 100644 index 6d818c3fc57982..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-02-14-37-32.bpo-44806.wOW_Qn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Non-protocol subclasses of :class:`typing.Protocol` ignore now the -``__init__`` method inherited from protocol base classes. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-20-37-45.bpo-44801.i49Aug.rst b/Misc/NEWS.d/next/Library/2021-08-03-20-37-45.bpo-44801.i49Aug.rst deleted file mode 100644 index 05e372a5fabb03..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-03-20-37-45.bpo-44801.i49Aug.rst +++ /dev/null @@ -1,3 +0,0 @@ -Ensure that the :class:`~typing.ParamSpec` variable in Callable -can only be substituted with a parameters expression (a list of types, -an ellipsis, ParamSpec or Concatenate). diff --git a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst deleted file mode 100644 index d078142886d2e0..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`sqlite3` user-defined functions and aggregators returning -:class:`strings ` with embedded NUL characters are no longer -truncated. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-08-05-14-59-39.bpo-44839.MURNL9.rst b/Misc/NEWS.d/next/Library/2021-08-05-14-59-39.bpo-44839.MURNL9.rst deleted file mode 100644 index 62ad62c5d48d54..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-05-14-59-39.bpo-44839.MURNL9.rst +++ /dev/null @@ -1,4 +0,0 @@ -:class:`MemoryError` raised in user-defined functions will now produce a -``MemoryError`` in :mod:`sqlite3`. :class:`OverflowError` will now be converted -to :class:`~sqlite3.DataError`. Previously -:class:`~sqlite3.OperationalError` was produced in these cases. diff --git a/Misc/NEWS.d/next/Library/2021-08-05-18-20-17.bpo-44524.9T1tfe.rst b/Misc/NEWS.d/next/Library/2021-08-05-18-20-17.bpo-44524.9T1tfe.rst deleted file mode 100644 index 0c9e82050883d5..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-05-18-20-17.bpo-44524.9T1tfe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed an issue wherein the ``__name__`` and ``__qualname__`` attributes of -subscribed specialforms could be ``None``. diff --git a/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst b/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst deleted file mode 100644 index 93783923e15b3d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst +++ /dev/null @@ -1 +0,0 @@ -The @functools.total_ordering() decorator now works with metaclasses. diff --git a/Misc/NEWS.d/next/Library/2021-08-06-13-00-28.bpo-44849.O78F_f.rst b/Misc/NEWS.d/next/Library/2021-08-06-13-00-28.bpo-44849.O78F_f.rst deleted file mode 100644 index b1f225485ddef5..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-06-13-00-28.bpo-44849.O78F_f.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the :func:`os.set_inheritable` function on FreeBSD 14 for file descriptor -opened with the :data:`~os.O_PATH` flag: ignore the :data:`~errno.EBADF` -error on ``ioctl()``, fallback on the ``fcntl()`` implementation. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-08-06-19-15-52.bpo-44581.oFDBTB.rst b/Misc/NEWS.d/next/Library/2021-08-06-19-15-52.bpo-44581.oFDBTB.rst deleted file mode 100644 index 99f08065b9d2f0..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-06-19-15-52.bpo-44581.oFDBTB.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade bundled pip to 21.2.3 and setuptools to 57.4.0 diff --git a/Misc/NEWS.d/next/Library/2021-08-07-17-28-56.bpo-44859.CCopjk.rst b/Misc/NEWS.d/next/Library/2021-08-07-17-28-56.bpo-44859.CCopjk.rst deleted file mode 100644 index ec9f774d66b8c4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-07-17-28-56.bpo-44859.CCopjk.rst +++ /dev/null @@ -1,8 +0,0 @@ -Improve error handling in :mod:`sqlite3` and raise more accurate exceptions. - -* :exc:`MemoryError` is now raised instead of :exc:`sqlite3.Warning` when memory is not enough for encoding a statement to UTF-8 in ``Connection.__call__()`` and ``Cursor.execute()``. -* :exc:`UnicodEncodeError` is now raised instead of :exc:`sqlite3.Warning` when the statement contains surrogate characters in ``Connection.__call__()`` and ``Cursor.execute()``. -* :exc:`TypeError` is now raised instead of :exc:`ValueError` for non-string script argument in ``Cursor.executescript()``. -* :exc:`ValueError` is now raised for script containing the null character instead of truncating it in ``Cursor.executescript()``. -* Correctly handle exceptions raised when getting boolean value of the result of the progress handler. -* Add many tests covering different corner cases. diff --git a/Misc/NEWS.d/next/Library/2021-08-07-22-51-32.bpo-44860.PTvRrU.rst b/Misc/NEWS.d/next/Library/2021-08-07-22-51-32.bpo-44860.PTvRrU.rst deleted file mode 100644 index 70e0be0dab40c3..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-07-22-51-32.bpo-44860.PTvRrU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the ``posix_user`` scheme in :mod:`sysconfig` to not depend on -:data:`sys.platlibdir`. diff --git a/Misc/NEWS.d/next/Library/2021-08-09-13-17-10.bpo-38956.owWLNv.rst b/Misc/NEWS.d/next/Library/2021-08-09-13-17-10.bpo-38956.owWLNv.rst deleted file mode 100644 index 3f57c0ea5d5a38..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-09-13-17-10.bpo-38956.owWLNv.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`argparse.BooleanOptionalAction`'s default value is no longer printed twice when used with :class:`argparse.ArgumentDefaultsHelpFormatter`. diff --git a/Misc/NEWS.d/next/Library/2021-08-10-16-57-10.bpo-44524.dk9QX4.rst b/Misc/NEWS.d/next/Library/2021-08-10-16-57-10.bpo-44524.dk9QX4.rst deleted file mode 100644 index bc3659fca52098..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-10-16-57-10.bpo-44524.dk9QX4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make exception message more useful when subclass from typing special form -alias. Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Library/2021-08-12-16-22-16.bpo-41322.utscTd.rst b/Misc/NEWS.d/next/Library/2021-08-12-16-22-16.bpo-41322.utscTd.rst deleted file mode 100644 index e16efd2c7bd556..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-12-16-22-16.bpo-41322.utscTd.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added ``DeprecationWarning`` for tests and async tests that return a -value!=None (as this may indicate an improperly written test, for example a -test written as a generator function). diff --git a/Misc/NEWS.d/next/Library/2021-08-14-00-55-16.bpo-44911.uk3hYk.rst b/Misc/NEWS.d/next/Library/2021-08-14-00-55-16.bpo-44911.uk3hYk.rst deleted file mode 100644 index f8aed69a40a3be..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-14-00-55-16.bpo-44911.uk3hYk.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`~unittest.IsolatedAsyncioTestCase` will no longer throw an exception while cancelling leaked tasks. Patch by Bar Harel. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-08-17-16-01-44.bpo-44935.roUl0G.rst b/Misc/NEWS.d/next/Library/2021-08-17-16-01-44.bpo-44935.roUl0G.rst deleted file mode 100644 index 3d41c3be1403dc..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-17-16-01-44.bpo-44935.roUl0G.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`subprocess` on Solaris now also uses :func:`os.posix_spawn()` for -better performance. diff --git a/Misc/NEWS.d/next/Library/2021-08-19-15-03-54.bpo-44955.1mxFQS.rst b/Misc/NEWS.d/next/Library/2021-08-19-15-03-54.bpo-44955.1mxFQS.rst deleted file mode 100644 index 57d1da533cde0d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-19-15-03-54.bpo-44955.1mxFQS.rst +++ /dev/null @@ -1,5 +0,0 @@ -Method :meth:`~unittest.TestResult.stopTestRun` is now always called in pair -with method :meth:`~unittest.TestResult.startTestRun` for -:class:`~unittest.TestResult` objects implicitly created in -:meth:`~unittest.TestCase.run`. Previously it was not called for test -methods and classes decorated with a skipping decorator. diff --git a/Misc/NEWS.d/next/Library/2021-08-19-23-49-10.bpo-42255.ofe3ms.rst b/Misc/NEWS.d/next/Library/2021-08-19-23-49-10.bpo-42255.ofe3ms.rst deleted file mode 100644 index 84a02c4c3fb2b4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-19-23-49-10.bpo-42255.ofe3ms.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`webbrowser.MacOSX` is deprecated and will be removed in Python 3.13. -It is untested and undocumented and also not used by webbrowser itself. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-08-22-13-25-17.bpo-44019.BN8HDy.rst b/Misc/NEWS.d/next/Library/2021-08-22-13-25-17.bpo-44019.BN8HDy.rst deleted file mode 100644 index 37556d76905d73..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-22-13-25-17.bpo-44019.BN8HDy.rst +++ /dev/null @@ -1,2 +0,0 @@ -A new function ``operator.call`` has been added, such that -``operator.call(obj, *args, **kwargs) == obj(*args, **kwargs)``. diff --git a/Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst b/Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst deleted file mode 100644 index 81fdfeb6294569..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ensure that :class:`set` and :class:`frozenset` objects are always -:mod:`marshalled ` reproducibly. diff --git a/Misc/NEWS.d/next/Library/2021-08-25-10-28-49.bpo-43613.WkYmI0.rst b/Misc/NEWS.d/next/Library/2021-08-25-10-28-49.bpo-43613.WkYmI0.rst deleted file mode 100644 index d6af35c12b3c70..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-25-10-28-49.bpo-43613.WkYmI0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve the speed of :func:`gzip.compress` and :func:`gzip.decompress` by -compressing and decompressing at once in memory instead of in a streamed -fashion. diff --git a/Misc/NEWS.d/next/Library/2021-08-25-20-18-31.bpo-39218.BlO6jW.rst b/Misc/NEWS.d/next/Library/2021-08-25-20-18-31.bpo-39218.BlO6jW.rst deleted file mode 100644 index 7bcf9a222e33c3..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-25-20-18-31.bpo-39218.BlO6jW.rst +++ /dev/null @@ -1 +0,0 @@ -Improve accuracy of variance calculations by using x*x instead of x**2. diff --git a/Misc/NEWS.d/next/Library/2021-08-26-09-54-14.bpo-45010.Cn23bQ.rst b/Misc/NEWS.d/next/Library/2021-08-26-09-54-14.bpo-45010.Cn23bQ.rst deleted file mode 100644 index bdf1bfe1ffe06e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-26-09-54-14.bpo-45010.Cn23bQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove support of special method ``__div__`` in :mod:`unittest.mock`. It is -not used in Python 3. diff --git a/Misc/NEWS.d/next/Library/2021-08-26-16-25-48.bpo-45001.tn_dKp.rst b/Misc/NEWS.d/next/Library/2021-08-26-16-25-48.bpo-45001.tn_dKp.rst deleted file mode 100644 index 55cc409d0da30f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-26-16-25-48.bpo-45001.tn_dKp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Made email date parsing more robust against malformed input, namely a -whitespace-only ``Date:`` header. Patch by Wouter Bolsterlee. diff --git a/Misc/NEWS.d/next/Library/2021-08-27-19-01-23.bpo-45030.tAmBbY.rst b/Misc/NEWS.d/next/Library/2021-08-27-19-01-23.bpo-45030.tAmBbY.rst deleted file mode 100644 index dec8c88b155888..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-27-19-01-23.bpo-45030.tAmBbY.rst +++ /dev/null @@ -1 +0,0 @@ -Fix integer overflow in pickling and copying the range iterator. diff --git a/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst b/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst deleted file mode 100644 index cf3d5ee0e456f1..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst +++ /dev/null @@ -1,8 +0,0 @@ -Fix bugs in cleaning up classes and modules in :mod:`unittest`: - -* Functions registered with :func:`~unittest.addModuleCleanup` were not called unless the user defines ``tearDownModule()`` in their test module. -* Functions registered with :meth:`~unittest.TestCase.addClassCleanup` were not called if ``tearDownClass`` is set to ``None``. -* Buffering in :class:`~unittest.TestResult` did not work with functions registered with ``addClassCleanup()`` and ``addModuleCleanup()``. -* Errors in functions registered with ``addClassCleanup()`` and ``addModuleCleanup()`` were not handled correctly in buffered and debug modes. -* Errors in ``setUpModule()`` and functions registered with ``addModuleCleanup()`` were reported in wrong order. -* And several lesser bugs. diff --git a/Misc/NEWS.d/next/Library/2021-08-28-13-00-12.bpo-45021.rReeaj.rst b/Misc/NEWS.d/next/Library/2021-08-28-13-00-12.bpo-45021.rReeaj.rst deleted file mode 100644 index 54fd9109a9ae5f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-28-13-00-12.bpo-45021.rReeaj.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a potential deadlock at shutdown of forked children when using :mod:`concurrent.futures` module \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-08-29-14-49-22.bpo-41620.WJ6PFL.rst b/Misc/NEWS.d/next/Library/2021-08-29-14-49-22.bpo-41620.WJ6PFL.rst deleted file mode 100644 index 7674d4c9532a27..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-29-14-49-22.bpo-41620.WJ6PFL.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`~unittest.TestCase.run` now always return a -:class:`~unittest.TestResult` instance. Previously it returned ``None`` if -the test class or method was decorated with a skipping decorator. diff --git a/Misc/NEWS.d/next/Library/2021-08-30-00-19-23.bpo-24444.Ki4bgz.rst b/Misc/NEWS.d/next/Library/2021-08-30-00-19-23.bpo-24444.Ki4bgz.rst new file mode 100644 index 00000000000000..efcacb8f0eb593 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-30-00-19-23.bpo-24444.Ki4bgz.rst @@ -0,0 +1,2 @@ +Fixed an error raised in :mod:`argparse` help display when help for an +option is set to 1+ blank spaces or when *choices* arg is an empty container. diff --git a/Misc/NEWS.d/next/Library/2021-08-30-13-55-09.bpo-31299.9QzjZs.rst b/Misc/NEWS.d/next/Library/2021-08-30-13-55-09.bpo-31299.9QzjZs.rst deleted file mode 100644 index 1ffa0b15172ee7..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-30-13-55-09.bpo-31299.9QzjZs.rst +++ /dev/null @@ -1 +0,0 @@ -Add option to completely drop frames from a traceback by returning ``None`` from a :meth:`~traceback.StackSummary.format_frame` override. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-09-01-15-27-00.bpo-45075.9xUpvt.rst b/Misc/NEWS.d/next/Library/2021-09-01-15-27-00.bpo-45075.9xUpvt.rst deleted file mode 100644 index 369b4506066e5b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-01-15-27-00.bpo-45075.9xUpvt.rst +++ /dev/null @@ -1,5 +0,0 @@ -Rename :meth:`traceback.StackSummary.format_frame` to -:meth:`traceback.StackSummary.format_frame_summary`. This method was added -for 3.11 so it was not released yet. - -Updated code and docs to better distinguish frame and FrameSummary. diff --git a/Misc/NEWS.d/next/Library/2021-09-02-00-18-32.bpo-40360.9nmMtB.rst b/Misc/NEWS.d/next/Library/2021-09-02-00-18-32.bpo-40360.9nmMtB.rst deleted file mode 100644 index 4e9422dc06d7f4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-02-00-18-32.bpo-40360.9nmMtB.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :mod:`lib2to3` package is now deprecated and may not be able to parse -Python 3.10 or newer. See the :pep:`617` (New PEG parser for CPython). Patch -by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-09-02-00-47-14.bpo-45085.mMnaDv.rst b/Misc/NEWS.d/next/Library/2021-09-02-00-47-14.bpo-45085.mMnaDv.rst deleted file mode 100644 index 22eada24f0f5a2..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-02-00-47-14.bpo-45085.mMnaDv.rst +++ /dev/null @@ -1,10 +0,0 @@ -The ``binhex`` module, deprecated in Python 3.9, is now removed. The -following :mod:`binascii` functions, deprecated in Python 3.9, are now also -removed: - -* ``a2b_hqx()``, ``b2a_hqx()``; -* ``rlecode_hqx()``, ``rledecode_hqx()``. - -The :func:`binascii.crc_hqx` function remains available. - -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-09-02-12-42-25.bpo-45081.tOjJ1k.rst b/Misc/NEWS.d/next/Library/2021-09-02-12-42-25.bpo-45081.tOjJ1k.rst deleted file mode 100644 index 86d7182003bb93..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-02-12-42-25.bpo-45081.tOjJ1k.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue when dataclasses that inherit from ``typing.Protocol`` subclasses -have wrong ``__init__``. Patch provided by Yurii Karabas. diff --git a/Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst b/Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst deleted file mode 100644 index b0a036fae6cfa7..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`unittest` now always reports skipped and failed subtests separately: -separate characters in default mode and separate lines in verbose mode. Also -the test description is now output for errors in test method, class and -module cleanups. diff --git a/Misc/NEWS.d/next/Library/2021-09-05-20-33-25.bpo-45034.62NLD5.rst b/Misc/NEWS.d/next/Library/2021-09-05-20-33-25.bpo-45034.62NLD5.rst deleted file mode 100644 index 8d94821470a2ad..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-05-20-33-25.bpo-45034.62NLD5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changes how error is formatted for ``struct.pack`` with ``'H'`` and ``'h'`` modes and -too large / small numbers. Now it shows the actual numeric limits, while previously it was showing arithmetic expressions. diff --git a/Misc/NEWS.d/next/Library/2021-09-05-21-37-28.bpo-30856.jj86y0.rst b/Misc/NEWS.d/next/Library/2021-09-05-21-37-28.bpo-30856.jj86y0.rst deleted file mode 100644 index 1ac4eb672d2ec9..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-05-21-37-28.bpo-30856.jj86y0.rst +++ /dev/null @@ -1,6 +0,0 @@ -:class:`unittest.TestResult` methods -:meth:`~unittest.TestResult.addFailure`, -:meth:`~unittest.TestResult.addError`, :meth:`~unittest.TestResult.addSkip` -and :meth:`~unittest.TestResult.addSubTest` are now called immediately after -raising an exception in test or finishing a subtest. Previously they were -called only after finishing the test clean up. diff --git a/Misc/NEWS.d/next/Library/2021-09-07-09-13-27.bpo-45124.Kw5AUs.rst b/Misc/NEWS.d/next/Library/2021-09-07-09-13-27.bpo-45124.Kw5AUs.rst deleted file mode 100644 index 2f6ab411b3deb1..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-07-09-13-27.bpo-45124.Kw5AUs.rst +++ /dev/null @@ -1,5 +0,0 @@ -The ``bdist_msi`` command, deprecated in Python 3.9, is now removed. - -Use ``bdist_wheel`` (wheel packages) instead. - -Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2021-09-07-14-27-39.bpo-45129.vXH0gw.rst b/Misc/NEWS.d/next/Library/2021-09-07-14-27-39.bpo-45129.vXH0gw.rst deleted file mode 100644 index 5ba6721923dbdb..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-07-14-27-39.bpo-45129.vXH0gw.rst +++ /dev/null @@ -1,6 +0,0 @@ -Due to significant security concerns, the *reuse_address* parameter of -:meth:`asyncio.loop.create_datagram_endpoint`, disabled in Python 3.9, is -now entirely removed. This is because of the behavior of the socket option -``SO_REUSEADDR`` in UDP. - -Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2021-09-07-16-33-51.bpo-45132.WI9zQY.rst b/Misc/NEWS.d/next/Library/2021-09-07-16-33-51.bpo-45132.WI9zQY.rst deleted file mode 100644 index 10f24003dc71fe..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-07-16-33-51.bpo-45132.WI9zQY.rst +++ /dev/null @@ -1,5 +0,0 @@ -Remove :meth:`__getitem__` methods of -:class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` -and :class:`fileinput.FileInput`, deprecated since Python 3.9. - -Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2021-09-08-01-19-31.bpo-20499.tSxx8Y.rst b/Misc/NEWS.d/next/Library/2021-09-08-01-19-31.bpo-20499.tSxx8Y.rst deleted file mode 100644 index cbbe61ac4a2697..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-08-01-19-31.bpo-20499.tSxx8Y.rst +++ /dev/null @@ -1 +0,0 @@ -Improve the speed and accuracy of statistics.pvariance(). diff --git a/Misc/NEWS.d/next/Library/2021-09-08-13-19-29.bpo-38371.y1kEfP.rst b/Misc/NEWS.d/next/Library/2021-09-08-13-19-29.bpo-38371.y1kEfP.rst deleted file mode 100644 index 65f35488c6f255..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-08-13-19-29.bpo-38371.y1kEfP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove the deprecated ``split()`` method of :class:`_tkinter.TkappType`. -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-09-10-12-53-28.bpo-25625.SzcBCw.rst b/Misc/NEWS.d/next/Library/2021-09-10-12-53-28.bpo-25625.SzcBCw.rst new file mode 100644 index 00000000000000..c001683b657f58 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-09-10-12-53-28.bpo-25625.SzcBCw.rst @@ -0,0 +1,3 @@ +Added non parallel-safe :func:`~contextlib.chdir` context manager to change +the current working directory and then restore it on exit. Simple wrapper +around :func:`~os.chdir`. diff --git a/Misc/NEWS.d/next/Library/2021-09-10-13-20-53.bpo-45162.2Jh-lq.rst b/Misc/NEWS.d/next/Library/2021-09-10-13-20-53.bpo-45162.2Jh-lq.rst deleted file mode 100644 index b22269d65b041b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-10-13-20-53.bpo-45162.2Jh-lq.rst +++ /dev/null @@ -1,6 +0,0 @@ -Remove many old deprecated :mod:`unittest` features: - -* "``fail*``" and "``assert*``" aliases of :class:`~unittest.TestCase` methods. -* Broken from start :class:`~unittest.TestCase` method ``assertDictContainsSubset()``. -* Ignored :meth:` TestLoader.loadTestsFromModule` parameter *use_load_tests*. -* Old alias ``_TextTestResult`` of :class:`~unittest.TextTestResult`. diff --git a/Misc/NEWS.d/next/Library/2021-09-10-21-35-53.bpo-45166.UHipXF.rst b/Misc/NEWS.d/next/Library/2021-09-10-21-35-53.bpo-45166.UHipXF.rst deleted file mode 100644 index b7242d45ea9be8..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-10-21-35-53.bpo-45166.UHipXF.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`typing.get_type_hints` now works with :data:`~typing.Final` wrapped in -:class:`~typing.ForwardRef`. diff --git a/Misc/NEWS.d/next/Library/2021-09-11-10-45-12.bpo-35474.tEY3SD.rst b/Misc/NEWS.d/next/Library/2021-09-11-10-45-12.bpo-35474.tEY3SD.rst deleted file mode 100644 index f4dd3b947a4936..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-11-10-45-12.bpo-35474.tEY3SD.rst +++ /dev/null @@ -1,3 +0,0 @@ -Calling :func:`mimetypes.guess_all_extensions` with ``strict=False`` no -longer affects the result of the following call with ``strict=True``. -Also, mutating the returned list no longer affects the global state. diff --git a/Misc/NEWS.d/next/Library/2021-09-11-14-41-02.bpo-44987.Mt8DiX.rst b/Misc/NEWS.d/next/Library/2021-09-11-14-41-02.bpo-44987.Mt8DiX.rst deleted file mode 100644 index dec50d87c916c0..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-11-14-41-02.bpo-44987.Mt8DiX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Pure ASCII strings are now normalized in constant time by :func:`unicodedata.normalize`. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-09-11-17-46-20.bpo-45173.UptGAn.rst b/Misc/NEWS.d/next/Library/2021-09-11-17-46-20.bpo-45173.UptGAn.rst deleted file mode 100644 index 0b29ec2364edcf..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-11-17-46-20.bpo-45173.UptGAn.rst +++ /dev/null @@ -1,7 +0,0 @@ -Remove from the :mod:`configparser` module: -the :class:`SafeConfigParser` class, -the :attr:`filename` property of the :class:`ParsingError` class, -the :meth:`readfp` method of the :class:`ConfigParser` class, -deprecated since Python 3.2. - -Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2021-09-11-18-44-40.bpo-21302.QxHRpR.rst b/Misc/NEWS.d/next/Library/2021-09-11-18-44-40.bpo-21302.QxHRpR.rst deleted file mode 100644 index 86f0a5ac4c2a5a..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-11-18-44-40.bpo-21302.QxHRpR.rst +++ /dev/null @@ -1,2 +0,0 @@ -In Unix operating systems, :func:`time.sleep` now uses the ``clock_nanosleep()`` function, -if available, which allows to sleep for an interval specified with nanosecond precision. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-09-13-14-28-49.bpo-45168.Z1mfW4.rst b/Misc/NEWS.d/next/Library/2021-09-13-14-28-49.bpo-45168.Z1mfW4.rst deleted file mode 100644 index 4e12e7d51e2c79..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-13-14-28-49.bpo-45168.Z1mfW4.rst +++ /dev/null @@ -1 +0,0 @@ -Change :func:`dis.dis` output to omit op arg values that cannot be resolved due to ``co_consts``, ``co_names`` etc not being provided. Previously the oparg itself was repeated in the value field, which is not useful and can be confusing. diff --git a/Misc/NEWS.d/next/Library/2021-09-13-14-59-01.bpo-20524.PMQ1Fh.rst b/Misc/NEWS.d/next/Library/2021-09-13-14-59-01.bpo-20524.PMQ1Fh.rst deleted file mode 100644 index 9a41c8e64f7e34..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-13-14-59-01.bpo-20524.PMQ1Fh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improves error messages on ``.format()`` operation for ``str``, ``float``, -``int``, and ``complex``. New format now shows the problematic pattern and -the object type. diff --git a/Misc/NEWS.d/next/Library/2021-09-13-19-32-58.bpo-42135.1ZAHqR.rst b/Misc/NEWS.d/next/Library/2021-09-13-19-32-58.bpo-42135.1ZAHqR.rst deleted file mode 100644 index 983b46e140e747..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-13-19-32-58.bpo-42135.1ZAHqR.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix typo: ``importlib.find_loader`` is really slated for removal in Python 3.12 not 3.10, like the others in GH-25169. - -Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst b/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst new file mode 100644 index 00000000000000..7dd9795aaa1708 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst @@ -0,0 +1,5 @@ +Fix the ``tempfile._infer_return_type`` function so that the ``dir`` +argument of the :mod:`tempfile` functions accepts an object implementing the +``os.PathLike`` protocol. + +Patch by Kyungmin Lee. diff --git a/Misc/NEWS.d/next/Library/2021-09-15-10-21-10.bpo-45320.4qaf5x.rst b/Misc/NEWS.d/next/Library/2021-09-15-10-21-10.bpo-45320.4qaf5x.rst new file mode 100644 index 00000000000000..1f2236bd424124 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-09-15-10-21-10.bpo-45320.4qaf5x.rst @@ -0,0 +1,15 @@ +Removed from the :mod:`inspect` module: + +* the ``getargspec`` function, deprecated since Python 3.0; + use :func:`inspect.signature` or :func:`inspect.getfullargspec` instead. + +* the ``formatargspec`` function, deprecated since Python 3.5; + use the :func:`inspect.signature` function and :class:`Signature` object + directly. + +* the undocumented ``Signature.from_callable`` and ``Signature.from_function`` + functions, deprecated since Python 3.5; use the + :meth:`Signature.from_callable() ` method + instead. + +Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2021-09-16-19-02-14.bpo-45225.xmKV4i.rst b/Misc/NEWS.d/next/Library/2021-09-16-19-02-14.bpo-45225.xmKV4i.rst deleted file mode 100644 index 734fdd9b007d6b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-16-19-02-14.bpo-45225.xmKV4i.rst +++ /dev/null @@ -1 +0,0 @@ -use map function instead of genexpr in capwords. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-09-17-09-59-33.bpo-45228.WV1dcT.rst b/Misc/NEWS.d/next/Library/2021-09-17-09-59-33.bpo-45228.WV1dcT.rst deleted file mode 100644 index 9336c0aed92bc4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-17-09-59-33.bpo-45228.WV1dcT.rst +++ /dev/null @@ -1 +0,0 @@ -Fix stack buffer overflow in parsing J1939 network address. diff --git a/Misc/NEWS.d/next/Library/2021-09-17-11-20-55.bpo-45234.qUcTVt.rst b/Misc/NEWS.d/next/Library/2021-09-17-11-20-55.bpo-45234.qUcTVt.rst deleted file mode 100644 index 3817b5de6449d4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-17-11-20-55.bpo-45234.qUcTVt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a regression in :func:`~shutil.copyfile`, :func:`~shutil.copy`, -:func:`~shutil.copy2` raising :exc:`FileNotFoundError` when source is a -directory, which should raise :exc:`IsADirectoryError` diff --git a/Misc/NEWS.d/next/Library/2021-09-17-15-58-53.bpo-45183.Vv_vch.rst b/Misc/NEWS.d/next/Library/2021-09-17-15-58-53.bpo-45183.Vv_vch.rst deleted file mode 100644 index f3194b34318f40..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-17-15-58-53.bpo-45183.Vv_vch.rst +++ /dev/null @@ -1,3 +0,0 @@ -Have zipimport.zipimporter.find_spec() not raise an exception when the underlying zip -file has been deleted and the internal cache has been reset via -invalidate_cache(). diff --git a/Misc/NEWS.d/next/Library/2021-09-17-16-55-37.bpo-45235.sXnmPA.rst b/Misc/NEWS.d/next/Library/2021-09-17-16-55-37.bpo-45235.sXnmPA.rst deleted file mode 100644 index 871ec5281d3344..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-17-16-55-37.bpo-45235.sXnmPA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where argparse would not preserve values in a provided namespace -when using a subparser with defaults. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-09-18-13-14-57.bpo-36674.a2k5Zb.rst b/Misc/NEWS.d/next/Library/2021-09-18-13-14-57.bpo-36674.a2k5Zb.rst deleted file mode 100644 index bc8c9247b080d4..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-18-13-14-57.bpo-36674.a2k5Zb.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`unittest.TestCase.debug` raises now a :class:`unittest.SkipTest` if -the class or the test method are decorated with the skipping decorator. diff --git a/Misc/NEWS.d/next/Library/2021-09-18-16-56-33.bpo-45238.Hng_9V.rst b/Misc/NEWS.d/next/Library/2021-09-18-16-56-33.bpo-45238.Hng_9V.rst deleted file mode 100644 index 857f315c520bba..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-18-16-56-33.bpo-45238.Hng_9V.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`unittest.IsolatedAsyncioTestCase.debug`: it runs now asynchronous -methods and callbacks. diff --git a/Misc/NEWS.d/next/Library/2021-09-20-22-46-40.bpo-21302.h56430.rst b/Misc/NEWS.d/next/Library/2021-09-20-22-46-40.bpo-21302.h56430.rst deleted file mode 100644 index 22011b791e5f2d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-20-22-46-40.bpo-21302.h56430.rst +++ /dev/null @@ -1,3 +0,0 @@ -On Windows, :func:`time.sleep` now uses a waitable timer which has a resolution -of 100 ns (10^-7 sec). Previously, it had a solution of 1 ms (10^-3 sec). -Patch by Livius and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-09-22-23-56-15.bpo-21302.vvQ3Su.rst b/Misc/NEWS.d/next/Library/2021-09-22-23-56-15.bpo-21302.vvQ3Su.rst deleted file mode 100644 index 52ee8d7cc64f15..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-09-22-23-56-15.bpo-21302.vvQ3Su.rst +++ /dev/null @@ -1 +0,0 @@ -In Unix operating systems, :func:`time.sleep` now uses the ``nanosleep()`` function, if ``clock_nanosleep()`` is not available but ``nanosleep()`` is available. ``nanosleep()`` allows to sleep with nanosecond precision. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-09-30-08-22-44.bpo-45328.8Z-Q0B.rst b/Misc/NEWS.d/next/Library/2021-09-30-08-22-44.bpo-45328.8Z-Q0B.rst new file mode 100644 index 00000000000000..eeb49310e8f66a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-09-30-08-22-44.bpo-45328.8Z-Q0B.rst @@ -0,0 +1 @@ +Fixed :class:`http.client.HTTPConnection` to work properly in OSs that don't support the ``TCP_NODELAY`` socket option. diff --git a/Misc/NEWS.d/next/Library/2021-10-01-23-07-02.bpo-45343.ixmctD.rst b/Misc/NEWS.d/next/Library/2021-10-01-23-07-02.bpo-45343.ixmctD.rst new file mode 100644 index 00000000000000..8dac4e6a252714 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-01-23-07-02.bpo-45343.ixmctD.rst @@ -0,0 +1 @@ +Update bundled pip to 21.2.4 and setuptools to 58.1.0 diff --git a/Misc/NEWS.d/next/Library/2021-10-03-21-14-37.bpo-20028.zBA4RK.rst b/Misc/NEWS.d/next/Library/2021-10-03-21-14-37.bpo-20028.zBA4RK.rst new file mode 100644 index 00000000000000..e75612116e9427 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-03-21-14-37.bpo-20028.zBA4RK.rst @@ -0,0 +1,2 @@ +Improve error message of :class:`csv.Dialect` when initializing. +Patch by Vajrasky Kok and Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-10-07-00-05-05.bpo-45386.q9ORpA.rst b/Misc/NEWS.d/next/Library/2021-10-07-00-05-05.bpo-45386.q9ORpA.rst new file mode 100644 index 00000000000000..eec77ceccf9338 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-07-00-05-05.bpo-45386.q9ORpA.rst @@ -0,0 +1,3 @@ +Make :mod:`xmlrpc.client` more robust to C runtimes where the underlying C +``strftime`` function results in a ``ValueError`` when testing for year +formatting options. diff --git a/Misc/NEWS.d/next/Library/2021-10-07-14-04-10.bpo-45262.HqF71Z.rst b/Misc/NEWS.d/next/Library/2021-10-07-14-04-10.bpo-45262.HqF71Z.rst new file mode 100644 index 00000000000000..4cd949fe1ed5df --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-07-14-04-10.bpo-45262.HqF71Z.rst @@ -0,0 +1 @@ +Prevent use-after-free in asyncio. Make sure the cached running loop holder gets cleared on dealloc to prevent use-after-free in get_running_loop \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-10-07-21-11-48.bpo-27580.tGcBTH.rst b/Misc/NEWS.d/next/Library/2021-10-07-21-11-48.bpo-27580.tGcBTH.rst new file mode 100644 index 00000000000000..15a8ff2dd6add4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-07-21-11-48.bpo-27580.tGcBTH.rst @@ -0,0 +1 @@ +Add support of null characters in :mod:`csv`. diff --git a/Misc/NEWS.d/next/Library/2021-10-08-04-11-55.bpo-10716.QSRVK2.rst b/Misc/NEWS.d/next/Library/2021-10-08-04-11-55.bpo-10716.QSRVK2.rst new file mode 100644 index 00000000000000..8ec944960211a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-08-04-11-55.bpo-10716.QSRVK2.rst @@ -0,0 +1,3 @@ +Migrated pydoc to HTML5 (without changing the look of it). Side effect is to +update xmlrpc's ``ServerHTMLDoc`` which now uses the CSS too. cgitb now +relies less on pydoc (as it can't use the CSS file). diff --git a/Misc/NEWS.d/next/Library/2021-10-08-11-29-29.bpo-45411.4jR--U.rst b/Misc/NEWS.d/next/Library/2021-10-08-11-29-29.bpo-45411.4jR--U.rst new file mode 100644 index 00000000000000..5f774dcda98490 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-08-11-29-29.bpo-45411.4jR--U.rst @@ -0,0 +1 @@ +Add extensions for files containing subtitles - .srt & .vtt - to the mimetypes.py module. diff --git a/Misc/NEWS.d/next/Library/2021-10-10-00-25-36.bpo-20028.bPx4Z8.rst b/Misc/NEWS.d/next/Library/2021-10-10-00-25-36.bpo-20028.bPx4Z8.rst new file mode 100644 index 00000000000000..9db15bc39e7ca9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-10-00-25-36.bpo-20028.bPx4Z8.rst @@ -0,0 +1,2 @@ +Empty escapechar/quotechar is not allowed when initializing +:class:`csv.Dialect`. Patch by Vajrasky Kok and Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-10-10-09-42-34.bpo-45416.n35O0_.rst b/Misc/NEWS.d/next/Library/2021-10-10-09-42-34.bpo-45416.n35O0_.rst new file mode 100644 index 00000000000000..cf335d1bcd2c94 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-10-09-42-34.bpo-45416.n35O0_.rst @@ -0,0 +1,2 @@ +Fix use of :class:`asyncio.Condition` with explicit :class:`asyncio.Lock` objects, which was a regression due to removal of explicit loop arguments. +Patch by Joongi Kim. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst b/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst new file mode 100644 index 00000000000000..1d5a857e254355 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst @@ -0,0 +1,2 @@ +Fix the behaviour of :func:`traceback.print_exc` when displaying the caret +when the ``end_offset`` in the exception is set to 0. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2021-10-12-20-35-06.bpo-45417.gQM-O7.rst b/Misc/NEWS.d/next/Library/2021-10-12-20-35-06.bpo-45417.gQM-O7.rst new file mode 100644 index 00000000000000..a15c23917a7e62 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-12-20-35-06.bpo-45417.gQM-O7.rst @@ -0,0 +1,2 @@ +Fix quadratic behaviour in the enum module: Creation of enum classes with a +lot of entries was quadratic. diff --git a/Misc/NEWS.d/next/Library/2021-10-13-17-52-48.bpo-45239.7li1_0.rst b/Misc/NEWS.d/next/Library/2021-10-13-17-52-48.bpo-45239.7li1_0.rst new file mode 100644 index 00000000000000..9e5ec561c362af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-13-17-52-48.bpo-45239.7li1_0.rst @@ -0,0 +1,3 @@ +Fixed :func:`email.utils.parsedate_tz` crashing with +:exc:`UnboundLocalError` on certain invalid input instead of returning +``None``. Patch by Ben Hoyt. diff --git a/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst b/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst new file mode 100644 index 00000000000000..c1c4ed1ace2485 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst @@ -0,0 +1,2 @@ +Fix incremental decoder and stream reader in the "unicode-escape" codec. +Previously they failed if the escape sequence was split. diff --git a/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst b/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst new file mode 100644 index 00000000000000..f2c0ae4ae51fb7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst @@ -0,0 +1,2 @@ +Fix incremental decoder and stream reader in the "raw-unicode-escape" codec. +Previously they failed if the escape sequence was split. diff --git a/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst b/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst new file mode 100644 index 00000000000000..556eca43ed3c77 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst @@ -0,0 +1 @@ +Fix a regression in py_compile when reading filenames from standard input. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-10-15-11-30-11.bpo-42222.hdHyac.rst b/Misc/NEWS.d/next/Library/2021-10-15-11-30-11.bpo-42222.hdHyac.rst new file mode 100644 index 00000000000000..9b29fa44d7711e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-15-11-30-11.bpo-42222.hdHyac.rst @@ -0,0 +1 @@ +Removed deprecated support for float arguments in *randrange()*. diff --git a/Misc/NEWS.d/next/Library/2021-10-16-23-46-39.bpo-45489.QB0rhG.rst b/Misc/NEWS.d/next/Library/2021-10-16-23-46-39.bpo-45489.QB0rhG.rst new file mode 100644 index 00000000000000..3921437d97ac5e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-16-23-46-39.bpo-45489.QB0rhG.rst @@ -0,0 +1,2 @@ +Update :class:`~typing.ForwardRef` to support ``|`` operator. Patch by +Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst b/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst new file mode 100644 index 00000000000000..6fce894e6e4d49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst @@ -0,0 +1,4 @@ +Reverted optimization of iterating :class:`gzip.GzipFile`, +:class:`bz2.BZ2File`, and :class:`lzma.LZMAFile` (see bpo-43787) because it +caused regression when user iterate them without having reference of them. +Patch by Inada Naoki. diff --git a/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst b/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst new file mode 100644 index 00000000000000..382733ff918830 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst @@ -0,0 +1,3 @@ +Add references to :mod:`zoneinfo` in the :mod:`datetime` documentation, +mostly replacing outdated references to ``dateutil.tz``. Change by Paul +Ganssle. diff --git a/Misc/NEWS.d/next/Library/2021-10-18-18-12-47.bpo-35673.KOkHWe.rst b/Misc/NEWS.d/next/Library/2021-10-18-18-12-47.bpo-35673.KOkHWe.rst new file mode 100644 index 00000000000000..e7d6a5fa5a9107 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-18-18-12-47.bpo-35673.KOkHWe.rst @@ -0,0 +1,4 @@ +Improve the introspectability of the ``__loader__`` attribute for namespace +packages. :class:`importlib.machinery.NamespaceLoader` is now public, and +implements the :class:`importlib.abc.InspectLoader` interface. ``_NamespaceLoader`` +is kept for backward compatibility. diff --git a/Misc/NEWS.d/next/Library/2021-10-19-01-30-57.bpo-42174.O2w9bi.rst b/Misc/NEWS.d/next/Library/2021-10-19-01-30-57.bpo-42174.O2w9bi.rst new file mode 100644 index 00000000000000..412582ded3e12c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-19-01-30-57.bpo-42174.O2w9bi.rst @@ -0,0 +1,2 @@ +:meth:`shutil.get_terminal_size` now falls back to sane values if the column +or line count are 0. diff --git a/Misc/NEWS.d/next/Library/2021-10-21-10-14-22.bpo-44019.Xk4Ncr.rst b/Misc/NEWS.d/next/Library/2021-10-21-10-14-22.bpo-44019.Xk4Ncr.rst new file mode 100644 index 00000000000000..1793603637fbbb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-21-10-14-22.bpo-44019.Xk4Ncr.rst @@ -0,0 +1 @@ +Add :func:`operator.call` to ``operator.__all__``. Patch by Kreusada. diff --git a/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst b/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst deleted file mode 100644 index 9669fc5ef37dde..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`http.client` now avoids infinitely reading potential HTTP headers after a -``100 Continue`` status response from the server. diff --git a/Misc/NEWS.d/next/Security/2021-05-08-11-50-46.bpo-43124.2CTM6M.rst b/Misc/NEWS.d/next/Security/2021-05-08-11-50-46.bpo-43124.2CTM6M.rst deleted file mode 100644 index e897d6cd3641d7..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-05-08-11-50-46.bpo-43124.2CTM6M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Made the internal ``putcmd`` function in :mod:`smtplib` sanitize input for -presence of ``\r`` and ``\n`` characters to avoid (unlikely) command injection. diff --git a/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst b/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst deleted file mode 100644 index e32563d2535c7e..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix -for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used -on Windows and macOS. diff --git a/Misc/NEWS.d/next/Security/2021-06-29-23-40-22.bpo-41180.uTWHv_.rst b/Misc/NEWS.d/next/Security/2021-06-29-23-40-22.bpo-41180.uTWHv_.rst deleted file mode 100644 index 88b70c7cea2610..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-06-29-23-40-22.bpo-41180.uTWHv_.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add auditing events to the :mod:`marshal` module, and stop raising -``code.__init__`` events for every unmarshalled code object. Directly -instantiated code objects will continue to raise an event, and audit event -handlers should inspect or collect the raw marshal data. This reduces a -significant performance overhead when loading from ``.pyc`` files. diff --git a/Misc/NEWS.d/next/Security/2021-07-25-20-04-54.bpo-44600.0WMldg.rst b/Misc/NEWS.d/next/Security/2021-07-25-20-04-54.bpo-44600.0WMldg.rst deleted file mode 100644 index ea4e04f6da911c..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-07-25-20-04-54.bpo-44600.0WMldg.rst +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect line numbers while tracing some failed patterns in :ref:`match ` statements. Patch by Charles Burkland. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Security/2021-08-29-12-39-44.bpo-42278.jvmQz_.rst b/Misc/NEWS.d/next/Security/2021-08-29-12-39-44.bpo-42278.jvmQz_.rst deleted file mode 100644 index db880cd9026da4..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-08-29-12-39-44.bpo-42278.jvmQz_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Replaced usage of :func:`tempfile.mktemp` with -:class:`~tempfile.TemporaryDirectory` to avoid a potential race condition. diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst b/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst deleted file mode 100644 index 4a7cfd52fcea22..00000000000000 --- a/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add test for nested queues when using ``multiprocessing`` shared objects -``AutoProxy[Queue]`` inside ``ListProxy`` and ``DictProxy`` diff --git a/Misc/NEWS.d/next/Tests/2020-10-25-19-20-26.bpo-35753.2LT-hO.rst b/Misc/NEWS.d/next/Tests/2020-10-25-19-20-26.bpo-35753.2LT-hO.rst deleted file mode 100644 index eddfc25906da9e..00000000000000 --- a/Misc/NEWS.d/next/Tests/2020-10-25-19-20-26.bpo-35753.2LT-hO.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash in doctest when doctest parses modules that include unwrappable -functions by skipping those functions. diff --git a/Misc/NEWS.d/next/Tests/2021-05-04-18-10-57.bpo-42083.EMS2TK.rst b/Misc/NEWS.d/next/Tests/2021-05-04-18-10-57.bpo-42083.EMS2TK.rst deleted file mode 100644 index 8457508237a888..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-05-04-18-10-57.bpo-42083.EMS2TK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add test to check that ``PyStructSequence_NewType`` accepts a -``PyStructSequence_Desc`` with ``doc`` field set to ``NULL``. diff --git a/Misc/NEWS.d/next/Tests/2021-05-07-15-46-04.bpo-31904.8dk3la.rst b/Misc/NEWS.d/next/Tests/2021-05-07-15-46-04.bpo-31904.8dk3la.rst deleted file mode 100644 index 334878f91ddacd..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-05-07-15-46-04.bpo-31904.8dk3la.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore error string case in test_file_not_exists(). diff --git a/Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst b/Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst deleted file mode 100644 index a646acf8e44431..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add test_frozenmain to test_embed to test the :c:func:`Py_FrozenMain` C -function. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-06-02-17-41-42.bpo-43921.xP7yZ4.rst b/Misc/NEWS.d/next/Tests/2021-06-02-17-41-42.bpo-43921.xP7yZ4.rst deleted file mode 100644 index 83146c78524671..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-06-02-17-41-42.bpo-43921.xP7yZ4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix test_pha_required_nocert() of test_ssl: catch two more EOF cases (when -the ``recv()`` method returns an empty string). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-06-03-03-29-34.bpo-43921.nwH1FS.rst b/Misc/NEWS.d/next/Tests/2021-06-03-03-29-34.bpo-43921.nwH1FS.rst deleted file mode 100644 index 30e0fadd661258..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-06-03-03-29-34.bpo-43921.nwH1FS.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix test_ssl.test_wrong_cert_tls13(): use ``suppress_ragged_eofs=False``, -since ``read()`` can raise :exc:`ssl.SSLEOFError` on Windows. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-06-09-15-32-05.bpo-44364.zu9Zee.rst b/Misc/NEWS.d/next/Tests/2021-06-09-15-32-05.bpo-44364.zu9Zee.rst deleted file mode 100644 index 12b80e8e6533b8..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-06-09-15-32-05.bpo-44364.zu9Zee.rst +++ /dev/null @@ -1 +0,0 @@ -Add non integral tests for :func:`math.sqrt` function. diff --git a/Misc/NEWS.d/next/Tests/2021-06-10-11-19-43.bpo-44363.-K9jD0.rst b/Misc/NEWS.d/next/Tests/2021-06-10-11-19-43.bpo-44363.-K9jD0.rst deleted file mode 100644 index 28468cbd2b682b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-06-10-11-19-43.bpo-44363.-K9jD0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Account for address sanitizer in test_capi. test_capi now passes when run -GCC address sanitizer. diff --git a/Misc/NEWS.d/next/Tests/2021-06-18-15-19-35.bpo-44451.aj5pqE.rst b/Misc/NEWS.d/next/Tests/2021-06-18-15-19-35.bpo-44451.aj5pqE.rst deleted file mode 100644 index 0f635cfe18d141..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-06-18-15-19-35.bpo-44451.aj5pqE.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reset ``DeprecationWarning`` filters in -``test.test_importlib.test_metadata_api.APITests.test_entry_points_by_index`` -to avoid ``StopIteration`` error if ``DeprecationWarnings`` are ignored. diff --git a/Misc/NEWS.d/next/Tests/2021-06-21-17-53-41.bpo-44287.YON57s.rst b/Misc/NEWS.d/next/Tests/2021-06-21-17-53-41.bpo-44287.YON57s.rst deleted file mode 100644 index 66b3afe139aa8d..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-06-21-17-53-41.bpo-44287.YON57s.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix asyncio test_popen() of test_windows_utils by using a longer timeout. -Use military grade battle-tested :data:`test.support.SHORT_TIMEOUT` timeout -rather than a hardcoded timeout of 10 seconds: it's 30 seconds by default, but -it is made longer on slow buildbots. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-06-26-18-37-36.bpo-44515.e9fO6f.rst b/Misc/NEWS.d/next/Tests/2021-06-26-18-37-36.bpo-44515.e9fO6f.rst deleted file mode 100644 index d2867b6e89f872..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-06-26-18-37-36.bpo-44515.e9fO6f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Adjust recently added contextlib tests to avoid assuming the use of a -refcounted GC diff --git a/Misc/NEWS.d/next/Tests/2021-07-16-14-02-33.bpo-44647.5LzqIy.rst b/Misc/NEWS.d/next/Tests/2021-07-16-14-02-33.bpo-44647.5LzqIy.rst deleted file mode 100644 index e4b2be2b409997..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-07-16-14-02-33.bpo-44647.5LzqIy.rst +++ /dev/null @@ -1,4 +0,0 @@ -Added a permanent Unicode-valued environment variable to regression tests to -ensure they handle this use case in the future. If your test environment -breaks because of that, report a bug to us, and temporarily set -PYTHONREGRTEST_UNICODE_GUARD=0 in your test environment. diff --git a/Misc/NEWS.d/next/Tests/2021-07-17-11-41-20.bpo-42095.kCB7oj.rst b/Misc/NEWS.d/next/Tests/2021-07-17-11-41-20.bpo-42095.kCB7oj.rst deleted file mode 100644 index bf7bc5b2ff4491..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-07-17-11-41-20.bpo-42095.kCB7oj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added interop tests for Apple plists: generate plist files with Python -plistlib and parse with Apple plutil; and the other way round. diff --git a/Misc/NEWS.d/next/Tests/2021-07-22-16-38-39.bpo-44708.SYNaac.rst b/Misc/NEWS.d/next/Tests/2021-07-22-16-38-39.bpo-44708.SYNaac.rst deleted file mode 100644 index 8b26c4de64b983..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-07-22-16-38-39.bpo-44708.SYNaac.rst +++ /dev/null @@ -1,2 +0,0 @@ -Regression tests, when run with -w, are now re-running only the affected -test methods instead of re-running the entire test file. diff --git a/Misc/NEWS.d/next/Tests/2021-07-24-20-09-15.bpo-44734.KKsNOV.rst b/Misc/NEWS.d/next/Tests/2021-07-24-20-09-15.bpo-44734.KKsNOV.rst deleted file mode 100644 index 94e9ce08f4bf6b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-07-24-20-09-15.bpo-44734.KKsNOV.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed floating point precision issue in turtle tests. diff --git a/Misc/NEWS.d/next/Tests/2021-08-06-00-07-15.bpo-40928.aIwb6G.rst b/Misc/NEWS.d/next/Tests/2021-08-06-00-07-15.bpo-40928.aIwb6G.rst deleted file mode 100644 index c9a5e1b01e58aa..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-06-00-07-15.bpo-40928.aIwb6G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Notify users running test_decimal regression tests on macOS of potential -harmless "malloc can't allocate region" messages spewed by test_decimal. diff --git a/Misc/NEWS.d/next/Tests/2021-08-06-18-36-04.bpo-44852.sUL8YX.rst b/Misc/NEWS.d/next/Tests/2021-08-06-18-36-04.bpo-44852.sUL8YX.rst deleted file mode 100644 index 41b5c2fb51c5c7..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-06-18-36-04.bpo-44852.sUL8YX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ability to wholesale silence DeprecationWarnings while running the -regression test suite. diff --git a/Misc/NEWS.d/next/Tests/2021-08-13-12-11-06.bpo-44891.T9_mBT.rst b/Misc/NEWS.d/next/Tests/2021-08-13-12-11-06.bpo-44891.T9_mBT.rst deleted file mode 100644 index 2f83389ec15855..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-13-12-11-06.bpo-44891.T9_mBT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Tests were added to clarify :func:`id` is preserved when ``obj * 1`` is used -on :class:`str` and :class:`bytes` objects. Patch by Nikita Sobolev. diff --git a/Misc/NEWS.d/next/Tests/2021-08-18-18-30-12.bpo-44949.VE5ENv.rst b/Misc/NEWS.d/next/Tests/2021-08-18-18-30-12.bpo-44949.VE5ENv.rst deleted file mode 100644 index 7fdf1810b165e5..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-18-18-30-12.bpo-44949.VE5ENv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix auto history tests of test_readline: sometimes, the newline character is -not written at the end, so don't expect it in the output. diff --git a/Misc/NEWS.d/next/Tests/2021-08-26-14-20-44.bpo-45011.mQZdXU.rst b/Misc/NEWS.d/next/Tests/2021-08-26-14-20-44.bpo-45011.mQZdXU.rst deleted file mode 100644 index 64e701e2f29f49..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-26-14-20-44.bpo-45011.mQZdXU.rst +++ /dev/null @@ -1,3 +0,0 @@ -Made tests relying on the :mod:`_asyncio` C extension module optional to -allow running on alternative Python implementations. Patch by Serhiy -Storchaka. diff --git a/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst b/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst deleted file mode 100644 index 43ce68bef46099..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst +++ /dev/null @@ -1 +0,0 @@ -Add calls of :func:`gc.collect` in tests to support PyPy. diff --git a/Misc/NEWS.d/next/Tests/2021-08-30-11-54-14.bpo-45042.QMz3X8.rst b/Misc/NEWS.d/next/Tests/2021-08-30-11-54-14.bpo-45042.QMz3X8.rst deleted file mode 100644 index e2c0dffced96e7..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-30-11-54-14.bpo-45042.QMz3X8.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes that test classes decorated with ``@hashlib_helper.requires_hashdigest`` were skipped all the time. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Tests/2021-09-01-17-17-44.bpo-44895.kV7H77.rst b/Misc/NEWS.d/next/Tests/2021-09-01-17-17-44.bpo-44895.kV7H77.rst deleted file mode 100644 index 038466f8d6a4f3..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-01-17-17-44.bpo-44895.kV7H77.rst +++ /dev/null @@ -1,5 +0,0 @@ -libregrtest now clears the type cache later to reduce the risk of false alarm -when checking for reference leaks. Previously, the type cache was cleared too -early and libregrtest raised a false alarm about reference leaks under very -specific conditions. -Patch by Irit Katriel and Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-09-06-19-00-29.bpo-45052.yrOK3J.rst b/Misc/NEWS.d/next/Tests/2021-09-06-19-00-29.bpo-45052.yrOK3J.rst deleted file mode 100644 index 5c2e4f3e0e67bc..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-06-19-00-29.bpo-45052.yrOK3J.rst +++ /dev/null @@ -1,7 +0,0 @@ -``WithProcessesTestSharedMemory.test_shared_memory_basics`` test was -ignored, because ``self.assertEqual(sms.size, sms2.size)`` line was failing. -It is now removed and test is unskipped. - -The main motivation for this line to be removed from the test is that the -``size`` of ``SharedMemory`` is not ever guaranteed to be the same. It is -decided by the platform. diff --git a/Misc/NEWS.d/next/Tests/2021-09-08-13-01-37.bpo-44860.qXd0kx.rst b/Misc/NEWS.d/next/Tests/2021-09-08-13-01-37.bpo-44860.qXd0kx.rst deleted file mode 100644 index 153a9c55733fbb..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-08-13-01-37.bpo-44860.qXd0kx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update ``test_sysconfig.test_user_similar()`` for the posix_user scheme: -``platlib`` doesn't use :data:`sys.platlibdir`. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-09-13-00-28-17.bpo-45156.8oomV3.rst b/Misc/NEWS.d/next/Tests/2021-09-13-00-28-17.bpo-45156.8oomV3.rst deleted file mode 100644 index b2094b5765331c..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-13-00-28-17.bpo-45156.8oomV3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes infinite loop on :func:`unittest.mock.seal` of mocks created by -:func:`~unittest.create_autospec`. diff --git a/Misc/NEWS.d/next/Tests/2021-09-14-13-16-18.bpo-45195.EyQR1G.rst b/Misc/NEWS.d/next/Tests/2021-09-14-13-16-18.bpo-45195.EyQR1G.rst deleted file mode 100644 index 16a1f4440483ca..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-14-13-16-18.bpo-45195.EyQR1G.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix test_readline.test_nonascii(): sometimes, the newline character is not -written at the end, so don't expect it in the output. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-09-14-14-54-04.bpo-45185.qFx5I6.rst b/Misc/NEWS.d/next/Tests/2021-09-14-14-54-04.bpo-45185.qFx5I6.rst deleted file mode 100644 index e723f24759e9e9..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-14-14-54-04.bpo-45185.qFx5I6.rst +++ /dev/null @@ -1 +0,0 @@ -Enables ``TestEnumerations`` test cases in ``test_ssl`` suite. diff --git a/Misc/NEWS.d/next/Tests/2021-09-15-23-32-39.bpo-45209.55ntL5.rst b/Misc/NEWS.d/next/Tests/2021-09-15-23-32-39.bpo-45209.55ntL5.rst deleted file mode 100644 index 4c3bed0983b899..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-15-23-32-39.bpo-45209.55ntL5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``UserWarning: resource_tracker`` warning in -``_test_multiprocessing._TestSharedMemory.test_shared_memory_cleaned_after_process_termination`` diff --git a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst deleted file mode 100644 index b50eb32b3faa8e..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``test_multiprocessing_fork`` failure due to ``test_logging`` and -``sys.modules`` manipulation. diff --git a/Misc/NEWS.d/next/Tests/2021-10-07-13-11-45.bpo-45400.h3iT7V.rst b/Misc/NEWS.d/next/Tests/2021-10-07-13-11-45.bpo-45400.h3iT7V.rst new file mode 100644 index 00000000000000..61b6653320dfc1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-10-07-13-11-45.bpo-45400.h3iT7V.rst @@ -0,0 +1,3 @@ +Fix test_name_error_suggestions_do_not_trigger_for_too_many_locals() of +test_exceptions if a directory name contains "a1" (like "Python-3.11.0a1"): +use a stricter regular expression. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-10-07-13-27-12.bpo-45403.7QiDvw.rst b/Misc/NEWS.d/next/Tests/2021-10-07-13-27-12.bpo-45403.7QiDvw.rst new file mode 100644 index 00000000000000..e4d17099609801 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-10-07-13-27-12.bpo-45403.7QiDvw.rst @@ -0,0 +1,2 @@ +Fix test_sys.test_stdlib_dir() when Python is built outside the source tree: +compare normalized paths. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-10-07-13-43-01.bpo-45402.jlQvep.rst b/Misc/NEWS.d/next/Tests/2021-10-07-13-43-01.bpo-45402.jlQvep.rst new file mode 100644 index 00000000000000..d8c655306584c7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-10-07-13-43-01.bpo-45402.jlQvep.rst @@ -0,0 +1,3 @@ +Fix test_tools.test_sundry() when Python is built out of tree: fix how +the freeze_modules.py tool locates the _freeze_module program. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-10-08-14-03-20.bpo-45410.Ex9xe2.rst b/Misc/NEWS.d/next/Tests/2021-10-08-14-03-20.bpo-45410.Ex9xe2.rst new file mode 100644 index 00000000000000..5f7b8bf1d6c66a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-10-08-14-03-20.bpo-45410.Ex9xe2.rst @@ -0,0 +1,4 @@ +When libregrtest spawns a worker process, stderr is now written into stdout +to keep messages order. Use a single pipe for stdout and stderr, rather than +two pipes. Previously, messages were out of order which made analysis of +buildbot logs harder Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst b/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst new file mode 100644 index 00000000000000..b0d1b686392ef8 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst @@ -0,0 +1,2 @@ +Add more test cases for `@functools.singledispatchmethod` when combined with +`@classmethod` or `@staticmethod`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst b/Misc/NEWS.d/next/Tools-Demos/2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst deleted file mode 100644 index c64c5488bfaa1b..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst +++ /dev/null @@ -1 +0,0 @@ -Added support for variadic positional parameters in Argument Clinic. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-05-08-13-57-00.bpo-44074.F09kCK.rst b/Misc/NEWS.d/next/Tools-Demos/2021-05-08-13-57-00.bpo-44074.F09kCK.rst deleted file mode 100644 index 8bccb080a54186..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-05-08-13-57-00.bpo-44074.F09kCK.rst +++ /dev/null @@ -1 +0,0 @@ -Make patchcheck automatically detect the correct base branch name (previously it was hardcoded to 'master') \ No newline at end of file diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-07-01-22-21-25.bpo-43425.t65len.rst b/Misc/NEWS.d/next/Tools-Demos/2021-07-01-22-21-25.bpo-43425.t65len.rst deleted file mode 100644 index b9ce6c467f90b9..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-07-01-22-21-25.bpo-43425.t65len.rst +++ /dev/null @@ -1,3 +0,0 @@ -Removed the 'test2to3' demo project that demonstrated using lib2to3 -to support Python 2.x and Python 3.x from a single source in -a distutils package. Patch by Dong-hee Na diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-08-22-11-45-31.bpo-44978.QupKV3.rst b/Misc/NEWS.d/next/Tools-Demos/2021-08-22-11-45-31.bpo-44978.QupKV3.rst deleted file mode 100644 index c7a844c1230e2e..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-08-22-11-45-31.bpo-44978.QupKV3.rst +++ /dev/null @@ -1 +0,0 @@ -Allow the Argument Clinic tool to handle ``__complex__`` special methods. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-08-26-11-57-31.bpo-44967.UT1RMV.rst b/Misc/NEWS.d/next/Tools-Demos/2021-08-26-11-57-31.bpo-44967.UT1RMV.rst deleted file mode 100644 index 564a5c7d768943..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-08-26-11-57-31.bpo-44967.UT1RMV.rst +++ /dev/null @@ -1 +0,0 @@ -pydoc now returns a non-zero status code when a module cannot be found. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-09-14-11-44-26.bpo-44786.DU0LC0.rst b/Misc/NEWS.d/next/Tools-Demos/2021-09-14-11-44-26.bpo-44786.DU0LC0.rst deleted file mode 100644 index 96ebf2c33cc5ae..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-09-14-11-44-26.bpo-44786.DU0LC0.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a warning in regular expression in the c-analyzer script. diff --git a/Misc/NEWS.d/next/Windows/2020-04-13-15-20-28.bpo-40263.1KKEbu.rst b/Misc/NEWS.d/next/Windows/2020-04-13-15-20-28.bpo-40263.1KKEbu.rst deleted file mode 100644 index 0c31606d4928c8..00000000000000 --- a/Misc/NEWS.d/next/Windows/2020-04-13-15-20-28.bpo-40263.1KKEbu.rst +++ /dev/null @@ -1,3 +0,0 @@ -This is a follow-on bug from https://bugs.python.org/issue26903. Once that -is applied we run into an off-by-one assertion problem. The assert was not -correct. diff --git a/Misc/NEWS.d/next/Windows/2021-01-01-21-21-03.bpo-42686.G_f-TC.rst b/Misc/NEWS.d/next/Windows/2021-01-01-21-21-03.bpo-42686.G_f-TC.rst deleted file mode 100644 index 2fcf6e946ec838..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-01-01-21-21-03.bpo-42686.G_f-TC.rst +++ /dev/null @@ -1 +0,0 @@ -Build :mod:`sqlite3` with math functions enabled. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Windows/2021-04-15-01-23-10.bpo-43851.qgU0gy.rst b/Misc/NEWS.d/next/Windows/2021-04-15-01-23-10.bpo-43851.qgU0gy.rst new file mode 100644 index 00000000000000..e9555d56754cc5 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-04-15-01-23-10.bpo-43851.qgU0gy.rst @@ -0,0 +1 @@ +Build SQLite ``SQLITE_OMIT_AUTOINIT`` on Windows. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Windows/2021-06-06-16-36-13.bpo-41299.Rg-vb_.rst b/Misc/NEWS.d/next/Windows/2021-06-06-16-36-13.bpo-41299.Rg-vb_.rst deleted file mode 100644 index 71f700ffa1553e..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-06-06-16-36-13.bpo-41299.Rg-vb_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix 16ms jitter when using timeouts in :mod:`threading`, such as with :meth:`threading.Lock.acquire` or :meth:`threading.Condition.wait`. diff --git a/Misc/NEWS.d/next/Windows/2021-07-07-21-07-18.bpo-44582.4Mm6Hh.rst b/Misc/NEWS.d/next/Windows/2021-07-07-21-07-18.bpo-44582.4Mm6Hh.rst deleted file mode 100644 index f79c88931c5310..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-07-07-21-07-18.bpo-44582.4Mm6Hh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Accelerate speed of :mod:`mimetypes` initialization using a native -implementation of the registry scan. diff --git a/Misc/NEWS.d/next/Windows/2021-07-13-15-32-49.bpo-44572.gXvhDc.rst b/Misc/NEWS.d/next/Windows/2021-07-13-15-32-49.bpo-44572.gXvhDc.rst deleted file mode 100644 index 6e074c59b8445b..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-07-13-15-32-49.bpo-44572.gXvhDc.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid consuming standard input in the :mod:`platform` module \ No newline at end of file diff --git a/Misc/NEWS.d/next/Windows/2021-08-06-10-11-07.bpo-44848.ib3Jcz.rst b/Misc/NEWS.d/next/Windows/2021-08-06-10-11-07.bpo-44848.ib3Jcz.rst deleted file mode 100644 index 6eadfedda865e9..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-08-06-10-11-07.bpo-44848.ib3Jcz.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade Windows installer to use SQLite 3.36.0. diff --git a/Misc/NEWS.d/next/Windows/2021-08-27-23-50-02.bpo-45007.NIBlVG.rst b/Misc/NEWS.d/next/Windows/2021-08-27-23-50-02.bpo-45007.NIBlVG.rst deleted file mode 100644 index fa076ee4c8b8ad..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-08-27-23-50-02.bpo-45007.NIBlVG.rst +++ /dev/null @@ -1 +0,0 @@ -Update to OpenSSL 1.1.1l in Windows build diff --git a/Misc/NEWS.d/next/Windows/2021-09-03-18-05-21.bpo-45022.bgpD_r.rst b/Misc/NEWS.d/next/Windows/2021-09-03-18-05-21.bpo-45022.bgpD_r.rst deleted file mode 100644 index 8c19faad771763..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-09-03-18-05-21.bpo-45022.bgpD_r.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows release to include libffi 3.4.2 diff --git a/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst b/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst new file mode 100644 index 00000000000000..007ee87195d6e3 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst @@ -0,0 +1,4 @@ +venv now warns when the created environment may need to be accessed at a +different path, due to redirections, links or junctions. It also now +correctly installs or upgrades components when the alternate path is +required. diff --git a/Misc/NEWS.d/next/macOS/2021-03-29-21-11-23.bpo-34932.f3Hdyd.rst b/Misc/NEWS.d/next/macOS/2021-03-29-21-11-23.bpo-34932.f3Hdyd.rst deleted file mode 100644 index d3a52c9fc37b5f..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-03-29-21-11-23.bpo-34932.f3Hdyd.rst +++ /dev/null @@ -1 +0,0 @@ -Add socket.TCP_KEEPALIVE support for macOS. Patch by Shane Harvey. diff --git a/Misc/NEWS.d/next/macOS/2021-05-24-21-15-41.bpo-43109.npKJ9c.rst b/Misc/NEWS.d/next/macOS/2021-05-24-21-15-41.bpo-43109.npKJ9c.rst deleted file mode 100644 index bb4d24fa95513f..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-05-24-21-15-41.bpo-43109.npKJ9c.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow --with-lto configure option to work with Apple-supplied Xcode or -Command Line Tools. diff --git a/Misc/NEWS.d/next/macOS/2021-07-12-15-42-02.bpo-41972.yUjE8j.rst b/Misc/NEWS.d/next/macOS/2021-07-12-15-42-02.bpo-41972.yUjE8j.rst deleted file mode 100644 index 6c70c07669f9c1..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-07-12-15-42-02.bpo-41972.yUjE8j.rst +++ /dev/null @@ -1,2 +0,0 @@ -The framework build's user header path in sysconfig is changed to add a -'pythonX.Y' component to match distutils's behavior. diff --git a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst deleted file mode 100644 index b1e878d1ee44af..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst +++ /dev/null @@ -1,5 +0,0 @@ - :meth:`ctypes.util.find_library` now works correctly on macOS 11 Big Sur - even if Python is built on an older version of macOS. Previously, when - built on older macOS systems, ``find_library`` was not able to find - macOS system libraries when running on Big Sur due to changes in - how system libraries are stored. diff --git a/Misc/NEWS.d/next/macOS/2021-08-06-10-08-41.bpo-44848.0uYXsE.rst b/Misc/NEWS.d/next/macOS/2021-08-06-10-08-41.bpo-44848.0uYXsE.rst deleted file mode 100644 index 7e9c41a8e9a480..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-08-06-10-08-41.bpo-44848.0uYXsE.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use SQLite 3.36.0. diff --git a/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst deleted file mode 100644 index 29a6ff92554e1c..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -When building CPython on macOS with ``./configure ---with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now -quadrupled to allow for the entire test suite to pass. diff --git a/Misc/NEWS.d/next/macOS/2021-08-30-00-04-10.bpo-45007.pixqUB.rst b/Misc/NEWS.d/next/macOS/2021-08-30-00-04-10.bpo-45007.pixqUB.rst deleted file mode 100644 index e4f1ac6de313d4..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-08-30-00-04-10.bpo-45007.pixqUB.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer builds to use OpenSSL 1.1.1l. diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index 0dcf2dd9d4d3f8..c662c3ce16f109 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -285,7 +285,7 @@ macro Py_UNBLOCK_THREADS macro Py_END_ALLOW_THREADS added 3.2 -# The following were added in PC/python3.def in the intial stable ABI commit, +# The following were added in PC/python3.def in the initial stable ABI commit, # 4d0d471a8031de90a2b1ce99c4ac4780e60b3bc9, # and later amendments in 3.2: # 0d012f284be829c6217f60523db0e1671b7db9d9 @@ -2149,6 +2149,8 @@ function PyType_GetName added 3.11 function PyType_GetQualName added 3.11 +data PyStructSequence_UnnamedField + added 3.11 function PyStructSequence_FromModuleAndDesc added 3.11 diff --git a/Modules/Setup b/Modules/Setup index 5e26472446677a..2e44ec24dacdc3 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -171,8 +171,7 @@ _symtable symtablemodule.c #math mathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # math library functions, e.g. sin() #_contextvars _contextvarsmodule.c # Context Variables #_struct -DPy_BUILD_CORE_MODULE _struct.c # binary structure packing/unpacking -#_weakref _weakref.c # basic weak reference support -#_testcapi _testcapimodule.c # Python C API test module +#_testcapi _testcapimodule.c # Python C API test module; CANNOT be statically compiled! #_testinternalcapi _testinternalcapi.c -I$(srcdir)/Include/internal -DPy_BUILD_CORE_MODULE # Python internal C API test module #_random _randommodule.c -DPy_BUILD_CORE_MODULE # Random number generator #_elementtree -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI _elementtree.c # elementtree accelerator @@ -185,6 +184,9 @@ _symtable symtablemodule.c #_json -I$(srcdir)/Include/internal -DPy_BUILD_CORE_BUILTIN _json.c # _json speedups #_statistics _statisticsmodule.c # statistics accelerator #_typing _typingmodule.c # typing accelerator +#_lsprof _lsprof.c rotatingtree.c # cProfile accelerators +#_opcode _opcode.c +#_queue _queuemodule.c -DPy_BUILD_CORE_MODULE #unicodedata unicodedata.c -DPy_BUILD_CORE_BUILTIN # static Unicode character database @@ -197,6 +199,7 @@ _symtable symtablemodule.c #spwd spwdmodule.c # spwd(3) #grp grpmodule.c # grp(3) #select selectmodule.c # select(2); not on ancient System V +#ossaudiodev ossaudiodev.c # Memory-mapped files (also works on Win32). #mmap mmapmodule.c @@ -212,19 +215,23 @@ _symtable symtablemodule.c # OPENSSL=/path/to/openssl/directory # _ssl _ssl.c \ # -I$(OPENSSL)/include -L$(OPENSSL)/lib \ -# -lssl -lcrypto +# -lssl -lcrypto \ +# -DPy_BUILD_CORE_BUILTIN #_hashlib _hashopenssl.c \ # -I$(OPENSSL)/include -L$(OPENSSL)/lib \ -# -lcrypto +# -lcrypto \ +# -DPy_BUILD_CORE_BUILTIN # To statically link OpenSSL: # _ssl _ssl.c \ # -I$(OPENSSL)/include -L$(OPENSSL)/lib \ # -l:libssl.a -Wl,--exclude-libs,libssl.a \ -# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a +# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a \ +# -DPy_BUILD_CORE_BUILTIN #_hashlib _hashopenssl.c \ # -I$(OPENSSL)/include -L$(OPENSSL)/lib \ -# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a +# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a \ +# -DPy_BUILD_CORE_BUILTIN # The crypt module is now disabled by default because it breaks builds # on many systems (where -lcrypt is needed), e.g. Linux (I believe). @@ -255,18 +262,22 @@ _symtable symtablemodule.c # The _md5 module implements the RSA Data Security, Inc. MD5 # Message-Digest Algorithm, described in RFC 1321. -#_md5 md5module.c +#_md5 md5module.c -DPy_BUILD_CORE_BUILTIN # The _sha module implements the SHA checksum algorithms. # (NIST's Secure Hash Algorithms.) -#_sha1 sha1module.c +#_sha1 sha1module.c -DPy_BUILD_CORE_BUILTIN #_sha256 sha256module.c -DPy_BUILD_CORE_BUILTIN #_sha512 sha512module.c -DPy_BUILD_CORE_BUILTIN -#_sha3 _sha3/sha3module.c +#_sha3 _sha3/sha3module.c -DPy_BUILD_CORE_BUILTIN # _blake module -#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c +#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c -DPy_BUILD_CORE_BUILTIN + +# Compression +#_bz2 _bz2module.c -lbz2 +#_lzma _lzmamodule.c -llzma # The _tkinter module. # @@ -339,7 +350,7 @@ _symtable symtablemodule.c # Helper module for various ascii-encoders -#binascii binascii.c +#binascii binascii.c -DPy_BUILD_CORE_MODULE # Andrew Kuchling's zlib module. # This require zlib 1.1.3 (or later). @@ -369,6 +380,17 @@ _symtable symtablemodule.c # Another example -- the 'xxsubtype' module shows C-level subtyping in action xxsubtype xxsubtype.c +# Limited API examples +#xxlimited xxlimited.c +#xxlimited_35 xxlimited_35.c + +# For testing +#_xxsubinterpreters _xxsubinterpretersmodule.c +#_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c +#_testbuffer _testbuffer.c +#_testimportmultiple _testimportmultiple.c +#_testmultiphase _testmultiphase.c + # Uncommenting the following line tells makesetup that all following modules # are not built (see above for more detail). # diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index ecc73d1ca8bf0f..adc5ff9f796cca 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_pyerrors.h" // _PyErr_ClearExcState() +#include "pycore_pystate.h" // _PyThreadState_GET() #include // offsetof() @@ -225,7 +226,7 @@ get_running_loop(PyObject **loop) { PyObject *rl; - PyThreadState *ts = PyThreadState_Get(); + PyThreadState *ts = _PyThreadState_GET(); uint64_t ts_id = PyThreadState_GetID(ts); if (ts_id == cached_running_holder_tsid && cached_running_holder != NULL) { // Fast path, check the cache. @@ -287,7 +288,7 @@ set_running_loop(PyObject *loop) { PyObject *ts_dict = NULL; - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate != NULL) { ts_dict = _PyThreadState_GetDict(tstate); // borrowed } @@ -3239,6 +3240,9 @@ new_running_loop_holder(PyObject *loop) static void PyRunningLoopHolder_tp_dealloc(PyRunningLoopHolder *rl) { + if (cached_running_holder == (PyObject *)rl) { + cached_running_holder = NULL; + } Py_CLEAR(rl->rl_loop); PyObject_Free(rl); } diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index 8b0d60d02ac7c8..b16324e8f7aeb8 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -14,7 +14,7 @@ */ #include "Python.h" -#include "pystrhex.h" +#include "pycore_strhex.h" // _Py_strhex() #include "../hashlib.h" #include "blake2ns.h" diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 7ab3917aa1a655..6b31a363ea27b9 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -14,7 +14,7 @@ */ #include "Python.h" -#include "pystrhex.h" +#include "pycore_strhex.h" // _Py_strhex() #include "../hashlib.h" #include "blake2ns.h" diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 2e8cb97fe77c92..50afc097b35026 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -489,34 +489,40 @@ _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data, _codecs.unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None + final: bool(accept={int}) = True / [clinic start generated code]*/ static PyObject * _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors) -/*[clinic end generated code: output=3ca3c917176b82ab input=8328081a3a569bd6]*/ + const char *errors, int final) +/*[clinic end generated code: output=b284f97b12c635ee input=6154f039a9f7c639]*/ { - PyObject *decoded = PyUnicode_DecodeUnicodeEscape(data->buf, data->len, - errors); - return codec_tuple(decoded, data->len); + Py_ssize_t consumed = data->len; + PyObject *decoded = _PyUnicode_DecodeUnicodeEscapeStateful(data->buf, data->len, + errors, + final ? NULL : &consumed); + return codec_tuple(decoded, consumed); } /*[clinic input] _codecs.raw_unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None + final: bool(accept={int}) = True / [clinic start generated code]*/ static PyObject * _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors) -/*[clinic end generated code: output=c98eeb56028070a6 input=d2f5159ce3b3392f]*/ + const char *errors, int final) +/*[clinic end generated code: output=11dbd96301e2879e input=2d166191beb3235a]*/ { - PyObject *decoded = PyUnicode_DecodeRawUnicodeEscape(data->buf, data->len, - errors); - return codec_tuple(decoded, data->len); + Py_ssize_t consumed = data->len; + PyObject *decoded = _PyUnicode_DecodeRawUnicodeEscapeStateful(data->buf, data->len, + errors, + final ? NULL : &consumed); + return codec_tuple(decoded, consumed); } /*[clinic input] diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 01522cd4945875..c6de9636032c0a 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetZero() #include "structmember.h" // PyMemberDef @@ -2010,7 +2011,7 @@ defdict_missing(defdictobject *dd, PyObject *key) Py_DECREF(tup); return NULL; } - value = _PyObject_CallNoArg(factory); + value = _PyObject_CallNoArgs(factory); if (value == NULL) return value; if (PyObject_SetItem((PyObject *)dd, key, value) < 0) { diff --git a/Modules/_csv.c b/Modules/_csv.c index 3109fd16bc744b..1c2f504ea5c097 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -14,6 +14,9 @@ module instead. #include "structmember.h" // PyMemberDef #include +#define NOT_SET ((Py_UCS4)-1) +#define EOL ((Py_UCS4)-2) + typedef struct { PyObject *error_obj; /* CSV exception */ @@ -153,9 +156,9 @@ get_dialect_from_registry(PyObject *name_obj, _csvstate *module_state) } static PyObject * -get_nullchar_as_None(Py_UCS4 c) +get_char_or_None(Py_UCS4 c) { - if (c == '\0') { + if (c == NOT_SET) { Py_RETURN_NONE; } else @@ -172,19 +175,19 @@ Dialect_get_lineterminator(DialectObj *self, void *Py_UNUSED(ignored)) static PyObject * Dialect_get_delimiter(DialectObj *self, void *Py_UNUSED(ignored)) { - return get_nullchar_as_None(self->delimiter); + return get_char_or_None(self->delimiter); } static PyObject * Dialect_get_escapechar(DialectObj *self, void *Py_UNUSED(ignored)) { - return get_nullchar_as_None(self->escapechar); + return get_char_or_None(self->escapechar); } static PyObject * Dialect_get_quotechar(DialectObj *self, void *Py_UNUSED(ignored)) { - return get_nullchar_as_None(self->quotechar); + return get_char_or_None(self->quotechar); } static PyObject * @@ -229,35 +232,66 @@ _set_int(const char *name, int *target, PyObject *src, int dflt) } static int -_set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) +_set_char_or_none(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) { - if (src == NULL) + if (src == NULL) { *target = dflt; + } else { - *target = '\0'; + *target = NOT_SET; if (src != Py_None) { - Py_ssize_t len; if (!PyUnicode_Check(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be string, not %.200s", name, + "\"%s\" must be string or None, not %.200s", name, Py_TYPE(src)->tp_name); return -1; } - len = PyUnicode_GetLength(src); - if (len > 1) { + Py_ssize_t len = PyUnicode_GetLength(src); + if (len < 0) { + return -1; + } + if (len != 1) { PyErr_Format(PyExc_TypeError, "\"%s\" must be a 1-character string", name); return -1; } /* PyUnicode_READY() is called in PyUnicode_GetLength() */ - if (len > 0) - *target = PyUnicode_READ_CHAR(src, 0); + *target = PyUnicode_READ_CHAR(src, 0); } } return 0; } +static int +_set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) +{ + if (src == NULL) { + *target = dflt; + } + else { + if (!PyUnicode_Check(src)) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be string, not %.200s", name, + Py_TYPE(src)->tp_name); + return -1; + } + Py_ssize_t len = PyUnicode_GetLength(src); + if (len < 0) { + return -1; + } + if (len != 1) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be a 1-character string", + name); + return -1; + } + /* PyUnicode_READY() is called in PyUnicode_GetLength() */ + *target = PyUnicode_READ_CHAR(src, 0); + } + return 0; +} + static int _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) { @@ -445,9 +479,9 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto err DIASET(_set_char, "delimiter", &self->delimiter, delimiter, ','); DIASET(_set_bool, "doublequote", &self->doublequote, doublequote, true); - DIASET(_set_char, "escapechar", &self->escapechar, escapechar, 0); + DIASET(_set_char_or_none, "escapechar", &self->escapechar, escapechar, NOT_SET); DIASET(_set_str, "lineterminator", &self->lineterminator, lineterminator, "\r\n"); - DIASET(_set_char, "quotechar", &self->quotechar, quotechar, '"'); + DIASET(_set_char_or_none, "quotechar", &self->quotechar, quotechar, '"'); DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL); DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, false); DIASET(_set_bool, "strict", &self->strict, strict, false); @@ -455,19 +489,19 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) /* validate options */ if (dialect_check_quoting(self->quoting)) goto err; - if (self->delimiter == 0) { + if (self->delimiter == NOT_SET) { PyErr_SetString(PyExc_TypeError, "\"delimiter\" must be a 1-character string"); goto err; } if (quotechar == Py_None && quoting == NULL) self->quoting = QUOTE_NONE; - if (self->quoting != QUOTE_NONE && self->quotechar == 0) { + if (self->quoting != QUOTE_NONE && self->quotechar == NOT_SET) { PyErr_SetString(PyExc_TypeError, "quotechar must be set if quoting enabled"); goto err; } - if (self->lineterminator == 0) { + if (self->lineterminator == NULL) { PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); goto err; } @@ -489,7 +523,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } /* Since dialect is now a heap type, it inherits pickling method for - * protocol 0 and 1 from object, therefore it needs to be overriden */ + * protocol 0 and 1 from object, therefore it needs to be overridden */ PyDoc_STRVAR(dialect_reduce_doc, "raises an exception to avoid pickling"); @@ -634,7 +668,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) switch (self->state) { case START_RECORD: /* start of record */ - if (c == '\0') + if (c == EOL) /* empty line - return [] */ break; else if (c == '\n' || c == '\r') { @@ -646,11 +680,11 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) /* fallthru */ case START_FIELD: /* expecting field */ - if (c == '\n' || c == '\r' || c == '\0') { + if (c == '\n' || c == '\r' || c == EOL) { /* save empty field - return [fields] */ if (parse_save_field(self) < 0) return -1; - self->state = (c == '\0' ? START_RECORD : EAT_CRNL); + self->state = (c == EOL ? START_RECORD : EAT_CRNL); } else if (c == dialect->quotechar && dialect->quoting != QUOTE_NONE) { @@ -686,7 +720,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) self->state = AFTER_ESCAPED_CRNL; break; } - if (c == '\0') + if (c == EOL) c = '\n'; if (parse_add_char(self, module_state, c) < 0) return -1; @@ -694,17 +728,17 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) break; case AFTER_ESCAPED_CRNL: - if (c == '\0') + if (c == EOL) break; /*fallthru*/ case IN_FIELD: /* in unquoted field */ - if (c == '\n' || c == '\r' || c == '\0') { + if (c == '\n' || c == '\r' || c == EOL) { /* end of line - return [fields] */ if (parse_save_field(self) < 0) return -1; - self->state = (c == '\0' ? START_RECORD : EAT_CRNL); + self->state = (c == EOL ? START_RECORD : EAT_CRNL); } else if (c == dialect->escapechar) { /* possible escaped character */ @@ -725,7 +759,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) case IN_QUOTED_FIELD: /* in quoted field */ - if (c == '\0') + if (c == EOL) ; else if (c == dialect->escapechar) { /* Possible escape character */ @@ -750,7 +784,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) break; case ESCAPE_IN_QUOTED_FIELD: - if (c == '\0') + if (c == EOL) c = '\n'; if (parse_add_char(self, module_state, c) < 0) return -1; @@ -772,11 +806,11 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) return -1; self->state = START_FIELD; } - else if (c == '\n' || c == '\r' || c == '\0') { + else if (c == '\n' || c == '\r' || c == EOL) { /* end of line - return [fields] */ if (parse_save_field(self) < 0) return -1; - self->state = (c == '\0' ? START_RECORD : EAT_CRNL); + self->state = (c == EOL ? START_RECORD : EAT_CRNL); } else if (!dialect->strict) { if (parse_add_char(self, module_state, c) < 0) @@ -795,7 +829,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) case EAT_CRNL: if (c == '\n' || c == '\r') ; - else if (c == '\0') + else if (c == EOL) self->state = START_RECORD; else { PyErr_Format(module_state->error_obj, @@ -873,12 +907,6 @@ Reader_iternext(ReaderObj *self) linelen = PyUnicode_GET_LENGTH(lineobj); while (linelen--) { c = PyUnicode_READ(kind, data, pos); - if (c == '\0') { - Py_DECREF(lineobj); - PyErr_Format(module_state->error_obj, - "line contains NUL"); - goto err; - } if (parse_process_char(self, module_state, c) < 0) { Py_DECREF(lineobj); goto err; @@ -886,7 +914,7 @@ Reader_iternext(ReaderObj *self) pos++; } Py_DECREF(lineobj); - if (parse_process_char(self, module_state, 0) < 0) + if (parse_process_char(self, module_state, EOL) < 0) goto err; } while (self->state != START_RECORD); @@ -1091,7 +1119,7 @@ join_append_data(WriterObj *self, unsigned int field_kind, const void *field_dat *quoted = 1; } if (want_escape) { - if (!dialect->escapechar) { + if (dialect->escapechar == NOT_SET) { PyErr_Format(self->error_obj, "need to escape, but no escapechar set"); return -1; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 1d6358c9109824..f8940fdbed2424 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -102,11 +102,16 @@ bytes(cdata) #define PY_SSIZE_T_CLEAN #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "structmember.h" // PyMemberDef #include #ifdef MS_WIN32 -#include #include #ifndef IS_INTRESOURCE #define IS_INTRESOURCE(x) (((size_t)(x) >> 16) == 0) @@ -216,7 +221,7 @@ PyDict_SetItemProxy(PyObject *dict, PyObject *key, PyObject *item) PyObject *proxy; int result; - obj = _PyObject_CallNoArg((PyObject *)&DictRemover_Type); + obj = _PyObject_CallNoArgs((PyObject *)&DictRemover_Type); if (obj == NULL) return -1; @@ -506,7 +511,7 @@ StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isSt return NULL; } - dict = (StgDictObject *)_PyObject_CallNoArg((PyObject *)&PyCStgDict_Type); + dict = (StgDictObject *)_PyObject_CallNoArgs((PyObject *)&PyCStgDict_Type); if (!dict) { Py_DECREF(result); return NULL; @@ -1067,7 +1072,7 @@ PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) stgdict items size, align, length contain info about pointers itself, stgdict->proto has info about the pointed to type! */ - stgdict = (StgDictObject *)_PyObject_CallNoArg( + stgdict = (StgDictObject *)_PyObject_CallNoArgs( (PyObject *)&PyCStgDict_Type); if (!stgdict) return NULL; @@ -1550,7 +1555,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - stgdict = (StgDictObject *)_PyObject_CallNoArg( + stgdict = (StgDictObject *)_PyObject_CallNoArgs( (PyObject *)&PyCStgDict_Type); if (!stgdict) goto error; @@ -2006,7 +2011,7 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject if (result == NULL) return NULL; - stgdict = (StgDictObject *)_PyObject_CallNoArg( + stgdict = (StgDictObject *)_PyObject_CallNoArgs( (PyObject *)&PyCStgDict_Type); if (!stgdict) { Py_DECREF(result); @@ -2120,7 +2125,7 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - stgdict = (StgDictObject *)_PyObject_CallNoArg( + stgdict = (StgDictObject *)_PyObject_CallNoArgs( (PyObject *)&PyCStgDict_Type); if (!stgdict) goto error; @@ -2560,7 +2565,7 @@ PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyTypeObject *result; StgDictObject *stgdict; - stgdict = (StgDictObject *)_PyObject_CallNoArg( + stgdict = (StgDictObject *)_PyObject_CallNoArgs( (PyObject *)&PyCStgDict_Type); if (!stgdict) return NULL; @@ -3985,10 +3990,10 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, goto error; } if (PyCArrayTypeObject_Check(ob)) - ob = _PyObject_CallNoArg(ob); + ob = _PyObject_CallNoArgs(ob); else /* Create an instance of the pointed-to type */ - ob = _PyObject_CallNoArg(dict->proto); + ob = _PyObject_CallNoArgs(dict->proto); /* XXX Is the following correct any longer? We must not pass a byref() to the array then but @@ -5596,7 +5601,7 @@ cast(void *ptr, PyObject *src, PyObject *ctype) CDataObject *result; if (0 == cast_check_pointertype(ctype)) return NULL; - result = (CDataObject *)_PyObject_CallNoArg(ctype); + result = (CDataObject *)_PyObject_CallNoArgs(ctype); if (result == NULL) return NULL; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a33d15de9c0d41..770c96c60d1f7d 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -4,6 +4,8 @@ #include #endif +#include // qsort() + #define EXPORT(x) Py_EXPORTED_SYMBOL x /* some functions handy for testing */ diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 5a4d1c543f1009..c24f04ce3a0f8e 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,12 +1,15 @@ #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "frameobject.h" #include #include -#ifdef MS_WIN32 -#include -#endif #include "ctypes.h" /**************************************************************/ @@ -187,7 +190,7 @@ static void _CallPythonObject(void *mem, */ } else if (dict) { /* Hm, shouldn't we use PyCData_AtAddress() or something like that instead? */ - CDataObject *obj = (CDataObject *)_PyObject_CallNoArg(cnv); + CDataObject *obj = (CDataObject *)_PyObject_CallNoArgs(cnv); if (!obj) { PrintError("create argument %zd:\n", i); Py_DECREF(cnv); @@ -572,7 +575,7 @@ long Call_CanUnloadNow(void) return E_FAIL; } - result = _PyObject_CallNoArg(func); + result = _PyObject_CallNoArgs(func); Py_DECREF(func); if (!result) { PyErr_WriteUnraisable(context ? context : Py_None); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 17e82f90cf4740..e2204961070dbf 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -834,7 +834,7 @@ static int _call_function_pointer(int flags, # define HAVE_FFI_PREP_CIF_VAR_RUNTIME false # endif - /* Even on Apple-arm64 the calling convention for variadic functions conincides + /* Even on Apple-arm64 the calling convention for variadic functions coincides * with the standard calling convention in the case that the function called * only with its fixed arguments. Thus, we do not need a special flag to be * set on variadic functions. We treat a function as variadic if it is called diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a21a9daaed02cc..2cfd657028aca0 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -1,10 +1,14 @@ #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + #include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_floatobject.h" // _PyFloat_Pack8() #include -#ifdef MS_WIN32 -#include -#endif #include "ctypes.h" @@ -35,7 +39,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwds) * Expects the size, index and offset for the current field in *psize and * *poffset, stores the total size so far in *psize, the offset for the next * field in *poffset, the alignment requirements for the current field in - * *palign, and returns a field desriptor for this field. + * *palign, and returns a field descriptor for this field. */ /* * bitfields extension: @@ -61,7 +65,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, #define CONT_BITFIELD 2 #define EXPAND_BITFIELD 3 - self = (CFieldObject *)_PyObject_CallNoArg((PyObject *)&PyCField_Type); + self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); if (self == NULL) return NULL; dict = PyType_stgdict(desc); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 6110027980827c..9600ddc7413b2e 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -208,7 +208,7 @@ typedef struct { PyObject *checker; int flags; /* calling convention and such */ - /* pep3118 fields, pointers neeed PyMem_Free */ + /* pep3118 fields, pointers need PyMem_Free */ char *format; int ndim; Py_ssize_t *shape; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 747339dee352eb..43669d7152a7e9 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -1,8 +1,13 @@ #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + +#include "pycore_call.h" // _PyObject_CallNoArgs() #include #ifdef MS_WIN32 -#include -#include +# include #endif #include "ctypes.h" @@ -248,7 +253,7 @@ MakeFields(PyObject *type, CFieldObject *descr, } continue; } - new_descr = (CFieldObject *)_PyObject_CallNoArg((PyObject *)&PyCField_Type); + new_descr = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); if (new_descr == NULL) { Py_DECREF(fdescr); Py_DECREF(fieldlist); diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index e2979a577c96c7..237edd5191fd94 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -27,7 +27,7 @@ #include -#include "longintrepr.h" +#include "pycore_pystate.h" // _PyThreadState_GET() #include "complexobject.h" #include "mpdecimal.h" @@ -1511,18 +1511,20 @@ static PyGetSetDef context_getsets [] = static PyObject * current_context_from_dict(void) { - PyObject *dict; - PyObject *tl_context; - PyThreadState *tstate; + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + // The caller must hold the GIL + _Py_EnsureTstateNotNULL(tstate); +#endif - dict = PyThreadState_GetDict(); + PyObject *dict = _PyThreadState_GetDict(tstate); if (dict == NULL) { PyErr_SetString(PyExc_RuntimeError, "cannot get thread state"); return NULL; } - tl_context = PyDict_GetItemWithError(dict, tls_context_key); + PyObject *tl_context = PyDict_GetItemWithError(dict, tls_context_key); if (tl_context != NULL) { /* We already have a thread local context. */ CONTEXT_CHECK(tl_context); @@ -1548,11 +1550,8 @@ current_context_from_dict(void) /* Cache the context of the current thread, assuming that it * will be accessed several times before a thread switch. */ - tstate = PyThreadState_GET(); - if (tstate) { - cached_context = (PyDecContextObject *)tl_context; - cached_context->tstate = tstate; - } + cached_context = (PyDecContextObject *)tl_context; + cached_context->tstate = tstate; /* Borrowed reference with refcount==1 */ return tl_context; @@ -1562,9 +1561,7 @@ current_context_from_dict(void) static PyObject * current_context(void) { - PyThreadState *tstate; - - tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (cached_context && cached_context->tstate == tstate) { return (PyObject *)cached_context; } diff --git a/Modules/_decimal/tests/bench.py b/Modules/_decimal/tests/bench.py index 3726db194e032f..24e091b6887ccd 100644 --- a/Modules/_decimal/tests/bench.py +++ b/Modules/_decimal/tests/bench.py @@ -7,10 +7,7 @@ import time -try: - from test.support import import_fresh_module -except ImportError: - from test.test_support import import_fresh_module +from test.support.import_helper import import_fresh_module C = import_fresh_module('decimal', fresh=['_decimal']) P = import_fresh_module('decimal', blocked=['_decimal']) diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py index 98ecd502c03fe9..edf753f3704a18 100644 --- a/Modules/_decimal/tests/deccheck.py +++ b/Modules/_decimal/tests/deccheck.py @@ -47,7 +47,7 @@ from queue import Queue, Empty from threading import Thread, Event, Lock -from test.support import import_fresh_module +from test.support.import_helper import import_fresh_module from randdec import randfloat, all_unary, all_binary, all_ternary from randdec import unary_optarg, binary_optarg, ternary_optarg from formathelper import rand_format, rand_locale diff --git a/Modules/_decimal/tests/formathelper.py b/Modules/_decimal/tests/formathelper.py index 19b2aad4a503b1..c3daacfb7b44f4 100644 --- a/Modules/_decimal/tests/formathelper.py +++ b/Modules/_decimal/tests/formathelper.py @@ -31,7 +31,7 @@ import os, sys, locale, random import platform, subprocess -from test.support import import_fresh_module +from test.support.import_helper import import_fresh_module from distutils.spawn import find_executable C = import_fresh_module('decimal', fresh=['_decimal']) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index d6dfefdd87767c..1f6b852f6d99b5 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_GC_TRACK @@ -268,7 +269,7 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, static void partial_setvectorcall(partialobject *pto) { - if (PyVectorcall_Function(pto->fn) == NULL) { + if (_PyVectorcall_Function(pto->fn) == NULL) { /* Don't use vectorcall if the underlying function doesn't support it */ pto->vectorcall = NULL; } @@ -1440,7 +1441,7 @@ static int _functools_exec(PyObject *module) { _functools_state *state = get_functools_state(module); - state->kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); + state->kwd_mark = _PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); if (state->kwd_mark == NULL) { return -1; } diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index efbf331ca302d1..445500c7ee4c15 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -5,11 +5,12 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "gdbm.h" -#include -#include #include -#include "gdbm.h" +#include // free() +#include +#include #if defined(WIN32) && !defined(__CYGWIN__) #include "gdbmerrno.h" diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index b9e68c05c3edbe..b4ba60baaac93d 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -22,7 +22,7 @@ #include "Python.h" #include "hashlib.h" -#include "pystrhex.h" +#include "pycore_strhex.h" // _Py_strhex() /* EVP is the preferred interface to hashing in OpenSSL */ #include diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 5984d34cc08290..dc6371a572b9f8 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -9,6 +9,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_object.h" #include "structmember.h" // PyMemberDef #include "_iomodule.h" @@ -341,11 +342,10 @@ _enter_buffered_busy(buffered *self) : buffered_closed(self))) #define CHECK_CLOSED(self, error_msg) \ - if (IS_CLOSED(self)) { \ + if (IS_CLOSED(self) & (Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t) == 0)) { \ PyErr_SetString(PyExc_ValueError, error_msg); \ return NULL; \ - } - + } \ #define VALID_READ_BUFFER(self) \ (self->readable && self->read_end != -1) @@ -530,6 +530,9 @@ buffered_close(buffered *self, PyObject *args) Py_CLEAR(res); } + self->read_end = 0; + self->pos = 0; + end: LEAVE_BUFFERED(self) return res; @@ -1548,7 +1551,7 @@ _bufferedreader_read_all(buffered *self) goto cleanup; } if (readall) { - tmp = _PyObject_CallNoArg(readall); + tmp = _PyObject_CallNoArgs(readall); Py_DECREF(readall); if (tmp == NULL) goto cleanup; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index b9856b3b631657..dd215e89399442 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -2,7 +2,8 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_object.h" +#include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH +#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "structmember.h" // PyMemberDef #include #ifdef HAVE_SYS_TYPES_H diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 460f2d3fa071a8..4002d28fc9b353 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -8,7 +8,8 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_object.h" +#include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH +#include "pycore_object.h" // _PyObject_GC_UNTRACK() #ifdef MS_WINDOWS diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 703067cc743a55..2e27afcea1b799 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -1,4 +1,6 @@ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_pystate.h" // _PyThreadState_GET() #include "rotatingtree.h" /************************************************************/ @@ -76,7 +78,7 @@ _lsprof_get_state(PyObject *module) static _PyTime_t CallExternalTimer(ProfilerObject *pObj) { - PyObject *o = _PyObject_CallNoArg(pObj->externalTimer); + PyObject *o = _PyObject_CallNoArgs(pObj->externalTimer); if (o == NULL) { PyErr_WriteUnraisable(pObj->externalTimer); return 0; @@ -671,7 +673,7 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { return NULL; } @@ -705,7 +707,7 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { return NULL; } @@ -742,7 +744,7 @@ static void profiler_dealloc(ProfilerObject *op) { if (op->flags & POF_ENABLED) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { PyErr_WriteUnraisable((PyObject *)op); } diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 915c0c918f6443..e50f55b75c5dc5 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -10,7 +10,7 @@ #include "Python.h" #include "structmember.h" // PyMemberDef -#include +#include // free() #include #include diff --git a/Modules/_operator.c b/Modules/_operator.c index 12a5bf6371b459..b3a8bef2eaedd9 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -704,10 +704,8 @@ static PyObject * _operator_is__impl(PyObject *module, PyObject *a, PyObject *b) /*[clinic end generated code: output=bcd47a402e482e1d input=5fa9b97df03c427f]*/ { - PyObject *result; - result = (a == b) ? Py_True : Py_False; - Py_INCREF(result); - return result; + PyObject *result = Py_Is(a, b) ? Py_True : Py_False; + return Py_NewRef(result); } /*[clinic input] diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 3e74fafb384176..53ab57111967ce 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -9,9 +9,12 @@ #endif #include "Python.h" +#include "pycore_floatobject.h" // _PyFloat_Pack8() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef +#include // strtol() + PyDoc_STRVAR(pickle_module_doc, "Optimized C implementation for the Python pickle module."); @@ -4527,7 +4530,7 @@ dump(PicklerObject *self, PyObject *obj) * call when setting the reducer_override attribute of the Pickler instance * to a bound method of the same instance. This is important as the Pickler * instance holds a reference to each object it has pickled (through its - * memo): thus, these objects wont be garbage-collected as long as the + * memo): thus, these objects won't be garbage-collected as long as the * Pickler itself is not collected. */ Py_CLEAR(self->reducer_override); return status; @@ -6540,7 +6543,7 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) return 0; if ((len - x) % 2 != 0) { PickleState *st = _Pickle_GetGlobalState(); - /* Currupt or hostile pickle -- we never write one like this. */ + /* Corrupt or hostile pickle -- we never write one like this. */ PyErr_SetString(st->UnpicklingError, "odd number of items for SETITEMS"); return -1; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 58772d7f0f5234..eb61349b76581d 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -182,7 +182,7 @@ _queue.SimpleQueue.get cls: defining_class / block: bool = True - timeout: object = None + timeout as timeout_obj: object = None Remove and return an item from the queue. @@ -198,11 +198,11 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, - int block, PyObject *timeout) -/*[clinic end generated code: output=1969aefa7db63666 input=5fc4d56b9a54757e]*/ + int block, PyObject *timeout_obj) +/*[clinic end generated code: output=5c2cca914cd1e55b input=5b4047bfbc645ec1]*/ { _PyTime_t endtime = 0; - _PyTime_t timeout_val; + _PyTime_t timeout; PyObject *item; PyLockStatus r; PY_TIMEOUT_T microseconds; @@ -211,24 +211,25 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, /* Non-blocking */ microseconds = 0; } - else if (timeout != Py_None) { + else if (timeout_obj != Py_None) { /* With timeout */ - if (_PyTime_FromSecondsObject(&timeout_val, - timeout, _PyTime_ROUND_CEILING) < 0) + if (_PyTime_FromSecondsObject(&timeout, + timeout_obj, _PyTime_ROUND_CEILING) < 0) { return NULL; - if (timeout_val < 0) { + } + if (timeout < 0) { PyErr_SetString(PyExc_ValueError, "'timeout' must be a non-negative number"); return NULL; } - microseconds = _PyTime_AsMicroseconds(timeout_val, + microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); - if (microseconds >= PY_TIMEOUT_MAX) { + if (microseconds > PY_TIMEOUT_MAX) { PyErr_SetString(PyExc_OverflowError, "timeout value is too large"); return NULL; } - endtime = _PyTime_GetMonotonicClock() + timeout_val; + endtime = _PyDeadline_Init(timeout); } else { /* Infinitely blocking */ @@ -247,6 +248,7 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, r = PyThread_acquire_lock_timed(self->lock, microseconds, 1); Py_END_ALLOW_THREADS } + if (r == PY_LOCK_INTR && Py_MakePendingCalls() < 0) { return NULL; } @@ -258,12 +260,15 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, return NULL; } self->locked = 1; + /* Adjust timeout for next iteration (if any) */ - if (endtime > 0) { - timeout_val = endtime - _PyTime_GetMonotonicClock(); - microseconds = _PyTime_AsMicroseconds(timeout_val, _PyTime_ROUND_CEILING); + if (microseconds > 0) { + timeout = _PyDeadline_Get(endtime); + microseconds = _PyTime_AsMicroseconds(timeout, + _PyTime_ROUND_CEILING); } } + /* BEGIN GIL-protected critical section */ assert(self->lst_pos < PyList_GET_SIZE(self->lst)); item = simplequeue_pop_item(self); diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 3974e0b6b47faa..a033c4e4525456 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -16,7 +16,7 @@ */ #include "Python.h" -#include "pystrhex.h" +#include "pycore_strhex.h" // _Py_strhex() #include "../hashlib.h" /* ************************************************************************** diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index bf5a58d7756c9e..e9e3064ae0f899 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -204,57 +204,30 @@ PyDoc_STRVAR(pysqlite_connection_create_function__doc__, "Creates a new function. Non-standard."); #define PYSQLITE_CONNECTION_CREATE_FUNCTION_METHODDEF \ - {"create_function", (PyCFunction)(void(*)(void))pysqlite_connection_create_function, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_function__doc__}, + {"create_function", (PyCFunction)(void(*)(void))pysqlite_connection_create_function, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_function__doc__}, static PyObject * pysqlite_connection_create_function_impl(pysqlite_Connection *self, - const char *name, int narg, - PyObject *func, int deterministic); + PyTypeObject *cls, const char *name, + int narg, PyObject *func, + int deterministic); static PyObject * -pysqlite_connection_create_function(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"name", "narg", "func", "deterministic", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "create_function", 0}; - PyObject *argsbuf[4]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + static _PyArg_Parser _parser = {"siO|$p:create_function", _keywords, 0}; const char *name; int narg; PyObject *func; int deterministic = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("create_function", "argument 'name'", "str", args[0]); - goto exit; - } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); - if (name == NULL) { + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &name, &narg, &func, &deterministic)) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } - narg = _PyLong_AsInt(args[1]); - if (narg == -1 && PyErr_Occurred()) { - goto exit; - } - func = args[2]; - if (!noptargs) { - goto skip_optional_kwonly; - } - deterministic = PyObject_IsTrue(args[3]); - if (deterministic < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = pysqlite_connection_create_function_impl(self, name, narg, func, deterministic); + return_value = pysqlite_connection_create_function_impl(self, cls, name, narg, func, deterministic); exit: return return_value; @@ -267,47 +240,29 @@ PyDoc_STRVAR(pysqlite_connection_create_aggregate__doc__, "Creates a new aggregate. Non-standard."); #define PYSQLITE_CONNECTION_CREATE_AGGREGATE_METHODDEF \ - {"create_aggregate", (PyCFunction)(void(*)(void))pysqlite_connection_create_aggregate, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_aggregate__doc__}, + {"create_aggregate", (PyCFunction)(void(*)(void))pysqlite_connection_create_aggregate, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_aggregate__doc__}, static PyObject * pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, + PyTypeObject *cls, const char *name, int n_arg, PyObject *aggregate_class); static PyObject * -pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"name", "n_arg", "aggregate_class", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "create_aggregate", 0}; - PyObject *argsbuf[3]; + static _PyArg_Parser _parser = {"siO:create_aggregate", _keywords, 0}; const char *name; int n_arg; PyObject *aggregate_class; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("create_aggregate", "argument 'name'", "str", args[0]); - goto exit; - } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); - if (name == NULL) { + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &name, &n_arg, &aggregate_class)) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } - n_arg = _PyLong_AsInt(args[1]); - if (n_arg == -1 && PyErr_Occurred()) { - goto exit; - } - aggregate_class = args[2]; - return_value = pysqlite_connection_create_aggregate_impl(self, name, n_arg, aggregate_class); + return_value = pysqlite_connection_create_aggregate_impl(self, cls, name, n_arg, aggregate_class); exit: return return_value; @@ -320,27 +275,26 @@ PyDoc_STRVAR(pysqlite_connection_set_authorizer__doc__, "Sets authorizer callback. Non-standard."); #define PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF \ - {"set_authorizer", (PyCFunction)(void(*)(void))pysqlite_connection_set_authorizer, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_authorizer__doc__}, + {"set_authorizer", (PyCFunction)(void(*)(void))pysqlite_connection_set_authorizer, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_authorizer__doc__}, static PyObject * pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, + PyTypeObject *cls, PyObject *callable); static PyObject * -pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"authorizer_callback", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "set_authorizer", 0}; - PyObject *argsbuf[1]; + static _PyArg_Parser _parser = {"O:set_authorizer", _keywords, 0}; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &callable)) { goto exit; } - callable = args[0]; - return_value = pysqlite_connection_set_authorizer_impl(self, callable); + return_value = pysqlite_connection_set_authorizer_impl(self, cls, callable); exit: return return_value; @@ -353,32 +307,27 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, "Sets progress handler callback. Non-standard."); #define PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF \ - {"set_progress_handler", (PyCFunction)(void(*)(void))pysqlite_connection_set_progress_handler, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_progress_handler__doc__}, + {"set_progress_handler", (PyCFunction)(void(*)(void))pysqlite_connection_set_progress_handler, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_progress_handler__doc__}, static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, + PyTypeObject *cls, PyObject *callable, int n); static PyObject * -pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"progress_handler", "n", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "set_progress_handler", 0}; - PyObject *argsbuf[2]; + static _PyArg_Parser _parser = {"Oi:set_progress_handler", _keywords, 0}; PyObject *callable; int n; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - callable = args[0]; - n = _PyLong_AsInt(args[1]); - if (n == -1 && PyErr_Occurred()) { + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &callable, &n)) { goto exit; } - return_value = pysqlite_connection_set_progress_handler_impl(self, callable, n); + return_value = pysqlite_connection_set_progress_handler_impl(self, cls, callable, n); exit: return return_value; @@ -393,27 +342,26 @@ PyDoc_STRVAR(pysqlite_connection_set_trace_callback__doc__, "Non-standard."); #define PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF \ - {"set_trace_callback", (PyCFunction)(void(*)(void))pysqlite_connection_set_trace_callback, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_trace_callback__doc__}, + {"set_trace_callback", (PyCFunction)(void(*)(void))pysqlite_connection_set_trace_callback, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_trace_callback__doc__}, static PyObject * pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, + PyTypeObject *cls, PyObject *callable); static PyObject * -pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"trace_callback", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "set_trace_callback", 0}; - PyObject *argsbuf[1]; + static _PyArg_Parser _parser = {"O:set_trace_callback", _keywords, 0}; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &callable)) { goto exit; } - callable = args[0]; - return_value = pysqlite_connection_set_trace_callback_impl(self, callable); + return_value = pysqlite_connection_set_trace_callback_impl(self, cls, callable); exit: return return_value; @@ -720,38 +668,28 @@ PyDoc_STRVAR(pysqlite_connection_create_collation__doc__, "Creates a collation function. Non-standard."); #define PYSQLITE_CONNECTION_CREATE_COLLATION_METHODDEF \ - {"create_collation", (PyCFunction)(void(*)(void))pysqlite_connection_create_collation, METH_FASTCALL, pysqlite_connection_create_collation__doc__}, + {"create_collation", (PyCFunction)(void(*)(void))pysqlite_connection_create_collation, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_collation__doc__}, static PyObject * pysqlite_connection_create_collation_impl(pysqlite_Connection *self, + PyTypeObject *cls, const char *name, PyObject *callable); static PyObject * -pysqlite_connection_create_collation(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) +pysqlite_connection_create_collation(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = {"sO:create_collation", _keywords, 0}; const char *name; PyObject *callable; - if (!_PyArg_CheckPositional("create_collation", nargs, 2, 2)) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("create_collation", "argument 1", "str", args[0]); - goto exit; - } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); - if (name == NULL) { - goto exit; - } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &name, &callable)) { goto exit; } - callable = args[1]; - return_value = pysqlite_connection_create_collation_impl(self, name, callable); + return_value = pysqlite_connection_create_collation_impl(self, cls, name, callable); exit: return return_value; @@ -819,4 +757,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=5b7268875f33c016 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7567e5d716309258 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 02481d9fbd8bb7..e94c4cbb4e8c3a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -256,6 +256,7 @@ pysqlite_do_all_statements(pysqlite_Connection *self) do { \ if (ctx) { \ Py_VISIT(ctx->callable); \ + Py_VISIT(ctx->module); \ } \ } while (0) @@ -280,6 +281,7 @@ clear_callback_context(callback_context *ctx) { if (ctx != NULL) { Py_CLEAR(ctx->callable); + Py_CLEAR(ctx->module); } } @@ -667,7 +669,7 @@ set_sqlite_error(sqlite3_context *context, const char *msg) } static void -_pysqlite_func_callback(sqlite3_context *context, int argc, sqlite3_value **argv) +func_callback(sqlite3_context *context, int argc, sqlite3_value **argv) { PyGILState_STATE threadstate = PyGILState_Ensure(); @@ -710,7 +712,7 @@ step_callback(sqlite3_context *context, int argc, sqlite3_value **params) if (*aggregate_instance == NULL) { callback_context *ctx = (callback_context *)sqlite3_user_data(context); assert(ctx != NULL); - *aggregate_instance = _PyObject_CallNoArg(ctx->callable); + *aggregate_instance = PyObject_CallNoArgs(ctx->callable); if (!*aggregate_instance) { set_sqlite_error(context, "user-defined aggregate's '__init__' method raised error"); @@ -822,13 +824,21 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) Py_SETREF(self->cursors, new_list); } +/* Allocate a UDF/callback context structure. In order to ensure that the state + * pointer always outlives the callback context, we make sure it owns a + * reference to the module itself. create_callback_context() is always called + * from connection methods, so we use the defining class to fetch the module + * pointer. + */ static callback_context * -create_callback_context(pysqlite_state *state, PyObject *callable) +create_callback_context(PyTypeObject *cls, PyObject *callable) { callback_context *ctx = PyMem_Malloc(sizeof(callback_context)); if (ctx != NULL) { + PyObject *module = PyType_GetModule(cls); ctx->callable = Py_NewRef(callable); - ctx->state = state; + ctx->module = Py_NewRef(module); + ctx->state = pysqlite_get_state(module); } return ctx; } @@ -838,6 +848,7 @@ free_callback_context(callback_context *ctx) { assert(ctx != NULL); Py_XDECREF(ctx->callable); + Py_XDECREF(ctx->module); PyMem_Free(ctx); } @@ -867,6 +878,8 @@ destructor_callback(void *ctx) /*[clinic input] _sqlite3.Connection.create_function as pysqlite_connection_create_function + cls: defining_class + / name: str narg: int func: object @@ -878,9 +891,10 @@ Creates a new function. Non-standard. static PyObject * pysqlite_connection_create_function_impl(pysqlite_Connection *self, - const char *name, int narg, - PyObject *func, int deterministic) -/*[clinic end generated code: output=07d1877dd98c0308 input=f2edcf073e815beb]*/ + PyTypeObject *cls, const char *name, + int narg, PyObject *func, + int deterministic) +/*[clinic end generated code: output=8a811529287ad240 input=f0f99754bfeafd8d]*/ { int rc; int flags = SQLITE_UTF8; @@ -903,12 +917,12 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, flags |= SQLITE_DETERMINISTIC; #endif } - callback_context *ctx = create_callback_context(self->state, func); + callback_context *ctx = create_callback_context(cls, func); if (ctx == NULL) { return NULL; } rc = sqlite3_create_function_v2(self->db, name, narg, flags, ctx, - _pysqlite_func_callback, + func_callback, NULL, NULL, &destructor_callback); // will decref func @@ -924,6 +938,8 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, /*[clinic input] _sqlite3.Connection.create_aggregate as pysqlite_connection_create_aggregate + cls: defining_class + / name: str n_arg: int aggregate_class: object @@ -933,9 +949,10 @@ Creates a new aggregate. Non-standard. static PyObject * pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, + PyTypeObject *cls, const char *name, int n_arg, PyObject *aggregate_class) -/*[clinic end generated code: output=fbb2f858cfa4d8db input=c2e13bbf234500a5]*/ +/*[clinic end generated code: output=1b02d0f0aec7ff96 input=bd527067e6c2e33f]*/ { int rc; @@ -943,8 +960,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, return NULL; } - callback_context *ctx = create_callback_context(self->state, - aggregate_class); + callback_context *ctx = create_callback_context(cls, aggregate_class); if (ctx == NULL) { return NULL; } @@ -1008,7 +1024,7 @@ progress_callback(void *ctx) assert(ctx != NULL); PyObject *callable = ((callback_context *)ctx)->callable; - ret = _PyObject_CallNoArg(callable); + ret = PyObject_CallNoArgs(callable); if (!ret) { /* abort query if error occurred */ rc = -1; @@ -1075,6 +1091,8 @@ trace_callback(void *ctx, const char *statement_string) /*[clinic input] _sqlite3.Connection.set_authorizer as pysqlite_connection_set_authorizer + cls: defining_class + / authorizer_callback as callable: object Sets authorizer callback. Non-standard. @@ -1082,8 +1100,9 @@ Sets authorizer callback. Non-standard. static PyObject * pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, + PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=c193601e9e8a5116 input=ec104f130b82050b]*/ +/*[clinic end generated code: output=75fa60114fc971c3 input=9f3e90d3d642c4a0]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1095,7 +1114,7 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, set_callback_context(&self->authorizer_ctx, NULL); } else { - callback_context *ctx = create_callback_context(self->state, callable); + callback_context *ctx = create_callback_context(cls, callable); if (ctx == NULL) { return NULL; } @@ -1114,6 +1133,8 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, /*[clinic input] _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_handler + cls: defining_class + / progress_handler as callable: object n: int @@ -1122,8 +1143,9 @@ Sets progress handler callback. Non-standard. static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, + PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=ba14008a483d7a53 input=3cf56d045f130a84]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=83e8dcbb4ce183f7]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1135,7 +1157,7 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, set_callback_context(&self->progress_ctx, NULL); } else { - callback_context *ctx = create_callback_context(self->state, callable); + callback_context *ctx = create_callback_context(cls, callable); if (ctx == NULL) { return NULL; } @@ -1148,6 +1170,8 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, /*[clinic input] _sqlite3.Connection.set_trace_callback as pysqlite_connection_set_trace_callback + cls: defining_class + / trace_callback as callable: object Sets a trace callback called for each SQL statement (passed as unicode). @@ -1157,8 +1181,9 @@ Non-standard. static PyObject * pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, + PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=c9fd551e359165d3 input=d76eabbb633057bc]*/ +/*[clinic end generated code: output=d91048c03bfcee05 input=96f03acec3ec8044]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1180,7 +1205,7 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, set_callback_context(&self->trace_ctx, NULL); } else { - callback_context *ctx = create_callback_context(self->state, callable); + callback_context *ctx = create_callback_context(cls, callable); if (ctx == NULL) { return NULL; } @@ -1501,10 +1526,8 @@ pysqlite_connection_executescript(pysqlite_Connection *self, /* ------------------------- COLLATION CODE ------------------------ */ static int -pysqlite_collation_callback( - void* context, - int text1_length, const void* text1_data, - int text2_length, const void* text2_data) +collation_callback(void *context, int text1_length, const void *text1_data, + int text2_length, const void *text2_data) { PyGILState_STATE gilstate = PyGILState_Ensure(); @@ -1744,6 +1767,7 @@ pysqlite_connection_backup_impl(pysqlite_Connection *self, /*[clinic input] _sqlite3.Connection.create_collation as pysqlite_connection_create_collation + cls: defining_class name: str callback as callable: object / @@ -1753,9 +1777,10 @@ Creates a collation function. Non-standard. static PyObject * pysqlite_connection_create_collation_impl(pysqlite_Connection *self, + PyTypeObject *cls, const char *name, PyObject *callable) -/*[clinic end generated code: output=a4ceaff957fdef9a input=301647aab0f2fb1d]*/ +/*[clinic end generated code: output=32d339e97869c378 input=fee2c8e5708602ad]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1773,12 +1798,12 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, PyErr_SetString(PyExc_TypeError, "parameter must be callable"); return NULL; } - ctx = create_callback_context(self->state, callable); + ctx = create_callback_context(cls, callable); if (ctx == NULL) { return NULL; } rc = sqlite3_create_collation_v2(self->db, name, flags, ctx, - &pysqlite_collation_callback, + &collation_callback, &destructor_callback); } diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index c4cec857ddbfeb..6a2aa1c8e080ec 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -35,6 +35,7 @@ typedef struct _callback_context { PyObject *callable; + PyObject *module; pysqlite_state *state; } callback_context; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 9bac607e9c9a6b..ca74a68de4dba7 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -104,7 +104,12 @@ cursor_clear(pysqlite_Cursor *self) Py_CLEAR(self->row_cast_map); Py_CLEAR(self->lastrowid); Py_CLEAR(self->row_factory); - Py_CLEAR(self->statement); + if (self->statement) { + /* Reset the statement if the user has not closed the cursor */ + pysqlite_statement_reset(self->statement); + Py_CLEAR(self->statement); + } + return 0; } @@ -116,14 +121,6 @@ cursor_dealloc(pysqlite_Cursor *self) if (self->in_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)self); } - if (self->statement) { - /* A SELECT query will lock the affected database table(s), so we need - * to reset the statement to unlock the database before disappearing */ - sqlite3_stmt *stmt = self->statement->st; - if (sqlite3_stmt_readonly(stmt)) { - pysqlite_statement_reset(self->statement); - } - } tp->tp_clear((PyObject *)self); tp->tp_free(self); Py_DECREF(tp); @@ -518,19 +515,18 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation } } + if (self->statement != NULL) { + /* There is an active statement */ + pysqlite_statement_reset(self->statement); + } + /* reset description and rowcount */ Py_INCREF(Py_None); Py_SETREF(self->description, Py_None); self->rowcount = 0L; if (self->statement) { - /* A SELECT query will lock the affected database table(s), so we need - * to reset the statement to unlock the database before switching - * statements */ - sqlite3_stmt *stmt = self->statement->st; - if (sqlite3_stmt_readonly(stmt)) { - pysqlite_statement_reset(self->statement); - } + (void)pysqlite_statement_reset(self->statement); } PyObject *stmt = get_statement_from_cache(self, operation); @@ -553,6 +549,8 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation goto error; } } + + pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); /* We start a transaction implicitly before a DML statement. @@ -572,7 +570,6 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation break; } - pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); pysqlite_statement_bind_parameters(state, self->statement, parameters); @@ -590,6 +587,7 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation PyErr_Clear(); } } + (void)pysqlite_statement_reset(self->statement); _pysqlite_seterror(state, self->connection->db); goto error; } @@ -648,9 +646,13 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation } if (rc == SQLITE_DONE && !multiple) { + pysqlite_statement_reset(self->statement); Py_CLEAR(self->statement); } + if (multiple) { + pysqlite_statement_reset(self->statement); + } Py_XDECREF(parameters); } @@ -758,7 +760,7 @@ pysqlite_cursor_executescript_impl(pysqlite_Cursor *self, &tail); if (rc == SQLITE_OK) { do { - (void)sqlite3_step(stmt); + rc = sqlite3_step(stmt); } while (rc == SQLITE_ROW); rc = sqlite3_finalize(stmt); } @@ -802,6 +804,7 @@ pysqlite_cursor_iternext(pysqlite_Cursor *self) sqlite3_stmt *stmt = self->statement->st; assert(stmt != NULL); if (sqlite3_data_count(stmt) == 0) { + (void)pysqlite_statement_reset(self->statement); Py_CLEAR(self->statement); return NULL; } @@ -812,7 +815,7 @@ pysqlite_cursor_iternext(pysqlite_Cursor *self) } int rc = pysqlite_step(stmt); if (rc == SQLITE_DONE) { - Py_CLEAR(self->statement); + (void)pysqlite_statement_reset(self->statement); } else if (rc != SQLITE_ROW) { (void)_pysqlite_seterror(self->connection->state, @@ -982,7 +985,11 @@ pysqlite_cursor_close_impl(pysqlite_Cursor *self, PyTypeObject *cls) return NULL; } - Py_CLEAR(self->statement); + if (self->statement) { + (void)pysqlite_statement_reset(self->statement); + Py_CLEAR(self->statement); + } + self->closed = 1; Py_RETURN_NONE; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 3016fe5bb45ce7..b20c91da3179c4 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -360,31 +360,23 @@ pysqlite_statement_bind_parameters(pysqlite_state *state, } } -void -pysqlite_statement_reset(pysqlite_Statement *self) +int pysqlite_statement_reset(pysqlite_Statement* self) { - sqlite3_stmt *stmt = self->st; - if (stmt == NULL || self->in_use == 0) { - return; - } + int rc; -#if SQLITE_VERSION_NUMBER >= 3020000 - /* Check if the statement has been run (that is, sqlite3_step() has been - * called at least once). Third parameter is non-zero in order to reset the - * run count. */ - if (sqlite3_stmt_status(stmt, SQLITE_STMTSTATUS_RUN, 1) == 0) { - return; - } -#endif + rc = SQLITE_OK; - int rc; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_reset(stmt); - Py_END_ALLOW_THREADS + if (self->in_use && self->st) { + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_reset(self->st); + Py_END_ALLOW_THREADS - if (rc == SQLITE_OK) { - self->in_use = 0; + if (rc == SQLITE_OK) { + self->in_use = 0; + } } + + return rc; } void pysqlite_statement_mark_dirty(pysqlite_Statement* self) diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h index cce81ed910de04..b901c43c479ae2 100644 --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -44,7 +44,7 @@ void pysqlite_statement_bind_parameters(pysqlite_state *state, pysqlite_Statement *self, PyObject *parameters); -void pysqlite_statement_reset(pysqlite_Statement *self); +int pysqlite_statement_reset(pysqlite_Statement* self); void pysqlite_statement_mark_dirty(pysqlite_Statement* self); int pysqlite_statement_setup_types(PyObject *module); diff --git a/Modules/_sre.c b/Modules/_sre.c index f4ec862dcabf0d..213730860cfb53 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -15,7 +15,7 @@ * 2001-05-14 fl fixes for 1.5.2 compatibility * 2001-07-01 fl added BIGCHARSET support (from Martin von Loewis) * 2001-10-18 fl fixed group reset issue (from Matthew Mueller) - * 2001-10-20 fl added split primitive; reenable unicode for 1.6/2.0/2.1 + * 2001-10-20 fl added split primitive; re-enable unicode for 1.6/2.0/2.1 * 2001-10-21 fl added sub/subn primitive * 2001-10-24 fl added finditer primitive (for 2.2 only) * 2001-12-07 fl fixed memory leak in sub/subn (Guido van Rossum) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 6c63301b2a7d8d..b2e241a0a338eb 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -949,8 +949,9 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) timeout = GET_SOCKET_TIMEOUT(sock); has_timeout = (timeout > 0); - if (has_timeout) - deadline = _PyTime_GetMonotonicClock() + timeout; + if (has_timeout) { + deadline = _PyDeadline_Init(timeout); + } /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ @@ -965,7 +966,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) goto error; if (has_timeout) - timeout = deadline - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(deadline); if (err.ssl == SSL_ERROR_WANT_READ) { sockstate = PySSL_select(sock, 0, timeout); @@ -2264,7 +2265,7 @@ PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout) if (!_PyIsSelectable_fd(s->sock_fd)) return SOCKET_TOO_LARGE_FOR_SELECT; - _PyTime_AsTimeval_noraise(timeout, &tv, _PyTime_ROUND_CEILING); + _PyTime_AsTimeval_clamp(timeout, &tv, _PyTime_ROUND_CEILING); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); @@ -2326,8 +2327,9 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) timeout = GET_SOCKET_TIMEOUT(sock); has_timeout = (timeout > 0); - if (has_timeout) - deadline = _PyTime_GetMonotonicClock() + timeout; + if (has_timeout) { + deadline = _PyDeadline_Init(timeout); + } sockstate = PySSL_select(sock, 1, timeout); if (sockstate == SOCKET_HAS_TIMED_OUT) { @@ -2354,8 +2356,9 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) if (PyErr_CheckSignals()) goto error; - if (has_timeout) - timeout = deadline - _PyTime_GetMonotonicClock(); + if (has_timeout) { + timeout = _PyDeadline_Get(deadline); + } if (err.ssl == SSL_ERROR_WANT_READ) { sockstate = PySSL_select(sock, 0, timeout); @@ -2494,7 +2497,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, timeout = GET_SOCKET_TIMEOUT(sock); has_timeout = (timeout > 0); if (has_timeout) - deadline = _PyTime_GetMonotonicClock() + timeout; + deadline = _PyDeadline_Init(timeout); do { PySSL_BEGIN_ALLOW_THREADS @@ -2506,8 +2509,9 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, if (PyErr_CheckSignals()) goto error; - if (has_timeout) - timeout = deadline - _PyTime_GetMonotonicClock(); + if (has_timeout) { + timeout = _PyDeadline_Get(deadline); + } if (err.ssl == SSL_ERROR_WANT_READ) { sockstate = PySSL_select(sock, 0, timeout); @@ -2592,8 +2596,9 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) timeout = GET_SOCKET_TIMEOUT(sock); has_timeout = (timeout > 0); - if (has_timeout) - deadline = _PyTime_GetMonotonicClock() + timeout; + if (has_timeout) { + deadline = _PyDeadline_Init(timeout); + } while (1) { PySSL_BEGIN_ALLOW_THREADS @@ -2626,8 +2631,9 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) continue; } - if (has_timeout) - timeout = deadline - _PyTime_GetMonotonicClock(); + if (has_timeout) { + timeout = _PyDeadline_Get(deadline); + } /* Possibly retry shutdown until timeout or failure */ if (err.ssl == SSL_ERROR_WANT_READ) @@ -3767,7 +3773,7 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) } if (pw_info->callable) { - fn_ret = _PyObject_CallNoArg(pw_info->callable); + fn_ret = PyObject_CallNoArgs(pw_info->callable); if (!fn_ret) { /* TODO: It would be nice to move _ctypes_add_traceback() into the core python API, so we could use it to add a frame here */ diff --git a/Modules/_struct.c b/Modules/_struct.c index 872c30d659d824..a8003a90b2682c 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -6,6 +6,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_floatobject.h" // _PyFloat_Unpack2() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef #include @@ -1475,7 +1476,6 @@ Struct___init___impl(PyStructObject *self, PyObject *format) if (format == NULL) return -1; } - /* XXX support buffer interface, too */ else { Py_INCREF(format); } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index d0b0a1d7da100c..30f6b02abe02a1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -18,29 +18,32 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "datetime.h" -#include "marshal.h" +#include "frameobject.h" // PyFrame_Check() +#include "datetime.h" // PyDateTimeAPI +#include "marshal.h" // PyMarshal_WriteLongToFile #include "structmember.h" // PyMemberDef -#include +#include // FLT_MAX #include #ifdef MS_WINDOWS -# include /* struct timeval */ +# include // struct timeval #endif #ifdef HAVE_SYS_WAIT_H -#include /* For W_STOPCODE */ +#include // W_STOPCODE #endif #ifdef Py_BUILD_CORE # error "_testcapi must test the public Python C API, not CPython internal C API" #endif + +// Forward declarations static struct PyModuleDef _testcapimodule; static PyType_Spec HeapTypeNameType_Spec; - static PyObject *TestError; /* set to exception object in init */ + /* Raise TestError with test_name + ": " + msg, and return NULL. */ static PyObject * @@ -334,19 +337,6 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) return result; } -static PyObject* -dict_hassplittable(PyObject *self, PyObject *arg) -{ - if (!PyDict_Check(arg)) { - PyErr_Format(PyExc_TypeError, - "dict_hassplittable() argument must be dict, not '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - - return PyBool_FromLong(_PyDict_HasSplitTable((PyDictObject*)arg)); -} - /* Issue #4701: Check that PyObject_Hash implicitly calls * PyType_Ready if it hasn't already been called */ @@ -1173,6 +1163,126 @@ test_get_type_name(PyObject *self, PyObject *Py_UNUSED(ignored)) } +static PyObject * +simple_str(PyObject *self) { + return PyUnicode_FromString(""); +} + + +static PyObject * +test_type_from_ephemeral_spec(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // Test that a heap type can be created from a spec that's later deleted + // (along with all its contents). + // All necessary data must be copied and held by the class + PyType_Spec *spec = NULL; + char *name = NULL; + char *doc = NULL; + PyType_Slot *slots = NULL; + PyObject *class = NULL; + PyObject *instance = NULL; + PyObject *obj = NULL; + PyObject *result = NULL; + + /* create a spec (and all its contents) on the heap */ + + const char NAME[] = "testcapi._Test"; + const char DOC[] = "a test class"; + + spec = PyMem_New(PyType_Spec, 1); + if (spec == NULL) { + PyErr_NoMemory(); + goto finally; + } + name = PyMem_New(char, sizeof(NAME)); + if (name == NULL) { + PyErr_NoMemory(); + goto finally; + } + memcpy(name, NAME, sizeof(NAME)); + + doc = PyMem_New(char, sizeof(DOC)); + if (name == NULL) { + PyErr_NoMemory(); + goto finally; + } + memcpy(doc, DOC, sizeof(DOC)); + + spec->name = name; + spec->basicsize = sizeof(PyObject); + spec->itemsize = 0; + spec->flags = Py_TPFLAGS_DEFAULT; + slots = PyMem_New(PyType_Slot, 3); + if (slots == NULL) { + PyErr_NoMemory(); + goto finally; + } + slots[0].slot = Py_tp_str; + slots[0].pfunc = simple_str; + slots[1].slot = Py_tp_doc; + slots[1].pfunc = doc; + slots[2].slot = 0; + slots[2].pfunc = NULL; + spec->slots = slots; + + /* create the class */ + + class = PyType_FromSpec(spec); + if (class == NULL) { + goto finally; + } + + /* deallocate the spec (and all contents) */ + + // (Explicitly ovewrite memory before freeing, + // so bugs show themselves even without the debug allocator's help.) + memset(spec, 0xdd, sizeof(PyType_Spec)); + PyMem_Del(spec); + spec = NULL; + memset(name, 0xdd, sizeof(NAME)); + PyMem_Del(name); + name = NULL; + memset(doc, 0xdd, sizeof(DOC)); + PyMem_Del(doc); + doc = NULL; + memset(slots, 0xdd, 3 * sizeof(PyType_Slot)); + PyMem_Del(slots); + slots = NULL; + + /* check that everything works */ + + PyTypeObject *class_tp = (PyTypeObject *)class; + PyHeapTypeObject *class_ht = (PyHeapTypeObject *)class; + assert(strcmp(class_tp->tp_name, "testcapi._Test") == 0); + assert(strcmp(PyUnicode_AsUTF8(class_ht->ht_name), "_Test") == 0); + assert(strcmp(PyUnicode_AsUTF8(class_ht->ht_qualname), "_Test") == 0); + assert(strcmp(class_tp->tp_doc, "a test class") == 0); + + // call and check __str__ + instance = PyObject_CallNoArgs(class); + if (instance == NULL) { + goto finally; + } + obj = PyObject_Str(instance); + if (obj == NULL) { + goto finally; + } + assert(strcmp(PyUnicode_AsUTF8(obj), "") == 0); + Py_CLEAR(obj); + + result = Py_NewRef(Py_None); + finally: + PyMem_Del(spec); + PyMem_Del(name); + PyMem_Del(doc); + PyMem_Del(slots); + Py_XDECREF(class); + Py_XDECREF(instance); + Py_XDECREF(obj); + return result; +} + + static PyObject * test_get_type_qualname(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1180,8 +1290,8 @@ test_get_type_qualname(PyObject *self, PyObject *Py_UNUSED(ignored)) assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "int") == 0); Py_DECREF(tp_qualname); - tp_qualname = PyType_GetQualName(&_PyNamespace_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "SimpleNamespace") == 0); + tp_qualname = PyType_GetQualName(&PyODict_Type); + assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "OrderedDict") == 0); Py_DECREF(tp_qualname); PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); @@ -2879,7 +2989,7 @@ _make_call(void *callable) PyObject *rc; int success; PyGILState_STATE s = PyGILState_Ensure(); - rc = _PyObject_CallNoArg((PyObject *)callable); + rc = PyObject_CallNoArgs((PyObject *)callable); success = (rc != NULL); Py_XDECREF(rc); PyGILState_Release(s); @@ -2950,7 +3060,7 @@ static int _pending_callback(void *arg) { /* we assume the argument is callable object to which we own a reference */ PyObject *callable = (PyObject *)arg; - PyObject *r = _PyObject_CallNoArg(callable); + PyObject *r = PyObject_CallNoArgs(callable); Py_DECREF(callable); Py_XDECREF(r); return r != NULL ? 0 : -1; @@ -3742,7 +3852,7 @@ slot_tp_del(PyObject *self) /* Execute __del__ method, if any. */ del = _PyObject_LookupSpecial(self, &PyId___tp_del__); if (del != NULL) { - res = _PyObject_CallNoArg(del); + res = PyObject_CallNoArgs(del); if (res == NULL) PyErr_WriteUnraisable(del); else @@ -4395,7 +4505,7 @@ temporary_c_thread(void *data) /* Allocate a Python thread state for this thread */ state = PyGILState_Ensure(); - res = _PyObject_CallNoArg(test_c_thread->callback); + res = PyObject_CallNoArgs(test_c_thread->callback); Py_CLEAR(test_c_thread->callback); if (res == NULL) { @@ -4711,7 +4821,32 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (seconds == NULL) { return NULL; } - return Py_BuildValue("Nl", seconds, tv.tv_usec); + return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); +} + +static PyObject * +test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timeval tv; + _PyTime_AsTimeval_clamp(t, &tv, round); + + PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); + if (seconds == NULL) { + return NULL; + } + return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); } #ifdef HAVE_CLOCK_GETTIME @@ -4732,6 +4867,22 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) } return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); } + +static PyObject * +test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timespec ts; + _PyTime_AsTimespec_clamp(t, &ts); + return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); +} #endif static PyObject * @@ -4889,7 +5040,7 @@ check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) #ifdef _Py_ADDRESS_SANITIZER Py_RETURN_NONE; #else - PyObject *op = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); + PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); if (op == NULL) { return NULL; } @@ -5267,7 +5418,7 @@ bad_get(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return NULL; } - PyObject *res = _PyObject_CallNoArg(cls); + PyObject *res = PyObject_CallNoArgs(cls); if (res == NULL) { return NULL; } @@ -5319,7 +5470,7 @@ encode_locale_ex(PyObject *self, PyObject *args) PyErr_SetString(PyExc_ValueError, "unsupported error handler"); break; default: - PyErr_SetString(PyExc_ValueError, "unknow error code"); + PyErr_SetString(PyExc_ValueError, "unknown error code"); break; } return res; @@ -5362,7 +5513,7 @@ decode_locale_ex(PyObject *self, PyObject *args) PyErr_SetString(PyExc_ValueError, "unsupported error handler"); break; default: - PyErr_SetString(PyExc_ValueError, "unknow error code"); + PyErr_SetString(PyExc_ValueError, "unknown error code"); break; } return res; @@ -5670,6 +5821,57 @@ type_get_version(PyObject *self, PyObject *type) } +// Test PyThreadState C API +static PyObject * +test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // PyThreadState_Get() + PyThreadState *tstate = PyThreadState_Get(); + assert(tstate != NULL); + + // PyThreadState_GET() + PyThreadState *tstate2 = PyThreadState_Get(); + assert(tstate2 == tstate); + + // private _PyThreadState_UncheckedGet() + PyThreadState *tstate3 = _PyThreadState_UncheckedGet(); + assert(tstate3 == tstate); + + // PyThreadState_EnterTracing(), PyThreadState_LeaveTracing() + PyThreadState_EnterTracing(tstate); + PyThreadState_LeaveTracing(tstate); + + // PyThreadState_GetDict(): no tstate argument + PyObject *dict = PyThreadState_GetDict(); + // PyThreadState_GetDict() API can return NULL if PyDict_New() fails, + // but it should not occur in practice. + assert(dict != NULL); + assert(PyDict_Check(dict)); + // dict is a borrowed reference + + // private _PyThreadState_GetDict() + PyObject *dict2 = _PyThreadState_GetDict(tstate); + assert(dict2 == dict); + // dict2 is a borrowed reference + + // PyThreadState_GetInterpreter() + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + assert(interp != NULL); + + // PyThreadState_GetFrame() + PyFrameObject*frame = PyThreadState_GetFrame(tstate); + assert(frame != NULL); + assert(PyFrame_Check(frame)); + Py_DECREF(frame); + + // PyThreadState_GetID() + uint64_t id = PyThreadState_GetID(tstate); + assert(id >= 1); + + Py_RETURN_NONE; +} + + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); @@ -5704,7 +5906,6 @@ static PyMethodDef TestMethods[] = { {"test_list_api", test_list_api, METH_NOARGS}, {"test_dict_iteration", test_dict_iteration, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, - {"dict_hassplittable", dict_hassplittable, METH_O}, {"test_lazy_hash_inheritance", test_lazy_hash_inheritance,METH_NOARGS}, {"test_long_api", test_long_api, METH_NOARGS}, {"test_xincref_doesnt_leak",test_xincref_doesnt_leak, METH_NOARGS}, @@ -5749,6 +5950,7 @@ static PyMethodDef TestMethods[] = { {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, {"test_get_type_name", test_get_type_name, METH_NOARGS}, {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, + {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS}, {"get_kwargs", (PyCFunction)(void(*)(void))get_kwargs, METH_VARARGS|METH_KEYWORDS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, @@ -5898,8 +6100,10 @@ static PyMethodDef TestMethods[] = { {"PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS}, {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, {"PyTime_AsTimeval", test_PyTime_AsTimeval, METH_VARARGS}, + {"PyTime_AsTimeval_clamp", test_PyTime_AsTimeval_clamp, METH_VARARGS}, #ifdef HAVE_CLOCK_GETTIME {"PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, + {"PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, #endif {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, @@ -5954,6 +6158,7 @@ static PyMethodDef TestMethods[] = { {"fatal_error", test_fatal_error, METH_VARARGS, PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, + {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d5616fd59c6e50..3ba939651a4173 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -18,7 +18,8 @@ #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() -#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() +#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() +#include "pycore_pystate.h" // _PyThreadState_GET() static PyObject * @@ -31,7 +32,7 @@ get_configs(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject* get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args)) { - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); /* subtract one to ignore the frame of the get_recursion_depth() call */ return PyLong_FromLong(tstate->recursion_depth - 1); diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index ad60f32f7e7a62..2d25e16bd4d390 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -3,6 +3,7 @@ */ #include "Python.h" +#include "pycore_namespace.h" // _PyNamespace_New() /* State for testing module state access from methods */ @@ -844,6 +845,28 @@ PyInit__testmultiphase_meth_state_access(PyObject *spec) return PyModuleDef_Init(&def_meth_state_access); } +static PyModuleDef def_module_state_shared = { + PyModuleDef_HEAD_INIT, + .m_name = "_test_module_state_shared", + .m_doc = PyDoc_STR("Regression Test module for single-phase init."), + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit__test_module_state_shared(PyObject *spec) +{ + PyObject *module = PyModule_Create(&def_module_state_shared); + if (module == NULL) { + return NULL; + } + + if (PyModule_AddObjectRef(module, "Error", PyExc_Exception) < 0) { + Py_DECREF(module); + return NULL; + } + return module; +} + /*** Helper for imp test ***/ diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 5b5d2c5b03ec3f..39b116afcaa3e1 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -84,13 +84,12 @@ lock_dealloc(lockobject *self) static PyLockStatus acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) { - PyLockStatus r; _PyTime_t endtime = 0; - if (timeout > 0) { - endtime = _PyTime_GetMonotonicClock() + timeout; + endtime = _PyDeadline_Init(timeout); } + PyLockStatus r; do { _PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); @@ -114,7 +113,7 @@ acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) /* If we're using a timeout, recompute the timeout after processing * signals, since those can take time. */ if (timeout > 0) { - timeout = endtime - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(endtime); /* Check for negative values, since those mean block forever. */ @@ -164,7 +163,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, _PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_TIMEOUT); - if (microseconds >= PY_TIMEOUT_MAX) { + if (microseconds > PY_TIMEOUT_MAX) { PyErr_SetString(PyExc_OverflowError, "timeout value is too large"); return -1; @@ -1220,7 +1219,7 @@ where the corresponding signal handler will be executed.\n\ If *signum* is omitted, SIGINT is assumed.\n\ A subthread can use this function to interrupt the main thread.\n\ \n\ -Note: the default signal hander for SIGINT raises ``KeyboardInterrupt``." +Note: the default signal handler for SIGINT raises ``KeyboardInterrupt``." ); static lockobject *newlockobject(PyObject *module); @@ -1320,7 +1319,7 @@ static PyObject * thread__set_sentinel(PyObject *module, PyObject *Py_UNUSED(ignored)) { PyObject *wr; - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); lockobject *lock; if (tstate->on_delete_data != NULL) { diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 7be9b8c0385b97..aabf20b8d963c7 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -797,7 +797,7 @@ PyTclObject_str(PyTclObject *self) Py_INCREF(self->string); return self->string; } - /* XXX Could chache result if it is non-ASCII. */ + /* XXX Could cache result if it is non-ASCII. */ return unicodeFromTclObj(self->value); } diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index fc3d7f51ee29a1..09d273ad280bbe 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -5,7 +5,10 @@ #include "pycore_hashtable.h" #include +#include // malloc() + #include "clinic/_tracemalloc.c.h" + /*[clinic input] module _tracemalloc [clinic start generated code]*/ @@ -302,7 +305,7 @@ static void tracemalloc_get_frame(InterpreterFrame *pyframe, frame_t *frame) { frame->filename = unknown_filename; - int lineno = PyCode_Addr2Line(pyframe->f_code, pyframe->f_lasti*2); + int lineno = PyCode_Addr2Line(pyframe->f_code, pyframe->f_lasti*sizeof(_Py_CODEUNIT)); if (lineno < 0) { lineno = 0; } @@ -833,7 +836,7 @@ tracemalloc_clear_filename(void *value) static void tracemalloc_clear_traces(void) { - /* The GIL protects variables againt concurrent access */ + /* The GIL protects variables against concurrent access */ assert(PyGILState_Check()); TABLES_LOCK(); diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 11b55f7f0c7d1b..b5c0a632191146 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -5,7 +5,8 @@ #include "Python.h" #include "frameobject.h" #include "pycore_frame.h" -#include "interpreteridobject.h" +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_interpreteridobject.h" static char * @@ -2017,7 +2018,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) } // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); + PyThreadState *save_tstate = _PyThreadState_GET(); // XXX Possible GILState issues? PyThreadState *tstate = _Py_NewInterpreter(isolated); PyThreadState_Swap(save_tstate); diff --git a/Modules/_xxtestfuzz/fuzz_tests.txt b/Modules/_xxtestfuzz/fuzz_tests.txt index 053b77b41b111a..4e046ecf6d898c 100644 --- a/Modules/_xxtestfuzz/fuzz_tests.txt +++ b/Modules/_xxtestfuzz/fuzz_tests.txt @@ -6,3 +6,4 @@ fuzz_sre_compile fuzz_sre_match fuzz_csv_reader fuzz_struct_unpack +fuzz_ast_literal_eval diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index e1256f59cc9c28..366e81a54519a7 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -393,6 +393,51 @@ static int fuzz_csv_reader(const char* data, size_t size) { return 0; } +#define MAX_AST_LITERAL_EVAL_TEST_SIZE 0x10000 +PyObject* ast_literal_eval_method = NULL; +/* Called by LLVMFuzzerTestOneInput for initialization */ +static int init_ast_literal_eval(void) { + PyObject* ast_module = PyImport_ImportModule("ast"); + if (ast_module == NULL) { + return 0; + } + ast_literal_eval_method = PyObject_GetAttrString(ast_module, "literal_eval"); + return ast_literal_eval_method != NULL; +} +/* Fuzz ast.literal_eval(x) */ +static int fuzz_ast_literal_eval(const char* data, size_t size) { + if (size > MAX_AST_LITERAL_EVAL_TEST_SIZE) { + return 0; + } + /* Ignore non null-terminated strings since ast can't handle + embedded nulls */ + if (memchr(data, '\0', size) == NULL) { + return 0; + } + + PyObject* s = PyUnicode_FromString(data); + /* Ignore exceptions until we have a valid string */ + if (s == NULL) { + PyErr_Clear(); + return 0; + } + + PyObject* literal = PyObject_CallOneArg(ast_literal_eval_method, s); + /* Ignore some common errors thrown by ast.literal_eval */ + if (literal == NULL && (PyErr_ExceptionMatches(PyExc_ValueError) || + PyErr_ExceptionMatches(PyExc_TypeError) || + PyErr_ExceptionMatches(PyExc_SyntaxError) || + PyErr_ExceptionMatches(PyExc_MemoryError) || + PyErr_ExceptionMatches(PyExc_RecursionError)) + ) { + PyErr_Clear(); + } + + Py_XDECREF(literal); + Py_DECREF(s); + return 0; +} + /* Run fuzzer and abort on failure. */ static int _run_fuzz(const uint8_t *data, size_t size, int(*fuzzer)(const char* , size_t)) { int rv = fuzzer((const char*) data, size); @@ -507,6 +552,17 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } rv |= _run_fuzz(data, size, fuzz_csv_reader); +#endif +#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_ast_literal_eval) + static int AST_LITERAL_EVAL_INITIALIZED = 0; + if (!AST_LITERAL_EVAL_INITIALIZED && !init_ast_literal_eval()) { + PyErr_Print(); + abort(); + } else { + AST_LITERAL_EVAL_INITIALIZED = 1; + } + + rv |= _run_fuzz(data, size, fuzz_ast_literal_eval); #endif return rv; } diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index c722330ee491a9..04fa09422b2134 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -1342,7 +1342,7 @@ tzrule_transitions(_tzrule *rule, int year, int64_t *start, int64_t *end) * could technically be calculated from the timestamp, but given that the * callers of this function already have the year information accessible from * the datetime struct, it is taken as an additional parameter to reduce - * unncessary calculation. + * unnecessary calculation. * */ static _ttinfo * find_tzrule_ttinfo(_tzrule *rule, int64_t ts, unsigned char fold, int year) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index c9b6a92c227491..5e57fe116059de 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -5,6 +5,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_floatobject.h" // _PyFloat_Unpack4() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef #include // offsetof() diff --git a/Modules/binascii.c b/Modules/binascii.c index db960dcdcb0b60..7037d34dbe2dd5 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -56,9 +56,10 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pystrhex.h" +#include "pycore_long.h" // _PyLong_DigitValue +#include "pycore_strhex.h" // _Py_strhex_bytes_with_sep() #ifdef USE_ZLIB_CRC32 -#include "zlib.h" +# include "zlib.h" #endif typedef struct binascii_state { diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 43378f94f98450..855ac77a7f73fd 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -1063,7 +1063,7 @@ _codecs_utf_32_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar } PyDoc_STRVAR(_codecs_unicode_escape_decode__doc__, -"unicode_escape_decode($module, data, errors=None, /)\n" +"unicode_escape_decode($module, data, errors=None, final=True, /)\n" "--\n" "\n"); @@ -1072,7 +1072,7 @@ PyDoc_STRVAR(_codecs_unicode_escape_decode__doc__, static PyObject * _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors); + const char *errors, int final); static PyObject * _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1080,8 +1080,9 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *return_value = NULL; Py_buffer data = {NULL, NULL}; const char *errors = NULL; + int final = 1; - if (!_PyArg_CheckPositional("unicode_escape_decode", nargs, 1, 2)) { + if (!_PyArg_CheckPositional("unicode_escape_decode", nargs, 1, 3)) { goto exit; } if (PyUnicode_Check(args[0])) { @@ -1122,8 +1123,15 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ _PyArg_BadArgument("unicode_escape_decode", "argument 2", "str or None", args[1]); goto exit; } + if (nargs < 3) { + goto skip_optional; + } + final = _PyLong_AsInt(args[2]); + if (final == -1 && PyErr_Occurred()) { + goto exit; + } skip_optional: - return_value = _codecs_unicode_escape_decode_impl(module, &data, errors); + return_value = _codecs_unicode_escape_decode_impl(module, &data, errors, final); exit: /* Cleanup for data */ @@ -1135,7 +1143,7 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ } PyDoc_STRVAR(_codecs_raw_unicode_escape_decode__doc__, -"raw_unicode_escape_decode($module, data, errors=None, /)\n" +"raw_unicode_escape_decode($module, data, errors=None, final=True, /)\n" "--\n" "\n"); @@ -1144,7 +1152,7 @@ PyDoc_STRVAR(_codecs_raw_unicode_escape_decode__doc__, static PyObject * _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors); + const char *errors, int final); static PyObject * _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1152,8 +1160,9 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss PyObject *return_value = NULL; Py_buffer data = {NULL, NULL}; const char *errors = NULL; + int final = 1; - if (!_PyArg_CheckPositional("raw_unicode_escape_decode", nargs, 1, 2)) { + if (!_PyArg_CheckPositional("raw_unicode_escape_decode", nargs, 1, 3)) { goto exit; } if (PyUnicode_Check(args[0])) { @@ -1194,8 +1203,15 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss _PyArg_BadArgument("raw_unicode_escape_decode", "argument 2", "str or None", args[1]); goto exit; } + if (nargs < 3) { + goto skip_optional; + } + final = _PyLong_AsInt(args[2]); + if (final == -1 && PyErr_Occurred()) { + goto exit; + } skip_optional: - return_value = _codecs_raw_unicode_escape_decode_impl(module, &data, errors); + return_value = _codecs_raw_unicode_escape_decode_impl(module, &data, errors, final); exit: /* Cleanup for data */ @@ -2801,4 +2817,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=557c3b37e4c492ac input=a9049054013a1b77]*/ +/*[clinic end generated code: output=814dae36b6f885cb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index d9522562359e64..22d2e992b6398d 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -139,7 +139,7 @@ PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, - int block, PyObject *timeout); + int block, PyObject *timeout_obj); static PyObject * _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -148,13 +148,13 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *con static const char * const _keywords[] = {"block", "timeout", NULL}; static _PyArg_Parser _parser = {"|pO:get", _keywords, 0}; int block = 1; - PyObject *timeout = Py_None; + PyObject *timeout_obj = Py_None; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &block, &timeout)) { + &block, &timeout_obj)) { goto exit; } - return_value = _queue_SimpleQueue_get_impl(self, cls, block, timeout); + return_value = _queue_SimpleQueue_get_impl(self, cls, block, timeout_obj); exit: return return_value; @@ -248,4 +248,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=96cc57168d72aab1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=acfaf0191d8935db input=a9049054013a1b77]*/ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 5ba56eaea6357a..034a03c30851a1 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -3976,7 +3976,7 @@ initializeEncoding(XML_Parser parser) { const char *s; #ifdef XML_UNICODE char encodingBuf[128]; - /* See comments abount `protoclEncodingName` in parserInit() */ + /* See comments about `protoclEncodingName` in parserInit() */ if (! parser->m_protocolEncodingName) s = NULL; else { diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index f2b6b406067ea9..5b935718743add 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -42,16 +42,16 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include -#include /* memcpy */ -#include - #ifdef _WIN32 # include "winconfig.h" #endif #include +#include +#include /* memcpy */ +#include + #include "expat_external.h" #include "internal.h" #include "xmltok.h" diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 350f4cf6b8edff..cb2e2588e19b24 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -3,10 +3,13 @@ #include "pycore_pyerrors.h" // _Py_DumpExtensionModules #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_traceback.h" // _Py_DumpTracebackThreads -#include + +#include "frameobject.h" + #include -#include #include +#include +#include // abort() #if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) # include #endif @@ -710,7 +713,7 @@ faulthandler_dump_traceback_later(PyObject *self, return NULL; } /* Limit to LONG_MAX seconds for format_timeout() */ - if (timeout_us >= PY_TIMEOUT_MAX || timeout_us / SEC_TO_US >= LONG_MAX) { + if (timeout_us > PY_TIMEOUT_MAX || timeout_us / SEC_TO_US > LONG_MAX) { PyErr_SetString(PyExc_OverflowError, "timeout value is too large"); return NULL; diff --git a/Modules/gc_weakref.txt b/Modules/gc_weakref.txt index b5b9f7b2e3d4dc..6d07cce1236431 100644 --- a/Modules/gc_weakref.txt +++ b/Modules/gc_weakref.txt @@ -50,7 +50,7 @@ CT while gc is running. https://www.python.org/sf/1055820 shows how innocent it can be, and also how nasty. Variants of the three -focussed test cases attached to that bug report are now part of Python's +focused test cases attached to that bug report are now part of Python's standard Lib/test/test_gc.py. Jim Fulton gave the best nutshell summary of the new (in 2.4 and 2.3.5) diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 2592c397cf2f73..7d1a45bcaeabf8 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1211,7 +1211,7 @@ gc_collect_main(PyThreadState *tstate, int generation, if (gcstate->debug & DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_GetMonotonicClock(); + t1 = _PyTime_GetPerfCounter(); } if (PyDTrace_GC_START_ENABLED()) @@ -1307,7 +1307,7 @@ gc_collect_main(PyThreadState *tstate, int generation, debug_cycle("uncollectable", FROM_GC(gc)); } if (gcstate->debug & DEBUG_STATS) { - double d = _PyTime_AsSecondsDouble(_PyTime_GetMonotonicClock() - t1); + double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 5f941a26e1d54c..7cb7397a22c8ab 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -40,8 +40,9 @@ Py_GetBuildInfo(void) const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; const char *gitid = _Py_gitidentifier(); - if (!(*gitid)) - gitid = "default"; + if (!(*gitid)) { + gitid = "main"; + } PyOS_snprintf(buildinfo, sizeof(buildinfo), "%s%s%s, %.20s, %.9s", gitid, sep, revision, DATE, TIME); diff --git a/Modules/getpath.c b/Modules/getpath.c index 363d62a0657ebd..1405023b39b580 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -6,8 +6,9 @@ #include "pycore_pathconfig.h" #include "osdefs.h" // DELIM -#include +#include // getenv() #include +#include #ifdef __APPLE__ # include @@ -115,11 +116,6 @@ extern "C" { #define BUILD_LANDMARK L"Modules/Setup.local" -#define DECODE_LOCALE_ERR(NAME, LEN) \ - ((LEN) == (size_t)-2) \ - ? _PyStatus_ERR("cannot decode " NAME) \ - : _PyStatus_NO_MEMORY() - #define PATHLEN_ERR() _PyStatus_ERR("path configuration: path too long") typedef struct { @@ -149,23 +145,6 @@ static const wchar_t delimiter[2] = {DELIM, '\0'}; static const wchar_t separator[2] = {SEP, '\0'}; -/* Get file status. Encode the path to the locale encoding. */ -static int -_Py_wstat(const wchar_t* path, struct stat *buf) -{ - int err; - char *fname; - fname = _Py_EncodeLocaleRaw(path, NULL); - if (fname == NULL) { - errno = EINVAL; - return -1; - } - err = stat(fname, buf); - PyMem_RawFree(fname); - return err; -} - - static void reduce(wchar_t *dir) { @@ -235,28 +214,18 @@ isdir(const wchar_t *filename) static PyStatus joinpath(wchar_t *path, const wchar_t *path2, size_t path_len) { - size_t n; - if (!_Py_isabs(path2)) { - n = wcslen(path); - if (n >= path_len) { + if (_Py_isabs(path2)) { + if (wcslen(path2) >= path_len) { return PATHLEN_ERR(); } - - if (n > 0 && path[n-1] != SEP) { - path[n++] = SEP; - } + wcscpy(path, path2); } else { - n = 0; - } - - size_t k = wcslen(path2); - if (n + k >= path_len) { - return PATHLEN_ERR(); + if (_Py_add_relfile(path, path2, path_len) < 0) { + return PATHLEN_ERR(); + } + return _PyStatus_OK(); } - wcsncpy(path + n, path2, k); - path[n + k] = '\0'; - return _PyStatus_OK(); } @@ -283,23 +252,7 @@ joinpath2(const wchar_t *path, const wchar_t *path2) if (_Py_isabs(path2)) { return _PyMem_RawWcsdup(path2); } - - size_t len = wcslen(path); - int add_sep = (len > 0 && path[len - 1] != SEP); - len += add_sep; - len += wcslen(path2); - - wchar_t *new_path = PyMem_RawMalloc((len + 1) * sizeof(wchar_t)); - if (new_path == NULL) { - return NULL; - } - - wcscpy(new_path, path); - if (add_sep) { - wcscat(new_path, separator); - } - wcscat(new_path, path2); - return new_path; + return _Py_join_relfile(path, path2); } @@ -1505,7 +1458,7 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) } /* If a pyvenv.cfg configure file is found, - argv0_path is overriden with its 'home' variable. */ + argv0_path is overridden with its 'home' variable. */ status = calculate_read_pyenv(calculate); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1540,6 +1493,16 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) } } + if (pathconfig->stdlib_dir == NULL) { + if (calculate->prefix_found) { + /* This must be done *before* calculate_set_prefix() is called. */ + pathconfig->stdlib_dir = _PyMem_RawWcsdup(calculate->prefix); + if (pathconfig->stdlib_dir == NULL) { + return _PyStatus_NO_MEMORY(); + } + } + } + if (pathconfig->prefix == NULL) { status = calculate_set_prefix(calculate, pathconfig); if (_PyStatus_EXCEPTION(status)) { diff --git a/Modules/hashlib.h b/Modules/hashlib.h index 978593e2f1a0c2..56ae7a5e50bf58 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -8,7 +8,7 @@ #define GET_BUFFER_VIEW_OR_ERROR(obj, viewp, erraction) do { \ if (PyUnicode_Check((obj))) { \ PyErr_SetString(PyExc_TypeError, \ - "Unicode-objects must be encoded before hashing");\ + "Strings must be encoded before hashing");\ erraction; \ } \ if (!PyObject_CheckBuffer((obj))) { \ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 0df179673a1114..342a3e6555ba05 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,7 +1,6 @@ - - #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -1058,7 +1057,7 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) PyTuple_SET_ITEM(result, 0, copyable); for (i = 1; i < n; i++) { - copyable = _PyObject_CallNoArg(copyfunc); + copyable = _PyObject_CallNoArgs(copyfunc); if (copyable == NULL) { Py_DECREF(copyfunc); Py_DECREF(result); diff --git a/Modules/main.c b/Modules/main.c index 2684d230672b94..c537e6b678515e 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1,6 +1,7 @@ /* Python interpreter main program */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_initconfig.h" // _PyArgv #include "pycore_interp.h" // _PyInterpreterState.sysdict #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() @@ -457,7 +458,7 @@ pymain_run_interactive_hook(int *exitcode) goto error; } - result = _PyObject_CallNoArg(hook); + result = _PyObject_CallNoArgs(hook); Py_DECREF(hook); if (result == NULL) { goto error; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index bd97b03205b7ca..4fac0cc29e4e98 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -54,7 +54,8 @@ raised for division by zero and mod by zero. #include "Python.h" #include "pycore_bitutils.h" // _Py_bit_length() -#include "pycore_dtoa.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dtoa.h" // _Py_dg_infinity() #include "pycore_long.h" // _PyLong_GetZero() #include "_math.h" @@ -1206,7 +1207,7 @@ math_ceil(PyObject *module, PyObject *number) if (!PyFloat_CheckExact(number)) { PyObject *method = _PyObject_LookupSpecial(number, &PyId___ceil__); if (method != NULL) { - PyObject *result = _PyObject_CallNoArg(method); + PyObject *result = _PyObject_CallNoArgs(method); Py_DECREF(method); return result; } @@ -1275,7 +1276,7 @@ math_floor(PyObject *module, PyObject *number) { PyObject *method = _PyObject_LookupSpecial(number, &PyId___floor__); if (method != NULL) { - PyObject *result = _PyObject_CallNoArg(method); + PyObject *result = _PyObject_CallNoArgs(method); Py_DECREF(method); return result; } @@ -2130,7 +2131,7 @@ math_trunc(PyObject *module, PyObject *x) Py_TYPE(x)->tp_name); return NULL; } - result = _PyObject_CallNoArg(trunc); + result = _PyObject_CallNoArgs(trunc); Py_DECREF(trunc); return result; } @@ -3082,14 +3083,9 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (result == NULL) { - result = PyLong_FromLong(1); - if (result == NULL) { - Py_DECREF(iter); - return NULL; - } - } else { - Py_INCREF(result); + result = _PyLong_GetOne(); } + Py_INCREF(result); #ifndef SLOW_PROD /* Fast paths for integers keeping temporary products in C. * Assumes all inputs are the same type. @@ -3105,7 +3101,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } /* Loop over all the items in the iterable until we finish, we overflow * or we found a non integer element */ - while(result == NULL) { + while (result == NULL) { item = PyIter_Next(iter); if (item == NULL) { Py_DECREF(iter); diff --git a/Modules/md5module.c b/Modules/md5module.c index b50d2e5b0bdb71..4d03f6b844b332 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -18,7 +18,7 @@ #include "Python.h" #include "hashlib.h" -#include "pystrhex.h" +#include "pycore_strhex.h" // _Py_strhex() /*[clinic input] module _md5 diff --git a/Modules/nismodule.c b/Modules/nismodule.c index 6655451ebd2fbf..cdda1a6a2fb7d4 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -12,6 +12,7 @@ #include "Python.h" +#include // free() #include #include #include diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index 4f2d9cb8b7c9c7..4bab9a58eb1045 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -21,6 +21,7 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include // getenv() #ifdef HAVE_FCNTL_H #include #else diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bd38bea4c6c99d..667a3339f5ba8e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10,30 +10,25 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_fileutils.h" -#include "pycore_moduleobject.h" // _PyModule_GetState() +// Include before pycore internal headers. FSCTL_GET_REPARSE_POINT +// is not exported by if the WIN32_LEAN_AND_MEAN macro is defined, +// whereas pycore_condvar.h defines the WIN32_LEAN_AND_MEAN macro. #ifdef MS_WINDOWS - /* include early to avoid conflict with pycore_condvar.h: - - #define WIN32_LEAN_AND_MEAN - #include - - FSCTL_GET_REPARSE_POINT is not exported with WIN32_LEAN_AND_MEAN. */ # include # include #endif -#if !defined(EX_OK) && defined(EXIT_SUCCESS) -#define EX_OK EXIT_SUCCESS -#endif - #ifdef __VXWORKS__ # include "pycore_bitutils.h" // _Py_popcount32() #endif +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_fileutils.h" // _Py_closerange() +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_pystate.h" // _PyInterpreterState_GET() + #include "structmember.h" // PyMemberDef #ifndef MS_WINDOWS # include "posixmodule.h" @@ -41,13 +36,18 @@ # include "winreparse.h" #endif +#if !defined(EX_OK) && defined(EXIT_SUCCESS) +# define EX_OK EXIT_SUCCESS +#endif + /* On android API level 21, 'AT_EACCESS' is not declared although * HAVE_FACCESSAT is defined. */ #ifdef __ANDROID__ # undef HAVE_FACCESSAT #endif -#include /* needed for ctermid() */ +#include // ctermid() +#include // system() /* * A number of APIs are available on macOS from a certain macOS version. @@ -561,7 +561,7 @@ run_at_forkers(PyObject *lst, int reverse) for (i = 0; i < PyList_GET_SIZE(cpy); i++) { PyObject *func, *res; func = PyList_GET_ITEM(cpy, i); - res = _PyObject_CallNoArg(func); + res = _PyObject_CallNoArgs(func); if (res == NULL) PyErr_WriteUnraisable(func); else @@ -1183,7 +1183,7 @@ path_converter(PyObject *o, void *p) if (NULL == func) { goto error_format; } - res = _PyObject_CallNoArg(func); + res = _PyObject_CallNoArgs(func); Py_DECREF(func); if (NULL == res) { goto error_exit; @@ -14380,7 +14380,7 @@ PyOS_FSPath(PyObject *path) _PyType_Name(Py_TYPE(path))); } - path_repr = _PyObject_CallNoArg(func); + path_repr = _PyObject_CallNoArgs(func); Py_DECREF(func); if (NULL == path_repr) { return NULL; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index ec684638ead118..b3d9bdda7e7ac5 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1204,10 +1204,10 @@ static void xmlparse_dealloc(xmlparseobject *self) { PyObject_GC_UnTrack(self); + (void)xmlparse_clear(self); if (self->itself != NULL) XML_ParserFree(self->itself); self->itself = NULL; - (void)xmlparse_clear(self); if (self->handlers != NULL) { PyMem_Free(self->handlers); diff --git a/Modules/readline.c b/Modules/readline.c index c79d22f85f84ea..1b616fc4f3b4e1 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -6,9 +6,11 @@ /* Standard definitions */ #include "Python.h" -#include -#include + #include +#include +#include +#include // free() #include #if defined(HAVE_SETLOCALE) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 3ecd0c32b30389..ff1c028d0d6720 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -318,8 +318,9 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, if (omax > max) max = omax; if (emax > max) max = emax; - if (tvp) - deadline = _PyTime_GetMonotonicClock() + timeout; + if (tvp) { + deadline = _PyDeadline_Init(timeout); + } do { Py_BEGIN_ALLOW_THREADS @@ -335,7 +336,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, goto finally; if (tvp) { - timeout = deadline - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(deadline); if (timeout < 0) { /* bpo-35310: lists were unmodified -- clear them explicitly */ FD_ZERO(&ifdset); @@ -344,7 +345,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, n = 0; break; } - _PyTime_AsTimeval_noraise(timeout, &tv, _PyTime_ROUND_CEILING); + _PyTime_AsTimeval_clamp(timeout, &tv, _PyTime_ROUND_CEILING); /* retry select() with the recomputed timeout */ } } while (1); @@ -599,7 +600,7 @@ select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) } if (timeout >= 0) { - deadline = _PyTime_GetMonotonicClock() + timeout; + deadline = _PyDeadline_Init(timeout); } } @@ -646,7 +647,7 @@ select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) } if (timeout >= 0) { - timeout = deadline - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(deadline); if (timeout < 0) { poll_result = 0; break; @@ -938,8 +939,9 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) dvp.dp_nfds = self->max_n_fds; dvp.dp_timeout = (int)ms; - if (timeout >= 0) - deadline = _PyTime_GetMonotonicClock() + timeout; + if (timeout >= 0) { + deadline = _PyDeadline_Init(timeout); + } do { /* call devpoll() */ @@ -956,7 +958,7 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) return NULL; if (timeout >= 0) { - timeout = deadline - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(deadline); if (timeout < 0) { poll_result = 0; break; @@ -1550,7 +1552,7 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, } if (timeout >= 0) { - deadline = _PyTime_GetMonotonicClock() + timeout; + deadline = _PyDeadline_Init(timeout); } } @@ -1584,7 +1586,7 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, goto error; if (timeout >= 0) { - timeout = deadline - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(deadline); if (timeout < 0) { nfds = 0; break; @@ -2172,8 +2174,9 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, } } - if (ptimeoutspec) - deadline = _PyTime_GetMonotonicClock() + timeout; + if (ptimeoutspec) { + deadline = _PyDeadline_Init(timeout); + } do { Py_BEGIN_ALLOW_THREADS @@ -2190,7 +2193,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, goto error; if (ptimeoutspec) { - timeout = deadline - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(deadline); if (timeout < 0) { gotevents = 0; break; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 6980051dc08f8a..153bc12a8bd743 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -18,7 +18,7 @@ #include "Python.h" #include "hashlib.h" -#include "pystrhex.h" +#include "pycore_strhex.h" // _Py_strhex() /*[clinic input] module _sha1 @@ -269,7 +269,7 @@ sha1_done(struct sha1_state *sha1, unsigned char *out) sha1->curlen = 0; } - /* pad upto 56 bytes of zeroes */ + /* pad up to 56 bytes of zeroes */ while (sha1->curlen < 56) { sha1->buf[sha1->curlen++] = (unsigned char)0; } diff --git a/Modules/sha256module.c b/Modules/sha256module.c index fac7073d951d2d..5858071db4e133 100644 --- a/Modules/sha256module.c +++ b/Modules/sha256module.c @@ -18,9 +18,9 @@ #include "Python.h" #include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_strhex.h" // _Py_strhex() #include "structmember.h" // PyMemberDef #include "hashlib.h" -#include "pystrhex.h" /*[clinic input] module _sha256 diff --git a/Modules/sha512module.c b/Modules/sha512module.c index 4ea2fc16de0526..e50b69be4617d9 100644 --- a/Modules/sha512module.c +++ b/Modules/sha512module.c @@ -18,9 +18,9 @@ #include "Python.h" #include "pycore_bitutils.h" // _Py_bswap64() +#include "pycore_strhex.h" // _Py_strhex() #include "structmember.h" // PyMemberDef #include "hashlib.h" -#include "pystrhex.h" /*[clinic input] module _sha512 diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index fc58cfd2084c73..09f4aed9d5ca9a 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -7,7 +7,8 @@ #include "pycore_atomic.h" // _Py_atomic_int #include "pycore_call.h" // _PyObject_Call() #include "pycore_ceval.h" // _PyEval_SignalReceived() -#include "pycore_frame.h" +#include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH +#include "pycore_frame.h" // InterpreterFrame #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pylifecycle.h" // NSIG @@ -1221,11 +1222,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) /*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ { - struct timespec ts; - siginfo_t si; - int res; - _PyTime_t timeout, deadline, monotonic; - + _PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) return NULL; @@ -1235,12 +1232,16 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, return NULL; } - deadline = _PyTime_GetMonotonicClock() + timeout; + _PyTime_t deadline = _PyDeadline_Init(timeout); + siginfo_t si; do { - if (_PyTime_AsTimespec(timeout, &ts) < 0) + struct timespec ts; + if (_PyTime_AsTimespec(timeout, &ts) < 0) { return NULL; + } + int res; Py_BEGIN_ALLOW_THREADS res = sigtimedwait(&sigset, &si, &ts); Py_END_ALLOW_THREADS @@ -1259,10 +1260,10 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, if (PyErr_CheckSignals()) return NULL; - monotonic = _PyTime_GetMonotonicClock(); - timeout = deadline - monotonic; - if (timeout < 0) + timeout = _PyDeadline_Get(deadline); + if (timeout < 0) { break; + } } while (1); return fill_siginfo(&si); @@ -1595,7 +1596,7 @@ signal_get_set_handlers(signal_state_t *state, PyObject *mod_dict) Py_XDECREF(old_func); } - // Instal Python SIGINT handler which raises KeyboardInterrupt + // Install Python SIGINT handler which raises KeyboardInterrupt PyObject* sigint_func = get_handler(SIGINT); if (sigint_func == state->default_handler) { PyObject *int_handler = PyMapping_GetItemString(mod_dict, diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 83f05b72ecbd9e..50962c41cd3a68 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -758,7 +758,7 @@ internal_select(PySocketSockObject *s, int writing, _PyTime_t interval, Py_END_ALLOW_THREADS; #else if (interval >= 0) { - _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_CEILING); + _PyTime_AsTimeval_clamp(interval, &tv, _PyTime_ROUND_CEILING); tvp = &tv; } else @@ -840,18 +840,20 @@ sock_call_ex(PySocketSockObject *s, if (deadline_initialized) { /* recompute the timeout */ - interval = deadline - _PyTime_GetMonotonicClock(); + interval = _PyDeadline_Get(deadline); } else { deadline_initialized = 1; - deadline = _PyTime_GetMonotonicClock() + timeout; + deadline = _PyDeadline_Init(timeout); interval = timeout; } - if (interval >= 0) + if (interval >= 0) { res = internal_select(s, writing, interval, connect); - else + } + else { res = 1; + } } else { res = internal_select(s, writing, timeout, connect); @@ -927,7 +929,7 @@ sock_call_ex(PySocketSockObject *s, reading, but the data then discarded by the OS because of a wrong checksum. - Loop on select() to recheck for socket readyness. */ + Loop on select() to recheck for socket readiness. */ continue; } @@ -4176,7 +4178,7 @@ sock_sendall(PySocketSockObject *s, PyObject *args) Py_buffer pbuf; struct sock_send ctx; int has_timeout = (s->sock_timeout > 0); - _PyTime_t interval = s->sock_timeout; + _PyTime_t timeout = s->sock_timeout; _PyTime_t deadline = 0; int deadline_initialized = 0; PyObject *res = NULL; @@ -4195,14 +4197,14 @@ sock_sendall(PySocketSockObject *s, PyObject *args) if (has_timeout) { if (deadline_initialized) { /* recompute the timeout */ - interval = deadline - _PyTime_GetMonotonicClock(); + timeout = _PyDeadline_Get(deadline); } else { deadline_initialized = 1; - deadline = _PyTime_GetMonotonicClock() + s->sock_timeout; + deadline = _PyDeadline_Init(timeout); } - if (interval <= 0) { + if (timeout <= 0) { PyErr_SetString(PyExc_TimeoutError, "timed out"); goto done; } @@ -4211,7 +4213,7 @@ sock_sendall(PySocketSockObject *s, PyObject *args) ctx.buf = buf; ctx.len = len; ctx.flags = flags; - if (sock_call_ex(s, 1, sock_send_impl, &ctx, 0, NULL, interval) < 0) + if (sock_call_ex(s, 1, sock_send_impl, &ctx, 0, NULL, timeout) < 0) goto done; n = ctx.result; assert(n >= 0); diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index e4f375d5e8100a..aea599f0ee6c85 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -8,9 +8,7 @@ # include # endif # include -# if !defined(__CYGWIN__) -# include -# endif +# include #else /* MS_WINDOWS */ # include diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index cdc94a60a373d7..0f2fea15b3a6f0 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -183,7 +183,7 @@ syslog_syslog(PyObject * self, PyObject * args) PyObject *openargs; /* Continue even if PyTuple_New fails, because openlog(3) is optional. - * So, we can still do loggin in the unlikely event things are so hosed + * So, we can still do logging in the unlikely event things are so hosed * that we can't do this tuple. */ if ((openargs = PyTuple_New(0))) { diff --git a/Modules/termios.c b/Modules/termios.c index 38573e25f51dd8..354e5ca18d04d8 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -408,7 +408,7 @@ termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) } Py_XDECREF(tmp_item); tmp_item = PySequence_GetItem(winsz, 1); - winsz_1 = PyLong_AsLong(tmp_item); + winsz_1 = PyLong_AsLong(tmp_item); if (winsz_1 == -1 && PyErr_Occurred()) { Py_XDECREF(tmp_item); return NULL; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 4639afa590a4fa..0ef3b2ffaa8987 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1,33 +1,29 @@ /* Time module */ #include "Python.h" +#include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH +#include "pycore_namespace.h" // _PyNamespace_New() #include #ifdef HAVE_SYS_TIMES_H -#include +# include #endif - #ifdef HAVE_SYS_TYPES_H -#include +# include #endif - #if defined(HAVE_SYS_RESOURCE_H) -#include +# include #endif - #ifdef QUICKWIN -#include +# include #endif - #if defined(HAVE_PTHREAD_H) # include #endif - #if defined(_AIX) # include #endif - #if defined(__WATCOMC__) && !defined(__QNX__) # include #else @@ -38,17 +34,17 @@ #endif /* !__WATCOMC__ || __QNX__ */ #ifdef _Py_MEMORY_SANITIZER -# include +# include #endif #ifdef _MSC_VER -#define _Py_timezone _timezone -#define _Py_daylight _daylight -#define _Py_tzname _tzname +# define _Py_timezone _timezone +# define _Py_daylight _daylight +# define _Py_tzname _tzname #else -#define _Py_timezone timezone -#define _Py_daylight daylight -#define _Py_tzname tzname +# define _Py_timezone timezone +# define _Py_daylight daylight +# define _Py_tzname tzname #endif #if defined(__APPLE__ ) && defined(__has_builtin) @@ -60,8 +56,10 @@ # define HAVE_CLOCK_GETTIME_RUNTIME 1 #endif + #define SEC_TO_NS (1000 * 1000 * 1000) + /* Forward declarations */ static int pysleep(_PyTime_t timeout); diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 9bd9a5407d43c4..93895c4f1214c3 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -1,4 +1,3 @@ - /* Use this file as a template to start implementing a module that also declares object types. All occurrences of 'Xxo' should be changed to something reasonable for your objects. After that, all other @@ -55,7 +54,7 @@ pass */ -#define Py_LIMITED_API 0x030a0000 +#define Py_LIMITED_API 0x030b0000 #include "Python.h" diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 28fe8840fc0675..67bde701fa608b 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -82,7 +82,7 @@ typedef struct { Bytef *next_posi; } _Uint32Window; -/* Initialize the buffer with an inital buffer size. +/* Initialize the buffer with an initial buffer size. On success, return value >= 0 On failure, return value < 0 */ diff --git a/Objects/abstract.c b/Objects/abstract.c index c056c6698850c9..6f7b94600e278a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_object.h" // _Py_CheckSlotResult() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -9,7 +10,6 @@ #include "pycore_unionobject.h" // _PyUnion_Check() #include #include // offsetof() -#include "longintrepr.h" @@ -114,7 +114,7 @@ PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) } return defaultvalue; } - result = _PyObject_CallNoArg(hint); + result = _PyObject_CallNoArgs(hint); Py_DECREF(hint); if (result == NULL) { PyThreadState *tstate = _PyThreadState_GET(); @@ -1576,7 +1576,7 @@ PyNumber_Long(PyObject *o) } trunc_func = _PyObject_LookupSpecial(o, &PyId___trunc__); if (trunc_func) { - result = _PyObject_CallNoArg(trunc_func); + result = _PyObject_CallNoArgs(trunc_func); Py_DECREF(trunc_func); if (result == NULL || PyLong_CheckExact(result)) { return result; diff --git a/Objects/boolobject.c b/Objects/boolobject.c index c72243a160b880..53f81926057974 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -2,7 +2,6 @@ #include "Python.h" #include "pycore_pyerrors.h" // _Py_FatalRefcountError() -#include "longintrepr.h" /* We define bool_repr to return "False" or "True" */ diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 1ab9621b1f2656..a6009854221ff5 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -4,9 +4,9 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" -#include "pycore_object.h" +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_strhex.h" // _Py_strhex_with_sep() #include "bytesobject.h" -#include "pystrhex.h" /*[clinic input] class bytearray "PyByteArrayObject *" "&PyByteArray_Type" diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 27f766bef6ed84..66fd2ecc3c41de 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -5,12 +5,14 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" // _Py_bytes_startswith() +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_format.h" // F_LJUST #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_long.h" // _PyLong_DigitValue #include "pycore_object.h" // _PyObject_GC_TRACK #include "pycore_pymem.h" // PYMEM_CLEANBYTE +#include "pycore_strhex.h" // _Py_strhex_with_sep() -#include "pystrhex.h" #include /*[clinic input] @@ -572,7 +574,7 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) /* does it support __bytes__? */ func = _PyObject_LookupSpecial(v, &PyId___bytes__); if (func != NULL) { - result = _PyObject_CallNoArg(func); + result = _PyObject_CallNoArgs(func); Py_DECREF(func); if (result == NULL) return NULL; @@ -2622,7 +2624,7 @@ bytes_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, integer argument before deferring to PyBytes_FromObject, something PyObject_Bytes doesn't do. */ else if ((func = _PyObject_LookupSpecial(x, &PyId___bytes__)) != NULL) { - bytes = _PyObject_CallNoArg(func); + bytes = _PyObject_CallNoArgs(func); Py_DECREF(func); if (bytes == NULL) return NULL; diff --git a/Objects/call.c b/Objects/call.c index 960c37e1961f0a..5e55518b04cbfe 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgTstate() +#include "pycore_call.h" // _PyObject_CallNoArgsTstate() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -109,8 +109,7 @@ _Py_CheckSlotResult(PyObject *obj, const char *slot_name, int success) PyObject * PyObject_CallNoArgs(PyObject *func) { - PyThreadState *tstate = _PyThreadState_GET(); - return _PyObject_CallNoArgTstate(tstate, func); + return _PyObject_CallNoArgs(func); } @@ -131,7 +130,7 @@ _PyObject_FastCallDictTstate(PyThreadState *tstate, PyObject *callable, assert(nargs == 0 || args != NULL); assert(kwargs == NULL || PyDict_Check(kwargs)); - vectorcallfunc func = PyVectorcall_Function(callable); + vectorcallfunc func = _PyVectorcall_Function(callable); if (func == NULL) { /* Use tp_call instead */ return _PyObject_MakeTpCall(tstate, callable, args, nargs, kwargs); @@ -225,28 +224,18 @@ _PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, } -PyObject * -PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs) +vectorcallfunc +PyVectorcall_Function(PyObject *callable) { - PyThreadState *tstate = _PyThreadState_GET(); - vectorcallfunc func; + return _PyVectorcall_FunctionInline(callable); +} - /* get vectorcallfunc as in PyVectorcall_Function, but without - * the Py_TPFLAGS_HAVE_VECTORCALL check */ - Py_ssize_t offset = Py_TYPE(callable)->tp_vectorcall_offset; - if (offset <= 0) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support vectorcall", - Py_TYPE(callable)->tp_name); - return NULL; - } - memcpy(&func, (char *) callable + offset, sizeof(func)); - if (func == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support vectorcall", - Py_TYPE(callable)->tp_name); - return NULL; - } + +static PyObject * +_PyVectorcall_Call(PyThreadState *tstate, vectorcallfunc func, + PyObject *callable, PyObject *tuple, PyObject *kwargs) +{ + assert(func != NULL); Py_ssize_t nargs = PyTuple_GET_SIZE(tuple); @@ -272,6 +261,53 @@ PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs) } +PyObject * +PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs) +{ + PyThreadState *tstate = _PyThreadState_GET(); + + /* get vectorcallfunc as in _PyVectorcall_Function, but without + * the Py_TPFLAGS_HAVE_VECTORCALL check */ + Py_ssize_t offset = Py_TYPE(callable)->tp_vectorcall_offset; + if (offset <= 0) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support vectorcall", + Py_TYPE(callable)->tp_name); + return NULL; + } + assert(PyCallable_Check(callable)); + + vectorcallfunc func; + memcpy(&func, (char *) callable + offset, sizeof(func)); + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support vectorcall", + Py_TYPE(callable)->tp_name); + return NULL; + } + + return _PyVectorcall_Call(tstate, func, callable, tuple, kwargs); +} + + +PyObject * +PyObject_Vectorcall(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyObject_VectorcallTstate(tstate, callable, + args, nargsf, kwnames); +} + + +PyObject * +_PyObject_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyObject_FastCallTstate(tstate, func, args, nargs); +} + + PyObject * _PyObject_Call(PyThreadState *tstate, PyObject *callable, PyObject *args, PyObject *kwargs) @@ -286,8 +322,9 @@ _PyObject_Call(PyThreadState *tstate, PyObject *callable, assert(PyTuple_Check(args)); assert(kwargs == NULL || PyDict_Check(kwargs)); - if (PyVectorcall_Function(callable) != NULL) { - return PyVectorcall_Call(callable, args, kwargs); + vectorcallfunc vector_func = _PyVectorcall_Function(callable); + if (vector_func != NULL) { + return _PyVectorcall_Call(tstate, vector_func, callable, args, kwargs); } else { call = Py_TYPE(callable)->tp_call; @@ -326,6 +363,19 @@ PyCFunction_Call(PyObject *callable, PyObject *args, PyObject *kwargs) } +PyObject * +PyObject_CallOneArg(PyObject *func, PyObject *arg) +{ + assert(arg != NULL); + PyObject *_args[2]; + PyObject **args = _args + 1; // For PY_VECTORCALL_ARGUMENTS_OFFSET + args[0] = arg; + PyThreadState *tstate = _PyThreadState_GET(); + size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + return _PyObject_VectorcallTstate(tstate, func, args, nargsf, NULL); +} + + /* --- PyFunction call functions ---------------------------------- */ PyObject * @@ -389,7 +439,7 @@ PyObject_CallObject(PyObject *callable, PyObject *args) PyThreadState *tstate = _PyThreadState_GET(); assert(!_PyErr_Occurred(tstate)); if (args == NULL) { - return _PyObject_CallNoArgTstate(tstate, callable); + return _PyObject_CallNoArgsTstate(tstate, callable); } if (!PyTuple_Check(args)) { _PyErr_SetString(tstate, PyExc_TypeError, @@ -455,7 +505,7 @@ _PyObject_CallFunctionVa(PyThreadState *tstate, PyObject *callable, } if (!format || !*format) { - return _PyObject_CallNoArgTstate(tstate, callable); + return _PyObject_CallNoArgsTstate(tstate, callable); } if (is_size_t) { diff --git a/Objects/classobject.c b/Objects/classobject.c index af73be3d262411..9d4fc99f1f858c 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -1,6 +1,7 @@ /* Class object implementation (dead now except for methods) */ #include "Python.h" +#include "pycore_call.h" // _PyObject_VectorcallTstate() #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Objects/codeobject.c b/Objects/codeobject.c index ad8f13a781b94a..8de5c4d9c8a9d7 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -656,15 +656,13 @@ _PyCode_Addr2Offset(PyCodeObject* co, int addrq) if (co->co_columntable == Py_None || addrq < 0) { return -1; } - if (addrq % 2 == 1) { - --addrq; - } - if (addrq >= PyBytes_GET_SIZE(co->co_columntable)) { + addrq /= sizeof(_Py_CODEUNIT); + if (addrq*2 >= PyBytes_GET_SIZE(co->co_columntable)) { return -1; } unsigned char* bytes = (unsigned char*)PyBytes_AS_STRING(co->co_columntable); - return bytes[addrq] - 1; + return bytes[addrq*2] - 1; } int @@ -673,15 +671,13 @@ _PyCode_Addr2EndOffset(PyCodeObject* co, int addrq) if (co->co_columntable == Py_None || addrq < 0) { return -1; } - if (addrq % 2 == 0) { - ++addrq; - } - if (addrq >= PyBytes_GET_SIZE(co->co_columntable)) { + addrq /= sizeof(_Py_CODEUNIT); + if (addrq*2+1 >= PyBytes_GET_SIZE(co->co_columntable)) { return -1; } unsigned char* bytes = (unsigned char*)PyBytes_AS_STRING(co->co_columntable); - return bytes[addrq] - 1; + return bytes[addrq*2+1] - 1; } void diff --git a/Objects/complexobject.c b/Objects/complexobject.c index cfe6c737578a03..f658dbf336dbf5 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -6,8 +6,10 @@ /* Submitted by Jim Hugunin */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_pymath.h" // _Py_ADJUST_ERANGE2() #include "structmember.h" // PyMemberDef @@ -283,7 +285,7 @@ try_complex_special_method(PyObject *op) f = _PyObject_LookupSpecial(op, &PyId___complex__); if (f) { - PyObject *res = _PyObject_CallNoArg(f); + PyObject *res = _PyObject_CallNoArgs(f); Py_DECREF(f); if (!res || PyComplex_CheckExact(res)) { return res; @@ -525,7 +527,7 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) p = _Py_c_pow(a, b); } - Py_ADJUST_ERANGE2(p.real, p.imag); + _Py_ADJUST_ERANGE2(p.real, p.imag); if (errno == EDOM) { PyErr_SetString(PyExc_ZeroDivisionError, "0.0 to a negative or complex power"); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 4ed36e94b3f79a..946ea6aa80319a 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -72,13 +72,8 @@ wrapperdescr_repr(PyWrapperDescrObject *descr) } static int -descr_check(PyDescrObject *descr, PyObject *obj, PyObject **pres) +descr_check(PyDescrObject *descr, PyObject *obj) { - if (obj == NULL) { - Py_INCREF(descr); - *pres = (PyObject *)descr; - return 1; - } if (!PyObject_TypeCheck(obj, descr->d_type)) { PyErr_Format(PyExc_TypeError, "descriptor '%V' for '%.100s' objects " @@ -86,8 +81,7 @@ descr_check(PyDescrObject *descr, PyObject *obj, PyObject **pres) descr_name((PyDescrObject *)descr), "?", descr->d_type->tp_name, Py_TYPE(obj)->tp_name); - *pres = NULL; - return 1; + return -1; } return 0; } @@ -137,10 +131,12 @@ classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) static PyObject * method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) { - PyObject *res; - - if (descr_check((PyDescrObject *)descr, obj, &res)) - return res; + if (obj == NULL) { + return Py_NewRef(descr); + } + if (descr_check((PyDescrObject *)descr, obj) < 0) { + return NULL; + } if (descr->d_method->ml_flags & METH_METHOD) { if (PyType_Check(type)) { return PyCMethod_New(descr->d_method, obj, NULL, descr->d_common.d_type); @@ -159,10 +155,12 @@ method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) static PyObject * member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) { - PyObject *res; - - if (descr_check((PyDescrObject *)descr, obj, &res)) - return res; + if (obj == NULL) { + return Py_NewRef(descr); + } + if (descr_check((PyDescrObject *)descr, obj) < 0) { + return NULL; + } if (descr->d_member->flags & PY_AUDIT_READ) { if (PySys_Audit("object.__getattr__", "Os", @@ -177,10 +175,12 @@ member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) static PyObject * getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) { - PyObject *res; - - if (descr_check((PyDescrObject *)descr, obj, &res)) - return res; + if (obj == NULL) { + return Py_NewRef(descr); + } + if (descr_check((PyDescrObject *)descr, obj) < 0) { + return NULL; + } if (descr->d_getset->get != NULL) return descr->d_getset->get(obj, descr->d_getset->closure); PyErr_Format(PyExc_AttributeError, @@ -193,16 +193,17 @@ getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) static PyObject * wrapperdescr_get(PyWrapperDescrObject *descr, PyObject *obj, PyObject *type) { - PyObject *res; - - if (descr_check((PyDescrObject *)descr, obj, &res)) - return res; + if (obj == NULL) { + return Py_NewRef(descr); + } + if (descr_check((PyDescrObject *)descr, obj) < 0) { + return NULL; + } return PyWrapper_New((PyObject *)descr, obj); } static int -descr_setcheck(PyDescrObject *descr, PyObject *obj, PyObject *value, - int *pres) +descr_setcheck(PyDescrObject *descr, PyObject *obj, PyObject *value) { assert(obj != NULL); if (!PyObject_TypeCheck(obj, descr->d_type)) { @@ -212,8 +213,7 @@ descr_setcheck(PyDescrObject *descr, PyObject *obj, PyObject *value, descr_name(descr), "?", descr->d_type->tp_name, Py_TYPE(obj)->tp_name); - *pres = -1; - return 1; + return -1; } return 0; } @@ -221,23 +221,22 @@ descr_setcheck(PyDescrObject *descr, PyObject *obj, PyObject *value, static int member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) { - int res; - - if (descr_setcheck((PyDescrObject *)descr, obj, value, &res)) - return res; + if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { + return -1; + } return PyMember_SetOne((char *)obj, descr->d_member, value); } static int getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) { - int res; - - if (descr_setcheck((PyDescrObject *)descr, obj, value, &res)) - return res; - if (descr->d_getset->set != NULL) + if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { + return -1; + } + if (descr->d_getset->set != NULL) { return descr->d_getset->set(obj, value, descr->d_getset->closure); + } PyErr_Format(PyExc_AttributeError, "attribute '%V' of '%.100s' objects is not writable", descr_name((PyDescrObject *)descr), "?", @@ -264,8 +263,7 @@ method_check_args(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObj return -1; } PyObject *self = args[0]; - PyObject *dummy; - if (descr_check((PyDescrObject *)func, self, &dummy)) { + if (descr_check((PyDescrObject *)func, self) < 0) { return -1; } if (kwnames && PyTuple_GET_SIZE(kwnames)) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ae0098be5b5478..6dcd5a1d19715e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -60,7 +60,6 @@ The DictObject can be in one of two forms. ma_values != NULL, dk_refcnt >= 1 Values are stored in the ma_values array. Only string (unicode) keys are allowed. - All dicts sharing same key must have same insertion order. There are four kinds of slots in the table (slot is index, and DK_ENTRIES(keys)[index] if index >= 0): @@ -96,9 +95,10 @@ dk_nentries to achieve amortized O(1). Since there are DKIX_DUMMY remains in dk_indices, we can't increment dk_usable even though dk_nentries is decremented. -In split table, inserting into pending entry is allowed only for dk_entries[ix] -where ix == mp->ma_used. Inserting into other index and deleting item cause -converting the dict to the combined table. +To preserve the order in a split table, a bit vector is used to record the +insertion order. When a key is inserted the bit vector is shifted up by 4 bits +and the index of the key is stored in the low 4 bits. +As a consequence of this, split keys have a maximum size of 16. */ /* PyDict_MINSIZE is the starting size for any new dict. @@ -112,13 +112,14 @@ converting the dict to the combined table. #define PyDict_MINSIZE 8 #include "Python.h" -#include "pycore_bitutils.h" // _Py_bit_length -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_pyerrors.h" // _PyErr_Fetch() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_dict.h" -#include "stringlib/eq.h" // unicode_eq() +#include "pycore_bitutils.h" // _Py_bit_length +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // PyDictKeysObject +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "stringlib/eq.h" // unicode_eq() /*[clinic input] class dict "PyDictObject *" "&PyDict_Type" @@ -239,17 +240,20 @@ uint64_t _pydict_global_version = 0; #include "clinic/dictobject.c.h" +#if PyDict_MAXFREELIST > 0 static struct _Py_dict_state * get_dict_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->dict_state; } +#endif void _PyDict_ClearFreeList(PyInterpreterState *interp) { +#if PyDict_MAXFREELIST > 0 struct _Py_dict_state *state = &interp->dict_state; while (state->numfree) { PyDictObject *op = state->free_list[--state->numfree]; @@ -259,6 +263,7 @@ _PyDict_ClearFreeList(PyInterpreterState *interp) while (state->keys_numfree) { PyObject_Free(state->keys_free_list[--state->keys_numfree]); } +#endif } @@ -266,7 +271,7 @@ void _PyDict_Fini(PyInterpreterState *interp) { _PyDict_ClearFreeList(interp); -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0 struct _Py_dict_state *state = &interp->dict_state; state->numfree = -1; state->keys_numfree = -1; @@ -278,9 +283,11 @@ _PyDict_Fini(PyInterpreterState *interp) void _PyDict_DebugMallocStats(FILE *out) { +#if PyDict_MAXFREELIST > 0 struct _Py_dict_state *state = get_dict_state(); _PyDebugAllocatorStats(out, "free PyDictObject", state->numfree, sizeof(PyDictObject)); +#endif } #define DK_MASK(dk) (DK_SIZE(dk)-1) @@ -445,7 +452,9 @@ static PyDictKeysObject empty_keys_struct = { DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY}, /* dk_indices */ }; -static PyObject *empty_values[1] = { NULL }; + +static PyDictValues empty_values_struct = { 0, { NULL }}; +#define empty_values (&empty_values_struct) #define Py_EMPTY_KEYS &empty_keys_struct @@ -458,6 +467,13 @@ static PyObject *empty_values[1] = { NULL }; # define ASSERT_CONSISTENT(op) assert(_PyDict_CheckConsistency((PyObject *)(op), 0)) #endif +static inline int +get_index_from_order(PyDictObject *mp, Py_ssize_t i) +{ + assert(mp->ma_used <= 16); + int shift = (int)(mp->ma_used-1-i)*4; + return (int)(mp->ma_values->mv_order >> shift) & 15; +} int _PyDict_CheckConsistency(PyObject *op, int check_content) @@ -482,17 +498,19 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) /* combined table */ CHECK(keys->dk_refcnt == 1); } + else { + CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + } if (check_content) { PyDictKeyEntry *entries = DK_ENTRIES(keys); - Py_ssize_t i; - for (i=0; i < DK_SIZE(keys); i++) { + for (Py_ssize_t i=0; i < DK_SIZE(keys); i++) { Py_ssize_t ix = dictkeys_get_index(keys, i); CHECK(DKIX_DUMMY <= ix && ix <= usable); } - for (i=0; i < usable; i++) { + for (Py_ssize_t i=0; i < usable; i++) { PyDictKeyEntry *entry = &entries[i]; PyObject *key = entry->me_key; @@ -517,9 +535,14 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) } if (splitted) { + CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); /* splitted table */ - for (i=0; i < mp->ma_used; i++) { - CHECK(mp->ma_values[i] != NULL); + int duplicate_check = 0; + for (Py_ssize_t i=0; i < mp->ma_used; i++) { + int index = get_index_from_order(mp, i); + CHECK((duplicate_check & (1<ma_values->values[index] != NULL); } } } @@ -553,6 +576,7 @@ new_keys_object(uint8_t log2_size) es = sizeof(Py_ssize_t); } +#if PyDict_MAXFREELIST > 0 struct _Py_dict_state *state = get_dict_state(); #ifdef Py_DEBUG // new_keys_object() must not be called after _PyDict_Fini() @@ -562,6 +586,7 @@ new_keys_object(uint8_t log2_size) dk = state->keys_free_list[--state->keys_numfree]; } else +#endif { dk = PyObject_Malloc(sizeof(PyDictKeysObject) + (es<dk_refcnt = 1; dk->dk_log2_size = log2_size; - dk->dk_usable = usable; dk->dk_kind = DICT_KEYS_UNICODE; dk->dk_nentries = 0; + dk->dk_usable = usable; dk->dk_version = 0; memset(&dk->dk_indices[0], 0xff, es< 0 struct _Py_dict_state *state = get_dict_state(); #ifdef Py_DEBUG // free_keys_object() must not be called after _PyDict_Fini() @@ -603,18 +629,26 @@ free_keys_object(PyDictKeysObject *keys) state->keys_free_list[state->keys_numfree++] = keys; return; } +#endif PyObject_Free(keys); } -#define new_values(size) PyMem_NEW(PyObject *, size) +static inline PyDictValues* +new_values(Py_ssize_t size) +{ + Py_ssize_t n = sizeof(PyDictValues) + sizeof(PyObject *) * (size-1); + return (PyDictValues*)PyMem_Malloc(n); +} + #define free_values(values) PyMem_Free(values) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyDictKeysObject *keys, PyObject **values) +new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp; assert(keys != NULL); +#if PyDict_MAXFREELIST > 0 struct _Py_dict_state *state = get_dict_state(); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() @@ -626,11 +660,13 @@ new_dict(PyDictKeysObject *keys, PyObject **values) assert (Py_IS_TYPE(mp, &PyDict_Type)); _Py_NewReference((PyObject *)mp); } - else { + else +#endif + { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { dictkeys_decref(keys); - if (values != empty_values) { + if (free_values_on_failure) { free_values(values); } return NULL; @@ -638,29 +674,36 @@ new_dict(PyDictKeysObject *keys, PyObject **values) } mp->ma_keys = keys; mp->ma_values = values; - mp->ma_used = 0; + mp->ma_used = used; mp->ma_version_tag = DICT_NEXT_VERSION(); ASSERT_CONSISTENT(mp); return (PyObject *)mp; } +static inline Py_ssize_t +shared_keys_usable_size(PyDictKeysObject *keys) +{ + return keys->dk_nentries + keys->dk_usable; +} + /* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyDictKeysObject *keys) { - PyObject **values; + PyDictValues *values; Py_ssize_t i, size; - size = USABLE_FRACTION(DK_SIZE(keys)); + size = shared_keys_usable_size(keys); values = new_values(size); if (values == NULL) { dictkeys_decref(keys); return PyErr_NoMemory(); } + values->mv_order = 0; for (i = 0; i < size; i++) { - values[i] = NULL; + values->values[i] = NULL; } - return new_dict(keys, values); + return new_dict(keys, values, 0, 1); } @@ -709,7 +752,7 @@ PyObject * PyDict_New(void) { dictkeys_incref(Py_EMPTY_KEYS); - return new_dict(Py_EMPTY_KEYS, empty_values); + return new_dict(Py_EMPTY_KEYS, empty_values, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -829,7 +872,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu *value_addr = NULL; } else if (kind == DICT_KEYS_SPLIT) { - *value_addr = mp->ma_values[ix]; + *value_addr = mp->ma_values->values[ix]; } else { *value_addr = DK_ENTRIES(dk)[ix].me_value; @@ -879,7 +922,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu Py_UNREACHABLE(); found: if (dk->dk_kind == DICT_KEYS_SPLIT) { - *value_addr = mp->ma_values[ix]; + *value_addr = mp->ma_values->values[ix]; } else { *value_addr = ep0[ix].me_value; @@ -928,7 +971,7 @@ _PyDict_MaybeUntrack(PyObject *op) numentries = mp->ma_keys->dk_nentries; if (_PyDict_HasSplitTable(mp)) { for (i = 0; i < numentries; i++) { - if ((value = mp->ma_values[i]) == NULL) + if ((value = mp->ma_values->values[i]) == NULL) continue; if (_PyObject_GC_MAY_BE_TRACKED(value)) { assert(!_PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)); @@ -974,6 +1017,40 @@ insertion_resize(PyDictObject *mp) return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp))); } +static int +insert_into_dictkeys(PyDictKeysObject *keys, PyObject *name) +{ + assert(PyUnicode_CheckExact(name)); + Py_hash_t hash = ((PyASCIIObject *)name)->hash; + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(name); + if (hash == -1) { + PyErr_Clear(); + return DKIX_EMPTY; + } + } + Py_ssize_t ix = dictkeys_stringlookup(keys, name, hash); + if (ix == DKIX_EMPTY) { + if (keys->dk_usable <= 0) { + return DKIX_EMPTY; + } + Py_INCREF(name); + /* Insert into new slot. */ + keys->dk_version = 0; + Py_ssize_t hashpos = find_empty_slot(keys, hash); + ix = keys->dk_nentries; + PyDictKeyEntry *ep = &DK_ENTRIES(keys)[ix]; + dictkeys_set_index(keys, hashpos, ix); + assert(ep->me_key == NULL); + ep->me_key = name; + ep->me_hash = hash; + keys->dk_usable--; + keys->dk_nentries++; + } + assert (ix < SHARED_KEYS_MAX_SIZE); + return (int)ix; +} + /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. @@ -998,17 +1075,6 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) MAINTAIN_TRACKING(mp, key, value); - /* When insertion order is different from shared key, we can't share - * the key anymore. Convert this instance to combine table. - */ - if (_PyDict_HasSplitTable(mp) && - ((ix >= 0 && old_value == NULL && mp->ma_used != ix) || - (ix == DKIX_EMPTY && mp->ma_used != mp->ma_keys->dk_nentries))) { - if (insertion_resize(mp) < 0) - goto Fail; - ix = DKIX_EMPTY; - } - if (ix == DKIX_EMPTY) { /* Insert into new slot. */ mp->ma_keys->dk_version = 0; @@ -1027,8 +1093,12 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) ep->me_key = key; ep->me_hash = hash; if (mp->ma_values) { - assert (mp->ma_values[mp->ma_keys->dk_nentries] == NULL); - mp->ma_values[mp->ma_keys->dk_nentries] = value; + Py_ssize_t index = mp->ma_keys->dk_nentries; + assert(index < SHARED_KEYS_MAX_SIZE); + assert((mp->ma_values->mv_order >> 60) == 0); + mp->ma_values->mv_order = ((mp->ma_values->mv_order)<<4) | index; + assert (mp->ma_values->values[index] == NULL); + mp->ma_values->values[index] = value; } else { ep->me_value = value; @@ -1044,10 +1114,9 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) if (old_value != value) { if (_PyDict_HasSplitTable(mp)) { - mp->ma_values[ix] = value; + mp->ma_values->values[ix] = value; if (old_value == NULL) { - /* pending state */ - assert(ix == mp->ma_used); + mp->ma_values->mv_order = (mp->ma_values->mv_order << 4) | ix; mp->ma_used++; } } @@ -1128,15 +1197,14 @@ actually be smaller than the old one. If a table is split (its keys and hashes are shared, its values are not), then the values are temporarily copied into the table, it is resized as a combined table, then the me_value slots in the old table are NULLed out. -After resizing a table is always combined, -but can be resplit by make_keys_shared(). +After resizing a table is always combined. */ static int dictresize(PyDictObject *mp, uint8_t log2_newsize) { Py_ssize_t numentries; PyDictKeysObject *oldkeys; - PyObject **oldvalues; + PyDictValues *oldvalues; PyDictKeyEntry *oldentries, *newentries; if (log2_newsize >= SIZEOF_SIZE_T*8) { @@ -1170,18 +1238,16 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize) if (oldvalues != NULL) { /* Convert split table into new combined table. * We must incref keys; we can transfer values. - * Note that values of split table is always dense. */ for (Py_ssize_t i = 0; i < numentries; i++) { - assert(oldvalues[i] != NULL); - PyDictKeyEntry *ep = &oldentries[i]; - PyObject *key = ep->me_key; - Py_INCREF(key); - newentries[i].me_key = key; + int index = get_index_from_order(mp, i); + PyDictKeyEntry *ep = &oldentries[index]; + assert(oldvalues->values[index] != NULL); + Py_INCREF(ep->me_key); + newentries[i].me_key = ep->me_key; newentries[i].me_hash = ep->me_hash; - newentries[i].me_value = oldvalues[i]; + newentries[i].me_value = oldvalues->values[index]; } - dictkeys_decref(oldkeys); mp->ma_values = NULL; if (oldvalues != empty_values) { @@ -1206,6 +1272,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize) #ifdef Py_REF_DEBUG _Py_RefTotal--; #endif +#if PyDict_MAXFREELIST > 0 struct _Py_dict_state *state = get_dict_state(); #ifdef Py_DEBUG // dictresize() must not be called after _PyDict_Fini() @@ -1216,7 +1283,9 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize) { state->keys_free_list[state->keys_numfree++] = oldkeys; } - else { + else +#endif + { PyObject_Free(oldkeys); } } @@ -1224,53 +1293,10 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize) build_indices(mp->ma_keys, newentries, numentries); mp->ma_keys->dk_usable -= numentries; mp->ma_keys->dk_nentries = numentries; + ASSERT_CONSISTENT(mp); return 0; } -/* Returns NULL if unable to split table. - * A NULL return does not necessarily indicate an error */ -static PyDictKeysObject * -make_keys_shared(PyObject *op) -{ - Py_ssize_t i; - Py_ssize_t size; - PyDictObject *mp = (PyDictObject *)op; - - if (!PyDict_CheckExact(op)) - return NULL; - if (!_PyDict_HasSplitTable(mp)) { - PyDictKeyEntry *ep0; - PyObject **values; - assert(mp->ma_keys->dk_refcnt == 1); - if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { - return NULL; - } - else if (mp->ma_used > mp->ma_keys->dk_nentries) { - /* Remove dummy keys */ - if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) - return NULL; - } - assert(mp->ma_used == mp->ma_keys->dk_nentries); - /* Copy values into a new array */ - ep0 = DK_ENTRIES(mp->ma_keys); - size = USABLE_FRACTION(DK_SIZE(mp->ma_keys)); - values = new_values(size); - if (values == NULL) { - PyErr_SetString(PyExc_MemoryError, - "Not enough memory to allocate new values array"); - return NULL; - } - for (i = 0; i < size; i++) { - values[i] = ep0[i].me_value; - ep0[i].me_value = NULL; - } - mp->ma_keys->dk_kind = DICT_KEYS_SPLIT; - mp->ma_values = values; - } - dictkeys_incref(mp->ma_keys); - return mp->ma_keys; -} - PyObject * _PyDict_NewPresized(Py_ssize_t minused) { @@ -1296,7 +1322,7 @@ _PyDict_NewPresized(Py_ssize_t minused) new_keys = new_keys_object(log2_newsize); if (new_keys == NULL) return NULL; - return new_dict(new_keys, NULL); + return new_dict(new_keys, NULL, 0, 0); } /* Note that, for historical reasons, PyDict_GetItem() suppresses all errors @@ -1366,7 +1392,7 @@ _PyDict_GetItemHint(PyDictObject *mp, PyObject *key, if (ep->me_key == key) { if (mp->ma_keys->dk_kind == DICT_KEYS_SPLIT) { assert(mp->ma_values != NULL); - res = mp->ma_values[(size_t)hint]; + res = mp->ma_values->values[(size_t)hint]; } else { res = ep->me_value; @@ -1468,6 +1494,9 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * + * + * + * * Raise an exception and return NULL if an error occurred (ex: computing the * key hash failed, key comparison failed, ...). Return NULL if the key doesn't * exist. Return the value if the key exists. @@ -1555,6 +1584,21 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, return insertdict(mp, key, hash, value); } +static uint64_t +delete_index_from_order(uint64_t order, Py_ssize_t ix) +{ /* Update order */ + for (int i = 0;; i+= 4) { + assert (i < 64); + if (((order >> i) & 15) == (uint64_t)ix) { + /* Remove 4 bits at ith position */ + uint64_t high = ((order>>i)>>4)<= 0); mp->ma_used--; - mp->ma_keys->dk_version = 0; mp->ma_version_tag = DICT_NEXT_VERSION(); ep = &DK_ENTRIES(mp->ma_keys)[ix]; - dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); - old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; - Py_DECREF(old_key); + if (mp->ma_values) { + assert(old_value == mp->ma_values->values[ix]); + mp->ma_values->values[ix] = NULL; + assert(ix < SHARED_KEYS_MAX_SIZE); + /* Update order */ + mp->ma_values->mv_order = + delete_index_from_order(mp->ma_values->mv_order, ix); + ASSERT_CONSISTENT(mp); + } + else { + mp->ma_keys->dk_version = 0; + dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); + old_key = ep->me_key; + ep->me_key = NULL; + ep->me_value = NULL; + Py_DECREF(old_key); + } Py_DECREF(old_value); ASSERT_CONSISTENT(mp); @@ -1617,15 +1672,6 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return -1; } - // Split table doesn't allow deletion. Combine it. - if (_PyDict_HasSplitTable(mp)) { - if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { - return -1; - } - ix = _Py_dict_lookup(mp, key, hash, &old_value); - assert(ix >= 0); - } - return delitem_common(mp, hash, ix, old_value); } @@ -1660,15 +1706,6 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return -1; } - // Split table doesn't allow deletion. Combine it. - if (_PyDict_HasSplitTable(mp)) { - if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { - return -1; - } - ix = _Py_dict_lookup(mp, key, hash, &old_value); - assert(ix >= 0); - } - res = predicate(old_value); if (res == -1) return -1; @@ -1688,7 +1725,7 @@ PyDict_Clear(PyObject *op) { PyDictObject *mp; PyDictKeysObject *oldkeys; - PyObject **oldvalues; + PyDictValues *oldvalues; Py_ssize_t i, n; if (!PyDict_Check(op)) @@ -1708,7 +1745,7 @@ PyDict_Clear(PyObject *op) if (oldvalues != NULL) { n = oldkeys->dk_nentries; for (i = 0; i < n; i++) - Py_CLEAR(oldvalues[i]); + Py_CLEAR(oldvalues->values[i]); free_values(oldvalues); dictkeys_decref(oldkeys); } @@ -1738,11 +1775,12 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, mp = (PyDictObject *)op; i = *ppos; if (mp->ma_values) { + assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); if (i < 0 || i >= mp->ma_used) return 0; - /* values of split table is always dense */ - entry_ptr = &DK_ENTRIES(mp->ma_keys)[i]; - value = mp->ma_values[i]; + int index = get_index_from_order(mp, i); + entry_ptr = &DK_ENTRIES(mp->ma_keys)[index]; + value = mp->ma_values->values[index]; assert(value != NULL); } else { @@ -1796,9 +1834,8 @@ PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) PyObject * _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *deflt) { - Py_ssize_t ix, hashpos; - PyObject *old_value, *old_key; - PyDictKeyEntry *ep; + Py_ssize_t ix; + PyObject *old_value; PyDictObject *mp; assert(PyDict_Check(dict)); @@ -1823,29 +1860,9 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d _PyErr_SetKeyError(key); return NULL; } - - // Split table doesn't allow deletion. Combine it. - if (_PyDict_HasSplitTable(mp)) { - if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { - return NULL; - } - ix = _Py_dict_lookup(mp, key, hash, &old_value); - assert(ix >= 0); - } - - hashpos = lookdict_index(mp->ma_keys, hash, ix); - assert(hashpos >= 0); assert(old_value != NULL); - mp->ma_used--; - mp->ma_version_tag = DICT_NEXT_VERSION(); - mp->ma_keys->dk_version = 0; - dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); - ep = &DK_ENTRIES(mp->ma_keys)[ix]; - mp->ma_keys->dk_version = 0; - old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; - Py_DECREF(old_key); + Py_INCREF(old_value); + delitem_common(mp, hash, ix, old_value); ASSERT_CONSISTENT(mp); return old_value; @@ -1882,7 +1899,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *d; int status; - d = _PyObject_CallNoArg(cls); + d = _PyObject_CallNoArgs(cls); if (d == NULL) return NULL; @@ -1966,7 +1983,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { - PyObject **values = mp->ma_values; + PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; @@ -1976,7 +1993,7 @@ dict_dealloc(PyDictObject *mp) if (values != NULL) { if (values != empty_values) { for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { - Py_XDECREF(values[i]); + Py_XDECREF(values->values[i]); } free_values(values); } @@ -1986,6 +2003,7 @@ dict_dealloc(PyDictObject *mp) assert(keys->dk_refcnt == 1); dictkeys_decref(keys); } +#if PyDict_MAXFREELIST > 0 struct _Py_dict_state *state = get_dict_state(); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() @@ -1994,7 +2012,9 @@ dict_dealloc(PyDictObject *mp) if (state->numfree < PyDict_MAXFREELIST && Py_IS_TYPE(mp, &PyDict_Type)) { state->free_list[state->numfree++] = mp; } - else { + else +#endif + { Py_TYPE(mp)->tp_free((PyObject *)mp); } Py_TRASHCAN_END @@ -2165,7 +2185,7 @@ dict_keys(PyDictObject *mp) } ep = DK_ENTRIES(mp->ma_keys); if (mp->ma_values) { - value_ptr = mp->ma_values; + value_ptr = mp->ma_values->values; offset = sizeof(PyObject *); } else { @@ -2208,7 +2228,7 @@ dict_values(PyDictObject *mp) } ep = DK_ENTRIES(mp->ma_keys); if (mp->ma_values) { - value_ptr = mp->ma_values; + value_ptr = mp->ma_values->values; offset = sizeof(PyObject *); } else { @@ -2265,7 +2285,7 @@ dict_items(PyDictObject *mp) /* Nothing we do below makes any function calls. */ ep = DK_ENTRIES(mp->ma_keys); if (mp->ma_values) { - value_ptr = mp->ma_values; + value_ptr = mp->ma_values->values; offset = sizeof(PyObject *); } else { @@ -2534,7 +2554,7 @@ dict_merge(PyObject *a, PyObject *b, int override) key = entry->me_key; hash = entry->me_hash; if (other->ma_values) - value = other->ma_values[i]; + value = other->ma_values->values[i]; else value = entry->me_value; @@ -2676,8 +2696,8 @@ PyDict_Copy(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - Py_ssize_t size = USABLE_FRACTION(DK_SIZE(mp->ma_keys)); - PyObject **newvalues; + Py_ssize_t size = shared_keys_usable_size(mp->ma_keys); + PyDictValues *newvalues; newvalues = new_values(size); if (newvalues == NULL) return PyErr_NoMemory(); @@ -2686,15 +2706,16 @@ PyDict_Copy(PyObject *o) free_values(newvalues); return NULL; } + newvalues->mv_order = mp->ma_values->mv_order; split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; split_copy->ma_version_tag = DICT_NEXT_VERSION(); dictkeys_incref(mp->ma_keys); for (i = 0, n = size; i < n; i++) { - PyObject *value = mp->ma_values[i]; + PyObject *value = mp->ma_values->values[i]; Py_XINCREF(value); - split_copy->ma_values[i] = value; + split_copy->ma_values->values[i] = value; } if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); @@ -2723,7 +2744,7 @@ PyDict_Copy(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(keys, NULL); + PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -2806,7 +2827,7 @@ dict_equal(PyDictObject *a, PyDictObject *b) PyDictKeyEntry *ep = &DK_ENTRIES(a->ma_keys)[i]; PyObject *aval; if (a->ma_values) - aval = a->ma_values[i]; + aval = a->ma_values->values[i]; else aval = ep->me_value; if (aval != NULL) { @@ -2962,15 +2983,6 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) if (ix == DKIX_ERROR) return NULL; - if (_PyDict_HasSplitTable(mp) && - ((ix >= 0 && value == NULL && mp->ma_used != ix) || - (ix == DKIX_EMPTY && mp->ma_used != mp->ma_keys->dk_nentries))) { - if (insertion_resize(mp) < 0) { - return NULL; - } - ix = DKIX_EMPTY; - } - if (ix == DKIX_EMPTY) { mp->ma_keys->dk_version = 0; PyDictKeyEntry *ep, *ep0; @@ -2993,8 +3005,11 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) ep->me_key = key; ep->me_hash = hash; if (_PyDict_HasSplitTable(mp)) { - assert(mp->ma_values[mp->ma_keys->dk_nentries] == NULL); - mp->ma_values[mp->ma_keys->dk_nentries] = value; + int index = (int)mp->ma_keys->dk_nentries; + assert(index < SHARED_KEYS_MAX_SIZE); + assert(mp->ma_values->values[index] == NULL); + mp->ma_values->values[index] = value; + mp->ma_values->mv_order = (mp->ma_values->mv_order << 4) | index; } else { ep->me_value = value; @@ -3008,10 +3023,11 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) else if (value == NULL) { value = defaultobj; assert(_PyDict_HasSplitTable(mp)); - assert(ix == mp->ma_used); + assert(mp->ma_values->values[ix] == NULL); Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); - mp->ma_values[ix] = value; + mp->ma_values->values[ix] = value; + mp->ma_values->mv_order = (mp->ma_values->mv_order << 4) | ix; mp->ma_used++; mp->ma_version_tag = DICT_NEXT_VERSION(); } @@ -3159,7 +3175,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) else { if (mp->ma_values != NULL) { for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values[i]); + Py_VISIT(mp->ma_values->values[i]); } } else { @@ -3183,20 +3199,22 @@ static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); Py_ssize_t _PyDict_SizeOf(PyDictObject *mp) { - Py_ssize_t size, usable, res; + Py_ssize_t size, res; size = DK_SIZE(mp->ma_keys); - usable = USABLE_FRACTION(size); res = _PyObject_SIZE(Py_TYPE(mp)); - if (mp->ma_values) - res += usable * sizeof(PyObject*); + if (mp->ma_values) { + res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); + } /* If the dictionary is split, the keys portion is accounted-for in the type object. */ - if (mp->ma_keys->dk_refcnt == 1) + if (mp->ma_keys->dk_refcnt == 1) { + Py_ssize_t usable = USABLE_FRACTION(size); res += (sizeof(PyDictKeysObject) + DK_IXSIZE(mp->ma_keys) * size + sizeof(PyDictKeyEntry) * usable); + } return res; } @@ -3677,8 +3695,9 @@ dictiter_iternextkey(dictiterobject *di) if (d->ma_values) { if (i >= d->ma_used) goto fail; - key = DK_ENTRIES(k)[i].me_key; - assert(d->ma_values[i] != NULL); + int index = get_index_from_order(d, i); + key = DK_ENTRIES(k)[index].me_key; + assert(d->ma_values->values[index] != NULL); } else { Py_ssize_t n = k->dk_nentries; @@ -3764,7 +3783,8 @@ dictiter_iternextvalue(dictiterobject *di) if (d->ma_values) { if (i >= d->ma_used) goto fail; - value = d->ma_values[i]; + int index = get_index_from_order(d, i); + value = d->ma_values->values[index]; assert(value != NULL); } else { @@ -3851,8 +3871,9 @@ dictiter_iternextitem(dictiterobject *di) if (d->ma_values) { if (i >= d->ma_used) goto fail; - key = DK_ENTRIES(d->ma_keys)[i].me_key; - value = d->ma_values[i]; + int index = get_index_from_order(d, i); + key = DK_ENTRIES(d->ma_keys)[index].me_key; + value = d->ma_values->values[index]; assert(value != NULL); } else { @@ -3968,8 +3989,9 @@ dictreviter_iternext(dictiterobject *di) goto fail; } if (d->ma_values) { - key = DK_ENTRIES(k)[i].me_key; - value = d->ma_values[i]; + int index = get_index_from_order(d, i); + key = DK_ENTRIES(k)[index].me_key; + value = d->ma_values->values[index]; assert (value != NULL); } else { @@ -4894,11 +4916,14 @@ dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(void) { - PyDictKeysObject *keys = new_keys_object(PyDict_LOG_MINSIZE); + PyDictKeysObject *keys = new_keys_object(5); /* log2(32) */ if (keys == NULL) { PyErr_Clear(); } else { + assert(keys->dk_nentries == 0); + /* Set to max size+1 as it will shrink by one before each new object */ + keys->dk_usable = SHARED_KEYS_MAX_SIZE; keys->dk_kind = DICT_KEYS_SPLIT; } return keys; @@ -4906,15 +4931,42 @@ _PyDict_NewKeysForClass(void) #define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) +static int +init_inline_values(PyObject *obj, PyTypeObject *tp) +{ + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_dictoffset > 0); + assert(tp->tp_inline_values_offset > 0); + PyDictKeysObject *keys = CACHED_KEYS(tp); + assert(keys != NULL); + if (keys->dk_usable > 1) { + keys->dk_usable--; + } + Py_ssize_t size = shared_keys_usable_size(keys); + assert(size > 0); + PyDictValues *values = new_values(size); + if (values == NULL) { + PyErr_NoMemory(); + return -1; + } + values->mv_order = 0; + for (int i = 0; i < size; i++) { + values->values[i] = NULL; + } + *((PyDictValues **)((char *)obj + tp->tp_inline_values_offset)) = values; + return 0; +} + int _PyObject_InitializeDict(PyObject *obj) { - PyObject **dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if (tp->tp_dictoffset == 0) { return 0; } - assert(*dictptr == NULL); - PyTypeObject *tp = Py_TYPE(obj); + if (tp->tp_inline_values_offset) { + return init_inline_values(obj, tp); + } PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); @@ -4926,15 +4978,174 @@ _PyObject_InitializeDict(PyObject *obj) if (dict == NULL) { return -1; } + PyObject **dictptr = _PyObject_DictPointer(obj); *dictptr = dict; return 0; } +static PyObject * +make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) +{ + dictkeys_incref(keys); + Py_ssize_t used = 0; + Py_ssize_t track = 0; + for (Py_ssize_t i = 0; i < shared_keys_usable_size(keys); i++) { + PyObject *val = values->values[i]; + if (val != NULL) { + used += 1; + track += _PyObject_GC_MAY_BE_TRACKED(val); + } + } + PyObject *res = new_dict(keys, values, used, 0); + if (track && res) { + _PyObject_GC_TRACK(res); + } + return res; +} + +PyObject * +_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) +{ + assert(Py_TYPE(obj)->tp_inline_values_offset != 0); + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + return make_dict_from_instance_attributes(keys, values); +} + +int +_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, + PyObject *name, PyObject *value) +{ + assert(PyUnicode_CheckExact(name)); + PyTypeObject *tp = Py_TYPE(obj); + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + assert(keys != NULL); + assert(values != NULL); + int ix = insert_into_dictkeys(keys, name); + if (ix == DKIX_EMPTY) { + if (value == NULL) { + PyErr_SetObject(PyExc_AttributeError, name); + return -1; + } + PyObject *dict = make_dict_from_instance_attributes(keys, values); + if (dict == NULL) { + return -1; + } + *((PyDictValues **)((char *)obj + tp->tp_inline_values_offset)) = NULL; + *((PyObject **) ((char *)obj + tp->tp_dictoffset)) = dict; + return PyDict_SetItem(dict, name, value); + } + PyObject *old_value = values->values[ix]; + Py_XINCREF(value); + values->values[ix] = value; + if (old_value == NULL) { + if (value == NULL) { + PyErr_SetObject(PyExc_AttributeError, name); + return -1; + } + values->mv_order = (values->mv_order << 4) | ix; + } + else { + if (value == NULL) { + values->mv_order = delete_index_from_order(values->mv_order, ix); + } + Py_DECREF(old_value); + } + return 0; +} + +PyObject * +_PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, + PyObject *name) +{ + assert(PyUnicode_CheckExact(name)); + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + assert(keys != NULL); + Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name); + if (ix == DKIX_EMPTY) { + return NULL; + } + PyObject *value = values->values[ix]; + Py_XINCREF(value); + return value; +} + +int +_PyObject_IsInstanceDictEmpty(PyObject *obj) +{ + PyTypeObject *tp = Py_TYPE(obj); + if (tp->tp_dictoffset == 0) { + return 1; + } + PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); + if (values_ptr && *values_ptr) { + PyDictKeysObject *keys = CACHED_KEYS(tp); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + if ((*values_ptr)->values[i] != NULL) { + return 0; + } + } + return 1; + } + PyObject **dictptr = _PyObject_DictPointer(obj); + PyObject *dict = *dictptr; + if (dict == NULL) { + return 1; + } + return ((PyDictObject *)dict)->ma_used == 0; +} + + +int +_PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg) +{ + PyTypeObject *tp = Py_TYPE(self); + assert(tp->tp_inline_values_offset); + PyDictValues **values_ptr = _PyObject_ValuesPointer(self); + if (*values_ptr == NULL) { + return 0; + } + PyDictKeysObject *keys = CACHED_KEYS(tp); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + Py_VISIT((*values_ptr)->values[i]); + } + return 0; +} + +void +_PyObject_ClearInstanceAttributes(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + assert(tp->tp_inline_values_offset); + PyDictValues **values_ptr = _PyObject_ValuesPointer(self); + if (*values_ptr == NULL) { + return; + } + PyDictKeysObject *keys = CACHED_KEYS(tp); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + Py_CLEAR((*values_ptr)->values[i]); + } +} + +void +_PyObject_FreeInstanceAttributes(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + assert(tp->tp_inline_values_offset); + PyDictValues **values_ptr = _PyObject_ValuesPointer(self); + if (*values_ptr == NULL) { + return; + } + PyDictKeysObject *keys = CACHED_KEYS(tp); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + Py_XDECREF((*values_ptr)->values[i]); + } + free_values(*values_ptr); +} PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject **dictptr = _PyObject_GetDictPtr(obj); + PyObject **dictptr = _PyObject_DictPointer(obj); if (dictptr == NULL) { PyErr_SetString(PyExc_AttributeError, "This object has no __dict__"); @@ -4943,7 +5154,14 @@ PyObject_GenericGetDict(PyObject *obj, void *context) PyObject *dict = *dictptr; if (dict == NULL) { PyTypeObject *tp = Py_TYPE(obj); - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { + PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); + if (values_ptr && *values_ptr) { + *dictptr = dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), *values_ptr); + if (dict != NULL) { + *values_ptr = NULL; + } + } + else if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); } @@ -4976,43 +5194,9 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, } if (value == NULL) { res = PyDict_DelItem(dict, key); - // Since key sharing dict doesn't allow deletion, PyDict_DelItem() - // always converts dict to combined form. - if ((cached = CACHED_KEYS(tp)) != NULL) { - CACHED_KEYS(tp) = NULL; - dictkeys_decref(cached); - } } else { - int was_shared = (cached == ((PyDictObject *)dict)->ma_keys); res = PyDict_SetItem(dict, key, value); - if (was_shared && - (cached = CACHED_KEYS(tp)) != NULL && - cached != ((PyDictObject *)dict)->ma_keys) { - /* PyDict_SetItem() may call dictresize and convert split table - * into combined table. In such case, convert it to split - * table again and update type's shared key only when this is - * the only dict sharing key with the type. - * - * This is to allow using shared key in class like this: - * - * class C: - * def __init__(self): - * # one dict resize happens - * self.a, self.b, self.c = 1, 2, 3 - * self.d, self.e, self.f = 4, 5, 6 - * a = C() - */ - if (cached->dk_refcnt == 1) { - CACHED_KEYS(tp) = make_keys_shared(dict); - } - else { - CACHED_KEYS(tp) = NULL; - } - dictkeys_decref(cached); - if (CACHED_KEYS(tp) == NULL && PyErr_Occurred()) - return -1; - } } } else { dict = *dictptr; @@ -5028,6 +5212,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, res = PyDict_SetItem(dict, key, value); } } + ASSERT_CONSISTENT(dict); return res; } diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 0ebaf5ea1fc797..4513831545b9a3 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -1,6 +1,7 @@ /* enumerate object */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -298,7 +299,7 @@ reversed_new_impl(PyTypeObject *type, PyObject *seq) return NULL; } if (reversed_meth != NULL) { - PyObject *res = _PyObject_CallNoArg(reversed_meth); + PyObject *res = _PyObject_CallNoArgs(reversed_meth); Py_DECREF(reversed_meth); return res; } diff --git a/Objects/exception_handling_notes.txt b/Objects/exception_handling_notes.txt index 2183fa111f1050..e738c2781ad23f 100644 --- a/Objects/exception_handling_notes.txt +++ b/Objects/exception_handling_notes.txt @@ -105,7 +105,7 @@ All offsets and lengths are in instructions, not bytes. We want the format to be compact, but quickly searchable. For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. -Binary search typically assumes fixed size entries, but that is not necesary, as long as we can identify the start of an entry. +Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: start, size, target, depth, push-lasti diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 714039ee1abca1..a9ea42c98422d9 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -90,7 +90,7 @@ static void BaseException_dealloc(PyBaseExceptionObject *self) { PyObject_GC_UnTrack(self); - // bpo-44348: The trashcan mecanism prevents stack overflow when deleting + // bpo-44348: The trashcan mechanism prevents stack overflow when deleting // long chains of exceptions. For example, exceptions can be chained // through the __context__ attributes or the __traceback__ attribute. Py_TRASHCAN_BEGIN(self, BaseException_dealloc) diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 5a2816f55244b6..8ca56a802b9769 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -2,7 +2,8 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_runtime.h" // _PyRuntime +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_runtime.h" // _PyRuntime #if defined(HAVE_GETC_UNLOCKED) && !defined(_Py_MEMORY_SANITIZER) /* clang MemorySanitizer doesn't yet understand getc_unlocked. */ @@ -190,7 +191,7 @@ PyObject_AsFileDescriptor(PyObject *o) return -1; } else if (meth != NULL) { - PyObject *fno = _PyObject_CallNoArg(meth); + PyObject *fno = _PyObject_CallNoArgs(meth); Py_DECREF(meth); if (fno == NULL) return -1; @@ -238,13 +239,7 @@ _PyLong_FileDescriptor_Converter(PyObject *o, void *ptr) ** Py_UniversalNewlineFgets is an fgets variation that understands ** all of \r, \n and \r\n conventions. ** The stream should be opened in binary mode. -** If fobj is NULL the routine always does newline conversion, and -** it may peek one char ahead to gobble the second char in \r\n. -** If fobj is non-NULL it must be a PyFileObject. In this case there -** is no readahead but in stead a flag is used to skip a following -** \n on the next read. Also, if the file is open in binary mode -** the whole conversion is skipped. Finally, the routine keeps track of -** the different types of newlines seen. +** The fobj parameter exists solely for legacy reasons and must be NULL. ** Note that we need no error handling: fgets() treats error and eof ** identically. */ @@ -253,63 +248,28 @@ Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj) { char *p = buf; int c; - int newlinetypes = 0; - int skipnextlf = 0; if (fobj) { errno = ENXIO; /* What can you do... */ return NULL; } FLOCKFILE(stream); - c = 'x'; /* Shut up gcc warning */ while (--n > 0 && (c = GETC(stream)) != EOF ) { - if (skipnextlf ) { - skipnextlf = 0; - if (c == '\n') { - /* Seeing a \n here with skipnextlf true - ** means we saw a \r before. - */ - newlinetypes |= NEWLINE_CRLF; - c = GETC(stream); - if (c == EOF) break; - } else { - /* - ** Note that c == EOF also brings us here, - ** so we're okay if the last char in the file - ** is a CR. - */ - newlinetypes |= NEWLINE_CR; - } - } if (c == '\r') { - /* A \r is translated into a \n, and we skip - ** an adjacent \n, if any. We don't set the - ** newlinetypes flag until we've seen the next char. - */ - skipnextlf = 1; - c = '\n'; - } else if ( c == '\n') { - newlinetypes |= NEWLINE_LF; + // A \r is translated into a \n, and we skip an adjacent \n, if any. + c = GETC(stream); + if (c != '\n') { + ungetc(c, stream); + c = '\n'; + } } *p++ = c; - if (c == '\n') break; + if (c == '\n') { + break; + } } - /* if ( c == EOF && skipnextlf ) - newlinetypes |= NEWLINE_CR; */ FUNLOCKFILE(stream); *p = '\0'; - if ( skipnextlf ) { - /* If we have no file object we cannot save the - ** skipnextlf flag. We have to readahead, which - ** will cause a pause if we're reading from an - ** interactive stream, but that is very unlikely - ** unless we're doing something silly like - ** exec(open("/dev/tty").read()). - */ - c = GETC(stream); - if ( c != '\n' ) - ungetc(c, stream); - } if (p == buf) return NULL; return buf; diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 92faa7c1320237..7fc192e7201171 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -5,13 +5,16 @@ #include "Python.h" #include "pycore_dtoa.h" // _Py_dg_dtoa() +#include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() #include "pycore_interp.h" // _PyInterpreterState.float_state #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_pymath.h" // _Py_ADJUST_ERANGE1() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include #include +#include // strtol() /*[clinic input] class float "PyObject *" "&PyFloat_Type" @@ -25,12 +28,14 @@ class float "PyObject *" "&PyFloat_Type" #endif +#if PyFloat_MAXFREELIST > 0 static struct _Py_float_state * get_float_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->float_state; } +#endif double @@ -123,8 +128,10 @@ PyFloat_GetInfo(void) PyObject * PyFloat_FromDouble(double fval) { + PyFloatObject *op; +#if PyFloat_MAXFREELIST > 0 struct _Py_float_state *state = get_float_state(); - PyFloatObject *op = state->free_list; + op = state->free_list; if (op != NULL) { #ifdef Py_DEBUG // PyFloat_FromDouble() must not be called after _PyFloat_Fini() @@ -133,7 +140,9 @@ PyFloat_FromDouble(double fval) state->free_list = (PyFloatObject *) Py_TYPE(op); state->numfree--; } - else { + else +#endif + { op = PyObject_Malloc(sizeof(PyFloatObject)); if (!op) { return PyErr_NoMemory(); @@ -230,6 +239,7 @@ PyFloat_FromString(PyObject *v) static void float_dealloc(PyFloatObject *op) { +#if PyFloat_MAXFREELIST > 0 if (PyFloat_CheckExact(op)) { struct _Py_float_state *state = get_float_state(); #ifdef Py_DEBUG @@ -244,7 +254,9 @@ float_dealloc(PyFloatObject *op) Py_SET_TYPE(op, (PyTypeObject *)state->free_list); state->free_list = op; } - else { + else +#endif + { Py_TYPE(op)->tp_free((PyObject *)op); } } @@ -809,7 +821,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) */ errno = 0; ix = pow(iv, iw); - Py_ADJUST_ERANGE1(ix); + _Py_ADJUST_ERANGE1(ix); if (negate_result) ix = -ix; @@ -2033,6 +2045,7 @@ _PyFloat_InitTypes(void) void _PyFloat_ClearFreeList(PyInterpreterState *interp) { +#if PyFloat_MAXFREELIST > 0 struct _Py_float_state *state = &interp->float_state; PyFloatObject *f = state->free_list; while (f != NULL) { @@ -2042,13 +2055,14 @@ _PyFloat_ClearFreeList(PyInterpreterState *interp) } state->free_list = NULL; state->numfree = 0; +#endif } void _PyFloat_Fini(PyInterpreterState *interp) { _PyFloat_ClearFreeList(interp); -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && PyFloat_MAXFREELIST > 0 struct _Py_float_state *state = &interp->float_state; state->numfree = -1; #endif @@ -2058,10 +2072,12 @@ _PyFloat_Fini(PyInterpreterState *interp) void _PyFloat_DebugMallocStats(FILE *out) { +#if PyFloat_MAXFREELIST > 0 struct _Py_float_state *state = get_float_state(); _PyDebugAllocatorStats(out, "free PyFloatObject", state->numfree, sizeof(PyFloatObject)); +#endif } @@ -2350,7 +2366,7 @@ _PyFloat_Pack8(double x, unsigned char *p, int le) flo = 0; ++fhi; if (fhi >> 28) { - /* And it also progagated out of the next 28 bits. */ + /* And it also propagated out of the next 28 bits. */ fhi = 0; ++e; if (e >= 2047) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 00d6888ff2a2ac..ffe19b3d994007 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -19,12 +19,14 @@ static PyMemberDef frame_memberlist[] = { {NULL} /* Sentinel */ }; +#if PyFrame_MAXFREELIST > 0 static struct _Py_frame_state * get_frame_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->frame; } +#endif static PyObject * @@ -45,7 +47,7 @@ PyFrame_GetLineNumber(PyFrameObject *f) return f->f_lineno; } else { - return PyCode_Addr2Line(f->f_frame->f_code, f->f_frame->f_lasti*2); + return PyCode_Addr2Line(f->f_frame->f_code, f->f_frame->f_lasti*sizeof(_Py_CODEUNIT)); } } @@ -67,7 +69,7 @@ frame_getlasti(PyFrameObject *f, void *closure) if (f->f_frame->f_lasti < 0) { return PyLong_FromLong(-1); } - return PyLong_FromLong(f->f_frame->f_lasti*2); + return PyLong_FromLong(f->f_frame->f_lasti*sizeof(_Py_CODEUNIT)); } static PyObject * @@ -373,8 +375,8 @@ marklines(PyCodeObject *code, int len) } while (PyLineTable_NextAddressRange(&bounds)) { - assert(bounds.ar_start/2 < len); - linestarts[bounds.ar_start/2] = bounds.ar_line; + assert(bounds.ar_start/(int)sizeof(_Py_CODEUNIT) < len); + linestarts[bounds.ar_start/sizeof(_Py_CODEUNIT)] = bounds.ar_line; } return linestarts; } @@ -408,7 +410,7 @@ frame_stack_pop(PyFrameObject *f) * would still work without any stack errors), but there are some constructs * that limit jumping: * - * o Any excpetion handlers. + * o Any exception handlers. * o 'for' and 'async for' loops can't be jumped into because the * iterator needs to be on the stack. * o Jumps cannot be made from within a trace function invoked with a @@ -607,9 +609,6 @@ static PyGetSetDef frame_getsetlist[] = { f_back next item on free list, or NULL */ -/* max value for numfree */ -#define PyFrame_MAXFREELIST 200 - static void _Py_HOT_FUNCTION frame_dealloc(PyFrameObject *f) { @@ -638,6 +637,7 @@ frame_dealloc(PyFrameObject *f) } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); +#if PyFrame_MAXFREELIST > 0 struct _Py_frame_state *state = get_frame_state(); #ifdef Py_DEBUG // frame_dealloc() must not be called after _PyFrame_Fini() @@ -648,7 +648,9 @@ frame_dealloc(PyFrameObject *f) f->f_back = state->free_list; state->free_list = f; } - else { + else +#endif + { PyObject_GC_Del(f); } @@ -801,8 +803,10 @@ static inline PyFrameObject* frame_alloc(InterpreterFrame *frame, int owns) { PyFrameObject *f; +#if PyFrame_MAXFREELIST > 0 struct _Py_frame_state *state = get_frame_state(); if (state->free_list == NULL) +#endif { f = PyObject_GC_New(PyFrameObject, &PyFrame_Type); if (f == NULL) { @@ -816,7 +820,9 @@ frame_alloc(InterpreterFrame *frame, int owns) return NULL; } } - else { +#if PyFrame_MAXFREELIST > 0 + else + { #ifdef Py_DEBUG // frame_alloc() must not be called after _PyFrame_Fini() assert(state->numfree != -1); @@ -827,6 +833,7 @@ frame_alloc(InterpreterFrame *frame, int owns) state->free_list = state->free_list->f_back; _Py_NewReference((PyObject *)f); } +#endif f->f_frame = frame; f->f_own_locals_memory = owns; return f; @@ -1069,6 +1076,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) void _PyFrame_ClearFreeList(PyInterpreterState *interp) { +#if PyFrame_MAXFREELIST > 0 struct _Py_frame_state *state = &interp->frame; while (state->free_list != NULL) { PyFrameObject *f = state->free_list; @@ -1077,13 +1085,14 @@ _PyFrame_ClearFreeList(PyInterpreterState *interp) --state->numfree; } assert(state->numfree == 0); +#endif } void _PyFrame_Fini(PyInterpreterState *interp) { _PyFrame_ClearFreeList(interp); -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && PyFrame_MAXFREELIST > 0 struct _Py_frame_state *state = &interp->frame; state->numfree = -1; #endif @@ -1093,10 +1102,12 @@ _PyFrame_Fini(PyInterpreterState *interp) void _PyFrame_DebugMallocStats(FILE *out) { +#if PyFrame_MAXFREELIST > 0 struct _Py_frame_state *state = get_frame_state(); _PyDebugAllocatorStats(out, "free PyFrameObject", state->numfree, sizeof(PyFrameObject)); +#endif } diff --git a/Objects/genobject.c b/Objects/genobject.c index be9238d9b6cfd3..8bf55155eab1c4 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1,14 +1,15 @@ /* Generator object implementation */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() -#include "pycore_object.h" +#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "frameobject.h" -#include "pycore_frame.h" +#include "pycore_frame.h" // InterpreterFrame +#include "frameobject.h" // PyFrameObject #include "structmember.h" // PyMemberDef -#include "opcode.h" +#include "opcode.h" // YIELD_FROM static PyObject *gen_close(PyGenObject *, PyObject *); static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *); @@ -319,7 +320,7 @@ gen_close_iter(PyObject *yf) PyErr_WriteUnraisable(yf); } if (meth) { - retval = _PyObject_CallNoArg(meth); + retval = _PyObject_CallNoArgs(meth); Py_DECREF(meth); if (retval == NULL) return -1; @@ -1284,7 +1285,7 @@ compute_cr_origin(int origin_depth) PyCodeObject *code = frame->f_code; PyObject *frameinfo = Py_BuildValue("OiO", code->co_filename, - PyCode_Addr2Line(frame->f_code, frame->f_lasti*2), + PyCode_Addr2Line(frame->f_code, frame->f_lasti*sizeof(_Py_CODEUNIT)), code->co_name); if (!frameinfo) { Py_DECREF(cr_origin); @@ -1566,12 +1567,14 @@ PyTypeObject PyAsyncGen_Type = { }; +#if _PyAsyncGen_MAXFREELIST > 0 static struct _Py_async_gen_state * get_async_gen_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->async_gen; } +#endif PyObject * @@ -1594,6 +1597,7 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp) { +#if _PyAsyncGen_MAXFREELIST > 0 struct _Py_async_gen_state *state = &interp->async_gen; while (state->value_numfree) { @@ -1609,13 +1613,14 @@ _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp) assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); PyObject_GC_Del(o); } +#endif } void _PyAsyncGen_Fini(PyInterpreterState *interp) { _PyAsyncGen_ClearFreeLists(interp); -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && _PyAsyncGen_MAXFREELIST > 0 struct _Py_async_gen_state *state = &interp->async_gen; state->value_numfree = -1; state->asend_numfree = -1; @@ -1662,6 +1667,7 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); +#if _PyAsyncGen_MAXFREELIST > 0 struct _Py_async_gen_state *state = get_async_gen_state(); #ifdef Py_DEBUG // async_gen_asend_dealloc() must not be called after _PyAsyncGen_Fini() @@ -1671,7 +1677,9 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) assert(PyAsyncGenASend_CheckExact(o)); state->asend_freelist[state->asend_numfree++] = o; } - else { + else +#endif + { PyObject_GC_Del(o); } } @@ -1824,6 +1832,7 @@ static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { PyAsyncGenASend *o; +#if _PyAsyncGen_MAXFREELIST > 0 struct _Py_async_gen_state *state = get_async_gen_state(); #ifdef Py_DEBUG // async_gen_asend_new() must not be called after _PyAsyncGen_Fini() @@ -1834,7 +1843,9 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) o = state->asend_freelist[state->asend_numfree]; _Py_NewReference((PyObject *)o); } - else { + else +#endif + { o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); if (o == NULL) { return NULL; @@ -1862,6 +1873,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) { _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->agw_val); +#if _PyAsyncGen_MAXFREELIST > 0 struct _Py_async_gen_state *state = get_async_gen_state(); #ifdef Py_DEBUG // async_gen_wrapped_val_dealloc() must not be called after _PyAsyncGen_Fini() @@ -1871,7 +1883,9 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) assert(_PyAsyncGenWrappedValue_CheckExact(o)); state->value_freelist[state->value_numfree++] = o; } - else { + else +#endif + { PyObject_GC_Del(o); } } @@ -1935,6 +1949,7 @@ _PyAsyncGenValueWrapperNew(PyObject *val) _PyAsyncGenWrappedValue *o; assert(val); +#if _PyAsyncGen_MAXFREELIST > 0 struct _Py_async_gen_state *state = get_async_gen_state(); #ifdef Py_DEBUG // _PyAsyncGenValueWrapperNew() must not be called after _PyAsyncGen_Fini() @@ -1946,7 +1961,9 @@ _PyAsyncGenValueWrapperNew(PyObject *val) assert(_PyAsyncGenWrappedValue_CheckExact(o)); _Py_NewReference((PyObject*)o); } - else { + else +#endif + { o = PyObject_GC_New(_PyAsyncGenWrappedValue, &_PyAsyncGenWrappedValue_Type); if (o == NULL) { diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c index 46239100dcb7b7..7b3e31beded594 100644 --- a/Objects/interpreteridobject.c +++ b/Objects/interpreteridobject.c @@ -3,7 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_interp.h" // _PyInterpreterState_LookUpID() -#include "interpreteridobject.h" +#include "pycore_interpreteridobject.h" typedef struct interpid { diff --git a/Objects/iterobject.c b/Objects/iterobject.c index e493e41131b70c..5db6bc10fb3ff2 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -1,7 +1,8 @@ /* Iterator objects */ #include "Python.h" -#include "pycore_object.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_object.h" // _PyObject_GC_TRACK() typedef struct { PyObject_HEAD @@ -217,7 +218,7 @@ calliter_iternext(calliterobject *it) return NULL; } - result = _PyObject_CallNoArg(it->it_callable); + result = _PyObject_CallNoArgs(it->it_callable); if (result != NULL) { int ok; diff --git a/Objects/listobject.c b/Objects/listobject.c index e7c4742f1db456..e0cae87f45781d 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -19,13 +19,14 @@ class list "PyListObject *" "&PyList_Type" #include "clinic/listobject.c.h" - +#if PyList_MAXFREELIST > 0 static struct _Py_list_state * get_list_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->list; } +#endif /* Ensure ob_item has room for at least newsize elements, and set @@ -108,19 +109,21 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) void _PyList_ClearFreeList(PyInterpreterState *interp) { +#if PyList_MAXFREELIST > 0 struct _Py_list_state *state = &interp->list; while (state->numfree) { PyListObject *op = state->free_list[--state->numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } +#endif } void _PyList_Fini(PyInterpreterState *interp) { _PyList_ClearFreeList(interp); -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && PyList_MAXFREELIST > 0 struct _Py_list_state *state = &interp->list; state->numfree = -1; #endif @@ -130,32 +133,38 @@ _PyList_Fini(PyInterpreterState *interp) void _PyList_DebugMallocStats(FILE *out) { +#if PyList_MAXFREELIST > 0 struct _Py_list_state *state = get_list_state(); _PyDebugAllocatorStats(out, "free PyListObject", state->numfree, sizeof(PyListObject)); +#endif } PyObject * PyList_New(Py_ssize_t size) { + PyListObject *op; + if (size < 0) { PyErr_BadInternalCall(); return NULL; } +#if PyList_MAXFREELIST > 0 struct _Py_list_state *state = get_list_state(); - PyListObject *op; #ifdef Py_DEBUG // PyList_New() must not be called after _PyList_Fini() assert(state->numfree != -1); #endif - if (state->numfree) { + if (PyList_MAXFREELIST && state->numfree) { state->numfree--; op = state->free_list[state->numfree]; _Py_NewReference((PyObject *)op); } - else { + else +#endif + { op = PyObject_GC_New(PyListObject, &PyList_Type); if (op == NULL) { return NULL; @@ -344,6 +353,7 @@ list_dealloc(PyListObject *op) } PyMem_Free(op->ob_item); } +#if PyList_MAXFREELIST > 0 struct _Py_list_state *state = get_list_state(); #ifdef Py_DEBUG // list_dealloc() must not be called after _PyList_Fini() @@ -352,7 +362,9 @@ list_dealloc(PyListObject *op) if (state->numfree < PyList_MAXFREELIST && PyList_CheckExact(op)) { state->free_list[state->numfree++] = op; } - else { + else +#endif + { Py_TYPE(op)->tp_free((PyObject *)op); } Py_TRASHCAN_END @@ -1973,7 +1985,7 @@ powerloop(Py_ssize_t s1, Py_ssize_t n1, Py_ssize_t n2, Py_ssize_t n) * and merge adjacent runs on the stack with greater power. See listsort.txt * for more info. * - * It's the caller's responsibilty to push the new run on the stack when this + * It's the caller's responsibility to push the new run on the stack when this * returns. * * Returns 0 on success, -1 on error. @@ -2067,7 +2079,7 @@ safe_object_compare(PyObject *v, PyObject *w, MergeState *ms) return PyObject_RichCompareBool(v, w, Py_LT); } -/* Homogeneous compare: safe for any two compareable objects of the same type. +/* Homogeneous compare: safe for any two comparable objects of the same type. * (ms->key_richcompare is set to ob_type->tp_richcompare in the * pre-sort check.) */ diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 306e5e44d2ecfe..32a59e510f0754 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -382,7 +382,7 @@ things we don't have: extension on most platforms, but not all, and there's no uniform spelling on the platforms that support it. -- Integer divison on an integer type twice as wide as needed to hold the +- Integer division on an integer type twice as wide as needed to hold the list length. But the latter is Py_ssize_t for us, and is typically the widest native signed integer type the platform supports. @@ -797,6 +797,6 @@ OPTIMIZATION OF INDIVIDUAL COMPARISONS As noted above, even the simplest Python comparison triggers a large pile of C-level pointer dereferences, conditionals, and function calls. This can be partially mitigated by pre-scanning the data to determine whether the data is -homogenous with respect to type. If so, it is sometimes possible to +homogeneous with respect to type. If so, it is sometimes possible to substitute faster type-specific comparisons for the slower, generic PyObject_RichCompareBool. diff --git a/Objects/longobject.c b/Objects/longobject.c index 33fea6491b73d2..5325d1852bc029 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -8,11 +8,11 @@ #include "pycore_long.h" // __PyLong_GetSmallInt_internal() #include "pycore_object.h" // _PyObject_InitVar() #include "pycore_pystate.h" // _Py_IsMainInterpreter() -#include "longintrepr.h" -#include #include +#include #include +#include // abs() #include "clinic/longobject.c.h" /*[clinic input] @@ -3593,13 +3593,11 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) return NULL; } -static PyObject * -long_mul(PyLongObject *a, PyLongObject *b) +PyObject * +_PyLong_Multiply(PyLongObject *a, PyLongObject *b) { PyLongObject *z; - CHECK_BINOP(a, b); - /* fast path for single-digit multiplication */ if (IS_MEDIUM_VALUE(a) && IS_MEDIUM_VALUE(b)) { stwodigits v = medium_value(a) * medium_value(b); @@ -3616,6 +3614,13 @@ long_mul(PyLongObject *a, PyLongObject *b) return (PyObject *)z; } +static PyObject * +long_mul(PyLongObject *a, PyLongObject *b) +{ + CHECK_BINOP(a, b); + return _PyLong_Multiply(a, b); +} + /* Fast modulo division for single-digit longs. */ static PyObject * fast_mod(PyLongObject *a, PyLongObject *b) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 913d358062219b..6257455d34715f 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -11,10 +11,10 @@ */ #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_object.h" -#include "pystrhex.h" -#include +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_strhex.h" // _Py_strhex_with_sep() +#include // offsetof() /*[clinic input] class memoryview "PyMemoryViewObject *" "&PyMemoryView_Type" diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 1c37eb08c89a38..1d649a7932098b 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -2,6 +2,7 @@ /* Module object implementation */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_moduleobject.h" // _PyModule_GetDef() @@ -866,7 +867,7 @@ module_dir(PyObject *self, PyObject *args) if (PyDict_Check(dict)) { PyObject *dirfunc = _PyDict_GetItemIdWithError(dict, &PyId___dir__); if (dirfunc) { - result = _PyObject_CallNoArg(dirfunc); + result = _PyObject_CallNoArgs(dirfunc); } else if (!PyErr_Occurred()) { result = PyDict_Keys(dict); diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index fa37ed250d30a4..7875e7cafecb65 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -1,6 +1,7 @@ // namespace object implementation #include "Python.h" +#include "pycore_namespace.h" // _PyNamespace_Type #include "structmember.h" // PyMemberDef diff --git a/Objects/object.c b/Objects/object.c index aa84815e56e864..25f5a2133d5745 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2,18 +2,21 @@ /* Generic object operations; and implementation of None */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() -#include "pycore_context.h" -#include "pycore_initconfig.h" -#include "pycore_object.h" -#include "pycore_pyerrors.h" -#include "pycore_pylifecycle.h" +#include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() +#include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() +#include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_namespace.h" // _PyNamespace_Type +#include "pycore_object.h" // _PyType_CheckConsistency() +#include "pycore_pyerrors.h" // _PyErr_Occurred() +#include "pycore_pylifecycle.h" // _PyTypes_InitSlotDefs() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type #include "pycore_unionobject.h" // _PyUnion_Type -#include "frameobject.h" -#include "interpreteridobject.h" +#include "frameobject.h" // PyFrame_Type +#include "pycore_interpreteridobject.h" // _PyInterpreterID_Type #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() @@ -560,7 +563,7 @@ PyObject_Bytes(PyObject *v) func = _PyObject_LookupSpecial(v, &PyId___bytes__); if (func != NULL) { - result = _PyObject_CallNoArg(func); + result = _PyObject_CallNoArgs(func); Py_DECREF(func); if (result == NULL) return NULL; @@ -1064,10 +1067,8 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) return -1; } -/* Helper to get a pointer to an object's __dict__ slot, if any */ - PyObject ** -_PyObject_GetDictPtr(PyObject *obj) +_PyObject_DictPointer(PyObject *obj) { Py_ssize_t dictoffset; PyTypeObject *tp = Py_TYPE(obj); @@ -1089,6 +1090,35 @@ _PyObject_GetDictPtr(PyObject *obj) return (PyObject **) ((char *)obj + dictoffset); } +/* Helper to get a pointer to an object's __dict__ slot, if any. + * Creates the dict from inline attributes if necessary. + * Does not set an exception. */ +PyObject ** +_PyObject_GetDictPtr(PyObject *obj) +{ + PyObject **dict_ptr = _PyObject_DictPointer(obj); + if (dict_ptr == NULL) { + return NULL; + } + if (*dict_ptr != NULL) { + return dict_ptr; + } + PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); + if (values_ptr == NULL || *values_ptr == NULL) { + return dict_ptr; + } + PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, *values_ptr); + if (dict == NULL) { + PyErr_Clear(); + return NULL; + } + assert(*dict_ptr == NULL); + assert(*values_ptr != NULL); + *values_ptr = NULL; + *dict_ptr = dict; + return dict_ptr; +} + PyObject * PyObject_SelfIter(PyObject *obj) { @@ -1135,7 +1165,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } - if (tp->tp_getattro != PyObject_GenericGetAttr || !PyUnicode_Check(name)) { + if (tp->tp_getattro != PyObject_GenericGetAttr || !PyUnicode_CheckExact(name)) { *method = PyObject_GetAttr(obj, name); return 0; } @@ -1155,23 +1185,34 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } } - - PyObject **dictptr = _PyObject_GetDictPtr(obj); - PyObject *dict; - if (dictptr != NULL && (dict = *dictptr) != NULL) { - Py_INCREF(dict); - PyObject *attr = PyDict_GetItemWithError(dict, name); + PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); + if (values_ptr && *values_ptr) { + assert(*_PyObject_DictPointer(obj) == NULL); + PyObject *attr = _PyObject_GetInstanceAttribute(obj, *values_ptr, name); if (attr != NULL) { - *method = Py_NewRef(attr); - Py_DECREF(dict); + *method = attr; Py_XDECREF(descr); return 0; } - Py_DECREF(dict); + } + else { + PyObject **dictptr = _PyObject_DictPointer(obj); + PyObject *dict; + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + PyObject *attr = PyDict_GetItemWithError(dict, name); + if (attr != NULL) { + *method = Py_NewRef(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + return 0; + } + Py_DECREF(dict); - if (PyErr_Occurred()) { - Py_XDECREF(descr); - return 0; + if (PyErr_Occurred()) { + Py_XDECREF(descr); + return 0; + } } } @@ -1199,6 +1240,17 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } +PyDictValues ** +_PyObject_ValuesPointer(PyObject *obj) +{ + PyTypeObject *tp = Py_TYPE(obj); + Py_ssize_t offset = tp->tp_inline_values_offset; + if (offset == 0) { + return NULL; + } + return (PyDictValues **) ((char *)obj + offset); +} + /* Generic GetAttr functions - put these in your tp_[gs]etattro slot. */ PyObject * @@ -1246,25 +1298,46 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, goto done; } } - if (dict == NULL) { - /* Inline _PyObject_GetDictPtr */ - dictoffset = tp->tp_dictoffset; - if (dictoffset != 0) { - if (dictoffset < 0) { - Py_ssize_t tsize = Py_SIZE(obj); - if (tsize < 0) { - tsize = -tsize; + PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); + if (values_ptr && *values_ptr) { + if (PyUnicode_CheckExact(name)) { + assert(*_PyObject_DictPointer(obj) == NULL); + res = _PyObject_GetInstanceAttribute(obj, *values_ptr, name); + if (res != NULL) { + goto done; } - size_t size = _PyObject_VAR_SIZE(tp, tsize); - _PyObject_ASSERT(obj, size <= PY_SSIZE_T_MAX); - - dictoffset += (Py_ssize_t)size; - _PyObject_ASSERT(obj, dictoffset > 0); - _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0); } - dictptr = (PyObject **) ((char *)obj + dictoffset); - dict = *dictptr; + else { + dictptr = _PyObject_DictPointer(obj); + assert(dictptr != NULL && *dictptr == NULL); + *dictptr = dict = _PyObject_MakeDictFromInstanceAttributes(obj, *values_ptr); + if (dict == NULL) { + res = NULL; + goto done; + } + *values_ptr = NULL; + } + } + else { + /* Inline _PyObject_DictPointer */ + dictoffset = tp->tp_dictoffset; + if (dictoffset != 0) { + if (dictoffset < 0) { + Py_ssize_t tsize = Py_SIZE(obj); + if (tsize < 0) { + tsize = -tsize; + } + size_t size = _PyObject_VAR_SIZE(tp, tsize); + _PyObject_ASSERT(obj, size <= PY_SSIZE_T_MAX); + + dictoffset += (Py_ssize_t)size; + _PyObject_ASSERT(obj, dictoffset > 0); + _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0); + } + dictptr = (PyObject **) ((char *)obj + dictoffset); + dict = *dictptr; + } } } if (dict != NULL) { @@ -1327,7 +1400,6 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, PyTypeObject *tp = Py_TYPE(obj); PyObject *descr; descrsetfunc f; - PyObject **dictptr; int res = -1; if (!PyUnicode_Check(name)){ @@ -1353,30 +1425,30 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } } - /* XXX [Steve Dower] These are really noisy - worth it? */ - /*if (PyType_Check(obj) || PyModule_Check(obj)) { - if (value && PySys_Audit("object.__setattr__", "OOO", obj, name, value) < 0) - return -1; - if (!value && PySys_Audit("object.__delattr__", "OO", obj, name) < 0) - return -1; - }*/ - if (dict == NULL) { - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - if (descr == NULL) { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); + PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); + if (values_ptr && *values_ptr) { + res = _PyObject_StoreInstanceAttribute(obj, *values_ptr, name, value); + } + else { + PyObject **dictptr = _PyObject_DictPointer(obj); + if (dictptr == NULL) { + if (descr == NULL) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + } + else { + PyErr_Format(PyExc_AttributeError, + "'%.50s' object attribute '%U' is read-only", + tp->tp_name, name); + } + goto done; } else { - PyErr_Format(PyExc_AttributeError, - "'%.50s' object attribute '%U' is read-only", - tp->tp_name, name); + res = _PyObjectDict_SetItem(tp, dictptr, name, value); } - goto done; } - res = _PyObjectDict_SetItem(tp, dictptr, name, value); } else { Py_INCREF(dict); @@ -1406,8 +1478,15 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); + PyDictValues** values_ptr = _PyObject_ValuesPointer(obj); + if (values_ptr != NULL && *values_ptr != NULL) { + /* Was unable to convert to dict */ + PyErr_NoMemory(); + } + else { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + } return -1; } if (value == NULL) { @@ -1521,7 +1600,7 @@ _dir_object(PyObject *obj) return NULL; } /* use __dir__ */ - result = _PyObject_CallNoArg(dirfunc); + result = _PyObject_CallNoArgs(dirfunc); Py_DECREF(dirfunc); if (result == NULL) return NULL; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 903ca1c9e4b983..2eddb2cafd4983 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,6 +2,7 @@ #include "pycore_pymem.h" // _PyTraceMalloc_Config #include +#include // malloc() /* Defined in tracemalloc.c */ @@ -848,7 +849,7 @@ static int running_on_valgrind = -1; /* * Alignment of addresses returned to the user. 8-bytes alignment works - * on most current architectures (with 32-bit or 64-bit address busses). + * on most current architectures (with 32-bit or 64-bit address buses). * The alignment value is also used for grouping small requests in size * classes spaced ALIGNMENT bytes apart. * @@ -1340,7 +1341,7 @@ _Py_GetAllocatedBlocks(void) #define MAP_TOP_BITS INTERIOR_BITS #define MAP_TOP_LENGTH (1 << MAP_TOP_BITS) -#define MAP_TOP_MASK (MAP_BOT_LENGTH - 1) +#define MAP_TOP_MASK (MAP_TOP_LENGTH - 1) #define MAP_MID_BITS INTERIOR_BITS #define MAP_MID_LENGTH (1 << MAP_MID_BITS) diff --git a/Objects/odictobject.c b/Objects/odictobject.c index e5361da6dcdedb..9af45c685ab3b3 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -465,10 +465,10 @@ Potential Optimizations */ #include "Python.h" -#include "pycore_object.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_dict.h" // _Py_dict_lookup() #include // offsetof() -#include "pycore_dict.h" -#include #include "clinic/odictobject.c.h" @@ -1203,7 +1203,7 @@ odict_copy(register PyODictObject *od, PyObject *Py_UNUSED(ignored)) if (PyODict_CheckExact(od)) od_copy = PyODict_New(); else - od_copy = _PyObject_CallNoArg((PyObject *)Py_TYPE(od)); + od_copy = _PyObject_CallNoArgs((PyObject *)Py_TYPE(od)); if (od_copy == NULL) return NULL; @@ -2221,7 +2221,7 @@ mutablemapping_update_arg(PyObject *self, PyObject *arg) return -1; } if (func != NULL) { - PyObject *keys = _PyObject_CallNoArg(func); + PyObject *keys = _PyObject_CallNoArgs(func); Py_DECREF(func); if (keys == NULL) { return -1; @@ -2253,7 +2253,7 @@ mutablemapping_update_arg(PyObject *self, PyObject *arg) return -1; } if (func != NULL) { - PyObject *items = _PyObject_CallNoArg(func); + PyObject *items = _PyObject_CallNoArgs(func); Py_DECREF(func); if (items == NULL) { return -1; diff --git a/Objects/setobject.c b/Objects/setobject.c index a3cdd33664d089..0be067857d6946 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -16,7 +16,7 @@ reduces the cost of hash collisions because consecutive memory accesses tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, we then use more of the upper bits from the hash value and apply a simple - linear congruential random number genearator. This helps break-up long + linear congruential random number generator. This helps break-up long chains of collisions. All arithmetic on hash should ignore overflow. @@ -1409,7 +1409,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* Optimization: When the other set is more than 8 times larger than the base set, replace the other set with - interesection of the two sets. + intersection of the two sets. */ if ((PySet_GET_SIZE(other) >> 3) > PySet_GET_SIZE(so)) { other = set_intersection(so, other); diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index 7152ec6ebe7128..a4eea7b91988b9 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -2,6 +2,8 @@ unicode_format.h -- implementation of str.format(). */ +#include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() + /************************************************************************/ /*********** Global data structures and forward declarations *********/ /************************************************************************/ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index e64b93ba63e41a..e9d1b5926abb37 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -66,7 +66,8 @@ tuple_alloc(Py_ssize_t size) return NULL; } -#if PyTuple_MAXSAVESIZE > 0 +// Check for max save size > 1. Empty tuple singleton is special case. +#if PyTuple_MAXSAVESIZE > 1 struct _Py_tuple_state *state = get_tuple_state(); #ifdef Py_DEBUG // tuple_alloc() must not be called after _PyTuple_Fini() @@ -484,6 +485,28 @@ _PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return (PyObject *)tuple; } +PyObject * +_PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n) +{ + if (n == 0) { + return tuple_get_empty(); + } + PyTupleObject *tuple = tuple_alloc(n); + if (tuple == NULL) { + for (Py_ssize_t i = 0; i < n; i++) { + Py_DECREF(src[i]); + } + return NULL; + } + PyObject **dst = tuple->ob_item; + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *item = src[i]; + dst[i] = item; + } + _PyObject_GC_TRACK(tuple); + return (PyObject *)tuple; +} + static PyObject * tupleslice(PyTupleObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a81d071e31d23b..18bea476ea9f25 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4,14 +4,14 @@ #include "pycore_call.h" #include "pycore_code.h" // CO_FAST_FREE #include "pycore_compile.h" // _Py_Mangle() -#include "pycore_initconfig.h" +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_moduleobject.h" // _PyModule_GetDef() -#include "pycore_object.h" -#include "pycore_pyerrors.h" +#include "pycore_object.h" // _PyType_HasFeature() +#include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_unionobject.h" // _Py_union_type_or -#include "frameobject.h" -#include "pycore_frame.h" +#include "frameobject.h" // PyFrameObject +#include "pycore_frame.h" // InterpreterFrame #include "opcode.h" // MAKE_CELL #include "structmember.h" // PyMemberDef @@ -1232,8 +1232,16 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg) assert(base); } + if (type->tp_inline_values_offset) { + assert(type->tp_dictoffset); + int err = _PyObject_VisitInstanceAttributes(self, visit, arg); + if (err) { + return err; + } + } + if (type->tp_dictoffset != base->tp_dictoffset) { - PyObject **dictptr = _PyObject_GetDictPtr(self); + PyObject **dictptr = _PyObject_DictPointer(self); if (dictptr && *dictptr) Py_VISIT(*dictptr); } @@ -1293,8 +1301,11 @@ subtype_clear(PyObject *self) /* Clear the instance dict (if any), to break cycles involving only __dict__ slots (as in the case 'self.__dict__ is self'). */ + if (type->tp_inline_values_offset) { + _PyObject_ClearInstanceAttributes(self); + } if (type->tp_dictoffset != base->tp_dictoffset) { - PyObject **dictptr = _PyObject_GetDictPtr(self); + PyObject **dictptr = _PyObject_DictPointer(self); if (dictptr && *dictptr) Py_CLEAR(*dictptr); } @@ -1433,9 +1444,12 @@ subtype_dealloc(PyObject *self) assert(base); } - /* If we added a dict, DECREF it */ + /* If we added a dict, DECREF it, or free inline values. */ + if (type->tp_inline_values_offset) { + _PyObject_FreeInstanceAttributes(self); + } if (type->tp_dictoffset && !base->tp_dictoffset) { - PyObject **dictptr = _PyObject_GetDictPtr(self); + PyObject **dictptr = _PyObject_DictPointer(self); if (dictptr != NULL) { PyObject *dict = *dictptr; if (dict != NULL) { @@ -1640,7 +1654,7 @@ call_unbound_noarg(int unbound, PyObject *func, PyObject *self) return PyObject_CallOneArg(func, self); } else { - return _PyObject_CallNoArg(func); + return _PyObject_CallNoArgs(func); } } @@ -2159,7 +2173,6 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 1; } - /* Calculate the best base amongst multiple base classes. This is the first one that's on the path to the "solid base". */ @@ -2230,6 +2243,10 @@ extra_ivars(PyTypeObject *type, PyTypeObject *base) return t_size != b_size || type->tp_itemsize != base->tp_itemsize; } + if (type->tp_inline_values_offset && base->tp_inline_values_offset == 0 && + type->tp_inline_values_offset + sizeof(PyDictValues *) == t_size && + type->tp_flags & Py_TPFLAGS_HEAPTYPE) + t_size -= sizeof(PyDictValues *); if (type->tp_weaklistoffset && base->tp_weaklistoffset == 0 && type->tp_weaklistoffset + sizeof(PyObject *) == t_size && type->tp_flags & Py_TPFLAGS_HEAPTYPE) @@ -2238,7 +2255,6 @@ extra_ivars(PyTypeObject *type, PyTypeObject *base) type->tp_dictoffset + sizeof(PyObject *) == t_size && type->tp_flags & Py_TPFLAGS_HEAPTYPE) t_size -= sizeof(PyObject *); - return t_size != b_size; } @@ -2258,6 +2274,7 @@ solid_base(PyTypeObject *type) } static void object_dealloc(PyObject *); +static PyObject *object_new(PyTypeObject *, PyObject *, PyObject *); static int object_init(PyObject *, PyObject *, PyObject *); static int update_slot(PyTypeObject *, PyObject *); static void fixup_slot_dispatchers(PyTypeObject *); @@ -2761,6 +2778,7 @@ type_new_alloc(type_new_ctx *ctx) et->ht_name = Py_NewRef(ctx->name); et->ht_module = NULL; + et->_ht_tpname = NULL; return type; } @@ -2979,6 +2997,13 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) type->tp_weaklistoffset = slotoffset; slotoffset += sizeof(PyObject *); } + if (type->tp_dictoffset > 0) { + type->tp_inline_values_offset = slotoffset; + slotoffset += sizeof(PyDictValues *); + } + else { + type->tp_inline_values_offset = 0; + } type->tp_basicsize = slotoffset; type->tp_itemsize = ctx->base->tp_itemsize; @@ -3181,7 +3206,8 @@ type_new_impl(type_new_ctx *ctx) // Put the proper slots in place fixup_slot_dispatchers(type); - if (type->tp_dictoffset) { + if (type->tp_inline_values_offset) { + assert(type->tp_dictoffset > 0); PyHeapTypeObject *et = (PyHeapTypeObject*)type; et->ht_cached_keys = _PyDict_NewKeysForClass(); } @@ -3195,6 +3221,7 @@ type_new_impl(type_new_ctx *ctx) } assert(_PyType_CheckConsistency(type)); + return (PyObject *)type; error: @@ -3332,7 +3359,7 @@ type_vectorcall(PyObject *metatype, PyObject *const *args, } /* In other (much less common) cases, fall back to more flexible calling conventions. */ - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); return _PyObject_MakeTpCall(tstate, metatype, args, nargs, kwnames); } @@ -3409,25 +3436,42 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) goto fail; } + type = &res->ht_type; + /* The flags must be initialized early, before the GC traverses us */ + type->tp_flags = spec->flags | Py_TPFLAGS_HEAPTYPE; + /* Set the type name and qualname */ const char *s = strrchr(spec->name, '.'); - if (s == NULL) + if (s == NULL) { s = spec->name; - else + } + else { s++; + } - type = &res->ht_type; - /* The flags must be initialized early, before the GC traverses us */ - type->tp_flags = spec->flags | Py_TPFLAGS_HEAPTYPE; res->ht_name = PyUnicode_FromString(s); - if (!res->ht_name) + if (!res->ht_name) { + goto fail; + } + res->ht_qualname = Py_NewRef(res->ht_name); + + /* Copy spec->name to a buffer we own. + * + * Unfortunately, we can't use tp_name directly (with some + * flag saying that it should be deallocated with the type), + * because tp_name is public API and may be set independently + * of any such flag. + * So, we use a separate buffer, _ht_tpname, that's always + * deallocated with the type (if it's non-NULL). + */ + Py_ssize_t name_buf_len = strlen(spec->name) + 1; + res->_ht_tpname = PyMem_Malloc(name_buf_len); + if (res->_ht_tpname == NULL) { goto fail; - res->ht_qualname = res->ht_name; - Py_INCREF(res->ht_qualname); - type->tp_name = spec->name; + } + type->tp_name = memcpy(res->_ht_tpname, spec->name, name_buf_len); - Py_XINCREF(module); - res->ht_module = module; + res->ht_module = Py_XNewRef(module); /* Adjust for empty tuple bases */ if (!bases) { @@ -3550,7 +3594,8 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) if (PyType_Ready(type) < 0) goto fail; - if (type->tp_dictoffset) { + if (type->tp_inline_values_offset) { + assert(type->tp_dictoffset > 0); res->ht_cached_keys = _PyDict_NewKeysForClass(); } @@ -4055,6 +4100,7 @@ type_dealloc(PyTypeObject *type) _PyDictKeys_DecRef(et->ht_cached_keys); } Py_XDECREF(et->ht_module); + PyMem_Free(et->_ht_tpname); Py_TYPE(type)->tp_free((PyObject *)type); } @@ -4246,10 +4292,12 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) Py_VISIT(type->tp_base); Py_VISIT(((PyHeapTypeObject *)type)->ht_module); - /* There's no need to visit type->tp_subclasses or - ((PyHeapTypeObject *)type)->ht_slots, because they can't be involved - in cycles; tp_subclasses is a list of weak references, - and slots is a tuple of strings. */ + /* There's no need to visit others because they can't be involved + in cycles: + type->tp_subclasses is a list of weak references, + ((PyHeapTypeObject *)type)->ht_slots is a tuple of strings, + ((PyHeapTypeObject *)type)->ht_*name are strings. + */ return 0; } @@ -4257,7 +4305,6 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) static int type_clear(PyTypeObject *type) { - PyDictKeysObject *cached_keys; /* Because of type_is_gc(), the collector only calls this for heaptypes. */ _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -4292,11 +4339,6 @@ type_clear(PyTypeObject *type) */ PyType_Modified(type); - cached_keys = ((PyHeapTypeObject *)type)->ht_cached_keys; - if (cached_keys != NULL) { - ((PyHeapTypeObject *)type)->ht_cached_keys = NULL; - _PyDictKeys_DecRef(cached_keys); - } if (type->tp_dict) { PyDict_Clear(type->tp_dict); } @@ -4618,6 +4660,7 @@ compatible_with_tp_base(PyTypeObject *child) child->tp_itemsize == parent->tp_itemsize && child->tp_dictoffset == parent->tp_dictoffset && child->tp_weaklistoffset == parent->tp_weaklistoffset && + child->tp_inline_values_offset == parent->tp_inline_values_offset && ((child->tp_flags & Py_TPFLAGS_HAVE_GC) == (parent->tp_flags & Py_TPFLAGS_HAVE_GC)) && (child->tp_dealloc == subtype_dealloc || @@ -4637,6 +4680,8 @@ same_slots_added(PyTypeObject *a, PyTypeObject *b) size += sizeof(PyObject *); if (a->tp_weaklistoffset == size && b->tp_weaklistoffset == size) size += sizeof(PyObject *); + if (a->tp_inline_values_offset == size && b->tp_inline_values_offset == size) + size += sizeof(PyObject *); /* Check slots compliance */ if (!(a->tp_flags & Py_TPFLAGS_HEAPTYPE) || @@ -4781,6 +4826,17 @@ object_set_class(PyObject *self, PyObject *value, void *closure) } if (compatible_for_assignment(oldto, newto, "__class__")) { + /* Changing the class will change the implicit dict keys, + * so we must materialize the dictionary first. */ + assert(oldto->tp_inline_values_offset == newto->tp_inline_values_offset); + _PyObject_GetDictPtr(self); + PyDictValues** values_ptr = _PyObject_ValuesPointer(self); + if (values_ptr != NULL && *values_ptr != NULL) { + /* Was unable to convert to dict */ + PyErr_NoMemory(); + return -1; + } + assert(_PyObject_ValuesPointer(self) == NULL || *_PyObject_ValuesPointer(self) == NULL); if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); } @@ -4906,23 +4962,16 @@ _PyObject_GetState(PyObject *obj, int required) Py_TYPE(obj)->tp_name); return NULL; } - - { - PyObject **dict; - dict = _PyObject_GetDictPtr(obj); - /* It is possible that the object's dict is not initialized - yet. In this case, we will return None for the state. - We also return None if the dict is empty to make the behavior - consistent regardless whether the dict was initialized or not. - This make unit testing easier. */ - if (dict != NULL && *dict != NULL && PyDict_GET_SIZE(*dict)) { - state = *dict; - } - else { - state = Py_None; - } + if (_PyObject_IsInstanceDictEmpty(obj)) { + state = Py_None; Py_INCREF(state); } + else { + state = PyObject_GenericGetDict(obj, NULL); + if (state == NULL) { + return NULL; + } + } slotnames = _PyType_GetSlotNames(Py_TYPE(obj)); if (slotnames == NULL) { @@ -4933,12 +4982,18 @@ _PyObject_GetState(PyObject *obj, int required) assert(slotnames == Py_None || PyList_Check(slotnames)); if (required) { Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize; - if (Py_TYPE(obj)->tp_dictoffset) + if (Py_TYPE(obj)->tp_dictoffset) { basicsize += sizeof(PyObject *); - if (Py_TYPE(obj)->tp_weaklistoffset) + } + if (Py_TYPE(obj)->tp_weaklistoffset) { basicsize += sizeof(PyObject *); - if (slotnames != Py_None) + } + if (Py_TYPE(obj)->tp_inline_values_offset) { + basicsize += sizeof(PyDictValues *); + } + if (slotnames != Py_None) { basicsize += sizeof(PyObject *) * PyList_GET_SIZE(slotnames); + } if (Py_TYPE(obj)->tp_basicsize > basicsize) { Py_DECREF(slotnames); Py_DECREF(state); @@ -5019,7 +5074,7 @@ _PyObject_GetState(PyObject *obj, int required) Py_DECREF(slotnames); } else { /* getstate != NULL */ - state = _PyObject_CallNoArg(getstate); + state = _PyObject_CallNoArgs(getstate); Py_DECREF(getstate); if (state == NULL) return NULL; @@ -5044,7 +5099,7 @@ _PyObject_GetNewArguments(PyObject *obj, PyObject **args, PyObject **kwargs) __getnewargs_ex__ on the object. */ getnewargs_ex = _PyObject_LookupSpecial(obj, &PyId___getnewargs_ex__); if (getnewargs_ex != NULL) { - PyObject *newargs = _PyObject_CallNoArg(getnewargs_ex); + PyObject *newargs = _PyObject_CallNoArgs(getnewargs_ex); Py_DECREF(getnewargs_ex); if (newargs == NULL) { return -1; @@ -5097,7 +5152,7 @@ _PyObject_GetNewArguments(PyObject *obj, PyObject **args, PyObject **kwargs) __getnewargs__ instead. */ getnewargs = _PyObject_LookupSpecial(obj, &PyId___getnewargs__); if (getnewargs != NULL) { - *args = _PyObject_CallNoArg(getnewargs); + *args = _PyObject_CallNoArgs(getnewargs); Py_DECREF(getnewargs); if (*args == NULL) { return -1; @@ -5358,7 +5413,7 @@ object___reduce_ex___impl(PyObject *self, int protocol) override = (clsreduce != objreduce); Py_DECREF(clsreduce); if (override) { - res = _PyObject_CallNoArg(reduce); + res = _PyObject_CallNoArgs(reduce); Py_DECREF(reduce); return res; } @@ -5708,6 +5763,7 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) COPYVAL(tp_itemsize); COPYVAL(tp_weaklistoffset); COPYVAL(tp_dictoffset); + COPYVAL(tp_inline_values_offset); #undef COPYVAL /* Setup fast subclass flags */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 02bf56e681e56e..61fc34d71da3ce 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -46,6 +46,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_format.h" // F_LJUST #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec +#include "pycore_long.h" // _PyLong_FormatWriter() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_pathconfig.h" // _Py_DumpPathConfig() #include "pycore_pyerrors.h" // _Py_FatalRefcountError() @@ -6342,9 +6343,10 @@ PyUnicode_AsUTF16String(PyObject *unicode) static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; PyObject * -_PyUnicode_DecodeUnicodeEscape(const char *s, +_PyUnicode_DecodeUnicodeEscapeInternal(const char *s, Py_ssize_t size, const char *errors, + Py_ssize_t *consumed, const char **first_invalid_escape) { const char *starts = s; @@ -6357,6 +6359,9 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, *first_invalid_escape = NULL; if (size == 0) { + if (consumed) { + *consumed = 0; + } _Py_RETURN_UNICODE_EMPTY(); } /* Escaped strings will always be longer than the resulting @@ -6375,8 +6380,6 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, unsigned char c = (unsigned char) *s++; Py_UCS4 ch; int count; - Py_ssize_t startinpos; - Py_ssize_t endinpos; const char *message; #define WRITE_ASCII_CHAR(ch) \ @@ -6403,11 +6406,11 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, continue; } - startinpos = s - starts - 1; + Py_ssize_t startinpos = s - starts - 1; /* \ - Escapes */ if (s >= end) { message = "\\ at end of string"; - goto error; + goto incomplete; } c = (unsigned char) *s++; @@ -6461,7 +6464,10 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, count = 8; message = "truncated \\UXXXXXXXX escape"; hexescape: - for (ch = 0; count && s < end; ++s, --count) { + for (ch = 0; count; ++s, --count) { + if (s >= end) { + goto incomplete; + } c = (unsigned char)*s; ch <<= 4; if (c >= '0' && c <= '9') { @@ -6474,12 +6480,9 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, ch += c - ('A' - 10); } else { - break; + goto error; } } - if (count) { - goto error; - } /* when we get here, ch is a 32-bit unicode character */ if (ch > MAX_UNICODE) { @@ -6506,14 +6509,20 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, } message = "malformed \\N character escape"; - if (s < end && *s == '{') { + if (s >= end) { + goto incomplete; + } + if (*s == '{') { const char *start = ++s; size_t namelen; /* look for the closing brace */ while (s < end && *s != '}') s++; + if (s >= end) { + goto incomplete; + } namelen = s - start; - if (namelen && s < end) { + if (namelen) { /* found a name. look it up in the unicode database */ s++; ch = 0xffffffff; /* in case 'getcode' messes up */ @@ -6539,8 +6548,13 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, continue; } - error: - endinpos = s-starts; + incomplete: + if (consumed) { + *consumed = startinpos; + break; + } + error:; + Py_ssize_t endinpos = s-starts; writer.min_length = end - s + writer.pos; if (unicode_decode_call_errorhandler_writer( errors, &errorHandler, @@ -6567,12 +6581,14 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, } PyObject * -PyUnicode_DecodeUnicodeEscape(const char *s, +_PyUnicode_DecodeUnicodeEscapeStateful(const char *s, Py_ssize_t size, - const char *errors) + const char *errors, + Py_ssize_t *consumed) { const char *first_invalid_escape; - PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors, + PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal(s, size, errors, + consumed, &first_invalid_escape); if (result == NULL) return NULL; @@ -6587,6 +6603,14 @@ PyUnicode_DecodeUnicodeEscape(const char *s, return result; } +PyObject * +PyUnicode_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) +{ + return _PyUnicode_DecodeUnicodeEscapeStateful(s, size, errors, NULL); +} + /* Return a Unicode-Escape string version of the Unicode object. */ PyObject * @@ -6710,9 +6734,10 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) /* --- Raw Unicode Escape Codec ------------------------------------------- */ PyObject * -PyUnicode_DecodeRawUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors) +_PyUnicode_DecodeRawUnicodeEscapeStateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { const char *starts = s; _PyUnicodeWriter writer; @@ -6721,6 +6746,9 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, PyObject *exc = NULL; if (size == 0) { + if (consumed) { + *consumed = 0; + } _Py_RETURN_UNICODE_EMPTY(); } @@ -6739,8 +6767,6 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, unsigned char c = (unsigned char) *s++; Py_UCS4 ch; int count; - Py_ssize_t startinpos; - Py_ssize_t endinpos; const char *message; #define WRITE_CHAR(ch) \ @@ -6755,11 +6781,21 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, } while(0) /* Non-escape characters are interpreted as Unicode ordinals */ - if (c != '\\' || s >= end) { + if (c != '\\' || (s >= end && !consumed)) { WRITE_CHAR(c); continue; } + Py_ssize_t startinpos = s - starts - 1; + /* \ - Escapes */ + if (s >= end) { + assert(consumed); + // Set message to silent compiler warning. + // Actually it is never used. + message = "\\ at end of string"; + goto incomplete; + } + c = (unsigned char) *s++; if (c == 'u') { count = 4; @@ -6775,10 +6811,12 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, WRITE_CHAR(c); continue; } - startinpos = s - starts - 2; /* \uHHHH with 4 hex digits, \U00HHHHHH with 8 */ - for (ch = 0; count && s < end; ++s, --count) { + for (ch = 0; count; ++s, --count) { + if (s >= end) { + goto incomplete; + } c = (unsigned char)*s; ch <<= 4; if (c >= '0' && c <= '9') { @@ -6791,18 +6829,23 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, ch += c - ('A' - 10); } else { - break; + goto error; } } - if (!count) { - if (ch <= MAX_UNICODE) { - WRITE_CHAR(ch); - continue; - } + if (ch > MAX_UNICODE) { message = "\\Uxxxxxxxx out of range"; + goto error; } + WRITE_CHAR(ch); + continue; - endinpos = s-starts; + incomplete: + if (consumed) { + *consumed = startinpos; + break; + } + error:; + Py_ssize_t endinpos = s-starts; writer.min_length = end - s + writer.pos; if (unicode_decode_call_errorhandler_writer( errors, &errorHandler, @@ -6824,7 +6867,14 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; +} +PyObject * +PyUnicode_DecodeRawUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) +{ + return _PyUnicode_DecodeRawUnicodeEscapeStateful(s, size, errors, NULL); } @@ -7349,7 +7399,7 @@ PyUnicode_AsASCIIString(PyObject *unicode) #endif /* INT_MAX is the theoretical largest chunk (or INT_MAX / 2 when - transcoding from UTF-16), but INT_MAX / 4 perfoms better in + transcoding from UTF-16), but INT_MAX / 4 performs better in both cases also and avoids partial characters overrunning the length limit in MultiByteToWideChar on Windows */ #define DECODING_CHUNK_SIZE (INT_MAX/4) @@ -15876,7 +15926,7 @@ init_fs_codec(PyInterpreterState *interp) _Py_error_handler error_handler; error_handler = get_error_handler_wide(config->filesystem_errors); if (error_handler == _Py_ERROR_UNKNOWN) { - PyErr_SetString(PyExc_RuntimeError, "unknow filesystem error handler"); + PyErr_SetString(PyExc_RuntimeError, "unknown filesystem error handler"); return -1; } diff --git a/PC/WinMain.c b/PC/WinMain.c index e439bed9ed1137..07e21ce3e9f2d5 100644 --- a/PC/WinMain.c +++ b/PC/WinMain.c @@ -4,6 +4,7 @@ #define WIN32_LEAN_AND_MEAN #include +#include /* __argc, __wargv */ int WINAPI wWinMain( HINSTANCE hInstance, /* handle to current instance */ diff --git a/PC/getpathp.c b/PC/getpathp.c index 603a1eb13c4ff0..549353d78b116a 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -80,6 +80,7 @@ #include "Python.h" +#include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_initconfig.h" // PyStatus #include "pycore_pathconfig.h" // _PyPathConfig #include "osdefs.h" // SEP, ALTSEP @@ -115,9 +116,7 @@ * with a semicolon separated path prior to calling Py_Initialize. */ -#ifndef LANDMARK -# define LANDMARK L"lib\\os.py" -#endif +#define STDLIB_SUBDIR L"lib" #define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow") @@ -216,7 +215,7 @@ exists(const wchar_t *filename) Assumes 'filename' MAXPATHLEN+1 bytes long - may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) +ismodule(wchar_t *filename) { size_t n; @@ -231,9 +230,8 @@ ismodule(wchar_t *filename, int update_filename) filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); - if (!update_filename) { - filename[n] = L'\0'; - } + // Drop the 'c' we just added. + filename[n] = L'\0'; return exist; } return 0; @@ -253,7 +251,7 @@ ismodule(wchar_t *filename, int update_filename) static void join(wchar_t *buffer, const wchar_t *stuff) { - if (FAILED(PathCchCombineEx(buffer, MAXPATHLEN+1, buffer, stuff, 0))) { + if (_Py_add_relfile(buffer, stuff, MAXPATHLEN+1) < 0) { Py_FatalError("buffer overflow in getpathp.c's join()"); } } @@ -267,36 +265,60 @@ canonicalize(wchar_t *buffer, const wchar_t *path) return _PyStatus_NO_MEMORY(); } - if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, path, 0))) { + if (PathIsRelativeW(path)) { + wchar_t buff[MAXPATHLEN + 1]; + if (!GetCurrentDirectoryW(MAXPATHLEN, buff)) { + return _PyStatus_ERR("unable to find current working directory"); + } + if (FAILED(PathCchCombineEx(buff, MAXPATHLEN + 1, buff, path, PATHCCH_ALLOW_LONG_PATHS))) { + return INIT_ERR_BUFFER_OVERFLOW(); + } + if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, buff, PATHCCH_ALLOW_LONG_PATHS))) { + return INIT_ERR_BUFFER_OVERFLOW(); + } + return _PyStatus_OK(); + } + + if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, path, PATHCCH_ALLOW_LONG_PATHS))) { return INIT_ERR_BUFFER_OVERFLOW(); } return _PyStatus_OK(); } - -/* gotlandmark only called by search_for_prefix, which ensures - 'prefix' is null terminated in bounds. join() ensures - 'landmark' can not overflow prefix if too long. */ static int -gotlandmark(const wchar_t *prefix, const wchar_t *landmark) +is_stdlibdir(wchar_t *stdlibdir) { - wchar_t filename[MAXPATHLEN+1]; - memset(filename, 0, sizeof(filename)); - wcscpy_s(filename, Py_ARRAY_LENGTH(filename), prefix); - join(filename, landmark); - return ismodule(filename, FALSE); + wchar_t *filename = stdlibdir; +#ifndef LANDMARK +# define LANDMARK L"os.py" +#endif + /* join() ensures 'landmark' can not overflow prefix if too long. */ + join(filename, LANDMARK); + return ismodule(filename); } - /* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd. assumption provided by only caller, calculate_path() */ static int -search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path, const wchar_t *landmark) +search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path) { - /* Search from argv0_path, until landmark is found */ - wcscpy_s(prefix, MAXPATHLEN + 1, argv0_path); + /* Search from argv0_path, until LANDMARK is found. + We guarantee 'prefix' is null terminated in bounds. */ + wcscpy_s(prefix, MAXPATHLEN+1, argv0_path); + if (!prefix[0]) { + return 0; + } + wchar_t stdlibdir[MAXPATHLEN+1]; + wcscpy_s(stdlibdir, Py_ARRAY_LENGTH(stdlibdir), prefix); + /* We initialize with the longest possible path, in case it doesn't fit. + This also gives us an initial SEP at stdlibdir[wcslen(prefix)]. */ + join(stdlibdir, STDLIB_SUBDIR); do { - if (gotlandmark(prefix, landmark)) { + assert(stdlibdir[wcslen(prefix)] == SEP); + /* Due to reduce() and our initial value, this result + is guaranteed to fit. */ + wcscpy(&stdlibdir[wcslen(prefix) + 1], STDLIB_SUBDIR); + if (is_stdlibdir(stdlibdir)) { return 1; } reduce(prefix); @@ -332,7 +354,7 @@ extern const char *PyWin_DLLVersionString; Returns NULL, or a pointer that should be freed. XXX - this code is pretty strange, as it used to also - work on Win16, where the buffer sizes werent available + work on Win16, where the buffer sizes were not available in advance. It could be simplied now Win16/Win32s is dead! */ static wchar_t * @@ -758,7 +780,7 @@ calculate_home_prefix(PyCalculatePath *calculate, reduce(prefix); calculate->home = prefix; } - else if (search_for_prefix(prefix, argv0_path, LANDMARK)) { + else if (search_for_prefix(prefix, argv0_path)) { calculate->home = prefix; } else { @@ -920,6 +942,7 @@ calculate_module_search_path(PyCalculatePath *calculate, the parent of that. */ if (prefix[0] == L'\0') { + PyStatus status; wchar_t lookBuf[MAXPATHLEN+1]; const wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */ while (1) { @@ -934,9 +957,13 @@ calculate_module_search_path(PyCalculatePath *calculate, nchars = lookEnd-look; wcsncpy(lookBuf, look+1, nchars); lookBuf[nchars] = L'\0'; + status = canonicalize(lookBuf, lookBuf); + if (_PyStatus_EXCEPTION(status)) { + return status; + } /* Up one level to the parent */ reduce(lookBuf); - if (search_for_prefix(prefix, lookBuf, LANDMARK)) { + if (search_for_prefix(prefix, lookBuf)) { break; } /* If we are out of paths to search - give up */ @@ -1010,6 +1037,12 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) } done: + if (pathconfig->stdlib_dir == NULL) { + pathconfig->stdlib_dir = _Py_join_relfile(prefix, STDLIB_SUBDIR); + if (pathconfig->stdlib_dir == NULL) { + return _PyStatus_NO_MEMORY(); + } + } if (pathconfig->prefix == NULL) { pathconfig->prefix = _PyMem_RawWcsdup(prefix); if (pathconfig->prefix == NULL) { diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 0591497871cadc..1f78d99c790ff9 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -17,6 +17,7 @@ ***********************************************************/ #include "Python.h" +#include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "malloc.h" #include #include diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 012a89aab2bc8e..b3e73d4dabefed 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -58,7 +58,6 @@ WIN32 is still required for the locale module. #include -#define HAVE_HYPOT #define HAVE_STRFTIME #define DONT_HAVE_SIG_ALARM #define DONT_HAVE_SIG_PAUSE @@ -189,11 +188,6 @@ typedef _W64 int Py_ssize_t; typedef int pid_t; -#include -#define Py_IS_NAN _isnan -#define Py_IS_INFINITY(X) (!_finite(X) && !_isnan(X)) -#define Py_IS_FINITE(X) _finite(X) - /* define some ANSI types that are not defined in earlier Win headers */ #if _MSC_VER >= 1200 /* This file only exists in VC 6.0 or higher */ @@ -353,20 +347,6 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Fairly standard from here! */ -/* Define to 1 if you have the `copysign' function. */ -#define HAVE_COPYSIGN 1 - -/* Define to 1 if you have the `round' function. */ -#if _MSC_VER >= 1800 -#define HAVE_ROUND 1 -#endif - -/* Define to 1 if you have the `isinf' macro. */ -#define HAVE_DECL_ISINF 1 - -/* Define to 1 if you have the `isnan' function. */ -#define HAVE_DECL_ISNAN 1 - /* Define if on AIX 3. System headers sometimes define this. We just want to avoid a redefinition error message. */ @@ -483,6 +463,9 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Use Python's own small-block memory-allocator. */ #define WITH_PYMALLOC 1 +/* Define if you want to compile in object freelists optimization */ +#define WITH_FREELISTS 1 + /* Define if you have clock. */ /* #define HAVE_CLOCK */ diff --git a/PC/python3dll.c b/PC/python3dll.c index bc499a52d3373d..8d14cc493b7eae 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -843,6 +843,7 @@ EXPORT_DATA(PySeqIter_Type) EXPORT_DATA(PySet_Type) EXPORT_DATA(PySetIter_Type) EXPORT_DATA(PySlice_Type) +EXPORT_DATA(PyStructSequence_UnnamedField) EXPORT_DATA(PySuper_Type) EXPORT_DATA(PyTraceBack_Type) EXPORT_DATA(PyTuple_Type) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index ea6532d10d8ccc..12bdde2af84d9a 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -305,6 +305,31 @@ $(IntDir)__hello__.g.h $(PySourcePath)Python\frozen_modules\__hello__.h + + __phello__ + $(IntDir)__phello__.g.h + $(PySourcePath)Python\frozen_modules\__phello__.h + + + __phello__.ham + $(IntDir)__phello__.ham.g.h + $(PySourcePath)Python\frozen_modules\__phello__.ham.h + + + __phello__.ham.eggs + $(IntDir)__phello__.ham.eggs.g.h + $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h + + + __phello__.spam + $(IntDir)__phello__.spam.g.h + $(PySourcePath)Python\frozen_modules\__phello__.spam.h + + + frozen_only + $(IntDir)frozen_only.g.h + $(PySourcePath)Python\frozen_modules\frozen_only.h + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 4a1c90f668e64b..5894909e0fbe1e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -61,6 +61,21 @@ Python Files + + Python Files + + + Python Files + + + Python Files + + + Python Files + + + Python Files + diff --git a/PCbuild/lib.pyproj b/PCbuild/lib.pyproj index ee225131854de9..06e0a8b0d1c537 100644 --- a/PCbuild/lib.pyproj +++ b/PCbuild/lib.pyproj @@ -1424,7 +1424,6 @@ - @@ -1436,7 +1435,6 @@ - diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index b3cbd471c66ac9..f32422a0acf44a 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -1,5 +1,8 @@  + + + {CC9B93A2-439D-4058-9D29-6DCF43774405} Win32 @@ -15,9 +18,7 @@ - $(Platform) - Win32 - x64 + $(PreferredToolArchitecture) $(Configuration) Release diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b8cadf469355f4..1b484be5e7a2ff 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -115,28 +115,32 @@ - - - + + + + + + - + + @@ -157,19 +161,18 @@ + + - - - @@ -188,6 +191,7 @@ + @@ -197,9 +201,11 @@ + + @@ -209,6 +215,7 @@ + @@ -217,18 +224,15 @@ - - - @@ -248,7 +252,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4eccf4f43d1ad2..c1519760d2207f 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -51,15 +51,9 @@ Include - - Include - Include - - Include - Include @@ -72,9 +66,6 @@ Include - - Include - Include @@ -93,9 +84,6 @@ Include - - Include - Include @@ -108,12 +96,6 @@ Include - - Include - - - Include - Include @@ -126,9 +108,6 @@ Include - - Include - Include @@ -201,9 +180,6 @@ Include - - Include - Include @@ -351,12 +327,6 @@ Include - - Include - - - Include - Modules @@ -378,15 +348,24 @@ Include\cpython + + Include\cpython + Include\cpython + + Include\cpython + Include\cpython Include + + Include\cpython + Include\cpython @@ -396,18 +375,33 @@ Include\cpython + + Include\cpython + Include\cpython Include\cpython + + Include + + + Include + Include Include\cpython + + Include\cpython + + + Include\cpython + Include\cpython @@ -456,9 +450,12 @@ Include\cpython - + Include\cpython + + Include + Include\cpython @@ -525,6 +522,9 @@ Include\internal + + Include\internal + Include\internal @@ -552,6 +552,9 @@ Include\internal + + Include\cpython + Include\internal @@ -561,6 +564,9 @@ Include\internal + + Include + Include\internal @@ -588,6 +594,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 9492cff2d8c3ea..c0bde1ec6ba511 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -104,7 +104,9 @@ Condition="($(Platform) == 'Win32' or $(Platform) == 'x64') and $(Configuration) != 'PGInstrument' and $(Configuration) != 'PGUpdate'"> - diff --git a/PCbuild/sqlite3.vcxproj b/PCbuild/sqlite3.vcxproj index e39e2d9c226ca1..310795c4e4d202 100644 --- a/PCbuild/sqlite3.vcxproj +++ b/PCbuild/sqlite3.vcxproj @@ -98,7 +98,7 @@ $(sqlite3Dir);%(AdditionalIncludeDirectories) - SQLITE_ENABLE_MATH_FUNCTIONS;SQLITE_ENABLE_JSON1;SQLITE_ENABLE_FTS4;SQLITE_ENABLE_FTS5;SQLITE_ENABLE_RTREE;SQLITE_API=__declspec(dllexport);%(PreprocessorDefinitions) + SQLITE_ENABLE_MATH_FUNCTIONS;SQLITE_ENABLE_JSON1;SQLITE_ENABLE_FTS4;SQLITE_ENABLE_FTS5;SQLITE_ENABLE_RTREE;SQLITE_OMIT_AUTOINIT;SQLITE_API=__declspec(dllexport);%(PreprocessorDefinitions) Level1 diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 5f0b89bba44890..37925a5783d6d0 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -145,7 +145,7 @@ def __init__(self, *args, **kwargs): # names where all the constructors # belonging to that type lack of any # fields. - # - identifiers: All identifiers used in the AST decclarations + # - identifiers: All identifiers used in the AST declarations # - singletons: List of all constructors that originates from # simple sums. # - types: List of all top level type names diff --git a/Parser/myreadline.c b/Parser/myreadline.c index e5e2fb1099d033..b10d306255bb67 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -10,6 +10,7 @@ */ #include "Python.h" +#include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_pystate.h" // _PyThreadState_GET() #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN diff --git a/Parser/parser.c b/Parser/parser.c index 3cea370c5ad2d0..47722f3baf1e6e 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -946,7 +946,7 @@ file_rule(Parser *p) void *a; Token * endmarker_var; if ( - (a = statements_rule(p), 1) // statements? + (a = statements_rule(p), !p->error_indicator) // statements? && (endmarker_var = _PyPegen_expect_token(p, ENDMARKER)) // token='ENDMARKER' ) @@ -1085,7 +1085,7 @@ func_type_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = type_expressions_rule(p), 1) // type_expressions? + (a = type_expressions_rule(p), !p->error_indicator) // type_expressions? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -1441,7 +1441,7 @@ simple_stmts_rule(Parser *p) if ( (a = (asdl_stmt_seq*)_gather_4_rule(p)) // ';'.simple_stmt+ && - (_opt_var = _PyPegen_expect_token(p, 13), 1) // ';'? + (_opt_var = _PyPegen_expect_token(p, 13), !p->error_indicator) // ';'? && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -2061,7 +2061,7 @@ assignment_rule(Parser *p) && (b = expression_rule(p)) // expression && - (c = _tmp_11_rule(p), 1) // ['=' annotated_rhs] + (c = _tmp_11_rule(p), !p->error_indicator) // ['=' annotated_rhs] ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ':' expression ['=' annotated_rhs]")); @@ -2103,7 +2103,7 @@ assignment_rule(Parser *p) && (b = expression_rule(p)) // expression && - (c = _tmp_13_rule(p), 1) // ['=' annotated_rhs] + (c = _tmp_13_rule(p), !p->error_indicator) // ['=' annotated_rhs] ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('(' single_target ')' | single_subscript_attribute_target) ':' expression ['=' annotated_rhs]")); @@ -2144,7 +2144,7 @@ assignment_rule(Parser *p) && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT?")); @@ -2668,7 +2668,7 @@ return_stmt_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 519)) // token='return' && - (a = star_expressions_rule(p), 1) // star_expressions? + (a = star_expressions_rule(p), !p->error_indicator) // star_expressions? ) { D(fprintf(stderr, "%*c+ return_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'return' star_expressions?")); @@ -2733,7 +2733,7 @@ raise_stmt_rule(Parser *p) && (a = expression_rule(p)) // expression && - (b = _tmp_17_rule(p), 1) // ['from' expression] + (b = _tmp_17_rule(p), !p->error_indicator) // ['from' expression] ) { D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression ['from' expression]")); @@ -3097,7 +3097,7 @@ assert_stmt_rule(Parser *p) && (a = expression_rule(p)) // expression && - (b = _tmp_23_rule(p), 1) // [',' expression] + (b = _tmp_23_rule(p), !p->error_indicator) // [',' expression] ) { D(fprintf(stderr, "%*c+ assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression [',' expression]")); @@ -3400,7 +3400,7 @@ import_from_targets_rule(Parser *p) && (a = import_from_as_names_rule(p)) // import_from_as_names && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -3569,7 +3569,7 @@ import_from_as_name_rule(Parser *p) if ( (a = _PyPegen_name_token(p)) // NAME && - (b = _tmp_28_rule(p), 1) // ['as' NAME] + (b = _tmp_28_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ['as' NAME]")); @@ -3672,7 +3672,7 @@ dotted_as_name_rule(Parser *p) if ( (a = dotted_name_rule(p)) // dotted_name && - (b = _tmp_31_rule(p), 1) // ['as' NAME] + (b = _tmp_31_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name ['as' NAME]")); @@ -4055,7 +4055,7 @@ class_def_raw_rule(Parser *p) && (a = _PyPegen_name_token(p)) // NAME && - (b = _tmp_33_rule(p), 1) // ['(' arguments? ')'] + (b = _tmp_33_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -4217,15 +4217,15 @@ function_def_raw_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (params = params_rule(p), 1) // params? + (params = params_rule(p), !p->error_indicator) // params? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_34_rule(p), 1) // ['->' expression] + (a = _tmp_34_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = func_type_comment_rule(p), 1) // func_type_comment? + (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) @@ -4277,15 +4277,15 @@ function_def_raw_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (params = params_rule(p), 1) // params? + (params = params_rule(p), !p->error_indicator) // params? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_35_rule(p), 1) // ['->' expression] + (a = _tmp_35_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = func_type_comment_rule(p), 1) // func_type_comment? + (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) @@ -4406,7 +4406,7 @@ parameters_rule(Parser *p) && (c = _loop0_37_rule(p)) // param_with_default* && - (d = star_etc_rule(p), 1) // star_etc? + (d = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default param_no_default* param_with_default* star_etc?")); @@ -4436,7 +4436,7 @@ parameters_rule(Parser *p) && (b = _loop0_38_rule(p)) // param_with_default* && - (c = star_etc_rule(p), 1) // star_etc? + (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default param_with_default* star_etc?")); @@ -4466,7 +4466,7 @@ parameters_rule(Parser *p) && (b = _loop0_40_rule(p)) // param_with_default* && - (c = star_etc_rule(p), 1) // star_etc? + (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default+ param_with_default* star_etc?")); @@ -4493,7 +4493,7 @@ parameters_rule(Parser *p) if ( (a = _loop1_41_rule(p)) // param_with_default+ && - (b = star_etc_rule(p), 1) // star_etc? + (b = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+ star_etc?")); @@ -4731,7 +4731,7 @@ star_etc_rule(Parser *p) && (b = _loop0_48_rule(p)) // param_maybe_default* && - (c = kwds_rule(p), 1) // kwds? + (c = kwds_rule(p), !p->error_indicator) // kwds? ) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' param_no_default param_maybe_default* kwds?")); @@ -4764,7 +4764,7 @@ star_etc_rule(Parser *p) && (b = _loop1_49_rule(p)) // param_maybe_default+ && - (c = kwds_rule(p), 1) // kwds? + (c = kwds_rule(p), !p->error_indicator) // kwds? ) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' ',' param_maybe_default+ kwds?")); @@ -4898,7 +4898,7 @@ param_no_default_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ param_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param ',' TYPE_COMMENT?")); @@ -4925,7 +4925,7 @@ param_no_default_rule(Parser *p) if ( (a = param_rule(p)) // param && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -4977,7 +4977,7 @@ param_with_default_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default ',' TYPE_COMMENT?")); @@ -5007,7 +5007,7 @@ param_with_default_rule(Parser *p) && (c = default_rule(p)) // default && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -5057,11 +5057,11 @@ param_maybe_default_rule(Parser *p) if ( (a = param_rule(p)) // param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default? ',' TYPE_COMMENT?")); @@ -5089,9 +5089,9 @@ param_maybe_default_rule(Parser *p) if ( (a = param_rule(p)) // param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -5146,7 +5146,7 @@ param_rule(Parser *p) if ( (a = _PyPegen_name_token(p)) // NAME && - (b = annotation_rule(p), 1) // annotation? + (b = annotation_rule(p), !p->error_indicator) // annotation? ) { D(fprintf(stderr, "%*c+ param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME annotation?")); @@ -5372,7 +5372,7 @@ if_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (c = else_block_rule(p), 1) // else_block? + (c = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ if_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' named_expression ':' block else_block?")); @@ -5510,7 +5510,7 @@ elif_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (c = else_block_rule(p), 1) // else_block? + (c = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ elif_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'elif' named_expression ':' block else_block?")); @@ -5666,7 +5666,7 @@ while_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (c = else_block_rule(p), 1) // else_block? + (c = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ while_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'while' named_expression ':' block else_block?")); @@ -5768,11 +5768,11 @@ for_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block && - (el = else_block_rule(p), 1) // else_block? + (el = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ for_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for' star_targets 'in' ~ star_expressions &&':' TYPE_COMMENT? block else_block?")); @@ -5832,11 +5832,11 @@ for_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block && - (el = else_block_rule(p), 1) // else_block? + (el = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ for_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "ASYNC 'for' star_targets 'in' ~ star_expressions &&':' TYPE_COMMENT? block else_block?")); @@ -5956,7 +5956,7 @@ with_stmt_rule(Parser *p) && (a = (asdl_withitem_seq*)_gather_50_rule(p)) // ','.with_item+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -6005,7 +6005,7 @@ with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block ) @@ -6056,7 +6056,7 @@ with_stmt_rule(Parser *p) && (a = (asdl_withitem_seq*)_gather_54_rule(p)) // ','.with_item+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -6108,7 +6108,7 @@ with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block ) @@ -6360,9 +6360,9 @@ try_stmt_rule(Parser *p) && (ex = (asdl_excepthandler_seq*)_loop1_59_rule(p)) // except_block+ && - (el = else_block_rule(p), 1) // else_block? + (el = else_block_rule(p), !p->error_indicator) // else_block? && - (f = finally_block_rule(p), 1) // finally_block? + (f = finally_block_rule(p), !p->error_indicator) // finally_block? ) { D(fprintf(stderr, "%*c+ try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' &&':' block except_block+ else_block? finally_block?")); @@ -6452,7 +6452,7 @@ except_block_rule(Parser *p) && (e = expression_rule(p)) // expression && - (t = _tmp_60_rule(p), 1) // ['as' NAME] + (t = _tmp_60_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -6743,7 +6743,7 @@ subject_expr_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (values = star_named_expressions_rule(p), 1) // star_named_expressions? + (values = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { D(fprintf(stderr, "%*c+ subject_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); @@ -6839,7 +6839,7 @@ case_block_rule(Parser *p) && (pattern = patterns_rule(p)) // patterns && - (guard = guard_rule(p), 1) // guard? + (guard = guard_rule(p), !p->error_indicator) // guard? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -8592,7 +8592,7 @@ sequence_pattern_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (patterns = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? + (patterns = maybe_sequence_pattern_rule(p), !p->error_indicator) // maybe_sequence_pattern? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -8631,7 +8631,7 @@ sequence_pattern_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (patterns = open_sequence_pattern_rule(p), 1) // open_sequence_pattern? + (patterns = open_sequence_pattern_rule(p), !p->error_indicator) // open_sequence_pattern? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -8689,7 +8689,7 @@ open_sequence_pattern_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (patterns = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? + (patterns = maybe_sequence_pattern_rule(p), !p->error_indicator) // maybe_sequence_pattern? ) { D(fprintf(stderr, "%*c+ open_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern ',' maybe_sequence_pattern?")); @@ -8734,7 +8734,7 @@ maybe_sequence_pattern_rule(Parser *p) if ( (patterns = _gather_68_rule(p)) // ','.maybe_star_pattern+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ maybe_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.maybe_star_pattern+ ','?")); @@ -8985,7 +8985,7 @@ mapping_pattern_rule(Parser *p) && (rest = double_star_pattern_rule(p)) // double_star_pattern && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -9034,7 +9034,7 @@ mapping_pattern_rule(Parser *p) && (rest = double_star_pattern_rule(p)) // double_star_pattern && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_2 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -9077,7 +9077,7 @@ mapping_pattern_rule(Parser *p) && (items = items_pattern_rule(p)) // items_pattern && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -9320,7 +9320,7 @@ class_pattern_rule(Parser *p) && (patterns = positional_patterns_rule(p)) // positional_patterns && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -9366,7 +9366,7 @@ class_pattern_rule(Parser *p) && (keywords = keyword_patterns_rule(p)) // keyword_patterns && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -9418,7 +9418,7 @@ class_pattern_rule(Parser *p) && (keywords = keyword_patterns_rule(p)) // keyword_patterns && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_2 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -9629,7 +9629,7 @@ expressions_rule(Parser *p) && (b = _loop1_77_rule(p)) // ((',' expression))+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ((',' expression))+ ','?")); @@ -9922,7 +9922,7 @@ yield_expr_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 570)) // token='yield' && - (a = star_expressions_rule(p), 1) // star_expressions? + (a = star_expressions_rule(p), !p->error_indicator) // star_expressions? ) { D(fprintf(stderr, "%*c+ yield_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'yield' star_expressions?")); @@ -9991,7 +9991,7 @@ star_expressions_rule(Parser *p) && (b = _loop1_78_rule(p)) // ((',' star_expression))+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expression ((',' star_expression))+ ','?")); @@ -10186,7 +10186,7 @@ star_named_expressions_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_79_rule(p)) // ','.star_named_expression+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_named_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.star_named_expression+ ','?")); @@ -12864,7 +12864,7 @@ primary_raw(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (b = arguments_rule(p), 1) // arguments? + (b = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -13016,7 +13016,7 @@ slices_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_85_rule(p)) // ','.slice+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ slices[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.slice+ ','?")); @@ -13078,13 +13078,13 @@ slice_rule(Parser *p) void *b; void *c; if ( - (a = expression_rule(p), 1) // expression? + (a = expression_rule(p), !p->error_indicator) // expression? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (b = expression_rule(p), 1) // expression? + (b = expression_rule(p), !p->error_indicator) // expression? && - (c = _tmp_87_rule(p), 1) // [':' expression?] + (c = _tmp_87_rule(p), !p->error_indicator) // [':' expression?] ) { D(fprintf(stderr, "%*c+ slice[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression? ':' expression? [':' expression?]")); @@ -13528,7 +13528,7 @@ lambdef_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 583)) // token='lambda' && - (a = lambda_params_rule(p), 1) // lambda_params? + (a = lambda_params_rule(p), !p->error_indicator) // lambda_params? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -13651,7 +13651,7 @@ lambda_parameters_rule(Parser *p) && (c = _loop0_93_rule(p)) // lambda_param_with_default* && - (d = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (d = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default lambda_param_no_default* lambda_param_with_default* lambda_star_etc?")); @@ -13681,7 +13681,7 @@ lambda_parameters_rule(Parser *p) && (b = _loop0_94_rule(p)) // lambda_param_with_default* && - (c = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default lambda_param_with_default* lambda_star_etc?")); @@ -13711,7 +13711,7 @@ lambda_parameters_rule(Parser *p) && (b = _loop0_96_rule(p)) // lambda_param_with_default* && - (c = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default+ lambda_param_with_default* lambda_star_etc?")); @@ -13738,7 +13738,7 @@ lambda_parameters_rule(Parser *p) if ( (a = _loop1_97_rule(p)) // lambda_param_with_default+ && - (b = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (b = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+ lambda_star_etc?")); @@ -13978,7 +13978,7 @@ lambda_star_etc_rule(Parser *p) && (b = _loop0_104_rule(p)) // lambda_param_maybe_default* && - (c = lambda_kwds_rule(p), 1) // lambda_kwds? + (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) { D(fprintf(stderr, "%*c+ lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' lambda_param_no_default lambda_param_maybe_default* lambda_kwds?")); @@ -14011,7 +14011,7 @@ lambda_star_etc_rule(Parser *p) && (b = _loop1_105_rule(p)) // lambda_param_maybe_default+ && - (c = lambda_kwds_rule(p), 1) // lambda_kwds? + (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) { D(fprintf(stderr, "%*c+ lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' ',' lambda_param_maybe_default+ lambda_kwds?")); @@ -14289,7 +14289,7 @@ lambda_param_maybe_default_rule(Parser *p) if ( (a = lambda_param_rule(p)) // lambda_param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) @@ -14318,7 +14318,7 @@ lambda_param_maybe_default_rule(Parser *p) if ( (a = lambda_param_rule(p)) // lambda_param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 11) // token=':' ) @@ -14479,7 +14479,7 @@ list_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (a = star_named_expressions_rule(p), 1) // star_named_expressions? + (a = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -14544,7 +14544,7 @@ tuple_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_107_rule(p), 1) // [star_named_expression ',' star_named_expressions?] + (a = _tmp_107_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -14674,7 +14674,7 @@ dict_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (a = double_starred_kvpairs_rule(p), 1) // double_starred_kvpairs? + (a = double_starred_kvpairs_rule(p), !p->error_indicator) // double_starred_kvpairs? && (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -14755,7 +14755,7 @@ double_starred_kvpairs_rule(Parser *p) if ( (a = _gather_108_rule(p)) // ','.double_starred_kvpair+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ','?")); @@ -15433,7 +15433,7 @@ arguments_rule(Parser *p) if ( (a = args_rule(p)) // args && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -15510,7 +15510,7 @@ args_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_114_rule(p)) // ','.(starred_expression | (assigment_expression | expression !':=') !'=')+ && - (b = _tmp_116_rule(p), 1) // [',' kwargs] + (b = _tmp_116_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assigment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -16013,7 +16013,7 @@ star_targets_rule(Parser *p) && (b = _loop0_125_rule(p)) // ((',' star_target))* && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target ((',' star_target))* ','?")); @@ -16067,7 +16067,7 @@ star_targets_list_seq_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_126_rule(p)) // ','.star_target+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets_list_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.star_target+ ','?")); @@ -16115,7 +16115,7 @@ star_targets_tuple_seq_rule(Parser *p) && (b = _loop1_128_rule(p)) // ((',' star_target))+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets_tuple_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target ((',' star_target))+ ','?")); @@ -16478,7 +16478,7 @@ star_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = star_targets_tuple_seq_rule(p), 1) // star_targets_tuple_seq? + (a = star_targets_tuple_seq_rule(p), !p->error_indicator) // star_targets_tuple_seq? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -16517,7 +16517,7 @@ star_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (a = star_targets_list_seq_rule(p), 1) // star_targets_list_seq? + (a = star_targets_list_seq_rule(p), !p->error_indicator) // star_targets_list_seq? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -16950,7 +16950,7 @@ t_primary_raw(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (b = arguments_rule(p), 1) // arguments? + (b = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -17108,7 +17108,7 @@ del_targets_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_130_rule(p)) // ','.del_target+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ del_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.del_target+ ','?")); @@ -17354,7 +17354,7 @@ del_t_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = del_targets_rule(p), 1) // del_targets? + (a = del_targets_rule(p), !p->error_indicator) // del_targets? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -17393,7 +17393,7 @@ del_t_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (a = del_targets_rule(p), 1) // del_targets? + (a = del_targets_rule(p), !p->error_indicator) // del_targets? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -17824,7 +17824,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_141_rule(p), 1) // [args | expression for_if_clauses] + (_opt_var = _tmp_141_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -17880,15 +17880,15 @@ invalid_arguments_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args for_if_clauses")); expr_ty a; - asdl_comprehension_seq* for_if_clauses_var; + asdl_comprehension_seq* b; if ( (a = args_rule(p)) // args && - (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses + (b = for_if_clauses_rule(p)) // for_if_clauses ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args for_if_clauses")); - _res = _PyPegen_nonparen_genexp_in_call ( p , a ); + _res = _PyPegen_nonparen_genexp_in_call ( p , a , b ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -19434,7 +19434,7 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (_keyword = _PyPegen_expect_token(p, 630)) // token='for' && @@ -19612,7 +19612,7 @@ invalid_with_stmt_rule(Parser *p) void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (_keyword = _PyPegen_expect_token(p, 606)) // token='with' && @@ -19645,7 +19645,7 @@ invalid_with_stmt_rule(Parser *p) void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (_keyword = _PyPegen_expect_token(p, 606)) // token='with' && @@ -19653,7 +19653,7 @@ invalid_with_stmt_rule(Parser *p) && (_gather_164_var = _gather_164_rule(p)) // ','.(expressions ['as' star_target])+ && - (_opt_var_1 = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -19700,7 +19700,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 606)) // token='with' && @@ -19743,7 +19743,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 606)) // token='with' && @@ -19751,7 +19751,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_gather_168_var = _gather_168_rule(p)) // ','.(expressions ['as' star_target])+ && - (_opt_var_1 = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -19898,7 +19898,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_171_rule(p), 1) // ['as' NAME] + (_opt_var = _tmp_171_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -19932,7 +19932,7 @@ invalid_except_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_172_rule(p), 1) // ['as' NAME] + (_opt_var = _tmp_172_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -20062,7 +20062,7 @@ invalid_except_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_173_rule(p), 1) // ['as' NAME] + (_opt_var = _tmp_173_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20233,7 +20233,7 @@ invalid_case_block_rule(Parser *p) && (patterns_var = patterns_rule(p)) // patterns && - (_opt_var = guard_rule(p), 1) // guard? + (_opt_var = guard_rule(p), !p->error_indicator) // guard? && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 11) // token=':' ) @@ -20268,7 +20268,7 @@ invalid_case_block_rule(Parser *p) && (patterns_var = patterns_rule(p)) // patterns && - (_opt_var = guard_rule(p), 1) // guard? + (_opt_var = guard_rule(p), !p->error_indicator) // guard? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20446,7 +20446,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_174_rule(p), 1) // [positional_patterns ','] + (_opt_var = _tmp_174_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -20801,7 +20801,7 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 630)) // token='for' && @@ -20868,7 +20868,7 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 632)) // token='def' && @@ -20876,11 +20876,11 @@ invalid_def_raw_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_opt_var_1 = params_rule(p), 1) // params? + (_opt_var_1 = params_rule(p), !p->error_indicator) // params? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_2 = _tmp_175_rule(p), 1) // ['->' expression] + (_opt_var_2 = _tmp_175_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20936,7 +20936,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_176_rule(p), 1) // ['(' arguments? ')'] + (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23038,7 +23038,7 @@ _tmp_33_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (z = arguments_rule(p), 1) // arguments? + (z = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -26354,7 +26354,7 @@ _tmp_87_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (d = expression_rule(p), 1) // expression? + (d = expression_rule(p), !p->error_indicator) // expression? ) { D(fprintf(stderr, "%*c+ _tmp_87[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); @@ -27708,7 +27708,7 @@ _tmp_107_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (z = star_named_expressions_rule(p), 1) // star_named_expressions? + (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { D(fprintf(stderr, "%*c+ _tmp_107[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); @@ -31699,7 +31699,7 @@ _tmp_176_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_opt_var = arguments_rule(p), 1) // arguments? + (_opt_var = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -32708,7 +32708,7 @@ _tmp_197_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_202_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_202_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); @@ -32748,7 +32748,7 @@ _tmp_198_rule(Parser *p) if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_203_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_203_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); @@ -32788,7 +32788,7 @@ _tmp_199_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_204_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_204_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); @@ -32828,7 +32828,7 @@ _tmp_200_rule(Parser *p) if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_205_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_205_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); diff --git a/Parser/pegen.c b/Parser/pegen.c index c77c534ff090b2..b00eff3432decf 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -350,10 +350,18 @@ tokenizer_error(Parser *p) errtype = PyExc_IndentationError; msg = "too many levels of indentation"; break; - case E_LINECONT: - col_offset = strlen(strtok(p->tok->buf, "\n")) - 1; + case E_LINECONT: { + char* loc = strrchr(p->tok->buf, '\n'); + const char* last_char = p->tok->cur - 1; + if (loc != NULL && loc != last_char) { + col_offset = p->tok->cur - loc - 1; + p->tok->buf = loc; + } else { + col_offset = last_char - p->tok->buf - 1; + } msg = "unexpected character after line continuation character"; break; + } default: msg = "unknown parsing error"; } @@ -729,7 +737,7 @@ _PyPegen_fill_token(Parser *p) { const char *start; const char *end; - int type = PyTokenizer_Get(p->tok, &start, &end); + int type = _PyTokenizer_Get(p->tok, &start, &end); // Record and skip '# type: ignore' comments while (type == TYPE_IGNORE) { @@ -746,7 +754,7 @@ _PyPegen_fill_token(Parser *p) PyErr_NoMemory(); return -1; } - type = PyTokenizer_Get(p->tok, &start, &end); + type = _PyTokenizer_Get(p->tok, &start, &end); } // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing @@ -1306,7 +1314,7 @@ _PyPegen_check_tokenizer_errors(Parser *p) { for (;;) { const char *start; const char *end; - switch (PyTokenizer_Get(p->tok, &start, &end)) { + switch (_PyTokenizer_Get(p->tok, &start, &end)) { case ERRORTOKEN: if (p->tok->level != 0) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; @@ -1342,13 +1350,16 @@ _PyPegen_run_parser(Parser *p) { void *res = _PyPegen_parse(p); if (res == NULL) { + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) { + return NULL; + } Token *last_token = p->tokens[p->fill - 1]; reset_parser_state(p); _PyPegen_parse(p); if (PyErr_Occurred()) { // Prioritize tokenizer errors to custom syntax errors raised // on the second phase only if the errors come from the parser. - if (p->tok->done != E_ERROR && PyErr_ExceptionMatches(PyExc_SyntaxError)) { + if (p->tok->done == E_DONE && PyErr_ExceptionMatches(PyExc_SyntaxError)) { _PyPegen_check_tokenizer_errors(p); } return NULL; @@ -1408,7 +1419,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena const char *enc, const char *ps1, const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { - struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2); + struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2); if (tok == NULL) { if (PyErr_Occurred()) { raise_tokenizer_init_error(filename_ob); @@ -1438,7 +1449,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena _PyPegen_Parser_Free(p); error: - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return result; } @@ -1450,9 +1461,9 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen struct tok_state *tok; if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) { - tok = PyTokenizer_FromUTF8(str, exec_input); + tok = _PyTokenizer_FromUTF8(str, exec_input); } else { - tok = PyTokenizer_FromString(str, exec_input); + tok = _PyTokenizer_FromString(str, exec_input); } if (tok == NULL) { if (PyErr_Occurred()) { @@ -1480,7 +1491,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen _PyPegen_Parser_Free(p); error: - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return result; } @@ -2551,8 +2562,17 @@ void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) { return RAISE_SYNTAX_ERROR(msg); } + +static inline expr_ty +_PyPegen_get_last_comprehension_item(comprehension_ty comprehension) { + if (comprehension->ifs == NULL || asdl_seq_LEN(comprehension->ifs) == 0) { + return comprehension->iter; + } + return PyPegen_last_item(comprehension->ifs, expr_ty); +} + void * -_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args) +_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions) { /* The rule that calls this function is 'args for_if_clauses'. For the input f(L, x for x in y), L and x are in args and @@ -2566,8 +2586,11 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args) return NULL; } - return RAISE_SYNTAX_ERROR_STARTING_FROM( + comprehension_ty last_comprehension = PyPegen_last_item(comprehensions, comprehension_ty); + + return RAISE_SYNTAX_ERROR_KNOWN_RANGE( (expr_ty) asdl_seq_GET(args->v.Call.args, len - 1), + _PyPegen_get_last_comprehension_item(last_comprehension), "Generator expression must be parenthesized" ); } diff --git a/Parser/pegen.h b/Parser/pegen.h index 57d11779efafe7..8721d7e891005a 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -327,7 +327,7 @@ _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e) } void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); -void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args); +void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions); // Generated function in parse.c - function definition in python.gram diff --git a/Parser/string_parser.c b/Parser/string_parser.c index fb37d37553a552..c6fe99c885d691 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -115,7 +115,7 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) s = buf; const char *first_invalid_escape; - v = _PyUnicode_DecodeUnicodeEscape(s, len, NULL, &first_invalid_escape); + v = _PyUnicode_DecodeUnicodeEscapeInternal(s, len, NULL, NULL, &first_invalid_escape); if (v != NULL && first_invalid_escape != NULL) { if (warn_invalid_escape_sequence(parser, *first_invalid_escape, t) < 0) { @@ -386,7 +386,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, str[0] = '('; str[len+1] = ')'; - struct tok_state* tok = PyTokenizer_FromString(str, 1); + struct tok_state* tok = _PyTokenizer_FromString(str, 1); if (tok == NULL) { PyMem_Free(str); return NULL; @@ -409,7 +409,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, exit: PyMem_Free(str); _PyPegen_Parser_Free(p2); - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return result; } @@ -1127,9 +1127,7 @@ _PyPegen_FstringParser_ConcatFstring(Parser *p, FstringParser *state, const char /* We know we have an expression. Convert any existing string to a Constant node. */ - if (!state->last_str) { - /* Do nothing. No previous literal. */ - } else { + if (state->last_str) { /* Convert the existing last_str literal to a Constant node. */ expr_ty last_str = make_str_node_and_del(p, &state->last_str, first_token, last_token); if (!last_str || ExprList_Append(&state->expr_list, last_str) < 0) { diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 90eee28e126539..ae3874b09de48e 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -3,6 +3,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include #include @@ -107,7 +108,7 @@ static char * error_ret(struct tok_state *tok) /* XXX */ { tok->decoding_erred = 1; - if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */ + if (tok->fp != NULL && tok->buf != NULL) /* see _PyTokenizer_Free */ PyMem_Free(tok->buf); tok->buf = tok->cur = tok->inp = NULL; tok->start = NULL; @@ -477,7 +478,7 @@ fp_setreadl(struct tok_state *tok, const char* enc) Py_XSETREF(tok->decoding_readline, readline); if (pos > 0) { - PyObject *bufobj = _PyObject_CallNoArg(readline); + PyObject *bufobj = _PyObject_CallNoArgs(readline); if (bufobj == NULL) return 0; Py_DECREF(bufobj); @@ -701,7 +702,7 @@ decode_str(const char *input, int single, struct tok_state *tok) /* Set up tokenizer for string */ struct tok_state * -PyTokenizer_FromString(const char *str, int exec_input) +_PyTokenizer_FromString(const char *str, int exec_input) { struct tok_state *tok = tok_new(); char *decoded; @@ -710,7 +711,7 @@ PyTokenizer_FromString(const char *str, int exec_input) return NULL; decoded = decode_str(str, exec_input, tok); if (decoded == NULL) { - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return NULL; } @@ -722,7 +723,7 @@ PyTokenizer_FromString(const char *str, int exec_input) /* Set up tokenizer for UTF-8 string */ struct tok_state * -PyTokenizer_FromUTF8(const char *str, int exec_input) +_PyTokenizer_FromUTF8(const char *str, int exec_input) { struct tok_state *tok = tok_new(); char *translated; @@ -730,7 +731,7 @@ PyTokenizer_FromUTF8(const char *str, int exec_input) return NULL; tok->input = translated = translate_newlines(str, exec_input, tok); if (translated == NULL) { - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return NULL; } tok->decoding_state = STATE_NORMAL; @@ -738,7 +739,7 @@ PyTokenizer_FromUTF8(const char *str, int exec_input) tok->str = translated; tok->encoding = new_string("utf-8", 5, tok); if (!tok->encoding) { - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return NULL; } @@ -750,14 +751,14 @@ PyTokenizer_FromUTF8(const char *str, int exec_input) /* Set up tokenizer for file */ struct tok_state * -PyTokenizer_FromFile(FILE *fp, const char* enc, - const char *ps1, const char *ps2) +_PyTokenizer_FromFile(FILE *fp, const char* enc, + const char *ps1, const char *ps2) { struct tok_state *tok = tok_new(); if (tok == NULL) return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return NULL; } tok->cur = tok->inp = tok->buf; @@ -770,7 +771,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc, gets copied into the parse tree. */ tok->encoding = new_string(enc, strlen(enc), tok); if (!tok->encoding) { - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return NULL; } tok->decoding_state = STATE_NORMAL; @@ -781,7 +782,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc, /* Free a tok_state structure */ void -PyTokenizer_Free(struct tok_state *tok) +_PyTokenizer_Free(struct tok_state *tok) { if (tok->encoding != NULL) { PyMem_Free(tok->encoding); @@ -2048,7 +2049,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) } int -PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end) +_PyTokenizer_Get(struct tok_state *tok, + const char **p_start, const char **p_end) { int result = tok_get(tok, p_start, p_end); if (tok->decoding_erred) { @@ -2061,7 +2063,7 @@ PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end) /* Get the encoding of a Python file. Check for the coding cookie and check if the file starts with a BOM. - PyTokenizer_FindEncodingFilename() returns NULL when it can't find the + _PyTokenizer_FindEncodingFilename() returns NULL when it can't find the encoding in the first or second line of the file (in which case the encoding should be assumed to be UTF-8). @@ -2069,7 +2071,7 @@ PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end) by the caller. */ char * -PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) +_PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) { struct tok_state *tok; FILE *fp; @@ -2086,7 +2088,7 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) if (fp == NULL) { return NULL; } - tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL); + tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL); if (tok == NULL) { fclose(fp); return NULL; @@ -2099,12 +2101,12 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) tok->filename = PyUnicode_FromString(""); if (tok->filename == NULL) { fclose(fp); - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return encoding; } } while (tok->lineno < 2 && tok->done == E_OK) { - PyTokenizer_Get(tok, &p_start, &p_end); + _PyTokenizer_Get(tok, &p_start, &p_end); } fclose(fp); if (tok->encoding) { @@ -2113,18 +2115,11 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) strcpy(encoding, tok->encoding); } } - PyTokenizer_Free(tok); + _PyTokenizer_Free(tok); return encoding; } -char * -PyTokenizer_FindEncoding(int fd) -{ - return PyTokenizer_FindEncodingFilename(fd, NULL); -} - #ifdef Py_DEBUG - void tok_dump(int type, char *start, char *end) { @@ -2132,5 +2127,4 @@ tok_dump(int type, char *start, char *end) if (type == NAME || type == NUMBER || type == STRING || type == OP) printf("(%.*s)", (int)(end - start), start); } - -#endif +#endif // Py_DEBUG diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index a40f7d9687b440..1d1cfd639d9d50 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -23,7 +23,7 @@ enum interactive_underflow_t { /* Normal mode of operation: return a new token when asked in interactie mode */ IUNDERFLOW_NORMAL, /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This - * can be used to prevent the tokenizer to promt the user for new tokens */ + * can be used to prevent the tokenizer to prompt the user for new tokens */ IUNDERFLOW_STOP, }; @@ -86,12 +86,12 @@ struct tok_state { enum interactive_underflow_t interactive_underflow; }; -extern struct tok_state *PyTokenizer_FromString(const char *, int); -extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); -extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*, +extern struct tok_state *_PyTokenizer_FromString(const char *, int); +extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int); +extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*, const char *, const char *); -extern void PyTokenizer_Free(struct tok_state *); -extern int PyTokenizer_Get(struct tok_state *, const char **, const char **); +extern void _PyTokenizer_Free(struct tok_state *); +extern int _PyTokenizer_Get(struct tok_state *, const char **, const char **); #define tok_dump _Py_tok_dump diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 4b0633e7e7a3d8..316c70d2c7824f 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -9,8 +9,10 @@ #include #include +#include #include +#include // malloc() #include #include #ifndef MS_WINDOWS @@ -24,8 +26,12 @@ static const struct _frozen _PyImport_FrozenModules[] = { {0, 0, 0} /* sentinel */ }; +static const struct _module_alias aliases[] = { + {0, 0} /* sentinel */ +}; const struct _frozen *PyImport_FrozenModules; +const struct _module_alias *_PyImport_FrozenAliases; static const char header[] = "/* Auto-generated by Programs/_freeze_module.c */"; @@ -183,6 +189,7 @@ main(int argc, char *argv[]) const char *name, *inpath, *outpath; PyImport_FrozenModules = _PyImport_FrozenModules; + _PyImport_FrozenAliases = aliases; if (argc != 4) { fprintf(stderr, "need to specify the name, input and output paths\n"); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7628e1a17d9ff6..b61fe3461bdc98 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -11,6 +11,7 @@ #include #include #include +#include // putenv() #include /********************************************************* @@ -211,7 +212,7 @@ static int test_pre_initialization_sys_options(void) * relying on the caller to keep the passed in strings alive. */ const wchar_t *static_warnoption = L"once"; - const wchar_t *static_xoption = L"also_not_an_option=2"; + const wchar_t *static_xoption = L"utf8=1"; size_t warnoption_len = wcslen(static_warnoption); size_t xoption_len = wcslen(static_xoption); wchar_t *dynamic_once_warnoption = \ @@ -230,7 +231,7 @@ static int test_pre_initialization_sys_options(void) PySys_AddWarnOption(L"module"); PySys_AddWarnOption(L"default"); _Py_EMBED_PREINIT_CHECK("Checking PySys_AddXOption\n"); - PySys_AddXOption(L"not_an_option=1"); + PySys_AddXOption(L"dev=2"); PySys_AddXOption(dynamic_xoption); /* Delete the dynamic options early */ @@ -548,7 +549,7 @@ static int test_init_from_config(void) L"-W", L"cmdline_warnoption", L"-X", - L"cmdline_xoption", + L"dev", L"-c", L"pass", L"arg2", @@ -556,10 +557,9 @@ static int test_init_from_config(void) config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); config.parse_argv = 1; - wchar_t* xoptions[3] = { - L"config_xoption1=3", - L"config_xoption2=", - L"config_xoption3", + wchar_t* xoptions[2] = { + L"dev=3", + L"utf8", }; config_set_wide_string_list(&config, &config.xoptions, Py_ARRAY_LENGTH(xoptions), xoptions); @@ -1375,7 +1375,6 @@ static int test_init_read_set(void) static int test_init_sys_add(void) { - PySys_AddXOption(L"sysadd_xoption"); PySys_AddXOption(L"faulthandler"); PySys_AddWarnOption(L"ignore:::sysadd_warnoption"); @@ -1387,14 +1386,14 @@ static int test_init_sys_add(void) L"-W", L"ignore:::cmdline_warnoption", L"-X", - L"cmdline_xoption", + L"utf8", }; config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); config.parse_argv = 1; PyStatus status; status = PyWideStringList_Append(&config.xoptions, - L"config_xoption"); + L"dev"); if (PyStatus_Exception(status)) { goto fail; } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 2933b5b7b1e202..d3ebbe1331a153 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -47,7 +47,7 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source) if (filename == NULL) { return NULL; } - self->tok = PyTokenizer_FromUTF8(source, 1); + self->tok = _PyTokenizer_FromUTF8(source, 1); if (self->tok == NULL) { Py_DECREF(filename); return NULL; @@ -61,7 +61,7 @@ tokenizeriter_next(tokenizeriterobject *it) { const char *start; const char *end; - int type = PyTokenizer_Get(it->tok, &start, &end); + int type = _PyTokenizer_Get(it->tok, &start, &end); if (type == ERRORTOKEN && PyErr_Occurred()) { return NULL; } @@ -105,7 +105,7 @@ static void tokenizeriter_dealloc(tokenizeriterobject *it) { PyTypeObject *tp = Py_TYPE(it); - PyTokenizer_Free(it->tok); + _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); } @@ -150,7 +150,7 @@ static PyMethodDef tokenize_methods[] = { }; static PyModuleDef_Slot tokenizemodule_slots[] = { - {Py_mod_exec, tokenizemodule_exec}, + {Py_mod_exec, tokenizemodule_exec}, {0, NULL} }; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d0d31805b3018d..9e3b25c59a5505 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3,6 +3,7 @@ #include "Python.h" #include #include "pycore_ast.h" // _PyAST_Validate() +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_object.h" // _Py_AddToAllObjects() #include "pycore_pyerrors.h" // _PyErr_NoMemory() @@ -212,7 +213,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto error; } PyFrameConstructor *f = PyFunction_AS_FRAME_CONSTRUCTOR(func); - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); cell = _PyEval_Vector(tstate, f, ns, NULL, 0, NULL); if (cell != NULL) { if (bases != orig_bases) { @@ -2305,7 +2306,7 @@ builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) } if (ndigits == Py_None) - result = _PyObject_CallNoArg(round); + result = _PyObject_CallNoArgs(round); else result = PyObject_CallOneArg(round, ndigits); Py_DECREF(round); @@ -2485,7 +2486,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) switch (Py_SIZE(item)) { case -1: b = -(sdigit) ((PyLongObject*)item)->ob_digit[0]; break; // Note: the continue goes to the top of the "while" loop that iterates over the elements - case 0: Py_DECREF(item); continue; + case 0: Py_DECREF(item); continue; case 1: b = ((PyLongObject*)item)->ob_digit[0]; break; default: b = PyLong_AsLongAndOverflow(item, &overflow); break; } diff --git a/Python/ceval.c b/Python/ceval.c index ab692fd8ded157..adc7b536247b2e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -17,7 +17,7 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_moduleobject.h" +#include "pycore_moduleobject.h" // PyModuleObject #include "pycore_pyerrors.h" // _PyErr_Fetch() #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() @@ -50,9 +50,9 @@ _Py_IDENTIFIER(__name__); /* Forward declarations */ -Py_LOCAL_INLINE(PyObject *) call_function( - PyThreadState *tstate, PyObject ***pp_stack, - Py_ssize_t oparg, PyObject *kwnames, int use_tracing); +static PyObject *trace_call_function( + PyThreadState *tstate, PyObject *callable, PyObject **stack, + Py_ssize_t oparg, PyObject *kwnames); static PyObject * do_call_core( PyThreadState *tstate, PyObject *func, PyObject *callargs, PyObject *kwdict, int use_tracing); @@ -98,6 +98,12 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs); static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); +static InterpreterFrame * +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFrameConstructor *con, + PyObject *locals, PyObject* const* args, + size_t argcount, PyObject *kwnames, int steal_args); +static int +_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame); #define NAME_ERROR_MSG \ "name '%.200s' is not defined" @@ -861,7 +867,7 @@ match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys) goto fail; } // dummy = object() - dummy = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); + dummy = _PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); if (dummy == NULL) { goto fail; } @@ -1058,7 +1064,7 @@ static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **); PyObject * PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (locals == NULL) { locals = globals; } @@ -1218,7 +1224,7 @@ eval_frame_handle_pending(PyThreadState *tstate) #endif #ifdef WITH_DTRACE -#define OR_DTRACE_LINE || PyDTrace_LINE_ENABLED() +#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0) #else #define OR_DTRACE_LINE #endif @@ -1240,11 +1246,13 @@ eval_frame_handle_pending(PyThreadState *tstate) #define USE_COMPUTED_GOTOS 0 #endif +#define INSTRUCTION_START() frame->f_lasti = INSTR_OFFSET(); next_instr++ + #if USE_COMPUTED_GOTOS -#define TARGET(op) TARGET_##op +#define TARGET(op) TARGET_##op: INSTRUCTION_START(); #define DISPATCH_GOTO() goto *opcode_targets[opcode] #else -#define TARGET(op) case op +#define TARGET(op) case op: INSTRUCTION_START(); #define DISPATCH_GOTO() goto dispatch_opcode #endif @@ -1274,12 +1282,11 @@ eval_frame_handle_pending(PyThreadState *tstate) #endif #endif #ifndef PRE_DISPATCH_GOTO -#define PRE_DISPATCH_GOTO() do { LLTRACE_INSTR(); RECORD_DXPROFILE(); } while (0) +#define PRE_DISPATCH_GOTO() do { LLTRACE_INSTR(); RECORD_DXPROFILE(); } while (0) #endif #define NOTRACE_DISPATCH() \ { \ - frame->f_lasti = INSTR_OFFSET(); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ @@ -1288,10 +1295,11 @@ eval_frame_handle_pending(PyThreadState *tstate) /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ { \ - if (cframe.use_tracing OR_DTRACE_LINE) { \ - goto tracing_dispatch; \ - } \ - NOTRACE_DISPATCH(); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \ + opcode |= cframe.use_tracing OR_DTRACE_LINE; \ + DISPATCH_GOTO(); \ } #define CHECK_EVAL_BREAKER() \ @@ -1316,7 +1324,6 @@ eval_frame_handle_pending(PyThreadState *tstate) _Py_CODEUNIT word = *next_instr; \ opcode = _Py_OPCODE(word); \ oparg = _Py_OPARG(word); \ - next_instr++; \ } while (0) #define JUMPTO(x) (next_instr = first_instr + (x)) #define JUMPBY(x) (next_instr += (x)) @@ -1326,7 +1333,6 @@ eval_frame_handle_pending(PyThreadState *tstate) _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code))[INSTR_OFFSET()]; \ opcode = _Py_OPCODE(word); \ oparg = _Py_OPARG(word); \ - next_instr++; \ } while (0) /* OpCode prediction macros @@ -1363,10 +1369,10 @@ eval_frame_handle_pending(PyThreadState *tstate) #define PREDICT(op) \ do { \ _Py_CODEUNIT word = *next_instr; \ - opcode = _Py_OPCODE(word); \ + opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \ if (opcode == op) { \ oparg = _Py_OPARG(word); \ - next_instr++; \ + INSTRUCTION_START(); \ goto PREDICT_ID(op); \ } \ } while(0) @@ -1446,11 +1452,6 @@ eval_frame_handle_pending(PyThreadState *tstate) #define UPDATE_PREV_INSTR_OPARG(instr, oparg) ((uint8_t*)(instr))[-1] = (oparg) -static inline void -record_hit_inline(_Py_CODEUNIT *next_instr, int oparg) -{ - UPDATE_PREV_INSTR_OPARG(next_instr, saturating_increment(oparg)); -} #define GLOBALS() frame->f_globals #define BUILTINS() frame->f_builtins @@ -1474,7 +1475,6 @@ record_hit_inline(_Py_CODEUNIT *next_instr, int oparg) res = ep->me_value; \ DEOPT_IF(res == NULL, LOAD_##attr_or_method); \ STAT_INC(LOAD_##attr_or_method, hit); \ - record_cache_hit(cache0); \ Py_INCREF(res); static int @@ -1516,6 +1516,21 @@ trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame) return 0; } +static PyObject * +make_coro(PyThreadState *tstate, PyFrameConstructor *con, + PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames); + +static int +skip_backwards_over_extended_args(PyCodeObject *code, int offset) { + _Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code); + while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) { + offset--; + } + return offset; +} + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag) { @@ -1534,10 +1549,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *retval = NULL; /* Return value */ _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker; - if (_Py_EnterRecursiveCall(tstate, "")) { - return NULL; - } - CFrame cframe; /* WARNING: Because the CFrame lives on the C stack, @@ -1549,9 +1560,18 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr cframe.previous = prev_cframe; tstate->cframe = &cframe; + assert(frame->depth == 0); /* push frame */ tstate->frame = frame; +start_frame: + if (_Py_EnterRecursiveCall(tstate, "")) { + tstate->recursion_depth++; + goto exit_eval_frame; + } + + assert(frame == tstate->frame); + if (cframe.use_tracing) { if (trace_function_entry(tstate, frame)) { goto exit_eval_frame; @@ -1573,7 +1593,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - +resume_frame: + co = frame->f_code; PyObject *names = co->co_names; PyObject *consts = co->co_consts; _Py_CODEUNIT *first_instr = co->co_firstinstr; @@ -1585,12 +1606,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr multiple values. When the PREDICT() macros are enabled, some opcode pairs follow in - direct succession without updating frame->f_lasti. A successful - prediction effectively links the two codes together as if they - were a single new opcode; accordingly,frame->f_lasti will point to - the first code in the pair (for instance, GET_ITER followed by - FOR_ITER is effectively a single opcode and frame->f_lasti will point - to the beginning of the combined pair.) + direct succession. A successful prediction effectively links the two + codes together as if they were a single new opcode, but the value + of frame->f_lasti is correctly updated so potential inlined calls + or lookups of frame->f_lasti are aways correct when the macros are used. */ assert(frame->f_lasti >= -1); _Py_CODEUNIT *next_instr = first_instr + frame->f_lasti + 1; @@ -1616,6 +1635,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr #endif if (throwflag) { /* support for generator.throw() */ + throwflag = 0; goto error; } @@ -1668,42 +1688,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); - tracing_dispatch: - { - int instr_prev = frame->f_lasti; - frame->f_lasti = INSTR_OFFSET(); - TRACING_NEXTOPARG(); - - if (PyDTrace_LINE_ENABLED()) - maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); - - /* line-by-line tracing support */ - - if (cframe.use_tracing && - tstate->c_tracefunc != NULL && !tstate->tracing) { - int err; - /* see maybe_call_line_trace() - for expository comments */ - _PyFrame_SetStackPointer(frame, stack_pointer); - - err = maybe_call_line_trace(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, instr_prev); - if (err) { - /* trace function raised an exception */ - goto error; - } - /* Reload possibly changed frame fields */ - JUMPTO(frame->f_lasti); - - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; - TRACING_NEXTOPARG(); - } - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - /* Start instructions */ #if USE_COMPUTED_GOTOS { @@ -1712,17 +1696,21 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr switch (opcode) { #endif + /* Variables used for making calls */ + PyObject *kwnames; + int nargs; + int postcall_shrink; + /* BEWARE! It is essential that any operation that fails must goto error and that all operation that succeed call DISPATCH() ! */ - TARGET(NOP): { + TARGET(NOP) { DISPATCH(); } - /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ - TARGET(LOAD_CLOSURE): - TARGET(LOAD_FAST): { + TARGET(LOAD_CLOSURE) { + /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ PyObject *value = GETLOCAL(oparg); if (value == NULL) { goto unbound_local_error; @@ -1732,7 +1720,17 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_CONST): { + TARGET(LOAD_FAST) { + PyObject *value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value = GETITEM(consts, oparg); Py_INCREF(value); @@ -1740,19 +1738,20 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(STORE_FAST): { + TARGET(STORE_FAST) { PREDICTED(STORE_FAST); PyObject *value = POP(); SETLOCAL(oparg, value); DISPATCH(); } - TARGET(LOAD_FAST__LOAD_FAST): { + TARGET(LOAD_FAST__LOAD_FAST) { PyObject *value = GETLOCAL(oparg); if (value == NULL) { goto unbound_local_error; } NEXTOPARG(); + next_instr++; Py_INCREF(value); PUSH(value); value = GETLOCAL(oparg); @@ -1764,12 +1763,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NOTRACE_DISPATCH(); } - TARGET(LOAD_FAST__LOAD_CONST): { + TARGET(LOAD_FAST__LOAD_CONST) { PyObject *value = GETLOCAL(oparg); if (value == NULL) { goto unbound_local_error; } NEXTOPARG(); + next_instr++; Py_INCREF(value); PUSH(value); value = GETITEM(consts, oparg); @@ -1778,10 +1778,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NOTRACE_DISPATCH(); } - TARGET(STORE_FAST__LOAD_FAST): { + TARGET(STORE_FAST__LOAD_FAST) { PyObject *value = POP(); SETLOCAL(oparg, value); NEXTOPARG(); + next_instr++; value = GETLOCAL(oparg); if (value == NULL) { goto unbound_local_error; @@ -1791,18 +1792,20 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NOTRACE_DISPATCH(); } - TARGET(STORE_FAST__STORE_FAST): { + TARGET(STORE_FAST__STORE_FAST) { PyObject *value = POP(); SETLOCAL(oparg, value); NEXTOPARG(); + next_instr++; value = POP(); SETLOCAL(oparg, value); NOTRACE_DISPATCH(); } - TARGET(LOAD_CONST__LOAD_FAST): { + TARGET(LOAD_CONST__LOAD_FAST) { PyObject *value = GETITEM(consts, oparg); NEXTOPARG(); + next_instr++; Py_INCREF(value); PUSH(value); value = GETLOCAL(oparg); @@ -1814,13 +1817,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NOTRACE_DISPATCH(); } - TARGET(POP_TOP): { + TARGET(POP_TOP) { PyObject *value = POP(); Py_DECREF(value); DISPATCH(); } - TARGET(ROT_TWO): { + TARGET(ROT_TWO) { PyObject *top = TOP(); PyObject *second = SECOND(); SET_TOP(second); @@ -1828,7 +1831,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(ROT_THREE): { + TARGET(ROT_THREE) { PyObject *top = TOP(); PyObject *second = SECOND(); PyObject *third = THIRD(); @@ -1838,7 +1841,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(ROT_FOUR): { + TARGET(ROT_FOUR) { PyObject *top = TOP(); PyObject *second = SECOND(); PyObject *third = THIRD(); @@ -1850,14 +1853,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DUP_TOP): { + TARGET(DUP_TOP) { PyObject *top = TOP(); Py_INCREF(top); PUSH(top); DISPATCH(); } - TARGET(DUP_TOP_TWO): { + TARGET(DUP_TOP_TWO) { PyObject *top = TOP(); PyObject *second = SECOND(); Py_INCREF(top); @@ -1868,7 +1871,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(UNARY_POSITIVE): { + TARGET(UNARY_POSITIVE) { PyObject *value = TOP(); PyObject *res = PyNumber_Positive(value); Py_DECREF(value); @@ -1878,7 +1881,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(UNARY_NEGATIVE): { + TARGET(UNARY_NEGATIVE) { PyObject *value = TOP(); PyObject *res = PyNumber_Negative(value); Py_DECREF(value); @@ -1888,7 +1891,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(UNARY_NOT): { + TARGET(UNARY_NOT) { PyObject *value = TOP(); int err = PyObject_IsTrue(value); Py_DECREF(value); @@ -1906,7 +1909,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto error; } - TARGET(UNARY_INVERT): { + TARGET(UNARY_INVERT) { PyObject *value = TOP(); PyObject *res = PyNumber_Invert(value); Py_DECREF(value); @@ -1916,7 +1919,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_POWER): { + TARGET(BINARY_POWER) { PyObject *exp = POP(); PyObject *base = TOP(); PyObject *res = PyNumber_Power(base, exp, Py_None); @@ -1928,19 +1931,76 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_MULTIPLY): { + TARGET(BINARY_MULTIPLY) { + PREDICTED(BINARY_MULTIPLY); + STAT_INC(BINARY_MULTIPLY, unquickened); PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_Multiply(left, right); Py_DECREF(left); Py_DECREF(right); SET_TOP(res); - if (res == NULL) + if (res == NULL) { goto error; + } DISPATCH(); } - TARGET(BINARY_MATRIX_MULTIPLY): { + TARGET(BINARY_MULTIPLY_ADAPTIVE) { + if (oparg == 0) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + next_instr--; + if (_Py_Specialize_BinaryMultiply(left, right, next_instr) < 0) { + goto error; + } + DISPATCH(); + } + else { + STAT_INC(BINARY_MULTIPLY, deferred); + UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1); + STAT_DEC(BINARY_MULTIPLY, unquickened); + JUMP_TO_INSTRUCTION(BINARY_MULTIPLY); + } + } + + TARGET(BINARY_MULTIPLY_INT) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_MULTIPLY); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_MULTIPLY); + STAT_INC(BINARY_MULTIPLY, hit); + PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + SET_SECOND(prod); + Py_DECREF(right); + Py_DECREF(left); + STACK_SHRINK(1); + if (prod == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(BINARY_MULTIPLY_FLOAT) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_MULTIPLY); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_MULTIPLY); + STAT_INC(BINARY_MULTIPLY, hit); + double dprod = ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + PyObject *prod = PyFloat_FromDouble(dprod); + SET_SECOND(prod); + Py_DECREF(right); + Py_DECREF(left); + STACK_SHRINK(1); + if (prod == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(BINARY_MATRIX_MULTIPLY) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_MatrixMultiply(left, right); @@ -1952,7 +2012,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_TRUE_DIVIDE): { + TARGET(BINARY_TRUE_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); PyObject *quotient = PyNumber_TrueDivide(dividend, divisor); @@ -1964,7 +2024,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_FLOOR_DIVIDE): { + TARGET(BINARY_FLOOR_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); PyObject *quotient = PyNumber_FloorDivide(dividend, divisor); @@ -1976,7 +2036,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_MODULO): { + TARGET(BINARY_MODULO) { PyObject *divisor = POP(); PyObject *dividend = TOP(); PyObject *res; @@ -1996,7 +2056,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_ADD): { + TARGET(BINARY_ADD) { PREDICTED(BINARY_ADD); STAT_INC(BINARY_ADD, unquickened); PyObject *right = POP(); @@ -2011,7 +2071,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_ADD_ADAPTIVE): { + TARGET(BINARY_ADD_ADAPTIVE) { if (oparg == 0) { PyObject *left = SECOND(); PyObject *right = TOP(); @@ -2029,13 +2089,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - TARGET(BINARY_ADD_UNICODE): { + TARGET(BINARY_ADD_UNICODE) { PyObject *left = SECOND(); PyObject *right = TOP(); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_ADD); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_ADD); STAT_INC(BINARY_ADD, hit); - record_hit_inline(next_instr, oparg); PyObject *res = PyUnicode_Concat(left, right); STACK_SHRINK(1); SET_TOP(res); @@ -2047,7 +2106,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_ADD_UNICODE_INPLACE_FAST): { + TARGET(BINARY_ADD_UNICODE_INPLACE_FAST) { PyObject *left = SECOND(); PyObject *right = TOP(); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_ADD); @@ -2064,7 +2123,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *var = GETLOCAL(next_oparg); DEOPT_IF(var != left, BINARY_ADD); STAT_INC(BINARY_ADD, hit); - record_hit_inline(next_instr, oparg); GETLOCAL(next_oparg) = NULL; Py_DECREF(left); STACK_SHRINK(1); @@ -2076,13 +2134,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_ADD_FLOAT): { + TARGET(BINARY_ADD_FLOAT) { PyObject *left = SECOND(); PyObject *right = TOP(); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_ADD); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_ADD); STAT_INC(BINARY_ADD, hit); - record_hit_inline(next_instr, oparg); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; PyObject *sum = PyFloat_FromDouble(dsum); @@ -2096,13 +2153,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_ADD_INT): { + TARGET(BINARY_ADD_INT) { PyObject *left = SECOND(); PyObject *right = TOP(); DEOPT_IF(!PyLong_CheckExact(left), BINARY_ADD); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_ADD); STAT_INC(BINARY_ADD, hit); - record_hit_inline(next_instr, oparg); PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sum); Py_DECREF(right); @@ -2114,7 +2170,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_SUBTRACT): { + TARGET(BINARY_SUBTRACT) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *diff = PyNumber_Subtract(left, right); @@ -2126,7 +2182,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_SUBSCR): { + TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, unquickened); PyObject *sub = POP(); @@ -2140,7 +2196,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_SUBSCR_ADAPTIVE): { + TARGET(BINARY_SUBSCR_ADAPTIVE) { if (oparg == 0) { PyObject *sub = TOP(); PyObject *container = SECOND(); @@ -2161,7 +2217,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - TARGET(BINARY_SUBSCR_LIST_INT): { + TARGET(BINARY_SUBSCR_LIST_INT) { PyObject *sub = TOP(); PyObject *list = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -2173,7 +2229,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); PyObject *res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -2185,7 +2240,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_SUBSCR_TUPLE_INT): { + TARGET(BINARY_SUBSCR_TUPLE_INT) { PyObject *sub = TOP(); PyObject *tuple = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -2197,7 +2252,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); PyObject *res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -2209,7 +2263,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_SUBSCR_DICT): { + TARGET(BINARY_SUBSCR_DICT) { PyObject *dict = SECOND(); DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -2226,7 +2280,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_LSHIFT): { + TARGET(BINARY_LSHIFT) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_Lshift(left, right); @@ -2238,7 +2292,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_RSHIFT): { + TARGET(BINARY_RSHIFT) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_Rshift(left, right); @@ -2250,7 +2304,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_AND): { + TARGET(BINARY_AND) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_And(left, right); @@ -2262,7 +2316,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_XOR): { + TARGET(BINARY_XOR) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_Xor(left, right); @@ -2274,7 +2328,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BINARY_OR): { + TARGET(BINARY_OR) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_Or(left, right); @@ -2286,7 +2340,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LIST_APPEND): { + TARGET(LIST_APPEND) { PyObject *v = POP(); PyObject *list = PEEK(oparg); int err; @@ -2298,7 +2352,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(SET_ADD): { + TARGET(SET_ADD) { PyObject *v = POP(); PyObject *set = PEEK(oparg); int err; @@ -2310,7 +2364,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_POWER): { + TARGET(INPLACE_POWER) { PyObject *exp = POP(); PyObject *base = TOP(); PyObject *res = PyNumber_InPlacePower(base, exp, Py_None); @@ -2322,7 +2376,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_MULTIPLY): { + TARGET(INPLACE_MULTIPLY) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_InPlaceMultiply(left, right); @@ -2334,7 +2388,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_MATRIX_MULTIPLY): { + TARGET(INPLACE_MATRIX_MULTIPLY) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_InPlaceMatrixMultiply(left, right); @@ -2346,7 +2400,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_TRUE_DIVIDE): { + TARGET(INPLACE_TRUE_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); PyObject *quotient = PyNumber_InPlaceTrueDivide(dividend, divisor); @@ -2358,7 +2412,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_FLOOR_DIVIDE): { + TARGET(INPLACE_FLOOR_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); PyObject *quotient = PyNumber_InPlaceFloorDivide(dividend, divisor); @@ -2370,7 +2424,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_MODULO): { + TARGET(INPLACE_MODULO) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *mod = PyNumber_InPlaceRemainder(left, right); @@ -2382,7 +2436,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_ADD): { + TARGET(INPLACE_ADD) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *sum; @@ -2401,7 +2455,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_SUBTRACT): { + TARGET(INPLACE_SUBTRACT) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *diff = PyNumber_InPlaceSubtract(left, right); @@ -2413,7 +2467,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_LSHIFT): { + TARGET(INPLACE_LSHIFT) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_InPlaceLshift(left, right); @@ -2425,7 +2479,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_RSHIFT): { + TARGET(INPLACE_RSHIFT) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_InPlaceRshift(left, right); @@ -2437,7 +2491,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_AND): { + TARGET(INPLACE_AND) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_InPlaceAnd(left, right); @@ -2449,7 +2503,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_XOR): { + TARGET(INPLACE_XOR) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_InPlaceXor(left, right); @@ -2461,7 +2515,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(INPLACE_OR): { + TARGET(INPLACE_OR) { PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_InPlaceOr(left, right); @@ -2473,7 +2527,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(STORE_SUBSCR): { + TARGET(STORE_SUBSCR) { PyObject *sub = TOP(); PyObject *container = SECOND(); PyObject *v = THIRD(); @@ -2489,7 +2543,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DELETE_SUBSCR): { + TARGET(DELETE_SUBSCR) { PyObject *sub = TOP(); PyObject *container = SECOND(); int err; @@ -2503,7 +2557,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(PRINT_EXPR): { + TARGET(PRINT_EXPR) { _Py_IDENTIFIER(displayhook); PyObject *value = POP(); PyObject *hook = _PySys_GetObjectId(&PyId_displayhook); @@ -2522,7 +2576,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(RAISE_VARARGS): { + TARGET(RAISE_VARARGS) { PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -2544,7 +2598,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto error; } - TARGET(RETURN_VALUE): { + TARGET(RETURN_VALUE) { retval = POP(); assert(EMPTY()); frame->f_state = FRAME_RETURNED; @@ -2552,7 +2606,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto exiting; } - TARGET(GET_AITER): { + TARGET(GET_AITER) { unaryfunc getter = NULL; PyObject *iter = NULL; PyObject *obj = TOP(); @@ -2596,7 +2650,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(GET_ANEXT): { + TARGET(GET_ANEXT) { unaryfunc getter = NULL; PyObject *next_iter = NULL; PyObject *awaitable = NULL; @@ -2647,7 +2701,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(GET_AWAITABLE): { + TARGET(GET_AWAITABLE) { PREDICTED(GET_AWAITABLE); PyObject *iterable = TOP(); PyObject *iter = _PyCoro_GetAwaitableIter(iterable); @@ -2688,7 +2742,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(YIELD_FROM): { + TARGET(YIELD_FROM) { PyObject *v = POP(); PyObject *receiver = TOP(); PySendResult gen_status; @@ -2740,7 +2794,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto exiting; } - TARGET(YIELD_VALUE): { + TARGET(YIELD_VALUE) { retval = POP(); if (co->co_flags & CO_ASYNC_GENERATOR) { @@ -2757,7 +2811,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto exiting; } - TARGET(GEN_START): { + TARGET(GEN_START) { PyObject *none = POP(); Py_DECREF(none); if (!Py_IsNone(none)) { @@ -2781,7 +2835,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(POP_EXCEPT): { + TARGET(POP_EXCEPT) { PyObject *type, *value, *traceback; _PyErr_StackItem *exc_info; exc_info = tstate->exc_info; @@ -2797,7 +2851,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(POP_EXCEPT_AND_RERAISE): { + TARGET(POP_EXCEPT_AND_RERAISE) { PyObject *lasti = PEEK(4); if (PyLong_Check(lasti)) { frame->f_lasti = PyLong_AsLong(lasti); @@ -2827,7 +2881,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto exception_unwind; } - TARGET(RERAISE): { + TARGET(RERAISE) { if (oparg) { PyObject *lasti = PEEK(oparg+3); if (PyLong_Check(lasti)) { @@ -2848,7 +2902,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto exception_unwind; } - TARGET(END_ASYNC_FOR): { + TARGET(END_ASYNC_FOR) { PyObject *exc = POP(); PyObject *val = POP(); PyObject *tb = POP(); @@ -2866,14 +2920,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - TARGET(LOAD_ASSERTION_ERROR): { + TARGET(LOAD_ASSERTION_ERROR) { PyObject *value = PyExc_AssertionError; Py_INCREF(value); PUSH(value); DISPATCH(); } - TARGET(LOAD_BUILD_CLASS): { + TARGET(LOAD_BUILD_CLASS) { _Py_IDENTIFIER(__build_class__); PyObject *bc; @@ -2904,7 +2958,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(STORE_NAME): { + TARGET(STORE_NAME) { PyObject *name = GETITEM(names, oparg); PyObject *v = POP(); PyObject *ns = LOCALS(); @@ -2925,7 +2979,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DELETE_NAME): { + TARGET(DELETE_NAME) { PyObject *name = GETITEM(names, oparg); PyObject *ns = LOCALS(); int err; @@ -2944,7 +2998,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(UNPACK_SEQUENCE): { + TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); PyObject *seq = POP(), *item, **items; if (PyTuple_CheckExact(seq) && @@ -2975,7 +3029,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(UNPACK_EX): { + TARGET(UNPACK_EX) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject *seq = POP(); @@ -2990,7 +3044,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(STORE_ATTR): { + TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); STAT_INC(STORE_ATTR, unquickened); PyObject *name = GETITEM(names, oparg); @@ -3006,7 +3060,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DELETE_ATTR): { + TARGET(DELETE_ATTR) { PyObject *name = GETITEM(names, oparg); PyObject *owner = POP(); int err; @@ -3017,7 +3071,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(STORE_GLOBAL): { + TARGET(STORE_GLOBAL) { PyObject *name = GETITEM(names, oparg); PyObject *v = POP(); int err; @@ -3028,7 +3082,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DELETE_GLOBAL): { + TARGET(DELETE_GLOBAL) { PyObject *name = GETITEM(names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -3042,7 +3096,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_NAME): { + TARGET(LOAD_NAME) { PyObject *name = GETITEM(names, oparg); PyObject *locals = LOCALS(); PyObject *v; @@ -3106,7 +3160,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_GLOBAL): { + TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); STAT_INC(LOAD_GLOBAL, unquickened); PyObject *name = GETITEM(names, oparg); @@ -3156,7 +3210,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_GLOBAL_ADAPTIVE): { + TARGET(LOAD_GLOBAL_ADAPTIVE) { assert(cframe.use_tracing == 0); SpecializedCacheEntry *cache = GET_CACHE(); if (cache->adaptive.counter == 0) { @@ -3176,7 +3230,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - TARGET(LOAD_GLOBAL_MODULE): { + TARGET(LOAD_GLOBAL_MODULE) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -3187,14 +3241,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + cache0->index; PyObject *res = ep->me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); - record_cache_hit(cache0); STAT_INC(LOAD_GLOBAL, hit); Py_INCREF(res); PUSH(res); DISPATCH(); } - TARGET(LOAD_GLOBAL_BUILTIN): { + TARGET(LOAD_GLOBAL_BUILTIN) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); @@ -3208,14 +3261,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyDictKeyEntry *ep = DK_ENTRIES(bdict->ma_keys) + cache0->index; PyObject *res = ep->me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); - record_cache_hit(cache0); STAT_INC(LOAD_GLOBAL, hit); Py_INCREF(res); PUSH(res); DISPATCH(); } - TARGET(DELETE_FAST): { + TARGET(DELETE_FAST) { PyObject *v = GETLOCAL(oparg); if (v != NULL) { SETLOCAL(oparg, NULL); @@ -3229,7 +3281,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto error; } - TARGET(MAKE_CELL): { + TARGET(MAKE_CELL) { // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -3241,7 +3293,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DELETE_DEREF): { + TARGET(DELETE_DEREF) { PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); if (oldobj != NULL) { @@ -3253,7 +3305,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto error; } - TARGET(LOAD_CLASSDEREF): { + TARGET(LOAD_CLASSDEREF) { PyObject *name, *value, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < co->co_nlocalsplus); @@ -3289,7 +3341,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_DEREF): { + TARGET(LOAD_DEREF) { PyObject *cell = GETLOCAL(oparg); PyObject *value = PyCell_GET(cell); if (value == NULL) { @@ -3301,7 +3353,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(STORE_DEREF): { + TARGET(STORE_DEREF) { PyObject *v = POP(); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); @@ -3310,7 +3362,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BUILD_STRING): { + TARGET(BUILD_STRING) { PyObject *str; PyObject *empty = PyUnicode_New(0, 0); if (empty == NULL) { @@ -3328,7 +3380,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BUILD_TUPLE): { + TARGET(BUILD_TUPLE) { PyObject *tup = PyTuple_New(oparg); if (tup == NULL) goto error; @@ -3340,7 +3392,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BUILD_LIST): { + TARGET(BUILD_LIST) { PyObject *list = PyList_New(oparg); if (list == NULL) goto error; @@ -3352,7 +3404,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LIST_TO_TUPLE): { + TARGET(LIST_TO_TUPLE) { PyObject *list = POP(); PyObject *tuple = PyList_AsTuple(list); Py_DECREF(list); @@ -3363,7 +3415,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LIST_EXTEND): { + TARGET(LIST_EXTEND) { PyObject *iterable = POP(); PyObject *list = PEEK(oparg); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); @@ -3384,7 +3436,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(SET_UPDATE): { + TARGET(SET_UPDATE) { PyObject *iterable = POP(); PyObject *set = PEEK(oparg); int err = _PySet_Update(set, iterable); @@ -3395,7 +3447,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BUILD_SET): { + TARGET(BUILD_SET) { PyObject *set = PySet_New(NULL); int err = 0; int i; @@ -3416,7 +3468,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BUILD_MAP): { + TARGET(BUILD_MAP) { Py_ssize_t i; PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg); if (map == NULL) @@ -3440,7 +3492,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(SETUP_ANNOTATIONS): { + TARGET(SETUP_ANNOTATIONS) { _Py_IDENTIFIER(__annotations__); int err; PyObject *ann_dict; @@ -3499,7 +3551,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BUILD_CONST_KEY_MAP): { + TARGET(BUILD_CONST_KEY_MAP) { Py_ssize_t i; PyObject *map; PyObject *keys = TOP(); @@ -3532,7 +3584,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DICT_UPDATE): { + TARGET(DICT_UPDATE) { PyObject *update = POP(); PyObject *dict = PEEK(oparg); if (PyDict_Update(dict, update) < 0) { @@ -3548,7 +3600,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(DICT_MERGE): { + TARGET(DICT_MERGE) { PyObject *update = POP(); PyObject *dict = PEEK(oparg); @@ -3562,7 +3614,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(MAP_ADD): { + TARGET(MAP_ADD) { PyObject *value = TOP(); PyObject *key = SECOND(); PyObject *map; @@ -3579,7 +3631,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_ATTR): { + TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); STAT_INC(LOAD_ATTR, unquickened); PyObject *name = GETITEM(names, oparg); @@ -3593,7 +3645,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_ATTR_ADAPTIVE): { + TARGET(LOAD_ATTR_ADAPTIVE) { assert(cframe.use_tracing == 0); SpecializedCacheEntry *cache = GET_CACHE(); if (cache->adaptive.counter == 0) { @@ -3614,7 +3666,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - TARGET(LOAD_ATTR_SPLIT_KEYS): { + TARGET(LOAD_ATTR_INSTANCE_VALUE) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; @@ -3625,21 +3677,19 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(cache1->tp_version != 0); DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR); assert(tp->tp_dictoffset > 0); - PyDictObject *dict = *(PyDictObject **)(((char *)owner) + tp->tp_dictoffset); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - DEOPT_IF(dict->ma_keys->dk_version != cache1->dk_version_or_hint, LOAD_ATTR); - res = dict->ma_values[cache0->index]; + assert(tp->tp_inline_values_offset > 0); + PyDictValues *values = *(PyDictValues **)(((char *)owner) + tp->tp_inline_values_offset); + DEOPT_IF(values == NULL, LOAD_ATTR); + res = values->values[cache0->index]; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - record_cache_hit(cache0); Py_INCREF(res); SET_TOP(res); Py_DECREF(owner); DISPATCH(); } - TARGET(LOAD_ATTR_MODULE): { + TARGET(LOAD_ATTR_MODULE) { assert(cframe.use_tracing == 0); // shared with LOAD_METHOD_MODULE PyObject *owner = TOP(); @@ -3650,7 +3700,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_ATTR_WITH_HINT): { + TARGET(LOAD_ATTR_WITH_HINT) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; @@ -3666,20 +3716,19 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, cache0->original_oparg); uint32_t hint = cache1->dk_version_or_hint; - DEOPT_IF(hint >= dict->ma_keys->dk_nentries, LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); res = ep->me_value; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - record_cache_hit(cache0); Py_INCREF(res); SET_TOP(res); Py_DECREF(owner); DISPATCH(); } - TARGET(LOAD_ATTR_SLOT): { + TARGET(LOAD_ATTR_SLOT) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; @@ -3693,14 +3742,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr res = *(PyObject **)addr; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - record_cache_hit(cache0); Py_INCREF(res); SET_TOP(res); Py_DECREF(owner); DISPATCH(); } - TARGET(STORE_ATTR_ADAPTIVE): { + TARGET(STORE_ATTR_ADAPTIVE) { assert(cframe.use_tracing == 0); SpecializedCacheEntry *cache = GET_CACHE(); if (cache->adaptive.counter == 0) { @@ -3721,7 +3769,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - TARGET(STORE_ATTR_SPLIT_KEYS): { + TARGET(STORE_ATTR_INSTANCE_VALUE) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); @@ -3731,35 +3779,27 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(cache1->tp_version != 0); DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR); assert(tp->tp_dictoffset > 0); - PyDictObject *dict = *(PyDictObject **)(((char *)owner) + tp->tp_dictoffset); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - DEOPT_IF(dict->ma_keys->dk_version != cache1->dk_version_or_hint, STORE_ATTR); - /* Need to maintain ordering of dicts */ - DEOPT_IF(cache0->index > 0 && dict->ma_values[cache0->index-1] == NULL, STORE_ATTR); + assert(tp->tp_inline_values_offset > 0); + PyDictValues *values = *(PyDictValues **)(((char *)owner) + tp->tp_inline_values_offset); + DEOPT_IF(values == NULL, STORE_ATTR); STAT_INC(STORE_ATTR, hit); - record_cache_hit(cache0); + int index = cache0->index; STACK_SHRINK(1); PyObject *value = POP(); - PyObject *old_value = dict->ma_values[cache0->index]; - dict->ma_values[cache0->index] = value; + PyObject *old_value = values->values[index]; + values->values[index] = value; if (old_value == NULL) { - dict->ma_used++; + assert(index < 16); + values->mv_order = (values->mv_order << 4) | index; } else { Py_DECREF(old_value); } - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); - } - /* PEP 509 */ - dict->ma_version_tag = DICT_NEXT_VERSION(); Py_DECREF(owner); DISPATCH(); } - TARGET(STORE_ATTR_WITH_HINT): { + TARGET(STORE_ATTR_WITH_HINT) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); @@ -3774,13 +3814,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, cache0->original_oparg); uint32_t hint = cache1->dk_version_or_hint; - DEOPT_IF(hint >= dict->ma_keys->dk_nentries, STORE_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); PyObject *old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); STAT_INC(STORE_ATTR, hit); - record_cache_hit(cache0); STACK_SHRINK(1); PyObject *value = POP(); ep->me_value = value; @@ -3795,7 +3834,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(STORE_ATTR_SLOT): { + TARGET(STORE_ATTR_SLOT) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); @@ -3806,7 +3845,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR); char *addr = (char *)owner + cache0->index; STAT_INC(STORE_ATTR, hit); - record_cache_hit(cache0); STACK_SHRINK(1); PyObject *value = POP(); PyObject *old_value = *(PyObject **)addr; @@ -3816,7 +3854,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(COMPARE_OP): { + TARGET(COMPARE_OP) { assert(oparg <= Py_GE); PyObject *right = POP(); PyObject *left = TOP(); @@ -3831,7 +3869,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(IS_OP): { + TARGET(IS_OP) { PyObject *right = POP(); PyObject *left = TOP(); int res = Py_Is(left, right) ^ oparg; @@ -3845,7 +3883,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(CONTAINS_OP): { + TARGET(CONTAINS_OP) { PyObject *right = POP(); PyObject *left = POP(); int res = PySequence_Contains(right, left); @@ -3862,10 +3900,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } -#define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ - "BaseException is not allowed" - - TARGET(JUMP_IF_NOT_EXC_MATCH): { + TARGET(JUMP_IF_NOT_EXC_MATCH) { + const char *cannot_catch_msg = "catching classes that do not " + "inherit from BaseException is not " + "allowed"; PyObject *right = POP(); PyObject *left = POP(); if (PyTuple_Check(right)) { @@ -3875,7 +3913,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *exc = PyTuple_GET_ITEM(right, i); if (!PyExceptionClass_Check(exc)) { _PyErr_SetString(tstate, PyExc_TypeError, - CANNOT_CATCH_MSG); + cannot_catch_msg); Py_DECREF(left); Py_DECREF(right); goto error; @@ -3885,7 +3923,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr else { if (!PyExceptionClass_Check(right)) { _PyErr_SetString(tstate, PyExc_TypeError, - CANNOT_CATCH_MSG); + cannot_catch_msg); Py_DECREF(left); Py_DECREF(right); goto error; @@ -3906,7 +3944,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(IMPORT_NAME): { + TARGET(IMPORT_NAME) { PyObject *name = GETITEM(names, oparg); PyObject *fromlist = POP(); PyObject *level = TOP(); @@ -3920,7 +3958,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(IMPORT_STAR): { + TARGET(IMPORT_STAR) { PyObject *from = POP(), *locals; int err; if (_PyFrame_FastToLocalsWithError(frame) < 0) { @@ -3943,7 +3981,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(IMPORT_FROM): { + TARGET(IMPORT_FROM) { PyObject *name = GETITEM(names, oparg); PyObject *from = TOP(); PyObject *res; @@ -3954,12 +3992,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(JUMP_FORWARD): { + TARGET(JUMP_FORWARD) { JUMPBY(oparg); DISPATCH(); } - TARGET(POP_JUMP_IF_FALSE): { + TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = POP(); int err; @@ -3986,7 +4024,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(POP_JUMP_IF_TRUE): { + TARGET(POP_JUMP_IF_TRUE) { PREDICTED(POP_JUMP_IF_TRUE); PyObject *cond = POP(); int err; @@ -4013,7 +4051,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(JUMP_IF_FALSE_OR_POP): { + TARGET(JUMP_IF_FALSE_OR_POP) { PyObject *cond = TOP(); int err; if (Py_IsTrue(cond)) { @@ -4037,7 +4075,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(JUMP_IF_TRUE_OR_POP): { + TARGET(JUMP_IF_TRUE_OR_POP) { PyObject *cond = TOP(); int err; if (Py_IsFalse(cond)) { @@ -4062,21 +4100,20 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(JUMP_ABSOLUTE): { + TARGET(JUMP_ABSOLUTE) { PREDICTED(JUMP_ABSOLUTE); - if (oparg < INSTR_OFFSET()) { - /* Increment the warmup counter and quicken if warm enough - * _Py_Quicken is idempotent so we don't worry about overflow */ - if (!PyCodeObject_IsWarmedUp(co)) { - PyCodeObject_IncrementWarmup(co); - if (PyCodeObject_IsWarmedUp(co)) { - if (_Py_Quicken(co)) { - goto error; - } - int nexti = INSTR_OFFSET(); - first_instr = co->co_firstinstr; - next_instr = first_instr + nexti; + assert(oparg < INSTR_OFFSET()); + /* Increment the warmup counter and quicken if warm enough + * _Py_Quicken is idempotent so we don't worry about overflow */ + if (!PyCodeObject_IsWarmedUp(co)) { + PyCodeObject_IncrementWarmup(co); + if (PyCodeObject_IsWarmedUp(co)) { + if (_Py_Quicken(co)) { + goto error; } + int nexti = INSTR_OFFSET(); + first_instr = co->co_firstinstr; + next_instr = first_instr + nexti; } } JUMPTO(oparg); @@ -4084,13 +4121,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(JUMP_ABSOLUTE_QUICK): { + TARGET(JUMP_ABSOLUTE_QUICK) { + assert(oparg < INSTR_OFFSET()); JUMPTO(oparg); CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(GET_LEN): { + TARGET(GET_LEN) { // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(TOP()); if (len_i < 0) { @@ -4104,7 +4142,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(MATCH_CLASS): { + TARGET(MATCH_CLASS) { // Pop TOS. On success, set TOS to True and TOS1 to a tuple of // attributes. On failure, set TOS to False. PyObject *names = POP(); @@ -4127,7 +4165,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(MATCH_MAPPING): { + TARGET(MATCH_MAPPING) { PyObject *subject = TOP(); int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; PyObject *res = match ? Py_True : Py_False; @@ -4136,7 +4174,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(MATCH_SEQUENCE): { + TARGET(MATCH_SEQUENCE) { PyObject *subject = TOP(); int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; PyObject *res = match ? Py_True : Py_False; @@ -4145,7 +4183,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(MATCH_KEYS): { + TARGET(MATCH_KEYS) { // On successful match for all keys, PUSH(values) and PUSH(True). // Otherwise, PUSH(None) and PUSH(False). PyObject *keys = TOP(); @@ -4166,7 +4204,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(COPY_DICT_WITHOUT_KEYS): { + TARGET(COPY_DICT_WITHOUT_KEYS) { // rest = dict(TOS1) // for key in TOS: // del rest[key] @@ -4192,7 +4230,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(GET_ITER): { + TARGET(GET_ITER) { /* before: [obj]; after [getiter(obj)] */ PyObject *iterable = TOP(); PyObject *iter = PyObject_GetIter(iterable); @@ -4205,7 +4243,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER): { + TARGET(GET_YIELD_FROM_ITER) { /* before: [obj]; after [getiter(obj)] */ PyObject *iterable = TOP(); PyObject *iter; @@ -4234,7 +4272,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(FOR_ITER): { + TARGET(FOR_ITER) { PREDICTED(FOR_ITER); /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); @@ -4261,7 +4299,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BEFORE_ASYNC_WITH): { + TARGET(BEFORE_ASYNC_WITH) { _Py_IDENTIFIER(__aenter__); _Py_IDENTIFIER(__aexit__); PyObject *mgr = TOP(); @@ -4290,7 +4328,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } SET_TOP(exit); Py_DECREF(mgr); - res = _PyObject_CallNoArg(enter); + res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) goto error; @@ -4299,7 +4337,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BEFORE_WITH): { + TARGET(BEFORE_WITH) { _Py_IDENTIFIER(__enter__); _Py_IDENTIFIER(__exit__); PyObject *mgr = TOP(); @@ -4328,7 +4366,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } SET_TOP(exit); Py_DECREF(mgr); - res = _PyObject_CallNoArg(enter); + res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { goto error; @@ -4337,7 +4375,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(WITH_EXCEPT_START): { + TARGET(WITH_EXCEPT_START) { /* At the top of the stack are 8 values: - (TOP, SECOND, THIRD) = exc_info() - (FOURTH, FIFTH, SIXTH) = previous exception @@ -4367,7 +4405,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(PUSH_EXC_INFO): { + TARGET(PUSH_EXC_INFO) { PyObject *type = TOP(); PyObject *value = SECOND(); PyObject *tb = THIRD(); @@ -4398,7 +4436,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_METHOD): { + TARGET(LOAD_METHOD) { PREDICTED(LOAD_METHOD); STAT_INC(LOAD_METHOD, unquickened); /* Designed to work in tandem with CALL_METHOD. */ @@ -4437,7 +4475,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_METHOD_ADAPTIVE): { + TARGET(LOAD_METHOD_ADAPTIVE) { assert(cframe.use_tracing == 0); SpecializedCacheEntry *cache = GET_CACHE(); if (cache->adaptive.counter == 0) { @@ -4458,34 +4496,40 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } } - TARGET(LOAD_METHOD_CACHED): { + TARGET(LOAD_METHOD_CACHED) { /* LOAD_METHOD, with cached method object */ assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); SpecializedCacheEntry *caches = GET_CACHE(); - _PyAdaptiveEntry *cache0 = &caches[0].adaptive; _PyAttrCache *cache1 = &caches[-1].attr; _PyObjectCache *cache2 = &caches[-2].obj; DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD); - assert(cache1->dk_version_or_hint != 0); - assert(cache1->tp_version != 0); - assert(self_cls->tp_dictoffset >= 0); - assert(Py_TYPE(self_cls)->tp_dictoffset > 0); - - // inline version of _PyObject_GetDictPtr for offset >= 0 - PyObject *dict = self_cls->tp_dictoffset != 0 ? - *(PyObject **) ((char *)self + self_cls->tp_dictoffset) : NULL; - - // Ensure self.__dict__ didn't modify keys. - // Don't care if self has no dict, it could be builtin or __slots__. - DEOPT_IF(dict != NULL && - ((PyDictObject *)dict)->ma_keys->dk_version != - cache1->dk_version_or_hint, LOAD_METHOD); + assert(self_cls->tp_dictoffset > 0); + assert(self_cls->tp_inline_values_offset > 0); + PyDictObject *dict = *(PyDictObject **)(((char *)self) + self_cls->tp_dictoffset); + DEOPT_IF(dict != NULL, LOAD_METHOD); + DEOPT_IF(((PyHeapTypeObject *)self_cls)->ht_cached_keys->dk_version != cache1->dk_version_or_hint, LOAD_METHOD); + STAT_INC(LOAD_METHOD, hit); + PyObject *res = cache2->obj; + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + Py_INCREF(res); + SET_TOP(res); + PUSH(self); + DISPATCH(); + } + TARGET(LOAD_METHOD_NO_DICT) { + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + SpecializedCacheEntry *caches = GET_CACHE(); + _PyAttrCache *cache1 = &caches[-1].attr; + _PyObjectCache *cache2 = &caches[-2].obj; + DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD); + assert(self_cls->tp_dictoffset == 0); STAT_INC(LOAD_METHOD, hit); - record_cache_hit(cache0); PyObject *res = cache2->obj; assert(res != NULL); assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -4495,7 +4539,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_METHOD_MODULE): { + TARGET(LOAD_METHOD_MODULE) { /* LOAD_METHOD, for module methods */ assert(cframe.use_tracing == 0); PyObject *owner = TOP(); @@ -4507,11 +4551,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(LOAD_METHOD_CLASS): { + TARGET(LOAD_METHOD_CLASS) { /* LOAD_METHOD, for class methods */ assert(cframe.use_tracing == 0); SpecializedCacheEntry *caches = GET_CACHE(); - _PyAdaptiveEntry *cache0 = &caches[0].adaptive; _PyAttrCache *cache1 = &caches[-1].attr; _PyObjectCache *cache2 = &caches[-2].obj; @@ -4522,10 +4565,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(cache1->tp_version != 0); STAT_INC(LOAD_METHOD, hit); - record_cache_hit(cache0); PyObject *res = cache2->obj; assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); Py_INCREF(res); SET_TOP(NULL); Py_DECREF(cls); @@ -4533,12 +4574,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(CALL_METHOD): { + TARGET(CALL_METHOD) { /* Designed to work in tamdem with LOAD_METHOD. */ - PyObject **sp, *res; - int meth_found; - - sp = stack_pointer; /* `meth` is NULL when LOAD_METHOD thinks that it's not a method call. @@ -4564,11 +4601,84 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr We'll be passing `oparg + 1` to call_function, to make it accept the `self` as a first argument. */ - meth_found = (PEEK(oparg + 2) != NULL); - res = call_function(tstate, &sp, oparg + meth_found, NULL, cframe.use_tracing); - stack_pointer = sp; + int is_method = (PEEK(oparg + 2) != NULL); + oparg += is_method; + nargs = oparg; + kwnames = NULL; + postcall_shrink = 2-is_method; + goto call_function; + } + + TARGET(CALL_METHOD_KW) { + /* Designed to work in tandem with LOAD_METHOD. Same as CALL_METHOD + but pops TOS to get a tuple of keyword names. */ + kwnames = POP(); + int is_method = (PEEK(oparg + 2) != NULL); + oparg += is_method; + nargs = oparg - (int)PyTuple_GET_SIZE(kwnames); + postcall_shrink = 2-is_method; + goto call_function; + } - STACK_SHRINK(1 - meth_found); + TARGET(CALL_FUNCTION_KW) { + kwnames = POP(); + nargs = oparg - (int)PyTuple_GET_SIZE(kwnames); + postcall_shrink = 1; + goto call_function; + } + + TARGET(CALL_FUNCTION) { + PREDICTED(CALL_FUNCTION); + STAT_INC(CALL_FUNCTION, unquickened); + PyObject *function; + nargs = oparg; + kwnames = NULL; + postcall_shrink = 1; + call_function: + // Check if the call can be inlined or not + function = PEEK(oparg + 1); + if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; + int is_generator = code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); + if (!is_generator) { + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function); + STACK_SHRINK(oparg); + InterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, PyFunction_AS_FRAME_CONSTRUCTOR(function), locals, + stack_pointer, + nargs, kwnames, 1); + STACK_SHRINK(postcall_shrink); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + Py_XDECREF(kwnames); + Py_DECREF(function); + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->depth = frame->depth + 1; + tstate->frame = frame = new_frame; + goto start_frame; + } + } + /* Callable is not a normal Python function */ + PyObject *res; + if (cframe.use_tracing) { + res = trace_call_function(tstate, function, stack_pointer-oparg, nargs, kwnames); + } + else { + res = PyObject_Vectorcall(function, stack_pointer-oparg, + nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(function); + Py_XDECREF(kwnames); + /* Clear the stack */ + STACK_SHRINK(oparg); + for (int i = 0; i < oparg; i++) { + Py_DECREF(stack_pointer[i]); + } + STACK_SHRINK(postcall_shrink); PUSH(res); if (res == NULL) { goto error; @@ -4577,65 +4687,183 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(CALL_METHOD_KW): { - /* Designed to work in tandem with LOAD_METHOD. Same as CALL_METHOD - but pops TOS to get a tuple of keyword names. */ - PyObject **sp, *res; - PyObject *names = NULL; - int meth_found; + TARGET(CALL_FUNCTION_ADAPTIVE) { + SpecializedCacheEntry *cache = GET_CACHE(); + nargs = cache->adaptive.original_oparg; + if (cache->adaptive.counter == 0) { + next_instr--; + if (_Py_Specialize_CallFunction( + PEEK(nargs + 1), next_instr, nargs, cache, BUILTINS()) < 0) { + goto error; + } + DISPATCH(); + } + else { + STAT_INC(CALL_FUNCTION, deferred); + cache->adaptive.counter--; + oparg = nargs; + kwnames = NULL; + postcall_shrink = 1; + goto call_function; + } + } - names = POP(); + TARGET(CALL_FUNCTION_PY_SIMPLE) { + SpecializedCacheEntry *caches = GET_CACHE(); + _PyAdaptiveEntry *cache0 = &caches[0].adaptive; + int argcount = cache0->original_oparg; + _PyCallCache *cache1 = &caches[-1].call; + PyObject *callable = PEEK(argcount+1); + DEOPT_IF(!PyFunction_Check(callable), CALL_FUNCTION); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != cache1->func_version, CALL_FUNCTION); + /* PEP 523 */ + DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_FUNCTION); + STAT_INC(CALL_FUNCTION, hit); + InterpreterFrame *new_frame = _PyThreadState_PushFrame( + tstate, PyFunction_AS_FRAME_CONSTRUCTOR(func), NULL); + if (new_frame == NULL) { + goto error; + } + STACK_SHRINK(argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = stack_pointer[i]; + } + int deflen = cache1->defaults_len; + for (int i = 0; i < deflen; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, cache1->defaults_start+i); + Py_INCREF(def); + new_frame->localsplus[argcount+i] = def; + } + STACK_SHRINK(1); + Py_DECREF(func); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = tstate->frame; + new_frame->depth = frame->depth + 1; + tstate->frame = frame = new_frame; + goto start_frame; + } - sp = stack_pointer; - meth_found = (PEEK(oparg + 2) != NULL); - res = call_function(tstate, &sp, oparg + meth_found, names, cframe.use_tracing); - stack_pointer = sp; + TARGET(CALL_FUNCTION_BUILTIN_O) { + assert(cframe.use_tracing == 0); + /* Builtin METH_O functions */ - STACK_SHRINK(1 - meth_found); - PUSH(res); - Py_DECREF(names); + PyObject *callable = SECOND(); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_FUNCTION); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_FUNCTION); + STAT_INC(CALL_FUNCTION, hit); + + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyObject *arg = POP(); + PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Clear the stack of the function object. */ + Py_DECREF(arg); + Py_DECREF(callable); + SET_TOP(res); if (res == NULL) { goto error; } - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_FUNCTION): { - PREDICTED(CALL_FUNCTION); - PyObject **sp, *res; - sp = stack_pointer; - res = call_function(tstate, &sp, oparg, NULL, cframe.use_tracing); - stack_pointer = sp; + TARGET(CALL_FUNCTION_BUILTIN_FAST) { + assert(cframe.use_tracing == 0); + /* Builtin METH_FASTCALL functions, without keywords */ + SpecializedCacheEntry *caches = GET_CACHE(); + _PyAdaptiveEntry *cache0 = &caches[0].adaptive; + int nargs = cache0->original_oparg; + PyObject **pfunc = &PEEK(nargs + 1); + PyObject *callable = *pfunc; + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_FUNCTION); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, + CALL_FUNCTION); + STAT_INC(CALL_FUNCTION, hit); + + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + &PEEK(nargs), + nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Clear the stack of the function object. */ + while (stack_pointer > pfunc) { + PyObject *x = POP(); + Py_DECREF(x); + } PUSH(res); if (res == NULL) { + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ goto error; } - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_FUNCTION_KW): { - PyObject **sp, *res, *names; + TARGET(CALL_FUNCTION_LEN) { + assert(cframe.use_tracing == 0); + /* len(o) */ + SpecializedCacheEntry *caches = GET_CACHE(); + assert(caches[0].adaptive.original_oparg == 1); + _PyObjectCache *cache1 = &caches[-1].obj; - names = POP(); - assert(PyTuple_Check(names)); - assert(PyTuple_GET_SIZE(names) <= oparg); - /* We assume without checking that names contains only strings */ - sp = stack_pointer; - res = call_function(tstate, &sp, oparg, names, cframe.use_tracing); - stack_pointer = sp; - PUSH(res); - Py_DECREF(names); + PyObject *callable = SECOND(); + DEOPT_IF(callable != cache1->obj, CALL_FUNCTION); + STAT_INC(CALL_FUNCTION, hit); + Py_ssize_t len_i = PyObject_Length(TOP()); + if (len_i < 0) { + goto error; + } + PyObject *res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Clear the stack of the function object. */ + Py_DECREF(POP()); + Py_DECREF(callable); + SET_TOP(res); if (res == NULL) { goto error; } - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_FUNCTION_EX): { + TARGET(CALL_FUNCTION_ISINSTANCE) { + assert(cframe.use_tracing == 0); + /* isinstance(o, o2) */ + SpecializedCacheEntry *caches = GET_CACHE(); + assert(caches[0].adaptive.original_oparg == 2); + _PyObjectCache *cache1 = &caches[-1].obj; + + PyObject *callable = THIRD(); + DEOPT_IF(callable != cache1->obj, CALL_FUNCTION); + STAT_INC(CALL_FUNCTION, hit); + + int retval = PyObject_IsInstance(SECOND(), TOP()); + if (retval < 0) { + goto error; + } + PyObject *res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Clear the stack of the function object. */ + Py_DECREF(POP()); + Py_DECREF(POP()); + Py_DECREF(callable); + SET_TOP(res); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); PyObject *func, *callargs, *kwargs = NULL, *result; if (oparg & 0x01) { @@ -4682,7 +4910,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(MAKE_FUNCTION): { + TARGET(MAKE_FUNCTION) { PyObject *codeobj = POP(); PyFunctionObject *func = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4713,7 +4941,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(BUILD_SLICE): { + TARGET(BUILD_SLICE) { PyObject *start, *stop, *step, *slice; if (oparg == 3) step = POP(); @@ -4731,7 +4959,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(FORMAT_VALUE): { + TARGET(FORMAT_VALUE) { /* Handles f-string value formatting. */ PyObject *result; PyObject *fmt_spec; @@ -4791,7 +5019,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(ROT_N): { + TARGET(ROT_N) { PyObject *top = TOP(); memmove(&PEEK(oparg - 1), &PEEK(oparg), sizeof(PyObject*) * (oparg - 1)); @@ -4799,7 +5027,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } - TARGET(EXTENDED_ARG): { + TARGET(EXTENDED_ARG) { int oldoparg = oparg; NEXTOPARG(); oparg |= oldoparg << 8; @@ -4807,6 +5035,45 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH_GOTO(); } +#if USE_COMPUTED_GOTOS + TARGET_DO_TRACING: { +#else + case DO_TRACING: { +#endif + int instr_prev = skip_backwards_over_extended_args(co, frame->f_lasti); + frame->f_lasti = INSTR_OFFSET(); + TRACING_NEXTOPARG(); + if (PyDTrace_LINE_ENABLED()) { + maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); + } + /* line-by-line tracing support */ + + if (cframe.use_tracing && + tstate->c_tracefunc != NULL && !tstate->tracing) { + int err; + /* see maybe_call_line_trace() + for expository comments */ + _PyFrame_SetStackPointer(frame, stack_pointer); + + err = maybe_call_line_trace(tstate->c_tracefunc, + tstate->c_traceobj, + tstate, frame, instr_prev); + if (err) { + /* trace function raised an exception */ + next_instr++; + goto error; + } + /* Reload possibly changed frame fields */ + JUMPTO(frame->f_lasti); + + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; + TRACING_NEXTOPARG(); + } + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + #if USE_COMPUTED_GOTOS _unknown_opcode: @@ -4815,7 +5082,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr #endif fprintf(stderr, "XXX lineno: %d, opcode: %d\n", - PyCode_Addr2Line(frame->f_code, frame->f_lasti*2), + PyCode_Addr2Line(frame->f_code, frame->f_lasti*sizeof(_Py_CODEUNIT)), opcode); _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode"); goto error; @@ -4833,8 +5100,8 @@ opname ## _miss: \ { \ STAT_INC(opname, miss); \ _PyAdaptiveEntry *cache = &GET_CACHE()->adaptive; \ - record_cache_miss(cache); \ - if (too_many_cache_misses(cache)) { \ + cache->counter--; \ + if (cache->counter == 0) { \ next_instr[-1] = _Py_MAKECODEUNIT(opname ## _ADAPTIVE, _Py_OPARG(next_instr[-1])); \ STAT_INC(opname, deopt); \ cache_backoff(cache); \ @@ -4848,10 +5115,10 @@ opname ## _miss: \ opname ## _miss: \ { \ STAT_INC(opname, miss); \ - uint8_t oparg = saturating_decrement(_Py_OPARG(next_instr[-1])); \ + uint8_t oparg = _Py_OPARG(next_instr[-1])-1; \ UPDATE_PREV_INSTR_OPARG(next_instr, oparg); \ assert(_Py_OPARG(next_instr[-1]) == oparg); \ - if (oparg == saturating_zero()) /* too many cache misses */ { \ + if (oparg == 0) /* too many cache misses */ { \ oparg = ADAPTIVE_CACHE_BACKOFF; \ next_instr[-1] = _Py_MAKECODEUNIT(opname ## _ADAPTIVE, oparg); \ STAT_INC(opname, deopt); \ @@ -4864,8 +5131,10 @@ MISS_WITH_CACHE(LOAD_ATTR) MISS_WITH_CACHE(STORE_ATTR) MISS_WITH_CACHE(LOAD_GLOBAL) MISS_WITH_CACHE(LOAD_METHOD) +MISS_WITH_CACHE(CALL_FUNCTION) MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR) MISS_WITH_OPARG_COUNTER(BINARY_ADD) +MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY) binary_subscr_dict_error: { @@ -4991,14 +5260,28 @@ MISS_WITH_OPARG_COUNTER(BINARY_ADD) /* pop frame */ exit_eval_frame: - /* Restore previous cframe */ - tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; - if (PyDTrace_FUNCTION_RETURN_ENABLED()) dtrace_function_return(frame); _Py_LeaveRecursiveCall(tstate); + + if (frame->depth) { + _PyFrame_StackPush(frame->previous, retval); + if (_PyEvalFrameClearAndPop(tstate, frame)) { + retval = NULL; + } + frame = tstate->frame; + if (retval == NULL) { + assert(_PyErr_Occurred(tstate)); + throwflag = 1; + } + retval = NULL; + goto resume_frame; + } tstate->frame = frame->previous; + + /* Restore previous cframe */ + tstate->cframe = cframe.previous; + tstate->cframe->use_tracing = cframe.use_tracing; return _Py_CheckFunctionResult(tstate, NULL, retval, __func__); } @@ -5309,7 +5592,7 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i static int initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, PyObject **localsplus, PyObject *const *args, - Py_ssize_t argcount, PyObject *kwnames) + Py_ssize_t argcount, PyObject *kwnames, int steal_args) { PyCodeObject *co = (PyCodeObject*)con->fc_code; const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; @@ -5319,8 +5602,9 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, Py_ssize_t i; if (co->co_flags & CO_VARKEYWORDS) { kwdict = PyDict_New(); - if (kwdict == NULL) - goto fail; + if (kwdict == NULL) { + goto fail_pre_positional; + } i = total_args; if (co->co_flags & CO_VARARGS) { i++; @@ -5342,20 +5626,35 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, } for (j = 0; j < n; j++) { PyObject *x = args[j]; - Py_INCREF(x); + if (!steal_args) { + Py_INCREF(x); + } assert(localsplus[j] == NULL); localsplus[j] = x; } /* Pack other positional arguments into the *args argument */ if (co->co_flags & CO_VARARGS) { - PyObject *u = _PyTuple_FromArray(args + n, argcount - n); + PyObject *u = NULL; + if (steal_args) { + u = _PyTuple_FromArraySteal(args + n, argcount - n); + } else { + u = _PyTuple_FromArray(args + n, argcount - n); + } if (u == NULL) { - goto fail; + goto fail_post_positional; } assert(localsplus[total_args] == NULL); localsplus[total_args] = u; } + else if (argcount > n) { + /* Too many postional args. Error is reported later */ + if (steal_args) { + for (j = n; j < argcount; j++) { + Py_DECREF(args[j]); + } + } + } /* Handle keyword arguments */ if (kwnames != NULL) { @@ -5370,7 +5669,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, _PyErr_Format(tstate, PyExc_TypeError, "%U() keywords must be strings", con->fc_qualname); - goto fail; + goto kw_fail; } /* Speed hack: do raw pointer compares. As names are @@ -5391,7 +5690,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, goto kw_found; } else if (cmp < 0) { - goto fail; + goto kw_fail; } } @@ -5403,28 +5702,42 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, kwcount, kwnames, con->fc_qualname)) { - goto fail; + goto kw_fail; } _PyErr_Format(tstate, PyExc_TypeError, "%U() got an unexpected keyword argument '%S'", con->fc_qualname, keyword); - goto fail; + goto kw_fail; } if (PyDict_SetItem(kwdict, keyword, value) == -1) { - goto fail; + goto kw_fail; + } + if (steal_args) { + Py_DECREF(value); } continue; + kw_fail: + if (steal_args) { + for (;i < kwcount; i++) { + PyObject *value = args[i+argcount]; + Py_DECREF(value); + } + } + goto fail_noclean; + kw_found: if (localsplus[j] != NULL) { _PyErr_Format(tstate, PyExc_TypeError, "%U() got multiple values for argument '%S'", con->fc_qualname, keyword); - goto fail; + goto kw_fail; + } + if (!steal_args) { + Py_INCREF(value); } - Py_INCREF(value); localsplus[j] = value; } } @@ -5433,7 +5746,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { too_many_positional(tstate, co, argcount, con->fc_defaults, localsplus, con->fc_qualname); - goto fail; + goto fail_noclean; } /* Add missing positional arguments (copy default values from defs) */ @@ -5449,7 +5762,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, if (missing) { missing_arguments(tstate, co, missing, defcount, localsplus, con->fc_qualname); - goto fail; + goto fail_noclean; } if (n > m) i = n - m; @@ -5482,7 +5795,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, continue; } else if (_PyErr_Occurred(tstate)) { - goto fail; + goto fail_noclean; } } missing++; @@ -5490,7 +5803,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, if (missing) { missing_arguments(tstate, co, missing, -1, localsplus, con->fc_qualname); - goto fail; + goto fail_noclean; } } @@ -5503,9 +5816,23 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con, return 0; -fail: /* Jump here from prelude on failure */ +fail_pre_positional: + if (steal_args) { + for (j = 0; j < argcount; j++) { + Py_DECREF(args[j]); + } + } + /* fall through */ +fail_post_positional: + if (steal_args) { + Py_ssize_t kwcount = kwnames != NULL ? PyTuple_GET_SIZE(kwnames) : 0; + for (j = argcount; j < argcount+kwcount; j++) { + Py_DECREF(args[j]); + } + } + /* fall through */ +fail_noclean: return -1; - } static InterpreterFrame * @@ -5528,7 +5855,7 @@ make_coro_frame(PyThreadState *tstate, } _PyFrame_InitializeSpecials(frame, con, locals, code->co_nlocalsplus); assert(frame->frame_obj == NULL); - if (initialize_locals(tstate, con, frame->localsplus, args, argcount, kwnames)) { + if (initialize_locals(tstate, con, frame->localsplus, args, argcount, kwnames, 0)) { _PyFrame_Clear(frame, 1); return NULL; } @@ -5554,17 +5881,20 @@ make_coro(PyThreadState *tstate, PyFrameConstructor *con, return gen; } +// If *steal_args* is set, the function will steal the references to all the arguments. +// In case of error, the function returns null and if *steal_args* is set, the caller +// will still own all the arguments. static InterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, int steal_args) { InterpreterFrame * frame = _PyThreadState_PushFrame(tstate, con, locals); if (frame == NULL) { return NULL; } PyObject **localsarray = _PyFrame_GetLocalsArray(frame); - if (initialize_locals(tstate, con, localsarray, args, argcount, kwnames)) { + if (initialize_locals(tstate, con, localsarray, args, argcount, kwnames, steal_args)) { _PyFrame_Clear(frame, 0); return NULL; } @@ -5579,9 +5909,9 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame) ++tstate->recursion_depth; assert(frame->frame_obj == NULL || frame->frame_obj->f_own_locals_memory == 0); if (_PyFrame_Clear(frame, 0)) { + --tstate->recursion_depth; return -1; } - assert(frame->frame_obj == NULL); --tstate->recursion_depth; tstate->frame = frame->previous; _PyThreadState_PopFrame(tstate, frame); @@ -5601,11 +5931,10 @@ _PyEval_Vector(PyThreadState *tstate, PyFrameConstructor *con, return make_coro(tstate, con, locals, args, argcount, kwnames); } InterpreterFrame *frame = _PyEvalFramePushAndInit( - tstate, con, locals, args, argcount, kwnames); + tstate, con, locals, args, argcount, kwnames, 0); if (frame == NULL) { return NULL; } - assert (tstate->interp->eval_frame != NULL); PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0); assert(_PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame)); if (_PyEvalFrameClearAndPop(tstate, frame)) { @@ -5732,7 +6061,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) if (PyExceptionClass_Check(exc)) { type = exc; - value = _PyObject_CallNoArg(exc); + value = _PyObject_CallNoArgs(exc); if (value == NULL) goto raise_error; if (!PyExceptionInstance_Check(value)) { @@ -5763,7 +6092,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) if (cause) { PyObject *fixed_cause; if (PyExceptionClass_Check(cause)) { - fixed_cause = _PyObject_CallNoArg(cause); + fixed_cause = _PyObject_CallNoArgs(cause); if (fixed_cause == NULL) goto raise_error; Py_DECREF(cause); @@ -5986,7 +6315,7 @@ call_trace(Py_tracefunc func, PyObject *obj, if (tstate->tracing) return 0; tstate->tracing++; - tstate->cframe->use_tracing = 0; + _PyThreadState_PauseTracing(tstate); PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f == NULL) { return -1; @@ -5996,12 +6325,11 @@ call_trace(Py_tracefunc func, PyObject *obj, } else { initialize_trace_info(&tstate->trace_info, frame); - f->f_lineno = _PyCode_CheckLineNumber(frame->f_lasti*2, &tstate->trace_info.bounds); + f->f_lineno = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &tstate->trace_info.bounds); } result = func(obj, f, what, arg); f->f_lineno = 0; - tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL) - || (tstate->c_profilefunc != NULL)); + _PyThreadState_ResumeTracing(tstate); tstate->tracing--; return result; } @@ -6015,8 +6343,7 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) PyObject *result; tstate->tracing = 0; - tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL) - || (tstate->c_profilefunc != NULL)); + _PyThreadState_ResumeTracing(tstate); result = PyObject_Call(func, args, NULL); tstate->tracing = save_tracing; tstate->cframe->use_tracing = save_use_tracing; @@ -6035,8 +6362,8 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, then call the trace function if we're tracing source lines. */ initialize_trace_info(&tstate->trace_info, frame); - int lastline = _PyCode_CheckLineNumber(instr_prev*2, &tstate->trace_info.bounds); - int line = _PyCode_CheckLineNumber(frame->f_lasti*2, &tstate->trace_info.bounds); + int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &tstate->trace_info.bounds); + int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &tstate->trace_info.bounds); PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f == NULL) { return -1; @@ -6073,7 +6400,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->c_profilefunc = NULL; tstate->c_profileobj = NULL; /* Must make sure that tracing is not ignored if 'profileobj' is freed */ - tstate->cframe->use_tracing = tstate->c_tracefunc != NULL; + _PyThreadState_ResumeTracing(tstate); Py_XDECREF(profileobj); Py_XINCREF(arg); @@ -6081,7 +6408,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->c_profilefunc = func; /* Flag that tracing or profiling is turned on */ - tstate->cframe->use_tracing = (func != NULL) || (tstate->c_tracefunc != NULL); + _PyThreadState_ResumeTracing(tstate); return 0; } @@ -6114,7 +6441,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->c_tracefunc = NULL; tstate->c_traceobj = NULL; /* Must make sure that profiling is not ignored if 'traceobj' is freed */ - tstate->cframe->use_tracing = (tstate->c_profilefunc != NULL); + _PyThreadState_ResumeTracing(tstate); Py_XDECREF(traceobj); Py_XINCREF(arg); @@ -6122,8 +6449,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->c_tracefunc = func; /* Flag that tracing or profiling is turned on */ - tstate->cframe->use_tracing = ((func != NULL) - || (tstate->c_profilefunc != NULL)); + _PyThreadState_ResumeTracing(tstate); return 0; } @@ -6395,40 +6721,6 @@ trace_call_function(PyThreadState *tstate, return PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); } -/* Issue #29227: Inline call_function() into _PyEval_EvalFrameDefault() - to reduce the stack consumption. */ -Py_LOCAL_INLINE(PyObject *) _Py_HOT_FUNCTION -call_function(PyThreadState *tstate, - PyObject ***pp_stack, - Py_ssize_t oparg, - PyObject *kwnames, - int use_tracing) -{ - PyObject **pfunc = (*pp_stack) - oparg - 1; - PyObject *func = *pfunc; - PyObject *x, *w; - Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); - Py_ssize_t nargs = oparg - nkwargs; - PyObject **stack = (*pp_stack) - nargs - nkwargs; - - if (use_tracing) { - x = trace_call_function(tstate, func, stack, nargs, kwnames); - } - else { - x = PyObject_Vectorcall(func, stack, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); - } - - assert((x != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Clear the stack of the function object. */ - while ((*pp_stack) > pfunc) { - w = EXT_POP(*pp_stack); - Py_DECREF(w); - } - - return x; -} - static PyObject * do_call_core(PyThreadState *tstate, PyObject *func, @@ -6871,6 +7163,7 @@ unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w, */ int opcode, oparg; NEXTOPARG(); + next_instr++; switch (opcode) { case STORE_FAST: { @@ -6978,7 +7271,7 @@ dtrace_function_entry(InterpreterFrame *frame) PyCodeObject *code = frame->f_code; filename = PyUnicode_AsUTF8(code->co_filename); funcname = PyUnicode_AsUTF8(code->co_name); - lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti*2); + lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti*sizeof(_Py_CODEUNIT)); PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); } @@ -6993,7 +7286,7 @@ dtrace_function_return(InterpreterFrame *frame) PyCodeObject *code = frame->f_code; filename = PyUnicode_AsUTF8(code->co_filename); funcname = PyUnicode_AsUTF8(code->co_name); - lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti*2); + lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti*sizeof(_Py_CODEUNIT)); PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); } @@ -7010,12 +7303,12 @@ maybe_dtrace_line(InterpreterFrame *frame, instruction window, reset the window. */ initialize_trace_info(trace_info, frame); - int lastline = _PyCode_CheckLineNumber(instr_prev*2, &trace_info->bounds); - int line = _PyCode_CheckLineNumber(frame->f_lasti*2, &trace_info->bounds); + int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds); + int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds); if (line != -1) { /* Trace backward edges or first instruction of a new line */ if (frame->f_lasti < instr_prev || - (line != lastline && frame->f_lasti*2 == trace_info->bounds.ar_start)) + (line != lastline && frame->f_lasti*sizeof(_Py_CODEUNIT) == (unsigned int)trace_info->bounds.ar_start)) { co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); if (!co_filename) { diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 438a348fa097fb..6052316cdd8e3e 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -169,33 +169,100 @@ _imp_init_frozen(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_imp_find_frozen__doc__, +"find_frozen($module, name, /, *, withdata=False)\n" +"--\n" +"\n" +"Return info about the corresponding frozen module (if there is one) or None.\n" +"\n" +"The returned info (a 2-tuple):\n" +"\n" +" * data the raw marshalled bytes\n" +" * is_package whether or not it is a package\n" +" * origname the originally frozen module\'s name, or None if not\n" +" a stdlib module (this will usually be the same as\n" +" the module\'s current name)"); + +#define _IMP_FIND_FROZEN_METHODDEF \ + {"find_frozen", (PyCFunction)(void(*)(void))_imp_find_frozen, METH_FASTCALL|METH_KEYWORDS, _imp_find_frozen__doc__}, + +static PyObject * +_imp_find_frozen_impl(PyObject *module, PyObject *name, int withdata); + +static PyObject * +_imp_find_frozen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "withdata", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "find_frozen", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *name; + int withdata = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("find_frozen", "argument 1", "str", args[0]); + goto exit; + } + if (PyUnicode_READY(args[0]) == -1) { + goto exit; + } + name = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + withdata = PyObject_IsTrue(args[1]); + if (withdata < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _imp_find_frozen_impl(module, name, withdata); + +exit: + return return_value; +} + PyDoc_STRVAR(_imp_get_frozen_object__doc__, -"get_frozen_object($module, name, /)\n" +"get_frozen_object($module, name, data=None, /)\n" "--\n" "\n" "Create a code object for a frozen module."); #define _IMP_GET_FROZEN_OBJECT_METHODDEF \ - {"get_frozen_object", (PyCFunction)_imp_get_frozen_object, METH_O, _imp_get_frozen_object__doc__}, + {"get_frozen_object", (PyCFunction)(void(*)(void))_imp_get_frozen_object, METH_FASTCALL, _imp_get_frozen_object__doc__}, static PyObject * -_imp_get_frozen_object_impl(PyObject *module, PyObject *name); +_imp_get_frozen_object_impl(PyObject *module, PyObject *name, + PyObject *dataobj); static PyObject * -_imp_get_frozen_object(PyObject *module, PyObject *arg) +_imp_get_frozen_object(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *name; + PyObject *dataobj = Py_None; - if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("get_frozen_object", "argument", "str", arg); + if (!_PyArg_CheckPositional("get_frozen_object", nargs, 1, 2)) { goto exit; } - if (PyUnicode_READY(arg) == -1) { + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("get_frozen_object", "argument 1", "str", args[0]); goto exit; } - name = arg; - return_value = _imp_get_frozen_object_impl(module, name); + if (PyUnicode_READY(args[0]) == -1) { + goto exit; + } + name = args[0]; + if (nargs < 2) { + goto skip_optional; + } + dataobj = args[1]; +skip_optional: + return_value = _imp_get_frozen_object_impl(module, name, dataobj); exit: return return_value; @@ -498,4 +565,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=96038c277119d6e3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=adcf787969a11353 input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index fa329ce2436427..b7c8db7e8b6189 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -9,6 +9,7 @@ Copyright (c) Corporation for National Research Initiatives. ------------------------------------------------------------------------ */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.codec_search_path #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI @@ -280,7 +281,7 @@ PyObject *codec_makeincrementalcodec(PyObject *codec_info, if (errors) ret = PyObject_CallFunction(inccodec, "s", errors); else - ret = _PyObject_CallNoArg(inccodec); + ret = _PyObject_CallNoArgs(inccodec); Py_DECREF(inccodec); return ret; } diff --git a/Python/compile.c b/Python/compile.c index fdc2ce61a8e035..0c025acec14914 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -6335,7 +6335,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // cases, though. assert(istores < icontrol); Py_ssize_t rotations = istores + 1; - // Perfom the same rotation on pc->stores: + // Perform the same rotation on pc->stores: PyObject *rotated = PyList_GetSlice(pc->stores, 0, rotations); if (rotated == NULL || @@ -7081,7 +7081,7 @@ assemble_line_range(struct assembler* a, int current, PyObject** table, int* prev, int* start, int* offset) { int ldelta, bdelta; - bdelta = (a->a_offset - *start) * 2; + bdelta = (a->a_offset - *start) * sizeof(_Py_CODEUNIT); if (bdelta == 0) { return 1; } @@ -7220,6 +7220,26 @@ assemble_emit(struct assembler *a, struct instr *i) return 1; } +static void +normalize_jumps(struct assembler *a) +{ + for (basicblock *b = a->a_entry; b != NULL; b = b->b_next) { + b->b_visited = 0; + } + for (basicblock *b = a->a_entry; b != NULL; b = b->b_next) { + b->b_visited = 1; + if (b->b_iused == 0) { + continue; + } + struct instr *last = &b->b_instr[b->b_iused-1]; + if (last->i_opcode == JUMP_ABSOLUTE && + last->i_target->b_visited == 0 + ) { + last->i_opcode = JUMP_FORWARD; + } + } +} + static void assemble_jump_offsets(struct assembler *a, struct compiler *c) { @@ -7306,7 +7326,7 @@ consts_dict_keys_inorder(PyObject *dict) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { i = PyLong_AS_LONG(v); - /* The keys of the dictionary can be tuples wrapping a contant. + /* The keys of the dictionary can be tuples wrapping a constant. * (see compiler_add_o and _PyCode_ConstantKey). In that case * the object we want is always second. */ if (PyTuple_CheckExact(k)) { @@ -7897,6 +7917,9 @@ assemble(struct compiler *c, int addNone) clean_basic_block(b); } + /* Order of basic blocks must have been determined by now */ + normalize_jumps(&a); + /* Can't modify the bytecode after computing jump offsets. */ assemble_jump_offsets(&a, c); diff --git a/Python/context.c b/Python/context.c index 9560fb3f6676e0..a20ec7123731c2 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1,5 +1,5 @@ #include "Python.h" - +#include "pycore_call.h" // _PyObject_VectorcallTstate() #include "pycore_context.h" #include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED() #include "pycore_hamt.h" @@ -9,9 +9,6 @@ #include "structmember.h" // PyMemberDef -#define CONTEXT_FREELIST_MAXLEN 255 - - #include "clinic/context.c.h" /*[clinic input] module _contextvars @@ -66,12 +63,14 @@ static int contextvar_del(PyContextVar *var); +#if PyContext_MAXFREELIST > 0 static struct _Py_context_state * get_context_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->context; } +#endif PyObject * @@ -340,8 +339,9 @@ class _contextvars.Context "PyContext *" "&PyContext_Type" static inline PyContext * _context_alloc(void) { - struct _Py_context_state *state = get_context_state(); PyContext *ctx; +#if PyContext_MAXFREELIST > 0 + struct _Py_context_state *state = get_context_state(); #ifdef Py_DEBUG // _context_alloc() must not be called after _PyContext_Fini() assert(state->numfree != -1); @@ -353,7 +353,9 @@ _context_alloc(void) ctx->ctx_weakreflist = NULL; _Py_NewReference((PyObject *)ctx); } - else { + else +#endif + { ctx = PyObject_GC_New(PyContext, &PyContext_Type); if (ctx == NULL) { return NULL; @@ -469,17 +471,20 @@ context_tp_dealloc(PyContext *self) } (void)context_tp_clear(self); +#if PyContext_MAXFREELIST > 0 struct _Py_context_state *state = get_context_state(); #ifdef Py_DEBUG // _context_alloc() must not be called after _PyContext_Fini() assert(state->numfree != -1); #endif - if (state->numfree < CONTEXT_FREELIST_MAXLEN) { + if (state->numfree < PyContext_MAXFREELIST) { state->numfree++; self->ctx_weakreflist = (PyObject *)state->freelist; state->freelist = self; } - else { + else +#endif + { Py_TYPE(self)->tp_free(self); } } @@ -728,7 +733,7 @@ static int contextvar_set(PyContextVar *var, PyObject *val) { var->var_cached = NULL; - PyThreadState *ts = PyThreadState_Get(); + PyThreadState *ts = _PyThreadState_GET(); PyContext *ctx = context_get(); if (ctx == NULL) { @@ -1289,6 +1294,7 @@ get_token_missing(void) void _PyContext_ClearFreeList(PyInterpreterState *interp) { +#if PyContext_MAXFREELIST > 0 struct _Py_context_state *state = &interp->context; for (; state->numfree; state->numfree--) { PyContext *ctx = state->freelist; @@ -1296,6 +1302,7 @@ _PyContext_ClearFreeList(PyInterpreterState *interp) ctx->ctx_weakreflist = NULL; PyObject_GC_Del(ctx); } +#endif } @@ -1306,7 +1313,7 @@ _PyContext_Fini(PyInterpreterState *interp) Py_CLEAR(_token_missing); } _PyContext_ClearFreeList(interp); -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && PyContext_MAXFREELIST > 0 struct _Py_context_state *state = &interp->context; state->numfree = -1; #endif diff --git a/Python/dtoa.c b/Python/dtoa.c index e629b296426f31..6c44f689651766 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -119,6 +119,7 @@ #include "Python.h" #include "pycore_dtoa.h" +#include // exit() /* if PY_NO_SHORT_FLOAT_REPR is defined, then don't even try to compile the following code */ diff --git a/Python/errors.c b/Python/errors.c index b2030f728a7ebd..519f2d459effd6 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -2,11 +2,12 @@ /* Error handling */ #include "Python.h" -#include "pycore_initconfig.h" -#include "pycore_pyerrors.h" -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_sysmodule.h" -#include "pycore_traceback.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_initconfig.h" // _PyStatus_ERR() +#include "pycore_pyerrors.h" // _PyErr_Format() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_traceback.h" // _PyTraceBack_FromFrame() #ifndef __STDC__ #ifndef MS_WINDOWS @@ -14,12 +15,13 @@ extern char *strerror(int); #endif #endif +#include #ifdef MS_WINDOWS -#include -#include +# include +# include +# include // _sys_nerr #endif -#include #ifdef __cplusplus extern "C" { @@ -91,7 +93,7 @@ _PyErr_CreateException(PyObject *exception_type, PyObject *value) PyObject *exc; if (value == NULL || value == Py_None) { - exc = _PyObject_CallNoArg(exception_type); + exc = _PyObject_CallNoArgs(exception_type); } else if (PyTuple_Check(value)) { exc = PyObject_Call(exception_type, value, NULL); diff --git a/Python/fileutils.c b/Python/fileutils.c index 9e732ddca55cec..3d8f3a4f16326c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -3,10 +3,12 @@ #include "pycore_runtime.h" // _PyRuntime #include "osdefs.h" // SEP #include +#include // mbstowcs() #ifdef MS_WINDOWS # include # include +# include // PathCchCombineEx extern int winerror_to_errno(int); #endif @@ -220,7 +222,7 @@ check_force_ascii(void) ch = (unsigned char)0xA7; res = _Py_mbstowcs(&wch, (char*)&ch, 1); if (res != DECODE_ERROR && wch == L'\xA7') { - /* On HP-UX withe C locale or the POSIX locale, + /* On HP-UX with C locale or the POSIX locale, nl_langinfo(CODESET) announces "roman8", whereas mbstowcs() uses Latin1 encoding in practice. Force ASCII in this case. @@ -1205,6 +1207,31 @@ _Py_fstat(int fd, struct _Py_stat_struct *status) return 0; } +/* Like _Py_stat() but with a raw filename. */ +int +_Py_wstat(const wchar_t* path, struct stat *buf) +{ + int err; +#ifdef MS_WINDOWS + struct _stat wstatbuf; + err = _wstat(path, &wstatbuf); + if (!err) { + buf->st_mode = wstatbuf.st_mode; + } +#else + char *fname; + fname = _Py_EncodeLocaleRaw(path, NULL); + if (fname == NULL) { + errno = EINVAL; + return -1; + } + err = stat(fname, buf); + PyMem_RawFree(fname); +#endif + return err; +} + + /* Call _wstat() on Windows, or encode the path to the filesystem encoding and call stat() otherwise. Only fill st_mode attribute on Windows. @@ -1216,7 +1243,6 @@ _Py_stat(PyObject *path, struct stat *statbuf) { #ifdef MS_WINDOWS int err; - struct _stat wstatbuf; #if USE_UNICODE_WCHAR_CACHE const wchar_t *wpath = _PyUnicode_AsUnicode(path); @@ -1226,9 +1252,7 @@ _Py_stat(PyObject *path, struct stat *statbuf) if (wpath == NULL) return -2; - err = _wstat(wpath, &wstatbuf); - if (!err) - statbuf->st_mode = wstatbuf.st_mode; + err = _Py_wstat(wpath, statbuf); #if !USE_UNICODE_WCHAR_CACHE PyMem_Free(wpath); #endif /* USE_UNICODE_WCHAR_CACHE */ @@ -2072,6 +2096,91 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) } +// The caller must ensure "buffer" is big enough. +static int +join_relfile(wchar_t *buffer, size_t bufsize, + const wchar_t *dirname, const wchar_t *relfile) +{ +#ifdef MS_WINDOWS + if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, 0))) { + return -1; + } +#else + assert(!_Py_isabs(relfile)); + size_t dirlen = wcslen(dirname); + size_t rellen = wcslen(relfile); + size_t maxlen = bufsize - 1; + if (maxlen > MAXPATHLEN || dirlen >= maxlen || rellen >= maxlen - dirlen) { + return -1; + } + if (dirlen == 0) { + // We do not add a leading separator. + wcscpy(buffer, relfile); + } + else { + if (dirname != buffer) { + wcscpy(buffer, dirname); + } + size_t relstart = dirlen; + if (dirlen > 1 && dirname[dirlen - 1] != SEP) { + buffer[dirlen] = SEP; + relstart += 1; + } + wcscpy(&buffer[relstart], relfile); + } +#endif + return 0; +} + +/* Join the two paths together, like os.path.join(). Return NULL + if memory could not be allocated. The caller is responsible + for calling PyMem_RawFree() on the result. */ +wchar_t * +_Py_join_relfile(const wchar_t *dirname, const wchar_t *relfile) +{ + assert(dirname != NULL && relfile != NULL); +#ifndef MS_WINDOWS + assert(!_Py_isabs(relfile)); +#endif + size_t maxlen = wcslen(dirname) + 1 + wcslen(relfile); + size_t bufsize = maxlen + 1; + wchar_t *filename = PyMem_RawMalloc(bufsize * sizeof(wchar_t)); + if (filename == NULL) { + return NULL; + } + assert(wcslen(dirname) < MAXPATHLEN); + assert(wcslen(relfile) < MAXPATHLEN - wcslen(dirname)); + join_relfile(filename, bufsize, dirname, relfile); + return filename; +} + +/* Join the two paths together, like os.path.join(). + dirname: the target buffer with the dirname already in place, + including trailing NUL + relfile: this must be a relative path + bufsize: total allocated size of the buffer + Return -1 if anything is wrong with the path lengths. */ +int +_Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize) +{ + assert(dirname != NULL && relfile != NULL); + assert(bufsize > 0); + return join_relfile(dirname, bufsize, dirname, relfile); +} + + +size_t +_Py_find_basename(const wchar_t *filename) +{ + for (size_t i = wcslen(filename); i > 0; --i) { + if (filename[i] == SEP) { + return i + 1; + } + } + return 0; +} + + /* Get the current directory. buflen is the buffer size in wide characters including the null character. Decode the path from the locale encoding. diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index 9071bf3a6589e5..0c21301fb9ed4b 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -3,7 +3,8 @@ of int.__float__, etc., that take and return unicode objects */ #include "Python.h" -#include "pycore_fileutils.h" +#include "pycore_fileutils.h" // _Py_GetLocaleconvNumeric() +#include "pycore_long.h" // _PyLong_FormatWriter() #include /* Raises an exception about an unknown presentation type for this diff --git a/Python/frozen.c b/Python/frozen.c index f9ad07c0b49e14..499b3b99570573 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -36,6 +36,7 @@ and __phello__.spam. Loading any will print some famous words... */ #include "Python.h" +#include "pycore_import.h" /* Includes for frozen modules: */ #include "frozen_modules/importlib._bootstrap.h" @@ -53,6 +54,11 @@ #include "frozen_modules/site.h" #include "frozen_modules/stat.h" #include "frozen_modules/__hello__.h" +#include "frozen_modules/__phello__.h" +#include "frozen_modules/__phello__.ham.h" +#include "frozen_modules/__phello__.ham.eggs.h" +#include "frozen_modules/__phello__.spam.h" +#include "frozen_modules/frozen_only.h" /* End includes */ /* Note that a negative size indicates a package. */ @@ -84,11 +90,37 @@ static const struct _frozen _PyImport_FrozenModules[] = { /* Test module */ {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__)}, - {"__phello__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__)}, - {"__phello__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__)}, - {0, 0, 0} /* sentinel */ + {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__)}, + {"__phello_alias__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__)}, + {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__)}, + {"__phello__", _Py_M____phello__, -(int)sizeof(_Py_M____phello__)}, + {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__)}, + {"__phello__.ham", _Py_M____phello___ham, -(int)sizeof(_Py_M____phello___ham)}, + {"__phello__.ham.__init__", _Py_M____phello___ham, + (int)sizeof(_Py_M____phello___ham)}, + {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, + (int)sizeof(_Py_M____phello___ham_eggs)}, + {"__phello__.spam", _Py_M____phello___spam, + (int)sizeof(_Py_M____phello___spam)}, + {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)}, + {0, 0, 0} /* modules sentinel */ }; +static const struct _module_alias aliases[] = { + {"_frozen_importlib", "importlib._bootstrap"}, + {"_frozen_importlib_external", "importlib._bootstrap_external"}, + {"os.path", "posixpath"}, + {"__hello_alias__", "__hello__"}, + {"__phello_alias__", "__hello__"}, + {"__phello_alias__.spam", "__hello__"}, + {"__phello__.__init__", "<__phello__"}, + {"__phello__.ham.__init__", "<__phello__.ham"}, + {"__hello_only__", NULL}, + {0, 0} /* aliases sentinel */ +}; +const struct _module_alias *_PyImport_FrozenAliases = aliases; + + /* Embedding apps may change this pointer to point to their favorite collection of frozen modules: */ diff --git a/Python/import.c b/Python/import.c index 317a836617c51d..fe4686cd56b3ba 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3,20 +3,19 @@ #include "Python.h" #include "pycore_import.h" // _PyImport_BootstrapImp() -#include "pycore_initconfig.h" -#include "pycore_pyerrors.h" -#include "pycore_pyhash.h" +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_interp.h" // _PyInterpreterState_ClearModules() +#include "pycore_namespace.h" // _PyNamespace_Type +#include "pycore_pyerrors.h" // _PyErr_SetString() +#include "pycore_pyhash.h" // _Py_KeyedHash() #include "pycore_pylifecycle.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() -#include "pycore_interp.h" // _PyInterpreterState_ClearModules() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_sysmodule.h" -#include "errcode.h" -#include "marshal.h" -#include "code.h" -#include "importdl.h" -#include "pydtrace.h" -#include +#include "pycore_sysmodule.h" // _PySys_Audit() +#include "marshal.h" // PyMarshal_ReadObjectFromString() +#include "importdl.h" // _PyImport_DynLoadFiletab +#include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED() +#include // bool #ifdef HAVE_FCNTL_H #include @@ -442,7 +441,9 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, return -1; } - if (_Py_IsMainInterpreter(tstate->interp)) { + // bpo-44050: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { if (def->m_size == -1) { if (def->m_base.m_copy) { /* Somebody already imported the module, @@ -885,10 +886,6 @@ _imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, } -/* Forward */ -static const struct _frozen * find_frozen(PyObject *); - - /* Helper to test for built-in module */ static int @@ -1048,6 +1045,29 @@ _imp_create_builtin(PyObject *module, PyObject *spec) } +/* Return true if the name is an alias. In that case, "alias" is set + to the original module name. If it is an alias but the original + module isn't known then "alias" is set to NULL while true is returned. */ +static bool +resolve_module_alias(const char *name, const struct _module_alias *aliases, + const char **alias) +{ + const struct _module_alias *entry; + for (entry = aliases; ; entry++) { + if (entry->name == NULL) { + /* It isn't an alias. */ + return false; + } + if (strcmp(name, entry->name) == 0) { + if (alias != NULL) { + *alias = entry->orig; + } + return true; + } + } +} + + /* Frozen modules */ static bool @@ -1117,75 +1137,127 @@ list_frozen_module_names() return names; } -static const struct _frozen * -find_frozen(PyObject *modname) +typedef enum { + FROZEN_OKAY, + FROZEN_BAD_NAME, // The given module name wasn't valid. + FROZEN_NOT_FOUND, // It wasn't in PyImport_FrozenModules. + FROZEN_DISABLED, // -X frozen_modules=off (and not essential) + FROZEN_EXCLUDED, // The PyImport_FrozenModules entry has NULL "code". + FROZEN_INVALID, // The PyImport_FrozenModules entry is bogus. +} frozen_status; + +static inline void +set_frozen_error(frozen_status status, PyObject *modname) +{ + const char *err = NULL; + switch (status) { + case FROZEN_BAD_NAME: + case FROZEN_NOT_FOUND: + case FROZEN_DISABLED: + err = "No such frozen object named %R"; + break; + case FROZEN_EXCLUDED: + err = "Excluded frozen object named %R"; + break; + case FROZEN_INVALID: + err = "Frozen object named %R is invalid"; + break; + case FROZEN_OKAY: + // There was no error. + break; + default: + Py_UNREACHABLE(); + } + if (err != NULL) { + PyObject *msg = PyUnicode_FromFormat(err, modname); + if (msg == NULL) { + PyErr_Clear(); + } + PyErr_SetImportError(msg, modname, NULL); + Py_XDECREF(msg); + } +} + +struct frozen_info { + PyObject *nameobj; + const char *data; + Py_ssize_t size; + bool is_package; + bool is_alias; + const char *origname; +}; + +static frozen_status +find_frozen(PyObject *nameobj, struct frozen_info *info) { - if (modname == NULL) { - return NULL; + if (info != NULL) { + memset(info, 0, sizeof(*info)); + } + + if (nameobj == NULL || nameobj == Py_None) { + return FROZEN_BAD_NAME; } - const char *name = PyUnicode_AsUTF8(modname); + const char *name = PyUnicode_AsUTF8(nameobj); if (name == NULL) { + // Note that this function previously used + // _PyUnicode_EqualToASCIIString(). We clear the error here + // (instead of propagating it) to match the earlier behavior + // more closely. PyErr_Clear(); - return NULL; + return FROZEN_BAD_NAME; } + if (!use_frozen() && !is_essential_frozen_module(name)) { - return NULL; + return FROZEN_DISABLED; } + const struct _frozen *p; for (p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) { - return NULL; + // We hit the end-of-list sentinel value. + return FROZEN_NOT_FOUND; } if (strcmp(name, p->name) == 0) { break; } } - return p; -} - -static PyObject * -get_frozen_object(PyObject *name) -{ - const struct _frozen *p = find_frozen(name); - int size; - - if (p == NULL) { - PyErr_Format(PyExc_ImportError, - "No such frozen object named %R", - name); - return NULL; + if (info != NULL) { + info->nameobj = nameobj; // borrowed + info->data = (const char *)p->code; + info->size = p->size < 0 ? -(p->size) : p->size; + info->is_package = p->size < 0 ? true : false; + info->origname = name; + info->is_alias = resolve_module_alias(name, _PyImport_FrozenAliases, + &info->origname); } + if (p->code == NULL) { - PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %R", - name); - return NULL; + /* It is frozen but marked as un-importable. */ + return FROZEN_EXCLUDED; } - size = p->size; - if (size < 0) - size = -size; - return PyMarshal_ReadObjectFromString((const char *)p->code, size); + if (p->code[0] == '\0' || p->size == 0) { + return FROZEN_INVALID; + } + return FROZEN_OKAY; } static PyObject * -is_frozen_package(PyObject *name) +unmarshal_frozen_code(struct frozen_info *info) { - const struct _frozen *p = find_frozen(name); - int size; - - if (p == NULL) { - PyErr_Format(PyExc_ImportError, - "No such frozen object named %R", - name); + PyObject *co = PyMarshal_ReadObjectFromString(info->data, info->size); + if (co == NULL) { + set_frozen_error(FROZEN_INVALID, info->nameobj); return NULL; } - - size = p->size; - - if (size < 0) - Py_RETURN_TRUE; - else - Py_RETURN_FALSE; + if (!PyCode_Check(co)) { + // We stick with TypeError for backward compatibility. + PyErr_Format(PyExc_TypeError, + "frozen object %R is not a code object", + info->nameobj); + Py_DECREF(co); + return NULL; + } + return co; } @@ -1198,38 +1270,28 @@ int PyImport_ImportFrozenModuleObject(PyObject *name) { PyThreadState *tstate = _PyThreadState_GET(); - const struct _frozen *p; - PyObject *co, *m, *d; - int ispackage; - int size; - - p = find_frozen(name); + PyObject *co, *m, *d = NULL; + int err; - if (p == NULL) + struct frozen_info info; + frozen_status status = find_frozen(name, &info); + if (status == FROZEN_NOT_FOUND || status == FROZEN_DISABLED) { return 0; - if (p->code == NULL) { - _PyErr_Format(tstate, PyExc_ImportError, - "Excluded frozen object named %R", - name); + } + else if (status == FROZEN_BAD_NAME) { + return 0; + } + else if (status != FROZEN_OKAY) { + set_frozen_error(status, name); return -1; } - size = p->size; - ispackage = (size < 0); - if (ispackage) - size = -size; - co = PyMarshal_ReadObjectFromString((const char *)p->code, size); - if (co == NULL) + co = unmarshal_frozen_code(&info); + if (co == NULL) { return -1; - if (!PyCode_Check(co)) { - _PyErr_Format(tstate, PyExc_TypeError, - "frozen object %R is not a code object", - name); - goto err_return; } - if (ispackage) { + if (info.is_package) { /* Set __path__ to the empty list */ PyObject *l; - int err; m = import_add_module(tstate, name); if (m == NULL) goto err_return; @@ -1250,15 +1312,33 @@ PyImport_ImportFrozenModuleObject(PyObject *name) goto err_return; } m = exec_code_in_module(tstate, name, d, co); - Py_DECREF(d); if (m == NULL) { goto err_return; } - Py_DECREF(co); Py_DECREF(m); + /* Set __origname__ (consumed in FrozenImporter._setup_module()). */ + PyObject *origname; + if (info.origname) { + origname = PyUnicode_FromString(info.origname); + if (origname == NULL) { + goto err_return; + } + } + else { + Py_INCREF(Py_None); + origname = Py_None; + } + err = PyDict_SetItemString(d, "__origname__", origname); + Py_DECREF(origname); + if (err != 0) { + goto err_return; + } + Py_DECREF(d); + Py_DECREF(co); return 1; err_return: + Py_XDECREF(d); Py_DECREF(co); return -1; } @@ -1964,20 +2044,116 @@ _imp_init_frozen_impl(PyObject *module, PyObject *name) return import_add_module(tstate, name); } +/*[clinic input] +_imp.find_frozen + + name: unicode + / + * + withdata: bool = False + +Return info about the corresponding frozen module (if there is one) or None. + +The returned info (a 2-tuple): + + * data the raw marshalled bytes + * is_package whether or not it is a package + * origname the originally frozen module's name, or None if not + a stdlib module (this will usually be the same as + the module's current name) +[clinic start generated code]*/ + +static PyObject * +_imp_find_frozen_impl(PyObject *module, PyObject *name, int withdata) +/*[clinic end generated code: output=8c1c3c7f925397a5 input=22a8847c201542fd]*/ +{ + struct frozen_info info; + frozen_status status = find_frozen(name, &info); + if (status == FROZEN_NOT_FOUND || status == FROZEN_DISABLED) { + Py_RETURN_NONE; + } + else if (status == FROZEN_BAD_NAME) { + Py_RETURN_NONE; + } + else if (status != FROZEN_OKAY) { + set_frozen_error(status, name); + return NULL; + } + + PyObject *data = NULL; + if (withdata) { + data = PyMemoryView_FromMemory((char *)info.data, info.size, PyBUF_READ); + if (data == NULL) { + return NULL; + } + } + + PyObject *origname = NULL; + if (info.origname != NULL && info.origname[0] != '\0') { + origname = PyUnicode_FromString(info.origname); + if (origname == NULL) { + Py_DECREF(data); + return NULL; + } + } + + PyObject *result = PyTuple_Pack(3, data ? data : Py_None, + info.is_package ? Py_True : Py_False, + origname ? origname : Py_None); + Py_XDECREF(origname); + Py_XDECREF(data); + return result; +} + /*[clinic input] _imp.get_frozen_object name: unicode + data as dataobj: object = None / Create a code object for a frozen module. [clinic start generated code]*/ static PyObject * -_imp_get_frozen_object_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=2568cc5b7aa0da63 input=ed689bc05358fdbd]*/ -{ - return get_frozen_object(name); +_imp_get_frozen_object_impl(PyObject *module, PyObject *name, + PyObject *dataobj) +/*[clinic end generated code: output=54368a673a35e745 input=034bdb88f6460b7b]*/ +{ + struct frozen_info info = {0}; + Py_buffer buf = {0}; + if (PyObject_CheckBuffer(dataobj)) { + if (PyObject_GetBuffer(dataobj, &buf, PyBUF_READ) != 0) { + return NULL; + } + info.data = (const char *)buf.buf; + info.size = buf.len; + } + else if (dataobj != Py_None) { + _PyArg_BadArgument("get_frozen_object", "argument 2", "bytes", dataobj); + return NULL; + } + else { + frozen_status status = find_frozen(name, &info); + if (status != FROZEN_OKAY) { + set_frozen_error(status, name); + return NULL; + } + } + + if (info.nameobj == NULL) { + info.nameobj = name; + } + if (info.size == 0) { + set_frozen_error(FROZEN_INVALID, name); + return NULL; + } + + PyObject *codeobj = unmarshal_frozen_code(&info); + if (dataobj != Py_None) { + PyBuffer_Release(&buf); + } + return codeobj; } /*[clinic input] @@ -1993,7 +2169,13 @@ static PyObject * _imp_is_frozen_package_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=e70cbdb45784a1c9 input=81b6cdecd080fbb8]*/ { - return is_frozen_package(name); + struct frozen_info info; + frozen_status status = find_frozen(name, &info); + if (status != FROZEN_OKAY && status != FROZEN_EXCLUDED) { + set_frozen_error(status, name); + return NULL; + } + return PyBool_FromLong(info.is_package); } /*[clinic input] @@ -2025,10 +2207,12 @@ static PyObject * _imp_is_frozen_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=01f408f5ec0f2577 input=7301dbca1897d66b]*/ { - const struct _frozen *p; - - p = find_frozen(name); - return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); + struct frozen_info info; + frozen_status status = find_frozen(name, &info); + if (status != FROZEN_OKAY) { + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; } /*[clinic input] @@ -2219,6 +2403,7 @@ static PyMethodDef imp_methods[] = { _IMP_LOCK_HELD_METHODDEF _IMP_ACQUIRE_LOCK_METHODDEF _IMP_RELEASE_LOCK_METHODDEF + _IMP_FIND_FROZEN_METHODDEF _IMP_GET_FROZEN_OBJECT_METHODDEF _IMP_IS_FROZEN_PACKAGE_METHODDEF _IMP_CREATE_BUILTIN_METHODDEF @@ -2289,7 +2474,7 @@ _PyImport_BootstrapImp(PyThreadState *tstate) // Mock a ModuleSpec object just good enough for PyModule_FromDefAndSpec(): // an object with just a name attribute. // - // _imp.__spec__ is overriden by importlib._bootstrap._instal() anyway. + // _imp.__spec__ is overridden by importlib._bootstrap._instal() anyway. PyObject *attrs = Py_BuildValue("{sO}", "name", name); if (attrs == NULL) { goto error; diff --git a/Python/importdl.c b/Python/importdl.c index 1847eba74aef45..6d2554741f9822 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -42,6 +42,9 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { /* Get the short name (substring after last dot) */ name_len = PyUnicode_GetLength(name); + if (name_len < 0) { + return NULL; + } lastdot = PyUnicode_FindChar(name, '.', 0, name_len, -1); if (lastdot < -1) { return NULL; @@ -121,7 +124,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) if (PySys_Audit("import", "OOOOO", name_unicode, path, Py_None, Py_None, Py_None) < 0) { - return NULL; + goto error; } #ifdef MS_WINDOWS @@ -204,7 +207,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) /* don't allow legacy init for non-ASCII module names */ PyErr_Format( PyExc_SystemError, - "initialization of * did not return PyModuleDef", + "initialization of %s did not return PyModuleDef", name_buf); goto error; } diff --git a/Python/initconfig.c b/Python/initconfig.c index 8740cc1cf7a2b3..c916e2f7c0714f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -10,7 +10,9 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "osdefs.h" // DELIM + #include // setlocale() +#include // getenv() #if defined(MS_WINDOWS) || defined(__CYGWIN__) # ifdef HAVE_IO_H # include @@ -98,7 +100,7 @@ static const char usage_3[] = "\ -X no_debug_ranges: disable the inclusion of the tables mapping extra location \n\ information (end line, start column offset and end column offset) to every \n\ instruction in code objects. This is useful when smaller code objects and pyc \n\ - files are desired as well as supressing the extra visual location indicators \n\ + files are desired as well as suppressing the extra visual location indicators \n\ when the interpreter displays tracebacks.\n\ -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ The default is \"on\" (or \"off\" if you are running a local build).\n\ @@ -142,7 +144,7 @@ static const char usage_6[] = "PYTHONNODEBUGRANGES: If this variable is set, it disables the inclusion of the \n" " tables mapping extra location information (end line, start column offset \n" " and end column offset) to every instruction in code objects. This is useful \n" -" when smaller cothe de objects and pyc files are desired as well as supressing the \n" +" when smaller cothe de objects and pyc files are desired as well as suppressing the \n" " extra visual location indicators when the interpreter displays tracebacks.\n"; #if defined(MS_WINDOWS) @@ -249,6 +251,14 @@ _Py_GetGlobalVariablesAsDict(void) #undef SET_ITEM_STR } +char* +Py_GETENV(const char *name) +{ + if (Py_IgnoreEnvironmentFlag) { + return NULL; + } + return getenv(name); +} /* --- PyStatus ----------------------------------------------- */ @@ -587,11 +597,6 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv) /* --- PyConfig ---------------------------------------------- */ -#define DECODE_LOCALE_ERR(NAME, LEN) \ - (((LEN) == -2) \ - ? _PyStatus_ERR("cannot decode " NAME) \ - : _PyStatus_NO_MEMORY()) - #define MAX_HASH_SEED 4294967295UL @@ -674,6 +679,7 @@ PyConfig_Clear(PyConfig *config) _PyWideStringList_Clear(&config->xoptions); _PyWideStringList_Clear(&config->module_search_paths); config->module_search_paths_set = 0; + CLEAR(config->stdlib_dir); CLEAR(config->executable); CLEAR(config->base_executable); @@ -733,6 +739,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + config->use_frozen_modules = -1; } @@ -914,6 +921,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTRLIST(xoptions); COPY_WSTRLIST(module_search_paths); COPY_ATTR(module_search_paths_set); + COPY_WSTR_ATTR(stdlib_dir); COPY_WSTR_ATTR(executable); COPY_WSTR_ATTR(base_executable); @@ -1020,6 +1028,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_WSTR(home); SET_ITEM_INT(module_search_paths_set); SET_ITEM_WSTRLIST(module_search_paths); + SET_ITEM_WSTR(stdlib_dir); SET_ITEM_WSTR(executable); SET_ITEM_WSTR(base_executable); SET_ITEM_WSTR(prefix); @@ -1323,6 +1332,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) // Path configuration output GET_UINT(module_search_paths_set); GET_WSTRLIST(module_search_paths); + GET_WSTR_OPT(stdlib_dir); GET_WSTR_OPT(executable); GET_WSTR_OPT(base_executable); GET_WSTR_OPT(prefix); @@ -2081,6 +2091,44 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } +/* Determine if the current build is a "development" build (e.g. running + out of the source tree) or not. + + A return value of -1 indicates that we do not know. + */ +static int +is_dev_env(PyConfig *config) +{ + // This should only ever get called early in runtime initialization, + // before the global path config is written. Otherwise we would + // use Py_GetProgramFullPath() and _Py_GetStdlibDir(). + assert(config != NULL); + + const wchar_t *executable = config->executable; + const wchar_t *stdlib = config->stdlib_dir; + if (executable == NULL || *executable == L'\0' || + stdlib == NULL || *stdlib == L'\0') { + // _PyPathConfig_Calculate() hasn't run yet. + return -1; + } + size_t len = _Py_find_basename(executable); + if (wcscmp(executable + len, L"python") != 0 && + wcscmp(executable + len, L"python.exe") != 0) { + return 0; + } + /* If dirname() is the same for both then it is a dev build. */ + if (len != _Py_find_basename(stdlib)) { + return 0; + } + // We do not bother normalizing the two filenames first since + // for config_init_import() is does the right thing as-is. + if (wcsncmp(stdlib, executable, len) != 0) { + return 0; + } + return 1; +} + + static PyStatus config_init_import(PyConfig *config, int compute_path_config) { @@ -2092,25 +2140,28 @@ config_init_import(PyConfig *config, int compute_path_config) } /* -X frozen_modules=[on|off] */ - const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); - if (value == NULL) { - // For now we always default to "off". - // In the near future we will be factoring in PGO and in-development. - config->use_frozen_modules = 0; - } - else if (wcscmp(value, L"on") == 0) { - config->use_frozen_modules = 1; - } - else if (wcscmp(value, L"off") == 0) { - config->use_frozen_modules = 0; - } - else if (wcslen(value) == 0) { - // "-X frozen_modules" and "-X frozen_modules=" both imply "on". - config->use_frozen_modules = 1; - } - else { - return PyStatus_Error("bad value for option -X frozen_modules " - "(expected \"on\" or \"off\")"); + if (config->use_frozen_modules < 0) { + const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); + if (value == NULL) { + int isdev = is_dev_env(config); + if (isdev >= 0) { + config->use_frozen_modules = !isdev; + } + } + else if (wcscmp(value, L"on") == 0) { + config->use_frozen_modules = 1; + } + else if (wcscmp(value, L"off") == 0) { + config->use_frozen_modules = 0; + } + else if (wcslen(value) == 0) { + // "-X frozen_modules" and "-X frozen_modules=" both imply "on". + config->use_frozen_modules = 1; + } + else { + return PyStatus_Error("bad value for option -X frozen_modules " + "(expected \"on\" or \"off\")"); + } } return _PyStatus_OK(); @@ -2122,6 +2173,49 @@ _PyConfig_InitImportConfig(PyConfig *config) return config_init_import(config, 1); } +// List of known xoptions to validate against the provided ones. Note that all +// options are listed, even if they are only available if a specific macro is +// set, like -X showrefcount which requires a debug build. In this case unknown +// options are silently ignored. +const wchar_t* known_xoptions[] = { + L"faulthandler", + L"showrefcount", + L"tracemalloc", + L"importtime", + L"dev", + L"utf8", + L"pycache_prefix", + L"warn_default_encoding", + L"no_debug_ranges", + L"frozen_modules", + NULL, +}; + +static const wchar_t* +_Py_check_xoptions(const PyWideStringList *xoptions, const wchar_t **names) +{ + for (Py_ssize_t i=0; i < xoptions->length; i++) { + const wchar_t *option = xoptions->items[i]; + size_t len; + wchar_t *sep = wcschr(option, L'='); + if (sep != NULL) { + len = (sep - option); + } + else { + len = wcslen(option); + } + int found = 0; + for (const wchar_t** name = names; *name != NULL; name++) { + if (wcsncmp(option, *name, len) == 0 && (*name)[len] == L'\0') { + found = 1; + } + } + if (found == 0) { + return option; + } + } + return NULL; +} static PyStatus config_read(PyConfig *config, int compute_path_config) @@ -2137,6 +2231,11 @@ config_read(PyConfig *config, int compute_path_config) } /* -X options */ + const wchar_t* option = _Py_check_xoptions(&config->xoptions, known_xoptions); + if (option != NULL) { + return PyStatus_Error("Unknown value for option -X"); + } + if (config_get_xoption(config, L"showrefcount")) { config->show_ref_count = 1; } @@ -2550,7 +2649,7 @@ warnoptions_append(PyConfig *config, PyWideStringList *options, { /* config_init_warnoptions() add existing config warnoptions at the end: ensure that the new option is not already present in this list to - prevent change the options order whne config_init_warnoptions() is + prevent change the options order when config_init_warnoptions() is called twice. */ if (_PyWideStringList_Find(&config->warnoptions, option)) { /* Already present: do nothing */ @@ -3099,6 +3198,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" environment = %i\n", config->use_environment); PySys_WriteStderr(" user site = %i\n", config->user_site_directory); PySys_WriteStderr(" import site = %i\n", config->site_import); + DUMP_CONFIG("stdlib dir", stdlib_dir); #undef DUMP_CONFIG #define DUMP_SYS(NAME) \ diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 189d72a8c84af3..3bf2e35ccb6dab 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -32,6 +32,7 @@ def write_contents(f): """ opcode = find_module('opcode') targets = ['_unknown_opcode'] * 256 + targets[255] = "TARGET_DO_TRACING" for opname, op in opcode.opmap.items(): targets[op] = "TARGET_%s" % opname next_op = 1 diff --git a/Python/marshal.c b/Python/marshal.c index 346384edea6180..51c77555d9ea97 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -9,11 +9,12 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "longintrepr.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_code.h" // _PyCode_New() +#include "pycore_floatobject.h" // _PyFloat_Pack8() +#include "pycore_hashtable.h" // _Py_hashtable_t #include "code.h" -#include "marshal.h" -#include "pycore_hashtable.h" -#include "pycore_code.h" // _PyCode_New() +#include "marshal.h" // Py_MARSHAL_VERSION /*[clinic input] module marshal @@ -1292,7 +1293,7 @@ r_object(RFILE *p) if (n == 0 && type == TYPE_FROZENSET) { /* call frozenset() to get the empty frozenset singleton */ - v = _PyObject_CallNoArg((PyObject*)&PyFrozenSet_Type); + v = _PyObject_CallNoArgs((PyObject*)&PyFrozenSet_Type); if (v == NULL) break; R_REF(v); diff --git a/Python/mystrtoul.c b/Python/mystrtoul.c index 19fa57aa144e2a..e6fe154eed611a 100644 --- a/Python/mystrtoul.c +++ b/Python/mystrtoul.c @@ -1,5 +1,5 @@ - #include "Python.h" +#include "pycore_long.h" // _PyLong_DigitValue #if defined(__sgi) && !defined(_SGI_MP_SOURCE) #define _SGI_MP_SOURCE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index f3bfae545bcd48..5c7d3ad544e563 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -20,7 +20,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_ADD_UNICODE_INPLACE_FAST, &&TARGET_BINARY_POWER, &&TARGET_BINARY_MULTIPLY, - &&TARGET_BINARY_SUBSCR_ADAPTIVE, + &&TARGET_BINARY_MULTIPLY_ADAPTIVE, &&TARGET_BINARY_MODULO, &&TARGET_BINARY_ADD, &&TARGET_BINARY_SUBTRACT, @@ -35,19 +35,19 @@ static void *opcode_targets[256] = { &&TARGET_MATCH_KEYS, &&TARGET_COPY_DICT_WITHOUT_KEYS, &&TARGET_PUSH_EXC_INFO, - &&TARGET_BINARY_SUBSCR_LIST_INT, + &&TARGET_BINARY_MULTIPLY_INT, &&TARGET_POP_EXCEPT_AND_RERAISE, + &&TARGET_BINARY_MULTIPLY_FLOAT, + &&TARGET_BINARY_SUBSCR_ADAPTIVE, + &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_BINARY_SUBSCR_DICT, - &&TARGET_JUMP_ABSOLUTE_QUICK, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ATTR_SPLIT_KEYS, - &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_CALL_FUNCTION_ADAPTIVE, + &&TARGET_CALL_FUNCTION_BUILTIN_O, + &&TARGET_CALL_FUNCTION_BUILTIN_FAST, + &&TARGET_CALL_FUNCTION_LEN, + &&TARGET_CALL_FUNCTION_ISINSTANCE, + &&TARGET_CALL_FUNCTION_PY_SIMPLE, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -57,7 +57,7 @@ static void *opcode_targets[256] = { &&TARGET_INPLACE_ADD, &&TARGET_INPLACE_SUBTRACT, &&TARGET_INPLACE_MULTIPLY, - &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_JUMP_ABSOLUTE_QUICK, &&TARGET_INPLACE_MODULO, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, @@ -79,15 +79,15 @@ static void *opcode_targets[256] = { &&TARGET_INPLACE_AND, &&TARGET_INPLACE_XOR, &&TARGET_INPLACE_OR, - &&TARGET_LOAD_METHOD_CACHED, - &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, &&TARGET_YIELD_VALUE, - &&TARGET_LOAD_METHOD_MODULE, - &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -119,46 +119,46 @@ static void *opcode_targets[256] = { &&TARGET_IS_OP, &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, - &&TARGET_STORE_ATTR_SPLIT_KEYS, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_JUMP_IF_NOT_EXC_MATCH, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_LOAD_GLOBAL_ADAPTIVE, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_GEN_START, &&TARGET_RAISE_VARARGS, &&TARGET_CALL_FUNCTION, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_METHOD_CACHED, &&TARGET_MAKE_CELL, &&TARGET_LOAD_CLOSURE, &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_METHOD_CLASS, &&TARGET_CALL_FUNCTION_KW, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_LOAD_METHOD_MODULE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_LOAD_METHOD_NO_DICT, + &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_MATCH_CLASS, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_METHOD, &&TARGET_CALL_METHOD, &&TARGET_LIST_EXTEND, @@ -166,6 +166,9 @@ static void *opcode_targets[256] = { &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_CALL_METHOD_KW, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -251,8 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode + &&TARGET_DO_TRACING }; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 470aba75bea969..ad22222e000fc0 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -54,6 +54,7 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->program_full_path); CLEAR(config->prefix); CLEAR(config->exec_prefix); + CLEAR(config->stdlib_dir); CLEAR(config->module_search_path); CLEAR(config->program_name); CLEAR(config->home); @@ -83,6 +84,7 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2) COPY_ATTR(prefix); COPY_ATTR(exec_prefix); COPY_ATTR(module_search_path); + COPY_ATTR(stdlib_dir); COPY_ATTR(program_name); COPY_ATTR(home); #ifdef MS_WINDOWS @@ -167,6 +169,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) COPY_CONFIG(program_full_path, executable); COPY_CONFIG(prefix, prefix); COPY_CONFIG(exec_prefix, exec_prefix); + COPY_CONFIG(stdlib_dir, stdlib_dir); COPY_CONFIG(program_name, program_name); COPY_CONFIG(home, home); #ifdef MS_WINDOWS @@ -218,6 +221,7 @@ _PyPathConfig_AsDict(void) SET_ITEM_STR(prefix); SET_ITEM_STR(exec_prefix); SET_ITEM_STR(module_search_path); + SET_ITEM_STR(stdlib_dir); SET_ITEM_STR(program_name); SET_ITEM_STR(home); #ifdef MS_WINDOWS @@ -311,6 +315,7 @@ config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig) - exec_prefix - module_search_path + - stdlib_dir - prefix - program_full_path @@ -401,11 +406,12 @@ config_init_pathconfig(PyConfig *config, int compute_path_config) COPY_ATTR(program_full_path, executable); COPY_ATTR(prefix, prefix); COPY_ATTR(exec_prefix, exec_prefix); + COPY_ATTR(stdlib_dir, stdlib_dir); #undef COPY_ATTR #ifdef MS_WINDOWS - /* If a ._pth file is found: isolated and site_import are overriden */ + /* If a ._pth file is found: isolated and site_import are overridden */ if (pathconfig.isolated != -1) { config->isolated = pathconfig.isolated; } @@ -486,16 +492,25 @@ Py_SetPath(const wchar_t *path) PyMem_RawFree(_Py_path_config.prefix); PyMem_RawFree(_Py_path_config.exec_prefix); + PyMem_RawFree(_Py_path_config.stdlib_dir); PyMem_RawFree(_Py_path_config.module_search_path); _Py_path_config.prefix = _PyMem_RawWcsdup(L""); _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L""); + // XXX Copy this from the new module_search_path? + if (_Py_path_config.home != NULL) { + _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(_Py_path_config.home); + } + else { + _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L""); + } _Py_path_config.module_search_path = _PyMem_RawWcsdup(path); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); if (_Py_path_config.prefix == NULL || _Py_path_config.exec_prefix == NULL + || _Py_path_config.stdlib_dir == NULL || _Py_path_config.module_search_path == NULL) { path_out_of_memory(__func__); @@ -572,6 +587,17 @@ Py_GetPath(void) } +wchar_t * +_Py_GetStdlibDir(void) +{ + wchar_t *stdlib_dir = _Py_path_config.stdlib_dir; + if (stdlib_dir != NULL && stdlib_dir[0] != L'\0') { + return stdlib_dir; + } + return NULL; +} + + wchar_t * Py_GetPrefix(void) { diff --git a/Python/preconfig.c b/Python/preconfig.c index ae1cc3f90fca7f..afa16cccf32e96 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -1,15 +1,12 @@ #include "Python.h" +#include "pycore_fileutils.h" // DECODE_LOCALE_ERR #include "pycore_getopt.h" // _PyOS_GetOpt() #include "pycore_initconfig.h" // _PyArgv #include "pycore_pymem.h" // _PyMem_GetAllocatorName() #include "pycore_runtime.h" // _PyRuntime_Initialize() -#include // setlocale() - -#define DECODE_LOCALE_ERR(NAME, LEN) \ - (((LEN) == -2) \ - ? _PyStatus_ERR("cannot decode " NAME) \ - : _PyStatus_NO_MEMORY()) +#include // setlocale() +#include // getenv() /* Forward declarations */ @@ -87,8 +84,7 @@ _PyArgv_AsWstrList(const _PyArgv *args, PyWideStringList *list) wchar_t *arg = Py_DecodeLocale(args->bytes_argv[i], &len); if (arg == NULL) { _PyWideStringList_Clear(&wargv); - return DECODE_LOCALE_ERR("command line arguments", - (Py_ssize_t)len); + return DECODE_LOCALE_ERR("command line arguments", len); } wargv.items[i] = arg; wargv.length++; diff --git a/Python/pyhash.c b/Python/pyhash.c index f0c82356f1e26c..d5ac9f83be61cc 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -358,19 +358,72 @@ static PyHash_FuncDef PyHash_Func = {fnv, "fnv", 8 * SIZEOF_PY_HASH_T, # define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) #endif -#define HALF_ROUND(a,b,c,d,s,t) \ - a += b; c += d; \ +#define HALF_ROUND(a,b,c,d,s,t) \ + a += b; c += d; \ b = ROTATE(b, s) ^ a; \ d = ROTATE(d, t) ^ c; \ a = ROTATE(a, 32); -#define DOUBLE_ROUND(v0,v1,v2,v3) \ - HALF_ROUND(v0,v1,v2,v3,13,16); \ - HALF_ROUND(v2,v1,v0,v3,17,21); \ - HALF_ROUND(v0,v1,v2,v3,13,16); \ +#define SINGLE_ROUND(v0,v1,v2,v3) \ + HALF_ROUND(v0,v1,v2,v3,13,16); \ HALF_ROUND(v2,v1,v0,v3,17,21); +#define DOUBLE_ROUND(v0,v1,v2,v3) \ + SINGLE_ROUND(v0,v1,v2,v3); \ + SINGLE_ROUND(v0,v1,v2,v3); + +static uint64_t +siphash13(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { + uint64_t b = (uint64_t)src_sz << 56; + const uint8_t *in = (const uint8_t*)src; + + uint64_t v0 = k0 ^ 0x736f6d6570736575ULL; + uint64_t v1 = k1 ^ 0x646f72616e646f6dULL; + uint64_t v2 = k0 ^ 0x6c7967656e657261ULL; + uint64_t v3 = k1 ^ 0x7465646279746573ULL; + + uint64_t t; + uint8_t *pt; + + while (src_sz >= 8) { + uint64_t mi; + memcpy(&mi, in, sizeof(mi)); + mi = _le64toh(mi); + in += sizeof(mi); + src_sz -= sizeof(mi); + v3 ^= mi; + SINGLE_ROUND(v0,v1,v2,v3); + v0 ^= mi; + } + + t = 0; + pt = (uint8_t *)&t; + switch (src_sz) { + case 7: pt[6] = in[6]; /* fall through */ + case 6: pt[5] = in[5]; /* fall through */ + case 5: pt[4] = in[4]; /* fall through */ + case 4: memcpy(pt, in, sizeof(uint32_t)); break; + case 3: pt[2] = in[2]; /* fall through */ + case 2: pt[1] = in[1]; /* fall through */ + case 1: pt[0] = in[0]; /* fall through */ + } + b |= _le64toh(t); + + v3 ^= b; + SINGLE_ROUND(v0,v1,v2,v3); + v0 ^= b; + v2 ^= 0xff; + SINGLE_ROUND(v0,v1,v2,v3); + SINGLE_ROUND(v0,v1,v2,v3); + SINGLE_ROUND(v0,v1,v2,v3); + + /* modified */ + t = (v0 ^ v1) ^ (v2 ^ v3); + return t; +} + +#if Py_HASH_ALGORITHM == Py_HASH_SIPHASH24 static uint64_t siphash24(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { uint64_t b = (uint64_t)src_sz << 56; @@ -419,14 +472,26 @@ siphash24(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { t = (v0 ^ v1) ^ (v2 ^ v3); return t; } +#endif uint64_t _Py_KeyedHash(uint64_t key, const void *src, Py_ssize_t src_sz) { - return siphash24(key, 0, src, src_sz); + return siphash13(key, 0, src, src_sz); } +#if Py_HASH_ALGORITHM == Py_HASH_SIPHASH13 +static Py_hash_t +pysiphash(const void *src, Py_ssize_t src_sz) { + return (Py_hash_t)siphash13( + _le64toh(_Py_HashSecret.siphash.k0), _le64toh(_Py_HashSecret.siphash.k1), + src, src_sz); +} + +static PyHash_FuncDef PyHash_Func = {pysiphash, "siphash13", 64, 128}; +#endif + #if Py_HASH_ALGORITHM == Py_HASH_SIPHASH24 static Py_hash_t pysiphash(const void *src, Py_ssize_t src_sz) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bfddc1922d25a9..c5a209abae61a3 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -16,6 +16,7 @@ #include "pycore_traceback.h" // _Py_DumpTracebackThreads() #include // setlocale() +#include // getenv() #if defined(__APPLE__) #include @@ -472,7 +473,7 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) int _PyInterpreterState_SetConfig(const PyConfig *src_config) { - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); int res = -1; PyConfig config; diff --git a/Python/pymath.c b/Python/pymath.c index 24b804223eef19..e7d0161ff94c2e 100644 --- a/Python/pymath.c +++ b/Python/pymath.c @@ -1,22 +1,9 @@ #include "Python.h" -#ifdef X87_DOUBLE_ROUNDING -/* On x86 platforms using an x87 FPU, this function is called from the - Py_FORCE_DOUBLE macro (defined in pymath.h) to force a floating-point - number out of an 80-bit x87 FPU register and into a 64-bit memory location, - thus rounding from extended precision to double precision. */ -double _Py_force_double(double x) -{ - volatile double y; - y = x; - return y; -} -#endif #ifdef HAVE_GCC_ASM_FOR_X87 - -/* inline assembly for getting and setting the 387 FPU control word on - gcc/x86 */ +// Inline assembly for getting and setting the 387 FPU control word on +// GCC/x86. #ifdef _Py_MEMORY_SANITIZER __attribute__((no_sanitize_memory)) #endif @@ -29,53 +16,4 @@ unsigned short _Py_get_387controlword(void) { void _Py_set_387controlword(unsigned short cw) { __asm__ __volatile__ ("fldcw %0" : : "m" (cw)); } - -#endif - - -#ifndef HAVE_HYPOT -double hypot(double x, double y) -{ - double yx; - - x = fabs(x); - y = fabs(y); - if (x < y) { - double temp = x; - x = y; - y = temp; - } - if (x == 0.) - return 0.; - else { - yx = y/x; - return x*sqrt(1.+yx*yx); - } -} -#endif /* HAVE_HYPOT */ - -#ifndef HAVE_COPYSIGN -double -copysign(double x, double y) -{ - /* use atan2 to distinguish -0. from 0. */ - if (y > 0. || (y == 0. && atan2(y, -1.) > 0.)) { - return fabs(x); - } else { - return -fabs(x); - } -} -#endif /* HAVE_COPYSIGN */ - -#ifndef HAVE_ROUND -double -round(double x) -{ - double absx, y; - absx = fabs(x); - y = floor(absx); - if (absx - y >= 0.5) - y += 1.0; - return copysign(y, x); -} -#endif /* HAVE_ROUND */ +#endif // HAVE_GCC_ASM_FOR_X87 diff --git a/Python/pystate.c b/Python/pystate.c index 45272142e41734..7804e17a064e15 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -230,7 +230,7 @@ PyInterpreterState_New(void) PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - interp->eval_frame = _PyEval_EvalFrameDefault; + interp->eval_frame = NULL; #ifdef HAVE_DLOPEN #if HAVE_DECL_RTLD_NOW interp->dlopenflags = RTLD_NOW; @@ -1201,6 +1201,22 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) } +void +PyThreadState_EnterTracing(PyThreadState *tstate) +{ + tstate->tracing++; + _PyThreadState_PauseTracing(tstate); +} + +void +PyThreadState_LeaveTracing(PyThreadState *tstate) +{ + tstate->tracing--; + _PyThreadState_ResumeTracing(tstate); +} + + + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ @@ -1671,8 +1687,11 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { - // PyThreadState_Get() aborts if tstate is NULL. - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + // The caller must hold the GIL + _Py_EnsureTstateNotNULL(tstate); +#endif PyInterpreterState *interp = tstate->interp; // Reset data before re-populating. @@ -1973,6 +1992,9 @@ _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) { + if (interp->eval_frame == NULL) { + return _PyEval_EvalFrameDefault; + } return interp->eval_frame; } @@ -1981,7 +2003,12 @@ void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) { - interp->eval_frame = eval_frame; + if (eval_frame == _PyEval_EvalFrameDefault) { + interp->eval_frame = NULL; + } + else { + interp->eval_frame = eval_frame; + } } diff --git a/Python/pystrhex.c b/Python/pystrhex.c index b74e57ad913b11..880af44ea0e94f 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -1,8 +1,8 @@ -/* bytes to hex implementation */ +/* Format bytes as hexadecimal */ #include "Python.h" - -#include "pystrhex.h" +#include "pycore_strhex.h" // _Py_strhex_with_sep() +#include // abs() static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, const PyObject* sep, int bytes_per_sep_group, diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 9145d4eba121e1..ab5814de21ba5c 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -2,6 +2,7 @@ #include #include "pycore_dtoa.h" +#include "pycore_pymath.h" // _Py_SET_53BIT_PRECISION_START #include /* Case-insensitive string match used for nan and inf detection; t should be diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 0e0262c0e8c69f..6cecef97932288 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -2,7 +2,7 @@ /* Top level execution of Python code (including in __main__) */ /* To help control the interfaces between the startup, execution and - * shutdown code, the phases are split across separate modules (boostrap, + * shutdown code, the phases are split across separate modules (bootstrap, * pythonrun, shutdown) */ @@ -943,7 +943,7 @@ print_exception(PyObject *f, PyObject *value) if (end_lineno > lineno) { end_offset = (error_line != NULL) ? line_size : -1; } - // Limit the ammount of '^' that we can display to + // Limit the amount of '^' that we can display to // the size of the text in the source line. if (error_line != NULL && end_offset > line_size + 1) { end_offset = line_size + 1; diff --git a/Python/pytime.c b/Python/pytime.c index b47a573488b6dc..9653662b0fb147 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,10 +1,11 @@ #include "Python.h" +#include "pycore_pymath.h" // _Py_InIntegralTypeRange() #ifdef MS_WINDOWS -#include /* struct timeval */ +# include // struct timeval #endif #if defined(__APPLE__) -#include /* mach_absolute_time(), mach_timebase_info() */ +# include // mach_absolute_time(), mach_timebase_info() #if defined(__APPLE__) && defined(__has_builtin) # if __has_builtin(__builtin_available) @@ -13,11 +14,6 @@ #endif #endif -#define _PyTime_check_mul_overflow(a, b) \ - (assert(b > 0), \ - (_PyTime_t)(a) < _PyTime_MIN / (_PyTime_t)(b) \ - || _PyTime_MAX / (_PyTime_t)(b) < (_PyTime_t)(a)) - /* To millisecond (10^-3) */ #define SEC_TO_MS 1000 @@ -35,6 +31,16 @@ #define NS_TO_US (1000) #define NS_TO_100NS (100) +#if SIZEOF_TIME_T == SIZEOF_LONG_LONG +# define PY_TIME_T_MAX LLONG_MAX +# define PY_TIME_T_MIN LLONG_MIN +#elif SIZEOF_TIME_T == SIZEOF_LONG +# define PY_TIME_T_MAX LONG_MAX +# define PY_TIME_T_MIN LONG_MIN +#else +# error "unsupported time_t size" +#endif + static void pytime_time_t_overflow(void) @@ -63,26 +69,92 @@ pytime_from_nanoseconds(_PyTime_t t) static inline _PyTime_t pytime_as_nanoseconds(_PyTime_t t) { - // _PyTime_t is a number of nanoseconds + // _PyTime_t is a number of nanoseconds: see pytime_from_nanoseconds() + return t; +} + + +// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +static inline int +pytime_add(_PyTime_t *t1, _PyTime_t t2) +{ + if (t2 > 0 && *t1 > _PyTime_MAX - t2) { + *t1 = _PyTime_MAX; + return -1; + } + else if (t2 < 0 && *t1 < _PyTime_MIN - t2) { + *t1 = _PyTime_MIN; + return -1; + } + else { + *t1 += t2; + return 0; + } +} + + +_PyTime_t +_PyTime_Add(_PyTime_t t1, _PyTime_t t2) +{ + (void)pytime_add(&t1, t2); + return t1; +} + + +static inline int +pytime_mul_check_overflow(_PyTime_t a, _PyTime_t b) +{ + if (b != 0) { + assert(b > 0); + return ((a < _PyTime_MIN / b) || (_PyTime_MAX / b < a)); + } + else { + return 0; + } +} + + +// Compute t * k. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +static inline int +pytime_mul(_PyTime_t *t, _PyTime_t k) +{ + assert(k >= 0); + if (pytime_mul_check_overflow(*t, k)) { + *t = (*t >= 0) ? _PyTime_MAX : _PyTime_MIN; + return -1; + } + else { + *t *= k; + return 0; + } +} + + +// Compute t * k. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +static inline _PyTime_t +_PyTime_Mul(_PyTime_t t, _PyTime_t k) +{ + (void)pytime_mul(&t, k); return t; } + + _PyTime_t _PyTime_MulDiv(_PyTime_t ticks, _PyTime_t mul, _PyTime_t div) { - _PyTime_t intpart, remaining; - /* Compute (ticks * mul / div) in two parts to prevent integer overflow: - compute integer part, and then the remaining part. + /* Compute (ticks * mul / div) in two parts to reduce the risk of integer + overflow: compute the integer part, and then the remaining part. (ticks * mul) / div == (ticks / div) * mul + (ticks % div) * mul / div - - The caller must ensure that "(div - 1) * mul" cannot overflow. */ + */ + _PyTime_t intpart, remaining; intpart = ticks / div; ticks %= div; - remaining = ticks * mul; - remaining /= div; - return intpart * mul + remaining; + remaining = _PyTime_Mul(ticks, mul) / div; + // intpart * mul + remaining + return _PyTime_Add(_PyTime_Mul(intpart, mul), remaining); } @@ -119,6 +191,48 @@ _PyLong_FromTime_t(time_t t) } +// Convert _PyTime_t to time_t. +// Return 0 on success. Return -1 and clamp the value on overflow. +static int +_PyTime_AsTime_t(_PyTime_t t, time_t *t2) +{ +#if SIZEOF_TIME_T < _SIZEOF_PYTIME_T + if ((_PyTime_t)PY_TIME_T_MAX < t) { + *t2 = PY_TIME_T_MAX; + return -1; + } + if (t < (_PyTime_t)PY_TIME_T_MIN) { + *t2 = PY_TIME_T_MIN; + return -1; + } +#endif + *t2 = (time_t)t; + return 0; +} + + +#ifdef MS_WINDOWS +// Convert _PyTime_t to long. +// Return 0 on success. Return -1 and clamp the value on overflow. +static int +_PyTime_AsLong(_PyTime_t t, long *t2) +{ +#if SIZEOF_LONG < _SIZEOF_PYTIME_T + if ((_PyTime_t)LONG_MAX < t) { + *t2 = LONG_MAX; + return -1; + } + if (t < (_PyTime_t)LONG_MIN) { + *t2 = LONG_MIN; + return -1; + } +#endif + *t2 = (long)t; + return 0; +} +#endif + + /* Round to nearest with ties going to nearest even integer (_PyTime_ROUND_HALF_EVEN) */ static double @@ -319,41 +433,25 @@ _PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) #ifdef HAVE_CLOCK_GETTIME static int -pytime_fromtimespec(_PyTime_t *tp, struct timespec *ts, int raise) +pytime_fromtimespec(_PyTime_t *tp, struct timespec *ts, int raise_exc) { _PyTime_t t, tv_nsec; - int res = 0; Py_BUILD_ASSERT(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); t = (_PyTime_t)ts->tv_sec; - if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { - if (raise) { - pytime_overflow(); - res = -1; - } - t = (t > 0) ? _PyTime_MAX : _PyTime_MIN; - } - else { - t = t * SEC_TO_NS; - } + int res1 = pytime_mul(&t, SEC_TO_NS); tv_nsec = ts->tv_nsec; - /* The following test is written for positive only tv_nsec */ - assert(tv_nsec >= 0); - if (t > _PyTime_MAX - tv_nsec) { - if (raise) { - pytime_overflow(); - res = -1; - } - t = _PyTime_MAX; - } - else { - t += tv_nsec; - } + int res2 = pytime_add(&t, tv_nsec); *tp = pytime_from_nanoseconds(t); - return res; + + if (raise_exc && (res1 < 0 || res2 < 0)) { + pytime_overflow(); + return -1; + } + return 0; } int @@ -364,43 +462,25 @@ _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts) #endif -#if !defined(MS_WINDOWS) +#ifndef MS_WINDOWS static int -pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise) +pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise_exc) { - _PyTime_t t, usec; - int res = 0; - Py_BUILD_ASSERT(sizeof(tv->tv_sec) <= sizeof(_PyTime_t)); - t = (_PyTime_t)tv->tv_sec; + _PyTime_t t = (_PyTime_t)tv->tv_sec; - if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { - if (raise) { - pytime_overflow(); - res = -1; - } - t = (t > 0) ? _PyTime_MAX : _PyTime_MIN; - } - else { - t = t * SEC_TO_NS; - } + int res1 = pytime_mul(&t, SEC_TO_NS); - usec = (_PyTime_t)tv->tv_usec * US_TO_NS; - /* The following test is written for positive only usec */ - assert(usec >= 0); - if (t > _PyTime_MAX - usec) { - if (raise) { - pytime_overflow(); - res = -1; - } - t = _PyTime_MAX; - } - else { - t += usec; - } + _PyTime_t usec = (_PyTime_t)tv->tv_usec * US_TO_NS; + int res2 = pytime_add(&t, usec); *tp = pytime_from_nanoseconds(t); - return res; + + if (raise_exc && (res1 < 0 || res2 < 0)) { + pytime_overflow(); + return -1; + } + return 0; } @@ -449,7 +529,6 @@ pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, return pytime_from_double(tp, d, round, unit_to_ns); } else { - Py_BUILD_ASSERT(sizeof(long long) <= sizeof(_PyTime_t)); long long sec = PyLong_AsLongLong(obj); if (sec == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) { @@ -458,11 +537,12 @@ pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, return -1; } - if (_PyTime_check_mul_overflow(sec, unit_to_ns)) { + Py_BUILD_ASSERT(sizeof(long long) <= sizeof(_PyTime_t)); + _PyTime_t ns = (_PyTime_t)sec; + if (pytime_mul(&ns, unit_to_ns) < 0) { pytime_overflow(); return -1; } - _PyTime_t ns = sec * unit_to_ns; *tp = pytime_from_nanoseconds(ns); return 0; @@ -514,16 +594,40 @@ _PyTime_AsNanosecondsObject(_PyTime_t t) } +static _PyTime_t +pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) +{ + assert(k > 1); + if (t >= 0) { + // Don't use (t + k - 1) / k to avoid integer overflow + // if t is equal to _PyTime_MAX + _PyTime_t q = t / k; + if (t % k) { + q += 1; + } + return q; + } + else { + // Don't use (t - (k - 1)) / k to avoid integer overflow + // if t is equals to _PyTime_MIN. + _PyTime_t q = t / k; + if (t % k) { + q -= 1; + } + return q; + } +} + + static _PyTime_t pytime_divide(const _PyTime_t t, const _PyTime_t k, const _PyTime_round_t round) { assert(k > 1); if (round == _PyTime_ROUND_HALF_EVEN) { - _PyTime_t x, r, abs_r; - x = t / k; - r = t % k; - abs_r = Py_ABS(r); + _PyTime_t x = t / k; + _PyTime_t r = t % k; + _PyTime_t abs_r = Py_ABS(r); if (abs_r > k / 2 || (abs_r == k / 2 && (Py_ABS(x) & 1))) { if (t >= 0) { x++; @@ -536,7 +640,7 @@ pytime_divide(const _PyTime_t t, const _PyTime_t k, } else if (round == _PyTime_ROUND_CEILING) { if (t >= 0) { - return (t + k - 1) / k; + return pytime_divide_round_up(t, k); } else { return t / k; @@ -547,18 +651,41 @@ pytime_divide(const _PyTime_t t, const _PyTime_t k, return t / k; } else { - return (t - (k - 1)) / k; + return pytime_divide_round_up(t, k); } } else { assert(round == _PyTime_ROUND_UP); - if (t >= 0) { - return (t + k - 1) / k; - } - else { - return (t - (k - 1)) / k; + return pytime_divide_round_up(t, k); + } +} + + +// Compute (t / k, t % k) in (pq, pr). +// Make sure that 0 <= pr < k. +// Return 0 on success. +// Return -1 on underflow and store (_PyTime_MIN, 0) in (pq, pr). +static int +pytime_divmod(const _PyTime_t t, const _PyTime_t k, + _PyTime_t *pq, _PyTime_t *pr) +{ + assert(k > 1); + _PyTime_t q = t / k; + _PyTime_t r = t % k; + if (r < 0) { + if (q == _PyTime_MIN) { + *pq = _PyTime_MIN; + *pr = 0; + return -1; } + r += k; + q -= 1; } + assert(0 <= r && r < k); + + *pq = q; + *pr = r; + return 0; } @@ -596,64 +723,41 @@ _PyTime_AsMilliseconds(_PyTime_t t, _PyTime_round_t round) static int -pytime_as_timeval(_PyTime_t t, _PyTime_t *p_secs, int *p_us, +pytime_as_timeval(_PyTime_t t, _PyTime_t *ptv_sec, int *ptv_usec, _PyTime_round_t round) { - _PyTime_t ns, tv_sec; - ns = pytime_as_nanoseconds(t); - tv_sec = ns / SEC_TO_NS; - ns = ns % SEC_TO_NS; - - int tv_usec = (int)pytime_divide(ns, US_TO_NS, round); - int res = 0; - if (tv_usec < 0) { - tv_usec += SEC_TO_US; - if (tv_sec != _PyTime_MIN) { - tv_sec -= 1; - } - else { - res = -1; - } - } - else if (tv_usec >= SEC_TO_US) { - tv_usec -= SEC_TO_US; - if (tv_sec != _PyTime_MAX) { - tv_sec += 1; - } - else { - res = -1; - } - } - assert(0 <= tv_usec && tv_usec < SEC_TO_US); - - *p_secs = tv_sec; - *p_us = tv_usec; + _PyTime_t ns = pytime_as_nanoseconds(t); + _PyTime_t us = pytime_divide(ns, US_TO_NS, round); + _PyTime_t tv_sec, tv_usec; + int res = pytime_divmod(us, SEC_TO_US, &tv_sec, &tv_usec); + *ptv_sec = tv_sec; + *ptv_usec = (int)tv_usec; return res; } static int pytime_as_timeval_struct(_PyTime_t t, struct timeval *tv, - _PyTime_round_t round, int raise) + _PyTime_round_t round, int raise_exc) { - _PyTime_t secs, secs2; - int us; - int res; - - res = pytime_as_timeval(t, &secs, &us, round); + _PyTime_t tv_sec; + int tv_usec; + int res = pytime_as_timeval(t, &tv_sec, &tv_usec, round); + int res2; #ifdef MS_WINDOWS - tv->tv_sec = (long)secs; + // On Windows, timeval.tv_sec type is long + res2 = _PyTime_AsLong(tv_sec, &tv->tv_sec); #else - tv->tv_sec = secs; + res2 = _PyTime_AsTime_t(tv_sec, &tv->tv_sec); #endif - tv->tv_usec = us; + if (res2 < 0) { + tv_usec = 0; + } + tv->tv_usec = tv_usec; - secs2 = (_PyTime_t)tv->tv_sec; - if (res < 0 || secs2 != secs) { - if (raise) { - pytime_time_t_overflow(); - } + if (raise_exc && (res < 0 || res2 < 0)) { + pytime_time_t_overflow(); return -1; } return 0; @@ -667,10 +771,10 @@ _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) } -int -_PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +void +_PyTime_AsTimeval_clamp(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) { - return pytime_as_timeval_struct(t, tv, round, 0); + (void)pytime_as_timeval_struct(t, tv, round, 0); } @@ -679,11 +783,12 @@ _PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, _PyTime_round_t round) { _PyTime_t secs; - int res = pytime_as_timeval(t, &secs, us, round); - - *p_secs = (time_t)secs; + if (pytime_as_timeval(t, &secs, us, round) < 0) { + pytime_time_t_overflow(); + return -1; + } - if (res < 0 || (_PyTime_t)*p_secs != secs) { + if (_PyTime_AsTime_t(secs, p_secs) < 0) { pytime_time_t_overflow(); return -1; } @@ -692,40 +797,49 @@ _PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) -int -_PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) +static int +pytime_as_timespec(_PyTime_t t, struct timespec *ts, int raise_exc) { + _PyTime_t ns = pytime_as_nanoseconds(t); _PyTime_t tv_sec, tv_nsec; + int res = pytime_divmod(ns, SEC_TO_NS, &tv_sec, &tv_nsec); - _PyTime_t ns = pytime_as_nanoseconds(t); - tv_sec = ns / SEC_TO_NS; - tv_nsec = ns % SEC_TO_NS; - if (tv_nsec < 0) { - tv_nsec += SEC_TO_NS; - tv_sec -= 1; - } - ts->tv_sec = (time_t)tv_sec; - assert(0 <= tv_nsec && tv_nsec < SEC_TO_NS); + int res2 = _PyTime_AsTime_t(tv_sec, &ts->tv_sec); + if (res2 < 0) { + tv_nsec = 0; + } ts->tv_nsec = tv_nsec; - if ((_PyTime_t)ts->tv_sec != tv_sec) { + if (raise_exc && (res < 0 || res2 < 0)) { pytime_time_t_overflow(); return -1; } return 0; } + +void +_PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts) +{ + (void)pytime_as_timespec(t, ts, 0); +} + +int +_PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) +{ + return pytime_as_timespec(t, ts, 1); +} #endif static int -py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) +py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { + assert(info == NULL || raise_exc); + #ifdef MS_WINDOWS FILETIME system_time; ULARGE_INTEGER large; - assert(info == NULL || raise); - GetSystemTimeAsFileTime(&system_time); large.u.LowPart = system_time.dwLowDateTime; large.u.HighPart = system_time.dwHighDateTime; @@ -760,8 +874,6 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) struct timeval tv; #endif - assert(info == NULL || raise); - #ifdef HAVE_CLOCK_GETTIME #ifdef HAVE_CLOCK_GETTIME_RUNTIME @@ -770,12 +882,12 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) err = clock_gettime(CLOCK_REALTIME, &ts); if (err) { - if (raise) { + if (raise_exc) { PyErr_SetFromErrno(PyExc_OSError); } return -1; } - if (pytime_fromtimespec(tp, &ts, raise) < 0) { + if (pytime_fromtimespec(tp, &ts, raise_exc) < 0) { return -1; } @@ -804,12 +916,12 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) /* test gettimeofday() */ err = gettimeofday(&tv, (struct timezone *)NULL); if (err) { - if (raise) { + if (raise_exc) { PyErr_SetFromErrno(PyExc_OSError); } return -1; } - if (pytime_fromtimeval(tp, &tv, raise) < 0) { + if (pytime_fromtimeval(tp, &tv, raise_exc) < 0) { return -1; } @@ -901,28 +1013,21 @@ py_mach_timebase_info(_PyTime_t *pnumer, _PyTime_t *pdenom, int raise) static int -py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) +py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { -#if defined(MS_WINDOWS) - ULONGLONG ticks; - _PyTime_t t; + assert(info == NULL || raise_exc); - assert(info == NULL || raise); - - ticks = GetTickCount64(); +#if defined(MS_WINDOWS) + ULONGLONG ticks = GetTickCount64(); Py_BUILD_ASSERT(sizeof(ticks) <= sizeof(_PyTime_t)); - t = (_PyTime_t)ticks; + _PyTime_t t = (_PyTime_t)ticks; - if (_PyTime_check_mul_overflow(t, MS_TO_NS)) { - if (raise) { - pytime_overflow(); - return -1; - } - // Truncate to _PyTime_MAX silently. - *tp = _PyTime_MAX; - } - else { - *tp = t * MS_TO_NS; + int res = pytime_mul(&t, MS_TO_NS); + *tp = t; + + if (raise_exc && res < 0) { + pytime_overflow(); + return -1; } if (info) { @@ -944,7 +1049,7 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) static _PyTime_t timebase_numer = 0; static _PyTime_t timebase_denom = 0; if (timebase_denom == 0) { - if (py_mach_timebase_info(&timebase_numer, &timebase_denom, raise) < 0) { + if (py_mach_timebase_info(&timebase_numer, &timebase_denom, raise_exc) < 0) { return -1; } } @@ -969,7 +1074,7 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) time = gethrtime(); if (time == -1) { - if (raise) { + if (raise_exc) { PyErr_SetFromErrno(PyExc_OSError); } return -1; @@ -985,7 +1090,7 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) } #else - struct timespec ts; + #ifdef CLOCK_HIGHRES const clockid_t clk_id = CLOCK_HIGHRES; const char *implementation = "clock_gettime(CLOCK_HIGHRES)"; @@ -994,30 +1099,30 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise) const char *implementation = "clock_gettime(CLOCK_MONOTONIC)"; #endif - assert(info == NULL || raise); - + struct timespec ts; if (clock_gettime(clk_id, &ts) != 0) { - if (raise) { + if (raise_exc) { PyErr_SetFromErrno(PyExc_OSError); return -1; } return -1; } + if (pytime_fromtimespec(tp, &ts, raise_exc) < 0) { + return -1; + } + if (info) { - struct timespec res; info->monotonic = 1; info->implementation = implementation; info->adjustable = 0; + struct timespec res; if (clock_getres(clk_id, &res) != 0) { PyErr_SetFromErrno(PyExc_OSError); return -1; } info->resolution = res.tv_sec + res.tv_nsec * 1e-9; } - if (pytime_fromtimespec(tp, &ts, raise) < 0) { - return -1; - } #endif return 0; } @@ -1081,11 +1186,13 @@ py_win_perf_counter_frequency(LONGLONG *pfrequency, int raise) static int -py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise) +py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { + assert(info == NULL || raise_exc); + static LONGLONG frequency = 0; if (frequency == 0) { - if (py_win_perf_counter_frequency(&frequency, raise) < 0) { + if (py_win_perf_counter_frequency(&frequency, raise_exc) < 0) { return -1; } } @@ -1111,7 +1218,7 @@ py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise) *tp = pytime_from_nanoseconds(ns); return 0; } -#endif +#endif // MS_WINDOWS int @@ -1209,3 +1316,19 @@ _PyTime_gmtime(time_t t, struct tm *tm) return 0; #endif /* MS_WINDOWS */ } + + +_PyTime_t +_PyDeadline_Init(_PyTime_t timeout) +{ + _PyTime_t now = _PyTime_GetMonotonicClock(); + return _PyTime_Add(now, timeout); +} + + +_PyTime_t +_PyDeadline_Get(_PyTime_t deadline) +{ + _PyTime_t now = _PyTime_GetMonotonicClock(); + return deadline - now; +} diff --git a/Python/specialize.c b/Python/specialize.c index 1ab79bf3ea0c5d..162728314e1003 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1,12 +1,14 @@ - #include "Python.h" #include "pycore_code.h" #include "pycore_dict.h" #include "pycore_long.h" #include "pycore_moduleobject.h" +#include "pycore_object.h" #include "opcode.h" #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX +#include // rand() + /* For guidance on adding or extending families of instructions see * ./adaptive.md */ @@ -123,8 +125,10 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); err += add_stat_dict(stats, LOAD_METHOD, "load_method"); err += add_stat_dict(stats, BINARY_ADD, "binary_add"); + err += add_stat_dict(stats, BINARY_MULTIPLY, "binary_multiply"); err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); + err += add_stat_dict(stats, CALL_FUNCTION, "call_function"); if (err < 0) { Py_DECREF(stats); return NULL; @@ -179,8 +183,10 @@ _Py_PrintSpecializationStats(void) print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global"); print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method"); print_stats(out, &_specialization_stats[BINARY_ADD], "binary_add"); + print_stats(out, &_specialization_stats[BINARY_MULTIPLY], "binary_multiply"); print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr"); print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr"); + print_stats(out, &_specialization_stats[CALL_FUNCTION], "call_function"); if (out != stderr) { fclose(out); } @@ -229,7 +235,9 @@ static uint8_t adaptive_opcodes[256] = { [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE, [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE, [BINARY_ADD] = BINARY_ADD_ADAPTIVE, + [BINARY_MULTIPLY] = BINARY_MULTIPLY_ADAPTIVE, [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, + [CALL_FUNCTION] = CALL_FUNCTION_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, }; @@ -239,7 +247,9 @@ static uint8_t cache_requirements[256] = { [LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */ [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */ [BINARY_ADD] = 0, + [BINARY_MULTIPLY] = 0, [BINARY_SUBSCR] = 0, + [CALL_FUNCTION] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */ }; @@ -402,6 +412,18 @@ _Py_Quicken(PyCodeObject *code) { return 0; } +static inline int +initial_counter_value(void) { + /* Starting value for the counter. + * This value needs to be not too low, otherwise + * it would cause excessive de-optimization. + * Neither should it be too high, or that would delay + * de-optimization excessively when it is needed. + * A value around 50 seems to work, and we choose a + * prime number to avoid artifacts. + */ + return 53; +} /* Common */ @@ -451,6 +473,20 @@ _Py_Quicken(PyCodeObject *code) { #define SPEC_FAIL_NON_FUNCTION_SCOPE 11 #define SPEC_FAIL_DIFFERENT_TYPES 12 +/* Calls */ +#define SPEC_FAIL_GENERATOR 7 +#define SPEC_FAIL_COMPLEX_PARAMETERS 8 +#define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 9 +#define SPEC_FAIL_CO_NOT_OPTIMIZED 10 +/* SPEC_FAIL_METHOD defined as 11 above */ +#define SPEC_FAIL_FREE_VARS 12 +#define SPEC_FAIL_PYCFUNCTION 13 +#define SPEC_FAIL_PYCFUNCTION_WITH_KEYWORDS 14 +#define SPEC_FAIL_PYCFUNCTION_FAST_WITH_KEYWORDS 15 +#define SPEC_FAIL_PYCFUNCTION_NOARGS 16 +#define SPEC_FAIL_BAD_CALL_FLAGS 17 +#define SPEC_FAIL_CLASS 18 + static int specialize_module_load_attr( @@ -462,6 +498,7 @@ specialize_module_load_attr( PyObject *value = NULL; PyObject *getattr; _Py_IDENTIFIER(__getattr__); + assert(owner->ob_type->tp_inline_values_offset == 0); PyDictObject *dict = (PyDictObject *)m->md_dict; if (dict == NULL) { SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT); @@ -584,28 +621,22 @@ specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, DesciptorClassification kind, PyObject *name, _PyAdaptiveEntry *cache0, _PyAttrCache *cache1, - int base_op, int split_op, int hint_op) + int base_op, int values_op, int hint_op) { assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD); - // No desciptor, or non overriding. + // No descriptor, or non overriding. if (type->tp_dictoffset < 0) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } if (type->tp_dictoffset > 0) { PyObject **dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset); - if (*dictptr == NULL || !PyDict_CheckExact(*dictptr)) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); - return 0; - } - // We found an instance with a __dict__. PyDictObject *dict = (PyDictObject *)*dictptr; - PyDictKeysObject *keys = dict->ma_keys; - if ((type->tp_flags & Py_TPFLAGS_HEAPTYPE) - && keys == ((PyHeapTypeObject*)type)->ht_cached_keys - ) { - // Keys are shared + if (type->tp_inline_values_offset && dict == NULL) { + // Virtual dictionary + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(type->tp_inline_values_offset > 0); assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); assert (index != DKIX_ERROR); @@ -613,18 +644,17 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys); - if (keys_version == 0) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_VERSIONS); - return 0; - } - cache1->dk_version_or_hint = keys_version; cache1->tp_version = type->tp_version_tag; cache0->index = (uint16_t)index; - *instr = _Py_MAKECODEUNIT(split_op, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(values_op, _Py_OPARG(*instr)); return 0; } else { + if (dict == NULL || !PyDict_CheckExact(dict)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); + return 0; + } + // We found an instance with a __dict__. PyObject *value = NULL; Py_ssize_t hint = _PyDict_GetItemHint(dict, name, -1, &value); @@ -736,7 +766,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp } int err = specialize_dict_access( owner, instr, type, kind, name, cache0, cache1, - LOAD_ATTR, LOAD_ATTR_SPLIT_KEYS, LOAD_ATTR_WITH_HINT + LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT ); if (err < 0) { return -1; @@ -752,7 +782,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp success: STAT_INC(LOAD_ATTR, specialization_success); assert(!PyErr_Occurred()); - cache0->counter = saturating_start(); + cache0->counter = initial_counter_value(); return 0; } @@ -818,7 +848,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S int err = specialize_dict_access( owner, instr, type, kind, name, cache0, cache1, - STORE_ATTR, STORE_ATTR_SPLIT_KEYS, STORE_ATTR_WITH_HINT + STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT ); if (err < 0) { return -1; @@ -834,7 +864,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S success: STAT_INC(STORE_ATTR, specialization_success); assert(!PyErr_Occurred()); - cache0->counter = saturating_start(); + cache0->counter = initial_counter_value(); return 0; } @@ -875,6 +905,27 @@ load_method_fail_kind(DesciptorClassification kind) } #endif +static int +specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, + _PyAttrCache *cache1, _PyObjectCache *cache2) +{ + + PyObject *descr = NULL; + DesciptorClassification kind = 0; + kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0); + switch (kind) { + case METHOD: + case NON_DESCRIPTOR: + cache1->tp_version = ((PyTypeObject *)owner)->tp_version_tag; + cache2->obj = descr; + *instr = _Py_MAKECODEUNIT(LOAD_METHOD_CLASS, _Py_OPARG(*instr)); + return 0; + default: + SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind)); + return -1; + } +} + // Please collect stats carefully before and after modifying. A subtle change // can cause a significant drop in cache hits. A possible test is // python.exe -m test_typing test_re test_dis test_zlib. @@ -886,7 +937,6 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, _PyObjectCache *cache2 = &cache[-2].obj; PyTypeObject *owner_cls = Py_TYPE(owner); - PyDictObject *owner_dict = NULL; if (PyModule_CheckExact(owner)) { int err = specialize_module_load_attr(owner, instr, name, cache0, cache1, LOAD_METHOD, LOAD_METHOD_MODULE); @@ -900,9 +950,12 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, return -1; } } - if (Py_TYPE(owner_cls)->tp_dictoffset < 0) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE); - goto fail; + if (PyType_Check(owner)) { + int err = specialize_class_load_method(owner, instr, name, cache1, cache2); + if (err) { + goto fail; + } + goto success; } // Technically this is fine for bound method calls, but it's uncommon and // slightly slower at runtime to get dict. @@ -910,66 +963,45 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - PyObject **owner_dictptr = _PyObject_GetDictPtr(owner); - int owner_has_dict = (owner_dictptr != NULL && *owner_dictptr != NULL); - owner_dict = owner_has_dict ? (PyDictObject *)*owner_dictptr : NULL; - // Make sure dict doesn't get GC-ed halfway. - Py_XINCREF(owner_dict); - // Check for classmethods. - int owner_is_class = PyType_Check(owner); - owner_cls = owner_is_class ? (PyTypeObject *)owner : owner_cls; - - if ((owner_cls->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG) == 0 || - owner_cls->tp_version_tag == 0) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_VERSIONS); - goto fail; - } PyObject *descr = NULL; DesciptorClassification kind = 0; kind = analyze_descriptor(owner_cls, name, &descr, 0); - // Store the version right away, in case it's modified halfway through. - cache1->tp_version = owner_cls->tp_version_tag; - assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); if (kind != METHOD) { SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind)); goto fail; } - // If o.__dict__ changes, the method might be found in o.__dict__ - // instead of old type lookup. So record o.__dict__'s keys. - uint32_t keys_version = UINT32_MAX; - if (owner_has_dict) { - // _PyDictKeys_GetVersionForCurrentState isn't accurate for - // custom dict subclasses at the moment. - if (!PyDict_CheckExact(owner_dict)) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_DICT_SUBCLASS); + if (owner_cls->tp_inline_values_offset) { + PyObject **owner_dictptr = _PyObject_DictPointer(owner); + assert(owner_dictptr); + if (*owner_dictptr) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_IS_ATTR); goto fail; } - assert(PyUnicode_CheckExact(name)); - Py_hash_t hash = PyObject_Hash(name); - if (hash == -1) { - return -1; - } - PyObject *value = NULL; - if (!owner_is_class) { - // Instance methods shouldn't be in o.__dict__. That makes - // it an attribute. - Py_ssize_t ix = _Py_dict_lookup(owner_dict, name, hash, &value); - assert(ix != DKIX_ERROR); - if (ix != DKIX_EMPTY) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_IS_ATTR); - goto fail; - } + PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; + Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); + if (index != DKIX_EMPTY) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_IS_ATTR); + goto fail; } - keys_version = _PyDictKeys_GetVersionForCurrentState(owner_dict->ma_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys); if (keys_version == 0) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } - // Fall through. - } // Else owner is maybe a builtin with no dict, or __slots__. Doesn't matter. - + cache1->dk_version_or_hint = keys_version; + *instr = _Py_MAKECODEUNIT(LOAD_METHOD_CACHED, _Py_OPARG(*instr)); + } + else { + if (owner_cls->tp_dictoffset == 0) { + *instr = _Py_MAKECODEUNIT(LOAD_METHOD_NO_DICT, _Py_OPARG(*instr)); + } + else { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_IS_ATTR); + goto fail; + } + } /* `descr` is borrowed. This is safe for methods (even inherited ones from * super classes!) as long as tp_version_tag is validated for two main reasons: * @@ -984,19 +1016,15 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, * PyType_Modified usages in typeobject.c). The MCACHE has been * working since Python 2.6 and it's battle-tested. */ + cache1->tp_version = owner_cls->tp_version_tag; cache2->obj = descr; - cache1->dk_version_or_hint = keys_version; - *instr = _Py_MAKECODEUNIT(owner_is_class ? LOAD_METHOD_CLASS : - LOAD_METHOD_CACHED, _Py_OPARG(*instr)); // Fall through. success: - Py_XDECREF(owner_dict); STAT_INC(LOAD_METHOD, specialization_success); assert(!PyErr_Occurred()); - cache0->counter = saturating_start(); + cache0->counter = initial_counter_value(); return 0; fail: - Py_XDECREF(owner_dict); STAT_INC(LOAD_METHOD, specialization_failure); assert(!PyErr_Occurred()); cache_backoff(cache0); @@ -1070,7 +1098,7 @@ _Py_Specialize_LoadGlobal( success: STAT_INC(LOAD_GLOBAL, specialization_success); assert(!PyErr_Occurred()); - cache0->counter = saturating_start(); + cache0->counter = initial_counter_value(); return 0; } @@ -1121,7 +1149,7 @@ _Py_Specialize_BinarySubscr( PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - *instr = _Py_MAKECODEUNIT(BINARY_SUBSCR_LIST_INT, saturating_start()); + *instr = _Py_MAKECODEUNIT(BINARY_SUBSCR_LIST_INT, initial_counter_value()); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1130,7 +1158,7 @@ _Py_Specialize_BinarySubscr( } if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { - *instr = _Py_MAKECODEUNIT(BINARY_SUBSCR_TUPLE_INT, saturating_start()); + *instr = _Py_MAKECODEUNIT(BINARY_SUBSCR_TUPLE_INT, initial_counter_value()); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1138,7 +1166,7 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - *instr = _Py_MAKECODEUNIT(BINARY_SUBSCR_DICT, saturating_start()); + *instr = _Py_MAKECODEUNIT(BINARY_SUBSCR_DICT, initial_counter_value()); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1166,19 +1194,19 @@ _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr) if (left_type == &PyUnicode_Type) { int next_opcode = _Py_OPCODE(instr[1]); if (next_opcode == STORE_FAST) { - *instr = _Py_MAKECODEUNIT(BINARY_ADD_UNICODE_INPLACE_FAST, saturating_start()); + *instr = _Py_MAKECODEUNIT(BINARY_ADD_UNICODE_INPLACE_FAST, initial_counter_value()); } else { - *instr = _Py_MAKECODEUNIT(BINARY_ADD_UNICODE, saturating_start()); + *instr = _Py_MAKECODEUNIT(BINARY_ADD_UNICODE, initial_counter_value()); } goto success; } else if (left_type == &PyLong_Type) { - *instr = _Py_MAKECODEUNIT(BINARY_ADD_INT, saturating_start()); + *instr = _Py_MAKECODEUNIT(BINARY_ADD_INT, initial_counter_value()); goto success; } else if (left_type == &PyFloat_Type) { - *instr = _Py_MAKECODEUNIT(BINARY_ADD_FLOAT, saturating_start()); + *instr = _Py_MAKECODEUNIT(BINARY_ADD_FLOAT, initial_counter_value()); goto success; } @@ -1195,3 +1223,228 @@ _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr) assert(!PyErr_Occurred()); return 0; } + +int +_Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr) +{ + if (!Py_IS_TYPE(left, Py_TYPE(right))) { + SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_DIFFERENT_TYPES); + goto fail; + } + if (PyLong_CheckExact(left)) { + *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_INT, initial_counter_value()); + goto success; + } + else if (PyFloat_CheckExact(left)) { + *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_FLOAT, initial_counter_value()); + goto success; + } + else { + SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_OTHER); + } +fail: + STAT_INC(BINARY_MULTIPLY, specialization_failure); + assert(!PyErr_Occurred()); + *instr = _Py_MAKECODEUNIT(_Py_OPCODE(*instr), ADAPTIVE_CACHE_BACKOFF); + return 0; +success: + STAT_INC(BINARY_MULTIPLY, specialization_success); + assert(!PyErr_Occurred()); + return 0; +} + +static int +specialize_class_call( + PyObject *callable, _Py_CODEUNIT *instr, + int nargs, SpecializedCacheEntry *cache) +{ + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_CLASS); + return -1; +} + +static int +specialize_py_call( + PyFunctionObject *func, _Py_CODEUNIT *instr, + int nargs, SpecializedCacheEntry *cache) +{ + _PyCallCache *cache1 = &cache[-1].call; + /* Exclude generator or coroutines for now */ + PyCodeObject *code = (PyCodeObject *)func->func_code; + int flags = code->co_flags; + if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_GENERATOR); + return -1; + } + if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_COMPLEX_PARAMETERS); + return -1; + } + if ((flags & CO_OPTIMIZED) == 0) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_CO_NOT_OPTIMIZED); + return -1; + } + if (code->co_nfreevars) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_FREE_VARS); + return -1; + } + int argcount = code->co_argcount; + int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= argcount); + int min_args = argcount-defcount; + if (nargs > argcount || nargs < min_args) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + assert(nargs <= argcount && nargs >= min_args); + int defstart = nargs - min_args; + int deflen = argcount - nargs; + assert(defstart >= 0 && deflen >= 0); + assert(deflen == 0 || func->func_defaults != NULL); + if (defstart > 0xffff || deflen > 0xffff) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_RANGE); + return -1; + } + int version = _PyFunction_GetVersionForCurrentState(func); + if (version == 0) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_VERSIONS); + return -1; + } + cache1->func_version = version; + cache1->defaults_start = defstart; + cache1->defaults_len = deflen; + *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_PY_SIMPLE, _Py_OPARG(*instr)); + return 0; +} + +#if COLLECT_SPECIALIZATION_STATS_DETAILED +static int +builtin_call_fail_kind(int ml_flags) +{ + switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | + METH_KEYWORDS | METH_METHOD)) { + case METH_VARARGS: + return SPEC_FAIL_PYCFUNCTION; + case METH_VARARGS | METH_KEYWORDS: + return SPEC_FAIL_PYCFUNCTION_WITH_KEYWORDS; + case METH_FASTCALL | METH_KEYWORDS: + return SPEC_FAIL_PYCFUNCTION_FAST_WITH_KEYWORDS; + case METH_NOARGS: + return SPEC_FAIL_PYCFUNCTION_NOARGS; + /* This case should never happen with PyCFunctionObject -- only + PyMethodObject. See zlib.compressobj()'s methods for an example. + */ + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + default: + return SPEC_FAIL_BAD_CALL_FLAGS; + } +} +#endif + +static int +specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, + SpecializedCacheEntry *cache, PyObject *builtins) +{ + _PyObjectCache *cache1 = &cache[-1].obj; + if (PyCFunction_GET_FUNCTION(callable) == NULL) { + return 1; + } + switch (PyCFunction_GET_FLAGS(callable) & + (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | + METH_KEYWORDS | METH_METHOD)) { + case METH_O: { + if (nargs != 1) { + SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_RANGE); + return 1; + } + /* len(o) */ + PyObject *builtin_len = PyDict_GetItemString(builtins, "len"); + if (callable == builtin_len) { + cache1->obj = builtin_len; // borrowed + *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_LEN, + _Py_OPARG(*instr)); + return 0; + } + *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_BUILTIN_O, + _Py_OPARG(*instr)); + return 0; + } + case METH_FASTCALL: { + if (nargs == 2) { + /* isinstance(o1, o2) */ + PyObject *builtin_isinstance = PyDict_GetItemString( + builtins, "isinstance"); + if (callable == builtin_isinstance) { + cache1->obj = builtin_isinstance; // borrowed + *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_ISINSTANCE, + _Py_OPARG(*instr)); + return 0; + } + } + *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_BUILTIN_FAST, + _Py_OPARG(*instr)); + return 0; + } + default: + SPECIALIZATION_FAIL(CALL_FUNCTION, + builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); + return 1; + } +} + +#if COLLECT_SPECIALIZATION_STATS_DETAILED +static int +call_fail_kind(PyObject *callable) +{ + if (PyInstanceMethod_Check(callable)) { + return SPEC_FAIL_METHOD; + } + else if (PyMethod_Check(callable)) { + return SPEC_FAIL_METHOD; + } + // builtin method + else if (PyCMethod_Check(callable)) { + return SPEC_FAIL_METHOD; + } + else if (PyType_Check(callable)) { + return SPEC_FAIL_CLASS; + } + return SPEC_FAIL_OTHER; +} +#endif + +/* TODO: + - Specialize calling classes. +*/ +int +_Py_Specialize_CallFunction( + PyObject *callable, _Py_CODEUNIT *instr, + int nargs, SpecializedCacheEntry *cache, + PyObject *builtins) +{ + int fail; + if (PyCFunction_CheckExact(callable)) { + fail = specialize_c_call(callable, instr, nargs, cache, builtins); + } + else if (PyFunction_Check(callable)) { + fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, cache); + } + else if (PyType_Check(callable)) { + fail = specialize_class_call(callable, instr, nargs, cache); + } + else { + SPECIALIZATION_FAIL(CALL_FUNCTION, call_fail_kind(callable)); + fail = -1; + } + _PyAdaptiveEntry *cache0 = &cache->adaptive; + if (fail) { + STAT_INC(CALL_FUNCTION, specialization_failure); + assert(!PyErr_Occurred()); + cache_backoff(cache0); + } + else { + STAT_INC(CALL_FUNCTION, specialization_success); + assert(!PyErr_Occurred()); + cache0->counter = initial_counter_value(); + } + return 0; +} diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5dfa917e8ffb20..5e663c17c79b85 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -15,21 +15,23 @@ Data members: */ #include "Python.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_RecursionLimitLowerWaterMark() +#include "pycore_code.h" // _Py_QuickenedCount +#include "pycore_frame.h" // InterpreterFrame #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IS_GC() -#include "pycore_code.h" // _Py_QuickenedCount #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pyerrors.h" // _PyErr_Fetch() #include "pycore_pylifecycle.h" // _PyErr_WriteUnraisableDefaultHook() #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_structseq.h" // PyStructSequence_InitType() +#include "pycore_tuple.h" // _PyTuple_FromArray() #include "code.h" #include "frameobject.h" // PyFrame_GetBack() -#include "pycore_frame.h" #include "pydtrace.h" #include "osdefs.h" // DELIM #include "stdlib_module_names.h" // _Py_stdlib_module_names @@ -254,8 +256,7 @@ sys_audit_tstate(PyThreadState *ts, const char *event, } /* Disallow tracing in hooks unless explicitly enabled */ - ts->tracing++; - ts->cframe->use_tracing = 0; + PyThreadState_EnterTracing(ts); while ((hook = PyIter_Next(hooks)) != NULL) { _Py_IDENTIFIER(__cantrace__); PyObject *o; @@ -268,14 +269,12 @@ sys_audit_tstate(PyThreadState *ts, const char *event, break; } if (canTrace) { - ts->cframe->use_tracing = (ts->c_tracefunc || ts->c_profilefunc); - ts->tracing--; + PyThreadState_LeaveTracing(ts); } PyObject* args[2] = {eventName, eventArgs}; o = _PyObject_FastCallTstate(ts, hook, args, 2); if (canTrace) { - ts->tracing++; - ts->cframe->use_tracing = 0; + PyThreadState_EnterTracing(ts); } if (!o) { break; @@ -283,8 +282,7 @@ sys_audit_tstate(PyThreadState *ts, const char *event, Py_DECREF(o); Py_CLEAR(hook); } - ts->cframe->use_tracing = (ts->c_tracefunc || ts->c_profilefunc); - ts->tracing--; + PyThreadState_LeaveTracing(ts); if (_PyErr_Occurred(ts)) { goto exit; } @@ -1676,7 +1674,7 @@ _PySys_GetSizeOf(PyObject *o) } } else { - res = _PyObject_CallNoArg(method); + res = _PyObject_CallNoArgs(method); Py_DECREF(method); } @@ -2974,6 +2972,14 @@ _PySys_UpdateConfig(PyThreadState *tstate) SET_SYS("_xoptions", sys_create_xoptions_dict(config)); + const wchar_t *stdlibdir = _Py_GetStdlibDir(); + if (stdlibdir != NULL) { + SET_SYS_FROM_WSTR("_stdlib_dir", stdlibdir); + } + else { + PyDict_SetItemString(sysdict, "_stdlib_dir", Py_None); + } + #undef SET_SYS_FROM_WSTR #undef COPY_LIST #undef COPY_WSTR diff --git a/Python/thread_nt.h b/Python/thread_nt.h index f8c098ca5238a8..0dde1a04097389 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -75,20 +75,20 @@ EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) } } } else if (milliseconds != 0) { - /* wait at least until the target */ - _PyTime_t now = _PyTime_GetPerfCounter(); + /* wait at least until the deadline */ _PyTime_t nanoseconds = _PyTime_FromNanoseconds((_PyTime_t)milliseconds * 1000000); - _PyTime_t target = now + nanoseconds; + _PyTime_t deadline = _PyTime_Add(_PyTime_GetPerfCounter(), nanoseconds); while (mutex->locked) { - _PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, _PyTime_ROUND_TIMEOUT); + _PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, + _PyTime_ROUND_TIMEOUT); if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { result = WAIT_FAILED; break; } - now = _PyTime_GetPerfCounter(); - if (target <= now) + nanoseconds = deadline - _PyTime_GetPerfCounter(); + if (nanoseconds <= 0) { break; - nanoseconds = target - now; + } } } if (!mutex->locked) { @@ -292,6 +292,11 @@ PyThread_free_lock(PyThread_type_lock aLock) FreeNonRecursiveMutex(aLock) ; } +// WaitForSingleObject() accepts timeout in milliseconds in the range +// [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no +// timeout. 0xFFFFFFFE milliseconds is around 49.7 days. +const DWORD TIMEOUT_MS_MAX = 0xFFFFFFFE; + /* * Return 1 on success if the lock was acquired * @@ -309,11 +314,20 @@ PyThread_acquire_lock_timed(PyThread_type_lock aLock, if (microseconds >= 0) { milliseconds = microseconds / 1000; - if (microseconds % 1000 > 0) - ++milliseconds; - if (milliseconds > PY_DWORD_MAX) { - Py_FatalError("Timeout larger than PY_TIMEOUT_MAX"); + // Round milliseconds away from zero + if (microseconds % 1000 > 0) { + milliseconds++; + } + if (milliseconds > (PY_TIMEOUT_T)TIMEOUT_MS_MAX) { + // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout + // overflow to the caller, so clamp the timeout to + // [0, TIMEOUT_MS_MAX] milliseconds. + // + // _thread.Lock.acquire() and _thread.RLock.acquire() raise an + // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. + milliseconds = TIMEOUT_MS_MAX; } + assert(milliseconds != INFINITE); } else { milliseconds = INFINITE; diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index a45d842ffe73d2..12dad7e9e44272 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -92,7 +92,7 @@ * mutexes and condition variables: */ #if (defined(_POSIX_SEMAPHORES) && !defined(HAVE_BROKEN_POSIX_SEMAPHORES) && \ - defined(HAVE_SEM_TIMEDWAIT)) + (defined(HAVE_SEM_TIMEDWAIT) || defined(HAVE_SEM_CLOCKWAIT))) # define USE_SEMAPHORES #else # undef USE_SEMAPHORES @@ -433,33 +433,62 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, PyLockStatus success; sem_t *thelock = (sem_t *)lock; int status, error = 0; - struct timespec ts; - _PyTime_t deadline = 0; (void) error; /* silence unused-but-set-variable warning */ dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n", lock, microseconds, intr_flag)); - if (microseconds > PY_TIMEOUT_MAX) { - Py_FatalError("Timeout larger than PY_TIMEOUT_MAX"); + _PyTime_t timeout; // relative timeout + if (microseconds >= 0) { + _PyTime_t ns; + if (microseconds <= _PyTime_MAX / 1000) { + ns = microseconds * 1000; + } + else { + // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout + // overflow to the caller, so clamp the timeout to + // [_PyTime_MIN, _PyTime_MAX]. + // + // _PyTime_MAX nanoseconds is around 292.3 years. + // + // _thread.Lock.acquire() and _thread.RLock.acquire() raise an + // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. + ns = _PyTime_MAX; + } + timeout = _PyTime_FromNanoseconds(ns); + } + else { + timeout = _PyTime_FromNanoseconds(-1); } - if (microseconds > 0) { - MICROSECONDS_TO_TIMESPEC(microseconds, ts); - - if (!intr_flag) { - /* cannot overflow thanks to (microseconds > PY_TIMEOUT_MAX) - check done above */ - _PyTime_t timeout = _PyTime_FromNanoseconds(microseconds * 1000); - deadline = _PyTime_GetMonotonicClock() + timeout; - } +#ifdef HAVE_SEM_CLOCKWAIT + struct timespec abs_timeout; + // Local scope for deadline + { + _PyTime_t deadline = _PyTime_Add(_PyTime_GetMonotonicClock(), timeout); + _PyTime_AsTimespec_clamp(deadline, &abs_timeout); + } +#else + _PyTime_t deadline = 0; + if (timeout > 0 && !intr_flag) { + deadline = _PyDeadline_Init(timeout); } +#endif while (1) { - if (microseconds > 0) { + if (timeout > 0) { +#ifdef HAVE_SEM_CLOCKWAIT + status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, + &abs_timeout)); +#else + _PyTime_t abs_time = _PyTime_Add(_PyTime_GetSystemClock(), + timeout); + struct timespec ts; + _PyTime_AsTimespec_clamp(abs_time, &ts); status = fix_status(sem_timedwait(thelock, &ts)); +#endif } - else if (microseconds == 0) { + else if (timeout == 0) { status = fix_status(sem_trywait(thelock)); } else { @@ -472,38 +501,35 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, break; } - if (microseconds > 0) { + // sem_clockwait() uses an absolute timeout, there is no need + // to recompute the relative timeout. +#ifndef HAVE_SEM_CLOCKWAIT + if (timeout > 0) { /* wait interrupted by a signal (EINTR): recompute the timeout */ - _PyTime_t dt = deadline - _PyTime_GetMonotonicClock(); - if (dt < 0) { + _PyTime_t timeout = _PyDeadline_Get(deadline); + if (timeout < 0) { status = ETIMEDOUT; break; } - else if (dt > 0) { - _PyTime_t realtime_deadline = _PyTime_GetSystemClock() + dt; - if (_PyTime_AsTimespec(realtime_deadline, &ts) < 0) { - /* Cannot occur thanks to (microseconds > PY_TIMEOUT_MAX) - check done above */ - Py_UNREACHABLE(); - } - /* no need to update microseconds value, the code only care - if (microseconds > 0 or (microseconds == 0). */ - } - else { - microseconds = 0; - } } +#endif } /* Don't check the status if we're stopping because of an interrupt. */ if (!(intr_flag && status == EINTR)) { - if (microseconds > 0) { - if (status != ETIMEDOUT) + if (timeout > 0) { + if (status != ETIMEDOUT) { +#ifdef HAVE_SEM_CLOCKWAIT + CHECK_STATUS("sem_clockwait"); +#else CHECK_STATUS("sem_timedwait"); +#endif + } } - else if (microseconds == 0) { - if (status != EAGAIN) + else if (timeout == 0) { + if (status != EAGAIN) { CHECK_STATUS("sem_trywait"); + } } else { CHECK_STATUS("sem_wait"); diff --git a/Python/traceback.c b/Python/traceback.c index 76280a35e3a5f0..b18cbb91ce8edc 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -4,18 +4,21 @@ #include "Python.h" #include "code.h" // PyCode_Addr2Line etc -#include "pycore_interp.h" // PyInterpreterState.gc #include "frameobject.h" // PyFrame_GetBack() -#include "pycore_frame.h" // _PyFrame_GetCode() -#include "pycore_pyarena.h" // _PyArena_Free() #include "pycore_ast.h" // asdl_seq_* #include "pycore_compile.h" // _PyAST_Optimize +#include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH +#include "pycore_frame.h" // _PyFrame_GetCode() +#include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_parser.h" // _PyParser_ASTFromString +#include "pycore_pyarena.h" // _PyArena_Free() +#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pystate.h" // _PyThreadState_GET() #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() #include "structmember.h" // PyMemberDef #include "osdefs.h" // SEP #ifdef HAVE_FCNTL_H -#include +# include #endif #define OFF(x) offsetof(PyTracebackObject, x) @@ -26,7 +29,7 @@ #define MAX_NTHREADS 100 /* Function from Parser/tokenizer.c */ -extern char * PyTokenizer_FindEncodingFilename(int, PyObject *); +extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *); _Py_IDENTIFIER(TextIOWrapper); _Py_IDENTIFIER(close); @@ -240,7 +243,7 @@ _PyTraceBack_FromFrame(PyObject *tb_next, PyFrameObject *frame) assert(tb_next == NULL || PyTraceBack_Check(tb_next)); assert(frame != NULL); - return tb_create_raw((PyTracebackObject *)tb_next, frame, frame->f_frame->f_lasti*2, + return tb_create_raw((PyTracebackObject *)tb_next, frame, frame->f_frame->f_lasti*sizeof(_Py_CODEUNIT), PyFrame_GetLineNumber(frame)); } @@ -267,11 +270,12 @@ void _PyTraceback_Add(const char *funcname, const char *filename, int lineno) PyCodeObject *code; PyFrameObject *frame; PyObject *exc, *val, *tb; + PyThreadState *tstate = _PyThreadState_GET(); /* Save and clear the current exception. Python functions must not be called with an exception set. Calling Python functions happens when the codec of the filesystem encoding is implemented in pure Python. */ - PyErr_Fetch(&exc, &val, &tb); + _PyErr_Fetch(tstate, &exc, &val, &tb); globals = PyDict_New(); if (!globals) @@ -281,14 +285,14 @@ void _PyTraceback_Add(const char *funcname, const char *filename, int lineno) Py_DECREF(globals); goto error; } - frame = PyFrame_New(PyThreadState_Get(), code, globals, NULL); + frame = PyFrame_New(tstate, code, globals, NULL); Py_DECREF(globals); Py_DECREF(code); if (!frame) goto error; frame->f_lineno = lineno; - PyErr_Restore(exc, val, tb); + _PyErr_Restore(tstate, exc, val, tb); PyTraceBack_Here(frame); Py_DECREF(frame); return; @@ -427,7 +431,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, i Py_DECREF(binary); return 0; } - found_encoding = PyTokenizer_FindEncodingFilename(fd, filename); + found_encoding = _PyTokenizer_FindEncodingFilename(fd, filename); if (found_encoding == NULL) PyErr_Clear(); encoding = (found_encoding != NULL) ? found_encoding : "utf-8"; @@ -1047,7 +1051,7 @@ dump_frame(int fd, InterpreterFrame *frame) PUTS(fd, "???"); } - int lineno = PyCode_Addr2Line(code, frame->f_lasti*2); + int lineno = PyCode_Addr2Line(code, frame->f_lasti*sizeof(_Py_CODEUNIT)); PUTS(fd, ", line "); if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); diff --git a/README.rst b/README.rst index 067d7d8ca2b818..c2b26a46515119 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.11.0 alpha 0 +This is Python version 3.11.0 alpha 1 ===================================== .. image:: https://travis-ci.com/python/cpython.svg?branch=main diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 24fc6cd182656b..5d89b29adf899e 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -482,7 +482,7 @@ def cmd_data(datacmd, filenames, known=None, *, cmd_analyze, ), 'data': ( - 'check/manage local data (e.g. knwon types, ignored vars, caches)', + 'check/manage local data (e.g. known types, ignored vars, caches)', [_cli_data], cmd_data, ), diff --git a/Tools/c-analyzer/c_analyzer/info.py b/Tools/c-analyzer/c_analyzer/info.py index b75918e5e7a687..27c3a5a4ee76f2 100644 --- a/Tools/c-analyzer/c_analyzer/info.py +++ b/Tools/c-analyzer/c_analyzer/info.py @@ -230,11 +230,11 @@ def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): return self def as_rowdata(self, columns=None): - # XXX finsih! + # XXX finish! return self.item.as_rowdata(columns) def render_rowdata(self, columns=None): - # XXX finsih! + # XXX finish! return self.item.render_rowdata(columns) def render(self, fmt='line', *, itemonly=False): diff --git a/Tools/c-analyzer/c_common/logging.py b/Tools/c-analyzer/c_common/logging.py index 12398f7e385fdb..10af852ec3c5e6 100644 --- a/Tools/c-analyzer/c_common/logging.py +++ b/Tools/c-analyzer/c_common/logging.py @@ -41,7 +41,7 @@ def configure_logger(logger, verbosity=VERBOSITY, *, def hide_emit_errors(): """Ignore errors while emitting log entries. - Rather than printing a message desribing the error, we show nothing. + Rather than printing a message describing the error, we show nothing. """ # For now we simply ignore all exceptions. If we wanted to ignore # specific ones (e.g. BrokenPipeError) then we would need to use diff --git a/Tools/c-analyzer/c_common/strutil.py b/Tools/c-analyzer/c_common/strutil.py index e7535d45bbba23..07193c091e4c3f 100644 --- a/Tools/c-analyzer/c_common/strutil.py +++ b/Tools/c-analyzer/c_common/strutil.py @@ -26,7 +26,7 @@ def parse_entries(entries, *, ignoresep=None): # We read the entire file here to ensure the file # gets closed sooner rather than later. Note that # the file would stay open if this iterator is never - # exchausted. + # exhausted. lines = infile.read().splitlines() for line in _iter_significant_lines(lines): yield line, filename diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py index 98ff511cfe64a0..697b1f26dc2151 100644 --- a/Tools/c-analyzer/c_parser/info.py +++ b/Tools/c-analyzer/c_parser/info.py @@ -1029,7 +1029,7 @@ def _resolve_data(cls, data): @classmethod def _raw_data(self, data): - # XXX finsh! + # XXX finish! return data @classmethod @@ -1255,7 +1255,7 @@ def _resolve_data(cls, data): @classmethod def _raw_data(self, data): - # XXX finsih! + # XXX finish! return data @classmethod @@ -1296,12 +1296,12 @@ class Statement(HighlevelParsedItem): @classmethod def _resolve_data(cls, data): - # XXX finsih! + # XXX finish! return data, None @classmethod def _raw_data(self, data): - # XXX finsih! + # XXX finish! return data @classmethod diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index 39056099f5e918..df70aae66b7760 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -7,7 +7,7 @@ Furthermore, the grammar rules for the C syntax (particularly as described in the K&R book) actually describe a superset, of which the -full C langage is a proper subset. Here are some of the extra +full C language is a proper subset. Here are some of the extra conditions that must be applied when parsing C code: * ... @@ -90,7 +90,7 @@ * no "inline" type declarations (struct, union, enum) in function parameters ~(including function pointers)~ * no "inline" type decls in function return types -* no superflous parentheses in declarators +* no superfluous parentheses in declarators * var decls in for loops are always "simple" (e.g. no inline types) * only inline struct/union/enum decls may be anonymouns (without a name) * no function pointers in function pointer parameters diff --git a/Tools/c-analyzer/c_parser/preprocessor/__init__.py b/Tools/c-analyzer/c_parser/preprocessor/__init__.py index 8da4d8cadf7d0e..e38176fee31fac 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/__init__.py +++ b/Tools/c-analyzer/c_parser/preprocessor/__init__.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) -# Supprted "source": +# Supported "source": # * filename (string) # * lines (iterable) # * text (string) @@ -156,7 +156,7 @@ def handling_errors(ignore_exc=None, *, log_err=None): # tools _COMPILERS = { - # matching disutils.ccompiler.compiler_class: + # matching distutils.ccompiler.compiler_class: 'unix': _gcc.preprocess, 'msvc': None, 'cygwin': None, diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index a11b687214d2f4..06ec871ba75e35 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -342,7 +342,7 @@ def cmd_capi(filenames=None, *, cmd_parse, ), 'data': ( - 'check/manage local data (e.g. knwon types, ignored vars, caches)', + 'check/manage local data (e.g. known types, ignored vars, caches)', [_cli_data], cmd_data, ), diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ef06a9fcb69033..8526b2af15a235 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -172,7 +172,6 @@ def clean_lines(text): Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 -Include/cpython/interpreteridobject.h Py_CPYTHON_INTERPRETERIDOBJECT_H 1 Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 302f10cb2bc466..f3fdf3b0d05d92 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -2106,7 +2106,7 @@ Python/import.c import_find_and_load header - #----------------------- # runtime state -# (look at the bottome of the file) +# (look at the bottom of the file) #----------------------- # modules diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index c11b23e74b9bed..54f72320cdd8b3 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -45,6 +45,7 @@ # compatible (2.6+ and 3.0+). See #19308. from __future__ import print_function + import gdb import os import locale @@ -444,10 +445,11 @@ def _write_instance_repr(out, visited, name, pyop_attrdict, address): out.write(name) # Write dictionary of instance attributes: - if isinstance(pyop_attrdict, PyDictObjectPtr): + if isinstance(pyop_attrdict, (PyKeysValuesPair, PyDictObjectPtr)): out.write('(') first = True - for pyop_arg, pyop_val in pyop_attrdict.iteritems(): + items = pyop_attrdict.iteritems() + for pyop_arg, pyop_val in items: if not first: out.write(', ') first = False @@ -519,6 +521,25 @@ def get_attr_dict(self): # Not found, or some kind of error: return None + def get_keys_values(self): + typeobj = self.type() + values_offset = int_from_int(typeobj.field('tp_inline_values_offset')) + if values_offset == 0: + return None + charptr = self._gdbval.cast(_type_char_ptr()) + values_offset + PyDictValuesPtrPtr = gdb.lookup_type("PyDictValues").pointer().pointer() + valuesptr = charptr.cast(PyDictValuesPtrPtr) + values = valuesptr.dereference() + if long(values) == 0: + return None + values = values['values'] + return PyKeysValuesPair(self.get_cached_keys(), values) + + def get_cached_keys(self): + typeobj = self.type() + HeapTypePtr = gdb.lookup_type("PyHeapTypeObject").pointer() + return typeobj._gdbval.cast(HeapTypePtr)['ht_cached_keys'] + def proxyval(self, visited): ''' Support for classes. @@ -532,7 +553,10 @@ def proxyval(self, visited): visited.add(self.as_address()) pyop_attr_dict = self.get_attr_dict() - if pyop_attr_dict: + keys_values = self.get_keys_values() + if keys_values: + attr_dict = keys_values.proxyval(visited) + elif pyop_attr_dict: attr_dict = pyop_attr_dict.proxyval(visited) else: attr_dict = {} @@ -548,9 +572,11 @@ def write_repr(self, out, visited): return visited.add(self.as_address()) - pyop_attrdict = self.get_attr_dict() + pyop_attrs = self.get_keys_values() + if not pyop_attrs: + pyop_attrs = self.get_attr_dict() _write_instance_repr(out, visited, - self.safe_tp_name(), pyop_attrdict, self.as_address()) + self.safe_tp_name(), pyop_attrs, self.as_address()) class ProxyException(Exception): def __init__(self, tp_name, args): @@ -672,6 +698,32 @@ def addr2line(self, addrq): assert False, "Unreachable" +def items_from_keys_and_values(keys, values): + entries, nentries = PyDictObjectPtr._get_entries(keys) + for i in safe_range(nentries): + ep = entries[i] + pyop_value = PyObjectPtr.from_pyobject_ptr(values[i]) + if not pyop_value.is_null(): + pyop_key = PyObjectPtr.from_pyobject_ptr(ep['me_key']) + yield (pyop_key, pyop_value) + +class PyKeysValuesPair: + + def __init__(self, keys, values): + self.keys = keys + self.values = values + + def iteritems(self): + return items_from_keys_and_values(self.keys, self.values) + + def proxyval(self, visited): + result = {} + for pyop_key, pyop_value in self.iteritems(): + proxy_key = pyop_key.proxyval(visited) + proxy_value = pyop_value.proxyval(visited) + result[proxy_key] = proxy_value + return result + class PyDictObjectPtr(PyObjectPtr): """ Class wrapping a gdb.Value that's a PyDictObject* i.e. a dict instance @@ -686,13 +738,17 @@ def iteritems(self): ''' keys = self.field('ma_keys') values = self.field('ma_values') + has_values = long(values) + if has_values: + values = values['values'] + if has_values: + for item in items_from_keys_and_values(keys, values): + yield item + return entries, nentries = self._get_entries(keys) for i in safe_range(nentries): ep = entries[i] - if long(values): - pyop_value = PyObjectPtr.from_pyobject_ptr(values[i]) - else: - pyop_value = PyObjectPtr.from_pyobject_ptr(ep['me_value']) + pyop_value = PyObjectPtr.from_pyobject_ptr(ep['me_value']) if not pyop_value.is_null(): pyop_key = PyObjectPtr.from_pyobject_ptr(ep['me_key']) yield (pyop_key, pyop_value) @@ -728,7 +784,8 @@ def write_repr(self, out, visited): pyop_value.write_repr(out, visited) out.write('}') - def _get_entries(self, keys): + @staticmethod + def _get_entries(keys): dk_nentries = int(keys['dk_nentries']) dk_size = 1< str: if self.arguments: parts.append(f"({', '.join(map(str, self.arguments))})") if self.force_true: - parts.append(", 1") + parts.append(", !p->error_indicator") if self.assigned_variable: if self.assigned_variable_type: parts = [ @@ -737,7 +737,7 @@ def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: Optional[str]) self.print( f'D(fprintf(stderr, "%*c+ {rulename}[%d-%d]: %s succeeded!\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' ) - # Prepare to emmit the rule action and do so + # Prepare to emit the rule action and do so if node.action and "EXTRA" in node.action: self._set_up_token_end_metadata_extraction() if self.skip_actions: diff --git a/Tools/peg_generator/pegen/first_sets.py b/Tools/peg_generator/pegen/first_sets.py index 611ef514d09bda..6d794ffa4bfa8b 100755 --- a/Tools/peg_generator/pegen/first_sets.py +++ b/Tools/peg_generator/pegen/first_sets.py @@ -56,7 +56,7 @@ def visit_Alt(self, item: Alt) -> Set[str]: result -= to_remove # If the set of new terminals can start with the empty string, - # it means that the item is completelly nullable and we should + # it means that the item is completely nullable and we should # also considering at least the next item in case the current # one fails to parse. diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index 4ce60e35dd0e0f..034e8e6017a22d 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -168,7 +168,7 @@ def __init__(self, tokenizer: Tokenizer, *, verbose: bool = False): self._verbose = verbose self._level = 0 self._cache: Dict[Tuple[Mark, str, Tuple[Any, ...]], Tuple[Any, Mark]] = {} - # Integer tracking wether we are in a left recursive rule or not. Can be useful + # Integer tracking whether we are in a left recursive rule or not. Can be useful # for error reporting. self.in_recursive_rule = 0 # Pass through common tokenizer methods. diff --git a/Tools/peg_generator/scripts/download_pypi_packages.py b/Tools/peg_generator/scripts/download_pypi_packages.py index 0af876c3ecad4d..180309d0bce628 100755 --- a/Tools/peg_generator/scripts/download_pypi_packages.py +++ b/Tools/peg_generator/scripts/download_pypi_packages.py @@ -73,7 +73,7 @@ def main() -> None: package_json = load_json(package_name) try: - print(f"Dowloading and compressing package {package_name} ... ", end="") + print(f"Downloading and compressing package {package_name} ... ", end="") download_package_code(package_name, package_json) print("Done") except (IndexError, KeyError): diff --git a/Tools/pynche/ColorDB.py b/Tools/pynche/ColorDB.py index eb76d4042d33b6..c013a608969089 100644 --- a/Tools/pynche/ColorDB.py +++ b/Tools/pynche/ColorDB.py @@ -9,7 +9,7 @@ trouble reading the file, None is returned. You can pass get_colordb() an optional filetype argument. -Supporte file types are: +Supported file types are: X_RGB_TXT -- X Consortium rgb.txt format files. Three columns of numbers from 0 .. 255 separated by whitespace. Arbitrary trailing diff --git a/Tools/scripts/dutree.doc b/Tools/scripts/dutree.doc index 97bd2e2e47cae9..490126b0182d1c 100644 --- a/Tools/scripts/dutree.doc +++ b/Tools/scripts/dutree.doc @@ -15,7 +15,7 @@ From the keyboard of flee@cs.psu.edu (Felix Lee): :And Perl is definitely awkward with data types. I haven't yet found a :pleasant way of shoving non-trivial data types into Perl's grammar. -Yes, it's pretty aweful at that, alright. Sometimes I write perl programs +Yes, it's pretty awful at that, alright. Sometimes I write perl programs that need them, and sometimes it just takes a little creativity. But sometimes it's not worth it. I actually wrote a C program the other day (gasp) because I didn't want to deal with a game matrix with six links per node. diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index aa799d763a55b9..5c7eee42952896 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -8,6 +8,7 @@ import os import ntpath import posixpath +import platform import subprocess import sys import textwrap @@ -18,6 +19,7 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) ROOT_DIR = os.path.abspath(ROOT_DIR) +FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py') STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') # If MODULES_DIR is changed then the .gitattributes and .gitignore files @@ -27,15 +29,19 @@ if sys.platform != "win32": TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') if not os.path.isfile(TOOL): - # When building out of the source tree, get the tool from the current - # directory - TOOL = os.path.join('Programs', '_freeze_module') + # When building out of the source tree, get the tool from directory + # of the Python executable + TOOL = os.path.dirname(sys.executable) + TOOL = os.path.join(TOOL, 'Programs', '_freeze_module') TOOL = os.path.abspath(TOOL) if not os.path.isfile(TOOL): sys.exit("ERROR: missing _freeze_module") else: def find_tool(): - for arch in ['amd64', 'win32']: + archs = ['amd64', 'win32'] + if platform.machine() == "ARM64": + archs.append('arm64') + for arch in archs: for exe in ['_freeze_module.exe', '_freeze_module_d.exe']: tool = os.path.join(ROOT_DIR, 'PCbuild', arch, exe) if os.path.isfile(tool): @@ -49,7 +55,6 @@ def find_tool(): MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters') -TEST_CTYPES = os.path.join(STDLIB_DIR, 'ctypes', 'test', 'test_values.py') OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath' @@ -91,8 +96,11 @@ def find_tool(): ]), ('Test module', [ '__hello__', - '__hello__ : <__phello__>', - '__hello__ : __phello__.spam', + '__hello__ : __hello_alias__', + '__hello__ : <__phello_alias__>', + '__hello__ : __phello_alias__.spam', + '<__phello__.**.*>', + f'frozen_only : __hello_only__ = {FROZEN_ONLY}', ]), ] ESSENTIAL = { @@ -131,14 +139,15 @@ def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None): seen = {} for section, specs in sectionalspecs: parsed = _parse_specs(specs, section, seen) - for frozenid, pyfile, modname, ispkg, section in parsed: + for item in parsed: + frozenid, pyfile, modname, ispkg, section = item try: source = seen[frozenid] except KeyError: source = FrozenSource.from_id(frozenid, pyfile, destdir) seen[frozenid] = source else: - assert not pyfile + assert not pyfile or pyfile == source.pyfile, item yield FrozenModule(modname, ispkg, section, source) @@ -217,13 +226,19 @@ def _parse_spec(spec, knownids=None, section=None): if ispkg: pkgid = frozenid pkgname = modname + pkgfiles = {pyfile: pkgid} def iter_subs(): for frozenid, pyfile, ispkg in resolved: - assert not knownids or frozenid not in knownids, (frozenid, spec) if pkgname: modname = frozenid.replace(pkgid, pkgname, 1) else: modname = frozenid + if pyfile: + if pyfile in pkgfiles: + frozenid = pkgfiles[pyfile] + pyfile = None + elif ispkg: + pkgfiles[pyfile] = frozenid yield frozenid, pyfile, modname, ispkg, section submodules = iter_subs() @@ -260,6 +275,15 @@ def symbol(self): name = self.frozenid.replace('.', '_') return '_Py_M__' + name + @property + def ispkg(self): + if not self.pyfile: + return False + elif self.frozenid.endswith('.__init__'): + return False + else: + return os.path.basename(self.pyfile) == '__init__.py' + def resolve_frozen_file(frozenid, destdir=MODULES_DIR): """Return the filename corresponding to the given frozen ID. @@ -291,6 +315,17 @@ def __getattr__(self, name): def modname(self): return self.name + @property + def orig(self): + return self.source.modname + + @property + def isalias(self): + orig = self.source.modname + if not orig: + return True + return self.name != orig + def summarize(self): source = self.source.modname if source: @@ -318,10 +353,10 @@ def _iter_sources(modules): # generic helpers def _get_checksum(filename): - with open(filename) as infile: - text = infile.read() + with open(filename, "rb") as infile: + contents = infile.read() m = hashlib.sha256() - m.update(text.encode('utf8')) + m.update(contents) return m.hexdigest() @@ -478,7 +513,7 @@ def regen_manifest(modules): modlines.append(' '.join(row).rstrip()) print(f'# Updating {os.path.relpath(MANIFEST)}') - with open(MANIFEST, 'w') as outfile: + with open(MANIFEST, 'w', encoding="utf-8") as outfile: lines = (l + '\n' for l in modlines) outfile.writelines(lines) @@ -493,6 +528,7 @@ def regen_frozen(modules): headerlines.append(f'#include "{header}"') deflines = [] + aliaslines = [] indent = ' ' lastsection = None for mod in modules: @@ -514,6 +550,15 @@ def regen_frozen(modules): deflines.append(line1) deflines.append(indent + line2) + if mod.isalias: + if not mod.orig: + entry = '{"%s", NULL},' % (mod.name,) + elif mod.source.ispkg: + entry = '{"%s", "<%s"},' % (mod.name, mod.orig) + else: + entry = '{"%s", "%s"},' % (mod.name, mod.orig) + aliaslines.append(indent + entry) + if not deflines[0]: del deflines[0] for i, line in enumerate(deflines): @@ -535,10 +580,17 @@ def regen_frozen(modules): lines = replace_block( lines, "static const struct _frozen _PyImport_FrozenModules[] =", - "/* sentinel */", + "/* modules sentinel */", deflines, FROZEN_FILE, ) + lines = replace_block( + lines, + "const struct _module_alias aliases[] =", + "/* aliases sentinel */", + aliaslines, + FROZEN_FILE, + ) outfile.writelines(lines) diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 48875d2a9dd391..dca9bbcfcbc839 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -72,7 +72,7 @@ def main(opcode_py, outfile='Include/opcode.h'): next_op += 1 fobj.write("#define %-23s %3s\n" % (name, next_op)) used[next_op] = True - + fobj.write("#define DO_TRACING 255\n") fobj.write("#ifdef NEED_OPCODE_JUMP_TABLES\n") write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj) write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj) diff --git a/Tools/scripts/generate_stdlib_module_names.py b/Tools/scripts/generate_stdlib_module_names.py index 325ae202b1d8c1..50042f14fb62e5 100644 --- a/Tools/scripts/generate_stdlib_module_names.py +++ b/Tools/scripts/generate_stdlib_module_names.py @@ -21,6 +21,9 @@ # Test modules and packages '__hello__', '__phello__', + '__hello_alias__', + '__phello_alias__', + '__hello_only__', '_ctypes_test', '_testbuffer', '_testcapi', diff --git a/Tools/scripts/gprof2html.py b/Tools/scripts/gprof2html.py index b14def4ef8482b..bf0530ef3e4337 100755 --- a/Tools/scripts/gprof2html.py +++ b/Tools/scripts/gprof2html.py @@ -24,7 +24,7 @@ """ def add_escapes(filename): - with open(filename) as fp: + with open(filename, encoding="utf-8") as fp: for line in fp: yield html.escape(line) @@ -79,7 +79,7 @@ def main(): filename = sys.argv[1] outputfilename = filename + ".html" input = add_escapes(filename) - with open(outputfilename, "w") as output: + with open(outputfilename, "w", encoding="utf-8") as output: gprof2html(input, output, filename) webbrowser.open("file:" + os.path.abspath(outputfilename)) diff --git a/Tools/scripts/stable_abi.py b/Tools/scripts/stable_abi.py index b7fd2c8583ba70..6cb310e5a31d5a 100755 --- a/Tools/scripts/stable_abi.py +++ b/Tools/scripts/stable_abi.py @@ -34,7 +34,6 @@ "datetime.h", "dtoa.h", "frameobject.h", - "funcobject.h", "genobject.h", "longintrepr.h", "parsetok.h", @@ -67,7 +66,7 @@ class Manifest: def add(self, item): if item.name in self.contents: # We assume that stable ABI items do not share names, - # even if they're diferent kinds (e.g. function vs. macro). + # even if they're different kinds (e.g. function vs. macro). raise ValueError(f'duplicate ABI item {item.name}') self.contents[item.name] = item @@ -295,7 +294,7 @@ def do_unixy_check(manifest, args): present_macros = gcc_get_limited_api_macros(['Include/Python.h']) feature_defines = manifest.feature_defines & present_macros - # Check that we have all neded macros + # Check that we have all needed macros expected_macros = set( item.name for item in manifest.select({'macro'}) ) @@ -412,7 +411,7 @@ def binutils_check_library(manifest, library, expected_symbols, dynamic): def gcc_get_limited_api_macros(headers): """Get all limited API macros from headers. - Runs the preprocesor over all the header files in "Include" setting + Runs the preprocessor over all the header files in "Include" setting "-DPy_LIMITED_API" to the correct value for the running version of the interpreter and extracting all macro definitions (via adding -dM to the compiler arguments). @@ -449,7 +448,7 @@ def gcc_get_limited_api_macros(headers): def gcc_get_limited_api_definitions(headers): """Get all limited API definitions from headers. - Run the preprocesor over all the header files in "Include" setting + Run the preprocessor over all the header files in "Include" setting "-DPy_LIMITED_API" to the correct value for the running version of the interpreter. diff --git a/aclocal.m4 b/aclocal.m4 index 987bfdf215ccb4..2f1bd37528c85d 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -275,3 +275,347 @@ AC_DEFUN([AX_CHECK_OPENSSL], [ AC_SUBST([OPENSSL_LDFLAGS]) ]) +# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +# serial 11 (pkg-config-0.29.1) + +dnl Copyright © 2004 Scott James Remnant . +dnl Copyright © 2012-2015 Dan Nicholson +dnl +dnl This program is free software; you can redistribute it and/or modify +dnl it under the terms of the GNU General Public License as published by +dnl the Free Software Foundation; either version 2 of the License, or +dnl (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, but +dnl WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +dnl General Public License for more details. +dnl +dnl You should have received a copy of the GNU General Public License +dnl along with this program; if not, write to the Free Software +dnl Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +dnl 02111-1307, USA. +dnl +dnl As a special exception to the GNU General Public License, if you +dnl distribute this file as part of a program that contains a +dnl configuration script generated by Autoconf, you may include it under +dnl the same distribution terms that you use for the rest of that +dnl program. + +dnl PKG_PREREQ(MIN-VERSION) +dnl ----------------------- +dnl Since: 0.29 +dnl +dnl Verify that the version of the pkg-config macros are at least +dnl MIN-VERSION. Unlike PKG_PROG_PKG_CONFIG, which checks the user's +dnl installed version of pkg-config, this checks the developer's version +dnl of pkg.m4 when generating configure. +dnl +dnl To ensure that this macro is defined, also add: +dnl m4_ifndef([PKG_PREREQ], +dnl [m4_fatal([must install pkg-config 0.29 or later before running autoconf/autogen])]) +dnl +dnl See the "Since" comment for each macro you use to see what version +dnl of the macros you require. +m4_defun([PKG_PREREQ], +[m4_define([PKG_MACROS_VERSION], [0.29.1]) +m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, + [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) +])dnl PKG_PREREQ + +dnl PKG_PROG_PKG_CONFIG([MIN-VERSION]) +dnl ---------------------------------- +dnl Since: 0.16 +dnl +dnl Search for the pkg-config tool and set the PKG_CONFIG variable to +dnl first found in the path. Checks that the version of pkg-config found +dnl is at least MIN-VERSION. If MIN-VERSION is not specified, 0.9.0 is +dnl used since that's the first version where most current features of +dnl pkg-config existed. +AC_DEFUN([PKG_PROG_PKG_CONFIG], +[m4_pattern_forbid([^_?PKG_[A-Z_]+$]) +m4_pattern_allow([^PKG_CONFIG(_(PATH|LIBDIR|SYSROOT_DIR|ALLOW_SYSTEM_(CFLAGS|LIBS)))?$]) +m4_pattern_allow([^PKG_CONFIG_(DISABLE_UNINSTALLED|TOP_BUILD_DIR|DEBUG_SPEW)$]) +AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility]) +AC_ARG_VAR([PKG_CONFIG_PATH], [directories to add to pkg-config's search path]) +AC_ARG_VAR([PKG_CONFIG_LIBDIR], [path overriding pkg-config's built-in search path]) + +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + AC_PATH_TOOL([PKG_CONFIG], [pkg-config]) +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=m4_default([$1], [0.9.0]) + AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version]) + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + PKG_CONFIG="" + fi +fi[]dnl +])dnl PKG_PROG_PKG_CONFIG + +dnl PKG_CHECK_EXISTS(MODULES, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ------------------------------------------------------------------- +dnl Since: 0.18 +dnl +dnl Check to see whether a particular set of modules exists. Similar to +dnl PKG_CHECK_MODULES(), but does not set variables or print errors. +dnl +dnl Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +dnl only at the first occurence in configure.ac, so if the first place +dnl it's called might be skipped (such as if it is within an "if", you +dnl have to call PKG_CHECK_EXISTS manually +AC_DEFUN([PKG_CHECK_EXISTS], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +if test -n "$PKG_CONFIG" && \ + AC_RUN_LOG([$PKG_CONFIG --exists --print-errors "$1"]); then + m4_default([$2], [:]) +m4_ifvaln([$3], [else + $3])dnl +fi]) + +dnl _PKG_CONFIG([VARIABLE], [COMMAND], [MODULES]) +dnl --------------------------------------------- +dnl Internal wrapper calling pkg-config via PKG_CONFIG and setting +dnl pkg_failed based on the result. +m4_define([_PKG_CONFIG], +[if test -n "$$1"; then + pkg_cv_[]$1="$$1" + elif test -n "$PKG_CONFIG"; then + PKG_CHECK_EXISTS([$3], + [pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes ], + [pkg_failed=yes]) + else + pkg_failed=untried +fi[]dnl +])dnl _PKG_CONFIG + +dnl _PKG_SHORT_ERRORS_SUPPORTED +dnl --------------------------- +dnl Internal check to see if pkg-config supports short errors. +AC_DEFUN([_PKG_SHORT_ERRORS_SUPPORTED], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi[]dnl +])dnl _PKG_SHORT_ERRORS_SUPPORTED + + +dnl PKG_CHECK_MODULES(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], +dnl [ACTION-IF-NOT-FOUND]) +dnl -------------------------------------------------------------- +dnl Since: 0.4.0 +dnl +dnl Note that if there is a possibility the first call to +dnl PKG_CHECK_MODULES might not happen, you should be sure to include an +dnl explicit call to PKG_PROG_PKG_CONFIG in your configure.ac +AC_DEFUN([PKG_CHECK_MODULES], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl +AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl + +pkg_failed=no +AC_MSG_CHECKING([for $1]) + +_PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) +_PKG_CONFIG([$1][_LIBS], [libs], [$2]) + +m4_define([_PKG_TEXT], [Alternatively, you may set the environment variables $1[]_CFLAGS +and $1[]_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details.]) + +if test $pkg_failed = yes; then + AC_MSG_RESULT([no]) + _PKG_SHORT_ERRORS_SUPPORTED + if test $_pkg_short_errors_supported = yes; then + $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` + else + $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD + + m4_default([$4], [AC_MSG_ERROR( +[Package requirements ($2) were not met: + +$$1_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +_PKG_TEXT])[]dnl + ]) +elif test $pkg_failed = untried; then + AC_MSG_RESULT([no]) + m4_default([$4], [AC_MSG_FAILURE( +[The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +_PKG_TEXT + +To get pkg-config, see .])[]dnl + ]) +else + $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS + $1[]_LIBS=$pkg_cv_[]$1[]_LIBS + AC_MSG_RESULT([yes]) + $3 +fi[]dnl +])dnl PKG_CHECK_MODULES + + +dnl PKG_CHECK_MODULES_STATIC(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], +dnl [ACTION-IF-NOT-FOUND]) +dnl --------------------------------------------------------------------- +dnl Since: 0.29 +dnl +dnl Checks for existence of MODULES and gathers its build flags with +dnl static libraries enabled. Sets VARIABLE-PREFIX_CFLAGS from --cflags +dnl and VARIABLE-PREFIX_LIBS from --libs. +dnl +dnl Note that if there is a possibility the first call to +dnl PKG_CHECK_MODULES_STATIC might not happen, you should be sure to +dnl include an explicit call to PKG_PROG_PKG_CONFIG in your +dnl configure.ac. +AC_DEFUN([PKG_CHECK_MODULES_STATIC], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +_save_PKG_CONFIG=$PKG_CONFIG +PKG_CONFIG="$PKG_CONFIG --static" +PKG_CHECK_MODULES($@) +PKG_CONFIG=$_save_PKG_CONFIG[]dnl +])dnl PKG_CHECK_MODULES_STATIC + + +dnl PKG_INSTALLDIR([DIRECTORY]) +dnl ------------------------- +dnl Since: 0.27 +dnl +dnl Substitutes the variable pkgconfigdir as the location where a module +dnl should install pkg-config .pc files. By default the directory is +dnl $libdir/pkgconfig, but the default can be changed by passing +dnl DIRECTORY. The user can override through the --with-pkgconfigdir +dnl parameter. +AC_DEFUN([PKG_INSTALLDIR], +[m4_pushdef([pkg_default], [m4_default([$1], ['${libdir}/pkgconfig'])]) +m4_pushdef([pkg_description], + [pkg-config installation directory @<:@]pkg_default[@:>@]) +AC_ARG_WITH([pkgconfigdir], + [AS_HELP_STRING([--with-pkgconfigdir], pkg_description)],, + [with_pkgconfigdir=]pkg_default) +AC_SUBST([pkgconfigdir], [$with_pkgconfigdir]) +m4_popdef([pkg_default]) +m4_popdef([pkg_description]) +])dnl PKG_INSTALLDIR + + +dnl PKG_NOARCH_INSTALLDIR([DIRECTORY]) +dnl -------------------------------- +dnl Since: 0.27 +dnl +dnl Substitutes the variable noarch_pkgconfigdir as the location where a +dnl module should install arch-independent pkg-config .pc files. By +dnl default the directory is $datadir/pkgconfig, but the default can be +dnl changed by passing DIRECTORY. The user can override through the +dnl --with-noarch-pkgconfigdir parameter. +AC_DEFUN([PKG_NOARCH_INSTALLDIR], +[m4_pushdef([pkg_default], [m4_default([$1], ['${datadir}/pkgconfig'])]) +m4_pushdef([pkg_description], + [pkg-config arch-independent installation directory @<:@]pkg_default[@:>@]) +AC_ARG_WITH([noarch-pkgconfigdir], + [AS_HELP_STRING([--with-noarch-pkgconfigdir], pkg_description)],, + [with_noarch_pkgconfigdir=]pkg_default) +AC_SUBST([noarch_pkgconfigdir], [$with_noarch_pkgconfigdir]) +m4_popdef([pkg_default]) +m4_popdef([pkg_description]) +])dnl PKG_NOARCH_INSTALLDIR + + +dnl PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE, +dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ------------------------------------------- +dnl Since: 0.28 +dnl +dnl Retrieves the value of the pkg-config variable for the given module. +AC_DEFUN([PKG_CHECK_VAR], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl + +_PKG_CONFIG([$1], [variable="][$3]["], [$2]) +AS_VAR_COPY([$1], [pkg_cv_][$1]) + +AS_VAR_IF([$1], [""], [$5], [$4])dnl +])dnl PKG_CHECK_VAR + +dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------ +dnl +dnl Prepare a "--with-" configure option using the lowercase +dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and +dnl PKG_CHECK_MODULES in a single macro. +AC_DEFUN([PKG_WITH_MODULES], +[ +m4_pushdef([with_arg], m4_tolower([$1])) + +m4_pushdef([description], + [m4_default([$5], [build with ]with_arg[ support])]) + +m4_pushdef([def_arg], [m4_default([$6], [auto])]) +m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) +m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) + +m4_case(def_arg, + [yes],[m4_pushdef([with_without], [--without-]with_arg)], + [m4_pushdef([with_without],[--with-]with_arg)]) + +AC_ARG_WITH(with_arg, + AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, + [AS_TR_SH([with_]with_arg)=def_arg]) + +AS_CASE([$AS_TR_SH([with_]with_arg)], + [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], + [auto],[PKG_CHECK_MODULES([$1],[$2], + [m4_n([def_action_if_found]) $3], + [m4_n([def_action_if_not_found]) $4])]) + +m4_popdef([with_arg]) +m4_popdef([description]) +m4_popdef([def_arg]) + +])dnl PKG_WITH_MODULES + +dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ----------------------------------------------- +dnl +dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES +dnl check._[VARIABLE-PREFIX] is exported as make variable. +AC_DEFUN([PKG_HAVE_WITH_MODULES], +[ +PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) + +AM_CONDITIONAL([HAVE_][$1], + [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) +])dnl PKG_HAVE_WITH_MODULES + +dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------------------ +dnl +dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after +dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make +dnl and preprocessor variable. +AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], +[ +PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) + +AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], + [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) +])dnl PKG_HAVE_DEFINE_WITH_MODULES + diff --git a/configure b/configure index 4acf91f22107f4..198b0703fd6af1 100755 --- a/configure +++ b/configure @@ -630,7 +630,6 @@ OPENSSL_RPATH OPENSSL_LDFLAGS OPENSSL_LIBS OPENSSL_INCLUDES -PKG_CONFIG ENSUREPIP SRCDIRS THREADHEADERS @@ -662,6 +661,9 @@ DTRACE TCLTK_LIBS TCLTK_INCLUDES LIBFFI_INCLUDEDIR +PKG_CONFIG_LIBDIR +PKG_CONFIG_PATH +PKG_CONFIG TZPATH SHLIBS CFLAGSFORSHARED @@ -844,6 +846,7 @@ with_dbmliborder enable_ipv6 with_doc_strings with_pymalloc +with_freelists with_c_locale_coercion with_valgrind with_dtrace @@ -873,7 +876,10 @@ LDFLAGS LIBS CPPFLAGS CPP -PROFILE_TASK' +PROFILE_TASK +PKG_CONFIG +PKG_CONFIG_PATH +PKG_CONFIG_LIBDIR' # Initialize some variables set by options. @@ -1556,9 +1562,9 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no) - --with-hash-algorithm=[fnv|siphash24] + --with-hash-algorithm=[fnv|siphash13|siphash24] select hash algorithm for use in Python/pyhash.c - (default is SipHash24) + (default is SipHash13) --with-tzpath= Select the default time zone search path for zoneinfo.TZPATH @@ -1583,6 +1589,7 @@ Optional Packages: names `ndbm', `gdbm' and `bdb'. --with-doc-strings enable documentation strings (default is yes) --with-pymalloc enable specialized mallocs (default is yes) + --with-freelists enable object freelists (default is yes) --with-c-locale-coercion enable C locale coercion to a UTF-8 based locale (default is yes) @@ -1638,6 +1645,11 @@ Some influential environment variables: CPP C preprocessor PROFILE_TASK Python args for PGO generation task + PKG_CONFIG path to pkg-config utility + PKG_CONFIG_PATH + directories to add to pkg-config's search path + PKG_CONFIG_LIBDIR + path overriding pkg-config's built-in search path Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -5217,9 +5229,6 @@ $as_echo "$as_me: fi -MULTIARCH=$($CC --print-multiarch 2>/dev/null) - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for the platform triplet based on compiler characteristics" >&5 $as_echo_n "checking for the platform triplet based on compiler characteristics... " >&6; } cat >> conftest.c <&6; } fi rm -f conftest.c conftest.out +if test x$PLATFORM_TRIPLET != xdarwin; then + MULTIARCH=$($CC --print-multiarch 2>/dev/null) +fi + + if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then as_fn_error $? "internal configure error for the platform triplet, please file a bug report" "$LINENO" 5 @@ -9764,7 +9778,7 @@ then BLDSHARED="$LDSHARED" fi ;; - Linux*|GNU*|QNX*|VxWorks*) + Linux*|GNU*|QNX*|VxWorks*|Haiku*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; FreeBSD*) @@ -9835,6 +9849,7 @@ then Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";; + Haiku*) CCSHARED="-fPIC";; OpenUNIX*|UnixWare*) if test "$GCC" = "yes" then CCSHARED="-fPIC" @@ -10420,6 +10435,10 @@ if test "${with_hash_algorithm+set}" = set; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 $as_echo "$withval" >&6; } case "$withval" in + siphash13) + $as_echo "#define Py_HASH_ALGORITHM 3" >>confdefs.h + + ;; siphash24) $as_echo "#define Py_HASH_ALGORITHM 1" >>confdefs.h @@ -10562,6 +10581,51 @@ if test "x$ac_cv_lib_socket_socket" = xyes; then : fi # SVR4 sockets +case $ac_sys_system/$ac_sys_release in + Haiku*) + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for socket in -lnetwork" >&5 +$as_echo_n "checking for socket in -lnetwork... " >&6; } +if ${ac_cv_lib_network_socket+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lnetwork $LIBS $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char socket (); +int +main () +{ +return socket (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_network_socket=yes +else + ac_cv_lib_network_socket=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_network_socket" >&5 +$as_echo "$ac_cv_lib_network_socket" >&6; } +if test "x$ac_cv_lib_network_socket" = xyes; then : + LIBS="-lnetwork $LIBS" +fi + + ;; +esac + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-libs" >&5 $as_echo_n "checking for --with-libs... " >&6; } @@ -10578,7 +10642,126 @@ $as_echo "no" >&6; } fi -PKG_PROG_PKG_CONFIG + + + + + + + +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}pkg-config", so it can be a program name with args. +set dummy ${ac_tool_prefix}pkg-config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PKG_CONFIG+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PKG_CONFIG=$ac_cv_path_PKG_CONFIG +if test -n "$PKG_CONFIG"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PKG_CONFIG" >&5 +$as_echo "$PKG_CONFIG" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_PKG_CONFIG"; then + ac_pt_PKG_CONFIG=$PKG_CONFIG + # Extract the first word of "pkg-config", so it can be a program name with args. +set dummy pkg-config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_PKG_CONFIG+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG +if test -n "$ac_pt_PKG_CONFIG"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_PKG_CONFIG" >&5 +$as_echo "$ac_pt_PKG_CONFIG" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_PKG_CONFIG" = x; then + PKG_CONFIG="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + PKG_CONFIG=$ac_pt_PKG_CONFIG + fi +else + PKG_CONFIG="$ac_cv_path_PKG_CONFIG" +fi + +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=0.9.0 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking pkg-config is at least version $_pkg_min_version" >&5 +$as_echo_n "checking pkg-config is at least version $_pkg_min_version... " >&6; } + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + PKG_CONFIG="" + fi +fi # Check for use of the system expat library { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-expat" >&5 @@ -11547,6 +11730,30 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pymalloc" >&5 $as_echo "$with_pymalloc" >&6; } +# Check whether objects such as float, tuple and dict are using +# freelists to optimization memory allocation. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-freelists" >&5 +$as_echo_n "checking for --with-freelists... " >&6; } + +# Check whether --with-freelists was given. +if test "${with_freelists+set}" = set; then : + withval=$with_freelists; +fi + + +if test -z "$with_freelists" +then + with_freelists="yes" +fi +if test "$with_freelists" != "no" +then + +$as_echo "#define WITH_FREELISTS 1" >>confdefs.h + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_freelists" >&5 +$as_echo "$with_freelists" >&6; } + # Check for --with-c-locale-coercion { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-c-locale-coercion" >&5 $as_echo_n "checking for --with-c-locale-coercion... " >&6; } @@ -11774,7 +11981,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ pthread_condattr_setclock pthread_init pthread_kill pwrite pwritev pwritev2 \ readlink readlinkat readv realpath renameat \ - sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ + sem_open sem_timedwait sem_clockwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ sched_get_priority_max sched_setaffinity sched_setscheduler sched_setparam \ @@ -13048,6 +13255,8 @@ done # We search for both crypt and crypt_r as one or the other may be defined # This gets us our -lcrypt in LIBS when required on the target platform. +# Save/restore LIBS to avoid linking libpython with libcrypt. +LIBS_SAVE=$LIBS { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing crypt" >&5 $as_echo_n "checking for library containing crypt... " >&6; } if ${ac_cv_search_crypt+:} false; then : @@ -13189,6 +13398,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi +LIBS=$LIBS_SAVE for ac_func in clock_gettime do : @@ -13252,19 +13462,19 @@ fi done -for ac_func in clock_nanosleep +for ac_func in clock_getres do : - ac_fn_c_check_func "$LINENO" "clock_nanosleep" "ac_cv_func_clock_nanosleep" -if test "x$ac_cv_func_clock_nanosleep" = xyes; then : + ac_fn_c_check_func "$LINENO" "clock_getres" "ac_cv_func_clock_getres" +if test "x$ac_cv_func_clock_getres" = xyes; then : cat >>confdefs.h <<_ACEOF -#define HAVE_CLOCK_NANOSLEEP 1 +#define HAVE_CLOCK_GETRES 1 _ACEOF else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 -$as_echo_n "checking for clock_nanosleep in -lrt... " >&6; } -if ${ac_cv_lib_rt_clock_nanosleep+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for clock_getres in -lrt" >&5 +$as_echo_n "checking for clock_getres in -lrt... " >&6; } +if ${ac_cv_lib_rt_clock_getres+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -13278,29 +13488,29 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char clock_nanosleep (); +char clock_getres (); int main () { -return clock_nanosleep (); +return clock_getres (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_rt_clock_nanosleep=yes + ac_cv_lib_rt_clock_getres=yes else - ac_cv_lib_rt_clock_nanosleep=no + ac_cv_lib_rt_clock_getres=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_nanosleep" >&5 -$as_echo "$ac_cv_lib_rt_clock_nanosleep" >&6; } -if test "x$ac_cv_lib_rt_clock_nanosleep" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_getres" >&5 +$as_echo "$ac_cv_lib_rt_clock_getres" >&6; } +if test "x$ac_cv_lib_rt_clock_getres" = xyes; then : - $as_echo "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h + $as_echo "#define HAVE_CLOCK_GETRES 1" >>confdefs.h fi @@ -13310,19 +13520,19 @@ fi done -for ac_func in nanosleep +for ac_func in clock_settime do : - ac_fn_c_check_func "$LINENO" "nanosleep" "ac_cv_func_nanosleep" -if test "x$ac_cv_func_nanosleep" = xyes; then : + ac_fn_c_check_func "$LINENO" "clock_settime" "ac_cv_func_clock_settime" +if test "x$ac_cv_func_clock_settime" = xyes; then : cat >>confdefs.h <<_ACEOF -#define HAVE_NANOSLEEP 1 +#define HAVE_CLOCK_SETTIME 1 _ACEOF else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nanosleep in -lrt" >&5 -$as_echo_n "checking for nanosleep in -lrt... " >&6; } -if ${ac_cv_lib_rt_nanosleep+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 +$as_echo_n "checking for clock_settime in -lrt... " >&6; } +if ${ac_cv_lib_rt_clock_settime+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -13336,29 +13546,29 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char nanosleep (); +char clock_settime (); int main () { -return nanosleep (); +return clock_settime (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_rt_nanosleep=yes + ac_cv_lib_rt_clock_settime=yes else - ac_cv_lib_rt_nanosleep=no + ac_cv_lib_rt_clock_settime=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_nanosleep" >&5 -$as_echo "$ac_cv_lib_rt_nanosleep" >&6; } -if test "x$ac_cv_lib_rt_nanosleep" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_settime" >&5 +$as_echo "$ac_cv_lib_rt_clock_settime" >&6; } +if test "x$ac_cv_lib_rt_clock_settime" = xyes; then : - $as_echo "#define HAVE_NANOSLEEP 1" >>confdefs.h + $as_echo "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h fi @@ -13368,19 +13578,19 @@ fi done -for ac_func in clock_getres +for ac_func in clock_nanosleep do : - ac_fn_c_check_func "$LINENO" "clock_getres" "ac_cv_func_clock_getres" -if test "x$ac_cv_func_clock_getres" = xyes; then : + ac_fn_c_check_func "$LINENO" "clock_nanosleep" "ac_cv_func_clock_nanosleep" +if test "x$ac_cv_func_clock_nanosleep" = xyes; then : cat >>confdefs.h <<_ACEOF -#define HAVE_CLOCK_GETRES 1 +#define HAVE_CLOCK_NANOSLEEP 1 _ACEOF else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for clock_getres in -lrt" >&5 -$as_echo_n "checking for clock_getres in -lrt... " >&6; } -if ${ac_cv_lib_rt_clock_getres+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 +$as_echo_n "checking for clock_nanosleep in -lrt... " >&6; } +if ${ac_cv_lib_rt_clock_nanosleep+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -13394,29 +13604,29 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char clock_getres (); +char clock_nanosleep (); int main () { -return clock_getres (); +return clock_nanosleep (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_rt_clock_getres=yes + ac_cv_lib_rt_clock_nanosleep=yes else - ac_cv_lib_rt_clock_getres=no + ac_cv_lib_rt_clock_nanosleep=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_getres" >&5 -$as_echo "$ac_cv_lib_rt_clock_getres" >&6; } -if test "x$ac_cv_lib_rt_clock_getres" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_nanosleep" >&5 +$as_echo "$ac_cv_lib_rt_clock_nanosleep" >&6; } +if test "x$ac_cv_lib_rt_clock_nanosleep" = xyes; then : - $as_echo "#define HAVE_CLOCK_GETRES 1" >>confdefs.h + $as_echo "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h fi @@ -13426,19 +13636,19 @@ fi done -for ac_func in clock_settime +for ac_func in nanosleep do : - ac_fn_c_check_func "$LINENO" "clock_settime" "ac_cv_func_clock_settime" -if test "x$ac_cv_func_clock_settime" = xyes; then : + ac_fn_c_check_func "$LINENO" "nanosleep" "ac_cv_func_nanosleep" +if test "x$ac_cv_func_nanosleep" = xyes; then : cat >>confdefs.h <<_ACEOF -#define HAVE_CLOCK_SETTIME 1 +#define HAVE_NANOSLEEP 1 _ACEOF else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 -$as_echo_n "checking for clock_settime in -lrt... " >&6; } -if ${ac_cv_lib_rt_clock_settime+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nanosleep in -lrt" >&5 +$as_echo_n "checking for nanosleep in -lrt... " >&6; } +if ${ac_cv_lib_rt_nanosleep+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -13452,29 +13662,29 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char clock_settime (); +char nanosleep (); int main () { -return clock_settime (); +return nanosleep (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_rt_clock_settime=yes + ac_cv_lib_rt_nanosleep=yes else - ac_cv_lib_rt_clock_settime=no + ac_cv_lib_rt_nanosleep=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_settime" >&5 -$as_echo "$ac_cv_lib_rt_clock_settime" >&6; } -if test "x$ac_cv_lib_rt_clock_settime" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_nanosleep" >&5 +$as_echo "$ac_cv_lib_rt_nanosleep" >&6; } +if test "x$ac_cv_lib_rt_nanosleep" = xyes; then : - $as_echo "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h + $as_echo "#define HAVE_NANOSLEEP 1" >>confdefs.h fi @@ -14882,7 +15092,7 @@ fi LIBS_SAVE=$LIBS LIBS="$LIBS $LIBM" -for ac_func in acosh asinh atanh copysign erf erfc expm1 finite gamma +for ac_func in acosh asinh atanh erf erfc expm1 finite gamma do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -14894,7 +15104,7 @@ _ACEOF fi done -for ac_func in hypot lgamma log1p log2 round tgamma +for ac_func in lgamma log1p log2 tgamma do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -14906,40 +15116,6 @@ _ACEOF fi done -ac_fn_c_check_decl "$LINENO" "isinf" "ac_cv_have_decl_isinf" "#include -" -if test "x$ac_cv_have_decl_isinf" = xyes; then : - ac_have_decl=1 -else - ac_have_decl=0 -fi - -cat >>confdefs.h <<_ACEOF -#define HAVE_DECL_ISINF $ac_have_decl -_ACEOF -ac_fn_c_check_decl "$LINENO" "isnan" "ac_cv_have_decl_isnan" "#include -" -if test "x$ac_cv_have_decl_isnan" = xyes; then : - ac_have_decl=1 -else - ac_have_decl=0 -fi - -cat >>confdefs.h <<_ACEOF -#define HAVE_DECL_ISNAN $ac_have_decl -_ACEOF -ac_fn_c_check_decl "$LINENO" "isfinite" "ac_cv_have_decl_isfinite" "#include -" -if test "x$ac_cv_have_decl_isfinite" = xyes; then : - ac_have_decl=1 -else - ac_have_decl=0 -fi - -cat >>confdefs.h <<_ACEOF -#define HAVE_DECL_ISFINITE $ac_have_decl -_ACEOF - # For multiprocessing module, check that sem_open # actually works. For FreeBSD versions <= 7.2, @@ -17799,6 +17975,66 @@ esac $as_echo "$OPENSSL_RPATH" >&6; } +# check if OpenSSL libraries work as expected +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether OpenSSL provides required APIs" >&5 +$as_echo_n "checking whether OpenSSL provides required APIs... " >&6; } +save_LIBS="$LIBS" +save_CFLAGS="$CFLAGS" +save_LDFLAGS="$LDFLAGS" +LIBS="$LIBS $OPENSSL_LIBS" +CFLAGS="$CFLAGS_NODIST $OPENSSL_INCLUDES" +LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS" + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +#include + +#if OPENSSL_VERSION_NUMBER < 0x10101000L +#error "OpenSSL >= 1.1.1 is required" +#endif + +static void keylog_cb(const SSL *ssl, const char *line) {} + +int +main () +{ + +/* SSL APIs */ +SSL_CTX *ctx = SSL_CTX_new(TLS_client_method()); +SSL_CTX_set_keylog_callback(ctx, keylog_cb); +SSL *ssl = SSL_new(ctx); +X509_VERIFY_PARAM *param = SSL_get0_param(ssl); +X509_VERIFY_PARAM_set1_host(param, "python.org", 0); +SSL_free(ssl); +SSL_CTX_free(ctx); + +/* hashlib APIs */ +OBJ_nid2sn(NID_md5); +OBJ_nid2sn(NID_sha1); +OBJ_nid2sn(NID_sha3_512); +OBJ_nid2sn(NID_blake2b512); +EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS="$save_LIBS" +CFLAGS="$save_CFLAGS" +LDFLAGS="$save_LDFLAGS" + # ssl module default cipher suite string diff --git a/configure.ac b/configure.ac index 48d86ef79199e1..edda08daac117e 100644 --- a/configure.ac +++ b/configure.ac @@ -717,9 +717,6 @@ then fi -MULTIARCH=$($CC --print-multiarch 2>/dev/null) -AC_SUBST(MULTIARCH) - AC_MSG_CHECKING([for the platform triplet based on compiler characteristics]) cat >> conftest.c </dev/null) +fi +AC_SUBST(MULTIARCH) + if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then AC_MSG_ERROR([internal configure error for the platform triplet, please file a bug report]) @@ -3036,16 +3038,19 @@ fi # str, bytes and memoryview hash algorithm AH_TEMPLATE(Py_HASH_ALGORITHM, [Define hash algorithm for str, bytes and memoryview. - SipHash24: 1, FNV: 2, externally defined: 0]) + SipHash24: 1, FNV: 2, SipHash13: 3, externally defined: 0]) AC_MSG_CHECKING(for --with-hash-algorithm) dnl quadrigraphs "@<:@" and "@:>@" produce "[" and "]" in the output AC_ARG_WITH(hash_algorithm, - AS_HELP_STRING([--with-hash-algorithm=@<:@fnv|siphash24@:>@], - [select hash algorithm for use in Python/pyhash.c (default is SipHash24)]), + AS_HELP_STRING([--with-hash-algorithm=@<:@fnv|siphash13|siphash24@:>@], + [select hash algorithm for use in Python/pyhash.c (default is SipHash13)]), [ AC_MSG_RESULT($withval) case "$withval" in + siphash13) + AC_DEFINE(Py_HASH_ALGORITHM, 3) + ;; siphash24) AC_DEFINE(Py_HASH_ALGORITHM, 1) ;; @@ -3099,8 +3104,11 @@ AC_SUBST(TZPATH) AC_CHECK_LIB(nsl, t_open, [LIBS="-lnsl $LIBS"]) # SVR4 AC_CHECK_LIB(socket, socket, [LIBS="-lsocket $LIBS"], [], $LIBS) # SVR4 sockets -# Haiku system library -AC_CHECK_LIB(network, socket, [LIBS="-lnetwork $LIBS"], [], $LIBS) +case $ac_sys_system/$ac_sys_release in + Haiku*) + AC_CHECK_LIB(network, socket, [LIBS="-lnetwork $LIBS"], [], $LIBS) + ;; +esac AC_MSG_CHECKING(for --with-libs) AC_ARG_WITH(libs, @@ -3608,6 +3616,23 @@ then fi AC_MSG_RESULT($with_pymalloc) +# Check whether objects such as float, tuple and dict are using +# freelists to optimization memory allocation. +AC_MSG_CHECKING(for --with-freelists) +AC_ARG_WITH(freelists, + AS_HELP_STRING([--with-freelists], [enable object freelists (default is yes)])) + +if test -z "$with_freelists" +then + with_freelists="yes" +fi +if test "$with_freelists" != "no" +then + AC_DEFINE(WITH_FREELISTS, 1, + [Define if you want to compile in object freelists optimization]) +fi +AC_MSG_RESULT($with_freelists) + # Check for --with-c-locale-coercion AC_MSG_CHECKING(for --with-c-locale-coercion) AC_ARG_WITH(c-locale-coercion, @@ -3744,7 +3769,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ pthread_condattr_setclock pthread_init pthread_kill pwrite pwritev pwritev2 \ readlink readlinkat readv realpath renameat \ - sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ + sem_open sem_timedwait sem_clockwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ sched_get_priority_max sched_setaffinity sched_setscheduler sched_setparam \ @@ -4079,6 +4104,8 @@ AC_CHECK_FUNCS(setpgrp, # We search for both crypt and crypt_r as one or the other may be defined # This gets us our -lcrypt in LIBS when required on the target platform. +# Save/restore LIBS to avoid linking libpython with libcrypt. +LIBS_SAVE=$LIBS AC_SEARCH_LIBS(crypt, crypt) AC_SEARCH_LIBS(crypt_r, crypt) @@ -4093,6 +4120,7 @@ char *r = crypt_r("", "", &d); [AC_DEFINE(HAVE_CRYPT_R, 1, [Define if you have the crypt_r() function.])], []) ) +LIBS=$LIBS_SAVE AC_CHECK_FUNCS(clock_gettime, [], [ AC_CHECK_LIB(rt, clock_gettime, [ @@ -4664,9 +4692,8 @@ fi LIBS_SAVE=$LIBS LIBS="$LIBS $LIBM" -AC_CHECK_FUNCS([acosh asinh atanh copysign erf erfc expm1 finite gamma]) -AC_CHECK_FUNCS([hypot lgamma log1p log2 round tgamma]) -AC_CHECK_DECLS([isinf, isnan, isfinite], [], [], [[#include ]]) +AC_CHECK_FUNCS([acosh asinh atanh erf erfc expm1 finite gamma]) +AC_CHECK_FUNCS([lgamma log1p log2 tgamma]) # For multiprocessing module, check that sem_open # actually works. For FreeBSD versions <= 7.2, @@ -5878,6 +5905,48 @@ AS_CASE($with_openssl_rpath, AC_MSG_RESULT($OPENSSL_RPATH) AC_SUBST([OPENSSL_RPATH]) +# check if OpenSSL libraries work as expected +AC_MSG_CHECKING(whether OpenSSL provides required APIs) +save_LIBS="$LIBS" +save_CFLAGS="$CFLAGS" +save_LDFLAGS="$LDFLAGS" +LIBS="$LIBS $OPENSSL_LIBS" +CFLAGS="$CFLAGS_NODIST $OPENSSL_INCLUDES" +LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS" + +AC_LINK_IFELSE([AC_LANG_PROGRAM([[ +#include +#include +#include + +#if OPENSSL_VERSION_NUMBER < 0x10101000L +#error "OpenSSL >= 1.1.1 is required" +#endif + +static void keylog_cb(const SSL *ssl, const char *line) {} +]], [[ +/* SSL APIs */ +SSL_CTX *ctx = SSL_CTX_new(TLS_client_method()); +SSL_CTX_set_keylog_callback(ctx, keylog_cb); +SSL *ssl = SSL_new(ctx); +X509_VERIFY_PARAM *param = SSL_get0_param(ssl); +X509_VERIFY_PARAM_set1_host(param, "python.org", 0); +SSL_free(ssl); +SSL_CTX_free(ctx); + +/* hashlib APIs */ +OBJ_nid2sn(NID_md5); +OBJ_nid2sn(NID_sha1); +OBJ_nid2sn(NID_sha3_512); +OBJ_nid2sn(NID_blake2b512); +EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); +]])], + [AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no)]) +LIBS="$save_LIBS" +CFLAGS="$save_CFLAGS" +LDFLAGS="$save_LDFLAGS" + # ssl module default cipher suite string AH_TEMPLATE(PY_SSL_DEFAULT_CIPHERS, [Default cipher suites list for ssl module. diff --git a/pyconfig.h.in b/pyconfig.h.in index 23d7111b9f77e7..081ea61bae8347 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -136,15 +136,15 @@ /* Define to 1 if you have the `clock' function. */ #undef HAVE_CLOCK -/* Define to 1 if you have the `clock_nanosleep' function. */ -#undef HAVE_CLOCK_NANOSLEEP - /* Define to 1 if you have the `clock_getres' function. */ #undef HAVE_CLOCK_GETRES /* Define to 1 if you have the `clock_gettime' function. */ #undef HAVE_CLOCK_GETTIME +/* Define to 1 if you have the `clock_nanosleep' function. */ +#undef HAVE_CLOCK_NANOSLEEP + /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME @@ -160,9 +160,6 @@ /* Define to 1 if you have the header file. */ #undef HAVE_CONIO_H -/* Define to 1 if you have the `copysign' function. */ -#undef HAVE_COPYSIGN - /* Define to 1 if you have the `copy_file_range' function. */ #undef HAVE_COPY_FILE_RANGE @@ -214,18 +211,6 @@ /* Define if you have the 'wchgat' function. */ #undef HAVE_CURSES_WCHGAT -/* Define to 1 if you have the declaration of `isfinite', and to 0 if you - don't. */ -#undef HAVE_DECL_ISFINITE - -/* Define to 1 if you have the declaration of `isinf', and to 0 if you don't. - */ -#undef HAVE_DECL_ISINF - -/* Define to 1 if you have the declaration of `isnan', and to 0 if you don't. - */ -#undef HAVE_DECL_ISNAN - /* Define to 1 if you have the declaration of `RTLD_DEEPBIND', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_DEEPBIND @@ -550,9 +535,6 @@ /* Define this if you have le64toh() */ #undef HAVE_HTOLE64 -/* Define to 1 if you have the `hypot' function. */ -#undef HAVE_HYPOT - /* Define to 1 if you have the header file. */ #undef HAVE_IEEEFP_H @@ -884,9 +866,6 @@ /* Define if you have readline 4.0 */ #undef HAVE_RL_RESIZE_TERMINAL -/* Define to 1 if you have the `round' function. */ -#undef HAVE_ROUND - /* Define to 1 if you have the `rtpSpawn' function. */ #undef HAVE_RTPSPAWN @@ -908,6 +887,9 @@ /* Define to 1 if you have the `sched_setscheduler' function. */ #undef HAVE_SCHED_SETSCHEDULER +/* Define to 1 if you have the `sem_clockwait' function. */ +#undef HAVE_SEM_CLOCKWAIT + /* Define to 1 if you have the `sem_getvalue' function. */ #undef HAVE_SEM_GETVALUE @@ -1436,7 +1418,7 @@ #undef Py_ENABLE_SHARED /* Define hash algorithm for str, bytes and memoryview. SipHash24: 1, FNV: 2, - externally defined: 0 */ + SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM /* Define if you want to enable tracing references for debugging purpose */ @@ -1573,6 +1555,9 @@ /* Define to build the readline module against Editline. */ #undef WITH_EDITLINE +/* Define if you want to compile in object freelists optimization */ +#undef WITH_FREELISTS + /* Define to 1 if libintl is needed for locale functions. */ #undef WITH_LIBINTL diff --git a/setup.py b/setup.py index 9e411cb35e00bd..5428cbde1cc9b0 100644 --- a/setup.py +++ b/setup.py @@ -801,6 +801,18 @@ def add_ldflags_cppflags(self): if env_val: parser = argparse.ArgumentParser() parser.add_argument(arg_name, dest="dirs", action="append") + + # To prevent argparse from raising an exception about any + # options in env_val that it mistakes for known option, we + # strip out all double dashes and any dashes followed by a + # character that is not for the option we are dealing with. + # + # Please note that order of the regex is important! We must + # strip out double-dashes first so that we don't end up with + # substituting "--Long" to "-Long" and thus lead to "ong" being + # used for a library directory. + env_val = re.sub(r'(^|\s+)-(-|(?!%s))' % arg_name[1], + ' ', env_val) options, _ = parser.parse_known_args(env_val.split()) if options.dirs: for directory in reversed(options.dirs): @@ -942,7 +954,8 @@ def detect_simple_extensions(self): extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # profiler (_lsprof is for cProfile.py) - self.add(Extension('_lsprof', ['_lsprof.c', 'rotatingtree.c'])) + self.add(Extension('_lsprof', ['_lsprof.c', 'rotatingtree.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # static Unicode character database self.add(Extension('unicodedata', ['unicodedata.c'], depends=['unicodedata_db.h', 'unicodename_db.h'], @@ -999,7 +1012,8 @@ def detect_simple_extensions(self): self.add(Extension('syslog', ['syslogmodule.c'])) # Python interface to subinterpreter C-API. - self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'])) + self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # # Here ends the simple stuff. From here on, modules need certain @@ -1041,7 +1055,8 @@ def detect_test_extensions(self): self.add(Extension('_testimportmultiple', ['_testimportmultiple.c'])) # Test multi-phase extension module init (PEP 489) - self.add(Extension('_testmultiphase', ['_testmultiphase.c'])) + self.add(Extension('_testmultiphase', ['_testmultiphase.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # Fuzz tests. self.add(Extension('_xxtestfuzz', @@ -1707,12 +1722,12 @@ def detect_compress_exts(self): # Helper module for various ascii-encoders. Uses zlib for an optimized # crc32 if we have it. Otherwise binascii uses its own. + extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] if have_zlib: - extra_compile_args = ['-DUSE_ZLIB_CRC32'] + extra_compile_args.append('-DUSE_ZLIB_CRC32') libraries = ['z'] extra_link_args = zlib_extra_link_args else: - extra_compile_args = [] libraries = [] extra_link_args = [] self.add(Extension('binascii', ['binascii.c'], @@ -1767,7 +1782,9 @@ def detect_expat_elementtree(self): ('XML_POOR_ENTROPY', '1'), ] extra_compile_args = [] - expat_lib = [] + # bpo-44394: libexpat uses isnan() of math.h and needs linkage + # against the libm + expat_lib = ['m'] expat_sources = ['expat/xmlparse.c', 'expat/xmlrole.c', 'expat/xmltok.c'] @@ -2307,7 +2324,7 @@ def detect_ctypes(self): def detect_decimal(self): # Stefan Krah's _decimal module - extra_compile_args = [] + extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] undef_macros = [] if '--with-system-libmpdec' in sysconfig.get_config_var("CONFIG_ARGS"): include_dirs = [] @@ -2465,6 +2482,7 @@ def split_var(name, sep): library_dirs=openssl_libdirs, libraries=openssl_libs, runtime_library_dirs=runtime_library_dirs, + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], ) # This static linking is NOT OFFICIALLY SUPPORTED. @@ -2526,27 +2544,29 @@ def detect_hash_builtins(self): if "sha256" in configured: self.add(Extension( '_sha256', ['sha256module.c'], + depends=['hashlib.h'], extra_compile_args=['-DPy_BUILD_CORE_MODULE'], - depends=['hashlib.h'] )) if "sha512" in configured: self.add(Extension( '_sha512', ['sha512module.c'], + depends=['hashlib.h'], extra_compile_args=['-DPy_BUILD_CORE_MODULE'], - depends=['hashlib.h'] )) if "md5" in configured: self.add(Extension( '_md5', ['md5module.c'], - depends=['hashlib.h'] + depends=['hashlib.h'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], )) if "sha1" in configured: self.add(Extension( '_sha1', ['sha1module.c'], - depends=['hashlib.h'] + depends=['hashlib.h'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], )) if "blake2" in configured: @@ -2561,7 +2581,8 @@ def detect_hash_builtins(self): '_blake2/blake2b_impl.c', '_blake2/blake2s_impl.c' ], - depends=blake2_deps + depends=blake2_deps, + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], )) if "sha3" in configured: @@ -2572,7 +2593,8 @@ def detect_hash_builtins(self): self.add(Extension( '_sha3', ['_sha3/sha3module.c'], - depends=sha3_deps + depends=sha3_deps, + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], )) def detect_nis(self):