From 228f275737615cc9be713a8c3f9325b359bf8aec Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 22 Dec 2024 01:47:41 -0800 Subject: [PATCH 01/44] gh-126664: revert: Use `else` instead of `finally` in docs explaining "with" (#128169) Revert "gh-126664: Use `else` instead of `finally` in "The with statement" documentation. (GH-126665)" This reverts commit 25257d61cfccc3b4189f96390a5c4db73fd5302c. --- Doc/reference/compound_stmts.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e73ce44270b082..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -534,15 +534,18 @@ is semantically equivalent to:: enter = type(manager).__enter__ exit = type(manager).__exit__ value = enter(manager) + hit_except = False try: TARGET = value SUITE except: + hit_except = True if not exit(manager, *sys.exc_info()): raise - else: - exit(manager, None, None, None) + finally: + if not hit_except: + exit(manager, None, None, None) With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: From b66a4ad9fc32b63da2ba10db24cbc8f4e29f781a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 22 Dec 2024 12:46:02 +0000 Subject: [PATCH 02/44] gh-127949: fix resource warnings in `test_tasks.py` (#128172) --- Lib/test/test_asyncio/test_tasks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 5b8979a8bbd13a..7d6d0564a9a9db 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2698,17 +2698,17 @@ def __str__(self): initial_refcount = sys.getrefcount(obj) coro = coroutine_function() - loop = asyncio.new_event_loop() - task = asyncio.Task.__new__(asyncio.Task) + with contextlib.closing(asyncio.EventLoop()) as loop: + task = asyncio.Task.__new__(asyncio.Task) - for _ in range(5): - with self.assertRaisesRegex(RuntimeError, 'break'): - task.__init__(coro, loop=loop, context=obj, name=Break()) + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) - coro.close() - del task + coro.close() + del task - self.assertEqual(sys.getrefcount(obj), initial_refcount) + self.assertEqual(sys.getrefcount(obj), initial_refcount) def add_subclass_tests(cls): From f420bdd29fbc1a97ad20d88075c38c937c1f8479 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sun, 22 Dec 2024 17:34:16 +0100 Subject: [PATCH 03/44] gh-119786: Fix typos in `InternalDocs/interpreter.md` (#128174) --- InternalDocs/interpreter.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index fa4a54fdc54fac..52702792c6cb7b 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -20,7 +20,7 @@ When the interpreter's [`PyEval_EvalCode()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) function is called to execute a `CodeObject`, it constructs a [`Frame`](frames.md) and calls [`_PyEval_EvalFrame()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) -to execute the code object in this frame. The frame hold the dynamic state of the +to execute the code object in this frame. The frame holds the dynamic state of the `CodeObject`'s execution, including the instruction pointer, the globals and builtins. It also has a reference to the `CodeObject` itself. @@ -153,9 +153,9 @@ More information about the use of inline caches can be found in Most instructions read or write some data in the form of object references (`PyObject *`). The CPython bytecode interpreter is a stack machine, meaning that its instructions operate by pushing data onto and popping it off the stack. -The stack is forms part of the frame for the code object. Its maximum depth is calculated +The stack forms part of the frame for the code object. Its maximum depth is calculated by the compiler and stored in the `co_stacksize` field of the code object, so that the -stack can be pre-allocated is a contiguous array of `PyObject*` pointers, when the frame +stack can be pre-allocated as a contiguous array of `PyObject*` pointers, when the frame is created. The stack effects of each instruction are also exposed through the @@ -462,7 +462,7 @@ set of values that allows them to: 2. Perform the operation quickly. This requires that the set of values is chosen such that membership can be -tested quickly and that membership is sufficient to allow the operation to +tested quickly and that membership is sufficient to allow the operation to be performed quickly. For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` From 9d3a8f494985e8bbef698c467099370e233fcbd4 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sun, 22 Dec 2024 13:01:45 -0600 Subject: [PATCH 04/44] gh-100384: Error on `unguarded-availability` in macOS builds (#128155) Generate a build error on ``unguarded-availability`` in portable macOS builds (i.e. using MACOSX_DEPLOYMENT_TARGET), preventing invalid use of symbols that are not available in older versions of the OS. --- .github/workflows/reusable-macos.yml | 1 + ...-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst | 2 + configure | 41 +++++++++++++++++++ configure.ac | 7 ++++ 4 files changed, 51 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 6fa389b2d66e5e..cdbe05e09fb8e7 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -45,6 +45,7 @@ jobs: brew link --overwrite tcl-tk@8 - name: Configure CPython run: | + MACOSX_DEPLOYMENT_TARGET=10.15 \ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ ./configure \ diff --git a/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst b/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst new file mode 100644 index 00000000000000..75c19fe3d8cef9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst @@ -0,0 +1,2 @@ +Error on ``unguarded-availability`` in macOS builds, preventing invalid +use of symbols that are not available in older versions of the OS. diff --git a/configure b/configure index e59c7046305d46..a697bc1f87b012 100755 --- a/configure +++ b/configure @@ -10406,6 +10406,47 @@ printf %s "checking which compiler should be used... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 printf "%s\n" "$CC" >&6; } + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wunguarded-availability" >&5 +printf %s "checking whether C compiler accepts -Wunguarded-availability... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wunguarded_availability+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wunguarded-availability" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wunguarded_availability=yes +else $as_nop + ax_cv_check_cflags__Werror__Wunguarded_availability=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wunguarded_availability" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wunguarded_availability" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wunguarded_availability" = xyes +then : + as_fn_append CFLAGS_NODIST " -Werror=unguarded-availability" +else $as_nop + : +fi + + LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" then diff --git a/configure.ac b/configure.ac index 074e2ce3dd3024..ebc15503f069cc 100644 --- a/configure.ac +++ b/configure.ac @@ -2603,6 +2603,13 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ esac AC_MSG_RESULT([$CC]) + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + AX_CHECK_COMPILE_FLAG([-Wunguarded-availability], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -Werror=unguarded-availability"])], + [], + [-Werror]) + LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" then From 831b6de6d725c697f2f61fd35c4448cd8a9354ff Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Mon, 23 Dec 2024 14:17:19 +1000 Subject: [PATCH 05/44] gh-126180: Remove getopt and optparse deprecation notices (GH-126227) * Remove getopt and optparse deprecation notices * Add new docs sections for command line app helper libraries * Add guidance on choosing a CLI parsing library to the optparse docs * Link to the new guidance from the argparse and getopt docs * Reword intro in docs section for superseded stdlib modules * Reframe the optparse->argparse guide as a migration guide rather than as an upgrade guide --------- Co-authored-by: Serhiy Storchaka --- Doc/howto/argparse-optparse.rst | 36 +++-- Doc/howto/argparse.rst | 15 +- Doc/library/allos.rst | 5 - Doc/library/argparse.rst | 12 ++ Doc/library/cmdlinelibs.rst | 21 +++ Doc/library/filesys.rst | 1 - Doc/library/getopt.rst | 54 +++++-- Doc/library/index.rst | 1 + Doc/library/optparse.rst | 139 ++++++++++++++++-- Doc/library/superseded.rst | 17 ++- Doc/whatsnew/3.13.rst | 24 ++- ...-10-31-14-31-36.gh-issue-126225.vTxGXm.rst | 6 + 12 files changed, 266 insertions(+), 65 deletions(-) create mode 100644 Doc/library/cmdlinelibs.rst create mode 100644 Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst index cef2d893b28a62..b684619885b4c7 100644 --- a/Doc/howto/argparse-optparse.rst +++ b/Doc/howto/argparse-optparse.rst @@ -1,20 +1,14 @@ .. currentmodule:: argparse .. _upgrading-optparse-code: +.. _migrating-optparse-code: -========================== -Upgrading optparse code -========================== +============================================ +Migrating ``optparse`` code to ``argparse`` +============================================ -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the :mod:`optparse` -module in a number of ways including: +The :mod:`argparse` module offers several higher level features not natively +provided by the :mod:`optparse` module, including: * Handling positional arguments. * Supporting subcommands. @@ -23,7 +17,23 @@ module in a number of ways including: * Producing more informative usage messages. * Providing a much simpler interface for custom ``type`` and ``action``. -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: +Originally, the :mod:`argparse` module attempted to maintain compatibility +with :mod:`optparse`. However, the fundamental design differences between +supporting declarative command line option processing (while leaving positional +argument processing to application code), and supporting both named options +and positional arguments in the declarative interface mean that the +API has diverged from that of ``optparse`` over time. + +As described in :ref:`choosing-an-argument-parser`, applications that are +currently using :mod:`optparse` and are happy with the way it works can +just continue to use ``optparse``. + +Application developers that are considering migrating should also review +the list of intrinsic behavioural differences described in that section +before deciding whether or not migration is desirable. + +For applications that do choose to migrate from :mod:`optparse` to :mod:`argparse`, +the following suggestions should be helpful: * Replace all :meth:`optparse.OptionParser.add_option` calls with :meth:`ArgumentParser.add_argument` calls. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 1efbee64d60bb3..902c50de00803c 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -13,11 +13,16 @@ recommended command-line parsing module in the Python standard library. .. note:: - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for ``getopt()`` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. + The standard library includes two other libraries directly related + to command-line parameter processing: the lower level :mod:`optparse` + module (which may require more code to configure for a given application, + but also allows an application to request behaviors that ``argparse`` + doesn't support), and the very low level :mod:`getopt` (which specifically + serves as an equivalent to the :c:func:`!getopt` family of functions + available to C programmers). + While neither of those modules is covered directly in this guide, many of + the core concepts in ``argparse`` first originated in ``optparse``, so + some aspects of this tutorial will also be relevant to ``optparse`` users. Concepts diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index 0223c1054ea5d8..1aed340b2527ac 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -15,14 +15,9 @@ but they are available on most other systems as well. Here's an overview: os.rst io.rst time.rst - argparse.rst logging.rst logging.config.rst logging.handlers.rst - getpass.rst - curses.rst - curses.ascii.rst - curses.panel.rst platform.rst errno.rst ctypes.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index da4071dee34b8c..8d0116d8c060b8 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -11,6 +11,18 @@ **Source code:** :source:`Lib/argparse.py` +.. note:: + + While :mod:`argparse` is the default recommended standard library module + for implementing basic command line applications, authors with more + exacting requirements for exactly how their command line applications + behave may find it doesn't provide the necessary level of control. + Refer to :ref:`choosing-an-argument-parser` for alternatives to + consider when ``argparse`` doesn't support behaviors that the application + requires (such as entirely disabling support for interspersed options and + positional arguments, or accepting option parameter values that start + with ``-`` even when they correspond to another defined option). + -------------- .. sidebar:: Tutorial diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst new file mode 100644 index 00000000000000..085d31af7bca1f --- /dev/null +++ b/Doc/library/cmdlinelibs.rst @@ -0,0 +1,21 @@ +.. _cmdlinelibs: + +******************************** +Command Line Interface Libraries +******************************** + +The modules described in this chapter assist with implementing +command line and terminal interfaces for applications. + +Here's an overview: + +.. toctree:: + :maxdepth: 1 + + argparse.rst + optparse.rst + getpass.rst + fileinput.rst + curses.rst + curses.ascii.rst + curses.panel.rst diff --git a/Doc/library/filesys.rst b/Doc/library/filesys.rst index 0ccf2b7bf59a0f..f1ea4761af7cb1 100644 --- a/Doc/library/filesys.rst +++ b/Doc/library/filesys.rst @@ -14,7 +14,6 @@ in this chapter is: pathlib.rst os.path.rst - fileinput.rst stat.rst filecmp.rst tempfile.rst diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index 891885d3afbf7a..5c63009e22d58c 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -7,18 +7,13 @@ **Source code:** :source:`Lib/getopt.py` -.. deprecated:: 3.13 - The :mod:`getopt` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - .. note:: - The :mod:`getopt` module is a parser for command line options whose API is - designed to be familiar to users of the C :c:func:`!getopt` function. Users who - are unfamiliar with the C :c:func:`!getopt` function or who would like to write - less code and get better help and error messages should consider using the - :mod:`argparse` module instead. + This module is considered feature complete. A more declarative and + extensible alternative to this API is provided in the :mod:`optparse` + module. Further functional enhancements for command line parameter + processing are provided either as third party modules on PyPI, + or else as features in the :mod:`argparse` module. -------------- @@ -28,6 +23,13 @@ the special meanings of arguments of the form '``-``' and '``--``'). Long options similar to those supported by GNU software may be used as well via an optional third argument. +Users who are unfamiliar with the Unix :c:func:`!getopt` function should consider +using the :mod:`argparse` module instead. Users who are familiar with the Unix +:c:func:`!getopt` function, but would like to get equivalent behavior while +writing less code and getting better help and error messages should consider +using the :mod:`optparse` module. See :ref:`choosing-an-argument-parser` for +additional details. + This module provides two functions and an exception: @@ -194,13 +196,27 @@ In a script, typical usage is something like this: output = a else: assert False, "unhandled option" - # ... + process(args, output=output, verbose=verbose) if __name__ == "__main__": main() Note that an equivalent command line interface could be produced with less code -and more informative help and error messages by using the :mod:`argparse` module: +and more informative help and error messages by using the :mod:`optparse` module: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +A roughly equivalent command line interface for this case can also be +produced by using the :mod:`argparse` module: .. testcode:: @@ -210,12 +226,18 @@ and more informative help and error messages by using the :mod:`argparse` module parser = argparse.ArgumentParser() parser.add_argument('-o', '--output') parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') args = parser.parse_args() - # ... do something with args.output ... - # ... do something with args.verbose .. + process(args.rest, output=args.output, verbose=args.verbose) + +See :ref:`choosing-an-argument-parser` for details on how the ``argparse`` +version of this code differs in behaviour from the ``optparse`` (and +``getopt``) version. .. seealso:: - Module :mod:`argparse` - Alternative command line option and argument parsing library. + Module :mod:`optparse` + Declarative command line option parsing. + Module :mod:`argparse` + More opinionated command line option and argument parsing library. diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 951fbcf13fbb13..44b218948d07e1 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -55,6 +55,7 @@ the `Python Package Index `_. fileformats.rst crypto.rst allos.rst + cmdlinelibs.rst concurrency.rst ipc.rst netdata.rst diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 74a49a8fb33666..ff327cf9162a8c 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -3,25 +3,135 @@ .. module:: optparse :synopsis: Command-line option parsing library. - :deprecated: .. moduleauthor:: Greg Ward .. sectionauthor:: Greg Ward **Source code:** :source:`Lib/optparse.py` -.. deprecated:: 3.2 - The :mod:`optparse` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - -------------- +.. _choosing-an-argument-parser: + +Choosing an argument parsing library +------------------------------------ + +The standard library includes three argument parsing libraries: + +* :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. + Included in the standard library since before the initial Python 1.0 release. +* :mod:`optparse`: a declarative replacement for ``getopt`` that + provides equivalent functionality without requiring each application + to implement its own procedural option parsing logic. Included + in the standard library since the Python 2.3 release. +* :mod:`argparse`: a more opinionated alternative to ``optparse`` that + provides more functionality by default, at the expense of reduced application + flexibility in controlling exactly how arguments are processed. Included in + the standard library since the Python 2.7 and Python 3.2 releases. + +In the absence of more specific argument parsing design constraints, :mod:`argparse` +is the recommended choice for implementing command line applications, as it offers +the highest level of baseline functionality with the least application level code. + +:mod:`getopt` is retained almost entirely for backwards compatibility reasons. +However, it also serves a niche use case as a tool for prototyping and testing +command line argument handling in ``getopt``-based C applications. + +:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +following cases: + +* an application is already using :mod:`optparse` and doesn't want to risk the + subtle behavioural changes that may arise when migrating to :mod:`argparse` +* the application requires additional control over the way options and + positional parameters are interleaved on the command line (including + the ability to disable the interleaving feature completely) +* the application requires additional control over the incremental parsing + of command line elements (while ``argparse`` does support this, the + exact way it works in practice is undesirable for some use cases) +* the application requires additional control over the handling of options + which accept parameter values that may start with ``-`` (such as delegated + options to be passed to invoked subprocesses) +* the application requires some other command line parameter processing + behavior which ``argparse`` does not support, but which can be implemented + in terms of the lower level interface offered by ``optparse`` + +These considerations also mean that :mod:`optparse` is likely to provide a +better foundation for library authors writing third party command line +argument processing libraries. + +As a concrete example, consider the following two command line argument +parsing configurations, the first using ``optparse``, and the second +using ``argparse``: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +.. testcode:: + + import argparse + + if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output') + parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') + args = parser.parse_args() + process(args.rest, output=args.output, verbose=args.verbose) + +The most obvious difference is that in the ``optparse`` version, the non-option +arguments are processed separately by the application after the option processing +is complete. In the ``argparse`` version, positional arguments are declared and +processed in the same way as the named options. + +However, the ``argparse`` version will also handle some parameter combination +differently from the way the ``optparse`` version would handle them. +For example (amongst other differences): + +* supplying ``-o -v`` gives ``output="-v"`` and ``verbose=False`` + when using ``optparse``, but a usage error with ``argparse`` + (complaining that no value has been supplied for ``-o/--output``, + since ``-v`` is interpreted as meaning the verbosity flag) +* similarly, supplying ``-o --`` gives ``output="--"`` and ``args=()`` + when using ``optparse``, but a usage error with ``argparse`` + (also complaining that no value has been supplied for ``-o/--output``, + since ``--`` is interpreted as terminating the option processing + and treating all remaining values as positional arguments) +* supplying ``-o=foo`` gives ``output="=foo"`` when using ``optparse``, + but gives ``output="foo"`` with ``argparse`` (since ``=`` is special + cased as an alternative separator for option parameter values) + +Whether these differing behaviors in the ``argparse`` version are +considered desirable or a problem will depend on the specific command line +application use case. + +.. seealso:: + + :pypi:`click` is a third party argument processing library (originally + based on ``optparse``), which allows command line applications to be + developed as a set of decorated command implementation functions. + + Other third party libraries, such as :pypi:`typer` or :pypi:`msgspec-click`, + allow command line interfaces to be specified in ways that more effectively + integrate with static checking of Python type annotations. + + +Introduction +------------ + :mod:`optparse` is a more convenient, flexible, and powerful library for parsing -command-line options than the old :mod:`getopt` module. :mod:`optparse` uses a -more declarative style of command-line parsing: you create an instance of -:class:`OptionParser`, populate it with options, and parse the command -line. :mod:`optparse` allows users to specify options in the conventional +command-line options than the minimalist :mod:`getopt` module. +:mod:`optparse` uses a more declarative style of command-line parsing: +you create an instance of :class:`OptionParser`, +populate it with options, and parse the command line. +:mod:`optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. Here's an example of using :mod:`optparse` in a simple script:: @@ -82,10 +192,11 @@ Background ---------- :mod:`optparse` was explicitly designed to encourage the creation of programs -with straightforward, conventional command-line interfaces. To that end, it -supports only the most common command-line syntax and semantics conventionally -used under Unix. If you are unfamiliar with these conventions, read this -section to acquaint yourself with them. +with straightforward command-line interfaces that follow the conventions +established by the :c:func:`!getopt` family of functions available to C developers. +To that end, it supports only the most common command-line syntax and semantics +conventionally used under Unix. If you are unfamiliar with these conventions, +reading this section will allow you to acquaint yourself with them. .. _optparse-terminology: diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index 17bfa66f043302..d120c6acf621e3 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -4,12 +4,23 @@ Superseded Modules ****************** -The modules described in this chapter are deprecated or :term:`soft deprecated` and only kept for -backwards compatibility. They have been superseded by other modules. +The modules described in this chapter have been superseded by other modules +for most use cases, and are retained primarily to preserve backwards compatibility. +Modules may appear in this chapter because they only cover a limited subset of +a problem space, and a more generally applicable solution is available elsewhere +in the standard library (for example, :mod:`getopt` covers the very specific +task of "mimic the C :c:func:`!getopt` API in Python", rather than the broader +command line option parsing and argument parsing capabilities offered by +:mod:`optparse` and :mod:`argparse`). + +Alternatively, modules may appear in this chapter because they are deprecated +outright, and awaiting removal in a future release, or they are +:term:`soft deprecated` and their use is actively discouraged in new projects. +With the removal of various obsolete modules through :pep:`594`, there are +currently no modules in this latter category. .. toctree:: :maxdepth: 1 getopt.rst - optparse.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c8e0f94f4246fb..6a0e483bd895d6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1648,6 +1648,22 @@ opcode (Contributed by Irit Katriel in :gh:`105481`.) +optparse +-------- + +* This module is no longer considered :term:`soft deprecated`. + While :mod:`argparse` remains preferred for new projects that + aren't using a third party command line argument processing + library, there are aspects of the way ``argparse`` works that + mean the lower level ``optparse`` module may provide a better + foundation for *writing* argument processing libraries, and + for implementing command line applications which adhere more + strictly than ``argparse`` does to various Unix command line + processing conventions that originate in the behaviour of the + C :c:func:`!getopt` function . + (Contributed by Alyssa Coghlan and Serhiy Storchaka in :gh:`126180`.) + + pathlib ------- @@ -1787,14 +1803,6 @@ New Deprecations Check membership in :data:`~dis.hasarg` instead. (Contributed by Irit Katriel in :gh:`109319`.) -* :mod:`getopt` and :mod:`optparse`: - - * Both modules are now :term:`soft deprecated`, - with :mod:`argparse` preferred for new projects. - This is a new soft-deprecation for the :mod:`!getopt` module, - whereas the :mod:`!optparse` module was already *de facto* soft deprecated. - (Contributed by Victor Stinner in :gh:`106535`.) - * :mod:`gettext`: * Deprecate non-integer numbers as arguments to functions and methods diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst new file mode 100644 index 00000000000000..13a1f213c7a58e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst @@ -0,0 +1,6 @@ +:mod:`getopt` and :mod:`optparse` are no longer marked as deprecated. +There are legitimate reasons to use one of these modules in preference to +:mod:`argparse`, and none of these modules are at risk of being removed +from the standard library. Of the three, ``argparse`` remains the +recommended default choice, *unless* one of the concerns noted at the top of +the ``optparse`` module documentation applies. From 180d417e9f9456defd4c5b53cae678c318287921 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Mon, 23 Dec 2024 04:31:33 -0800 Subject: [PATCH 06/44] gh-114203: Optimise simple recursive critical sections (#128126) Add a fast path to (single-mutex) critical section locking _iff_ the mutex is already held by the currently active, top-most critical section of this thread. This can matter a lot for indirectly recursive critical sections without intervening critical sections. --- Include/internal/pycore_critical_section.h | 14 +++++++++++ ...-12-20-23-07-33.gh-issue-114203.84NgoW.rst | 1 + Python/critical_section.c | 24 +++++++++++++------ 3 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 9ba2fce56d3c9c..e66d6d805c1b3b 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -145,6 +145,12 @@ _PyCriticalSection_Pop(PyCriticalSection *c) static inline void _PyCriticalSection_End(PyCriticalSection *c) { + // If the mutex is NULL, we used the fast path in + // _PyCriticalSection_BeginSlow for locks already held in the top-most + // critical section, and we shouldn't unlock or pop this critical section. + if (c->_cs_mutex == NULL) { + return; + } PyMutex_Unlock(c->_cs_mutex); _PyCriticalSection_Pop(c); } @@ -199,6 +205,14 @@ _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) static inline void _PyCriticalSection2_End(PyCriticalSection2 *c) { + // if mutex1 is NULL, we used the fast path in + // _PyCriticalSection_BeginSlow for mutexes that are already held, + // which should only happen when mutex1 and mutex2 were the same mutex, + // and mutex2 should also be NULL. + if (c->_cs_base._cs_mutex == NULL) { + assert(c->_cs_mutex2 == NULL); + return; + } if (c->_cs_mutex2) { PyMutex_Unlock(c->_cs_mutex2); } diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst new file mode 100644 index 00000000000000..6a9856e90c32bc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst @@ -0,0 +1 @@ +Optimize ``Py_BEGIN_CRITICAL_SECTION`` for simple recursive calls. diff --git a/Python/critical_section.c b/Python/critical_section.c index 62ed25523fd6dc..73857b85496316 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -8,11 +8,28 @@ static_assert(_Alignof(PyCriticalSection) >= 4, "critical section must be aligned to at least 4 bytes"); #endif +#ifdef Py_GIL_DISABLED +static PyCriticalSection * +untag_critical_section(uintptr_t tag) +{ + return (PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK); +} +#endif + void _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) { #ifdef Py_GIL_DISABLED PyThreadState *tstate = _PyThreadState_GET(); + // As an optimisation for locking the same object recursively, skip + // locking if the mutex is currently locked by the top-most critical + // section. + if (tstate->critical_section && + untag_critical_section(tstate->critical_section)->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } c->_cs_mutex = NULL; c->_cs_prev = (uintptr_t)tstate->critical_section; tstate->critical_section = (uintptr_t)c; @@ -42,13 +59,6 @@ _PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, #endif } -#ifdef Py_GIL_DISABLED -static PyCriticalSection * -untag_critical_section(uintptr_t tag) -{ - return (PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK); -} -#endif // Release all locks held by critical sections. This is called by // _PyThreadState_Detach. From c5b0c90b62f1a10b0742db4bcd17da080d4e9111 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 24 Dec 2024 02:08:34 +0900 Subject: [PATCH 07/44] gh-115999: Update test_opcache to test with nested method (gh-128166) gh-115999: Update test_opcace to test with nested method --- Lib/test/test_opcache.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index ba111b5117b41d..79f452f8068c7f 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -606,7 +606,7 @@ def assert_races_do_not_crash( for writer in writers: writer.join() - @requires_specialization + @requires_specialization_ft def test_binary_subscr_getitem(self): def get_items(): class C: @@ -1242,14 +1242,6 @@ def f(o, n): f(test_obj, 1) self.assertEqual(test_obj.b, 0) -# gh-127274: BINARY_SUBSCR_GETITEM will only cache __getitem__ methods that -# are deferred. We only defer functions defined at the top-level. -class CGetItem: - def __init__(self, val): - self.val = val - def __getitem__(self, item): - return self.val - class TestSpecializer(TestBase): @@ -1592,7 +1584,13 @@ def binary_subscr_str_int(): self.assert_no_opcode(binary_subscr_str_int, "BINARY_SUBSCR") def binary_subscr_getitems(): - items = [CGetItem(i) for i in range(100)] + class C: + def __init__(self, val): + self.val = val + def __getitem__(self, item): + return self.val + + items = [C(i) for i in range(100)] for i in range(100): self.assertEqual(items[i][i], i) From d61542b5ff1fe64705e5ce1bcc53048f14098dba Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Mon, 23 Dec 2024 17:22:15 +0000 Subject: [PATCH 08/44] pathlib tests: create test hierarchy without using class under test (#128200) In the pathlib tests, avoid using the path class under test (`self.cls`) in test setup. Instead we use `os` functions in `test_pathlib`, and direct manipulation of `DummyPath` internal data in `test_pathlib_abc`. --- Lib/test/test_pathlib/test_pathlib.py | 38 ++++++++++++++++++- Lib/test/test_pathlib/test_pathlib_abc.py | 46 ++++++++++------------- 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index ef482c311542fa..fac8cbdf65a122 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -578,10 +578,44 @@ def setUp(self): if name in _tests_needing_symlinks and not self.can_symlink: self.skipTest('requires symlinks') super().setUp() - os.chmod(self.parser.join(self.base, 'dirE'), 0) + + def createTestHierarchy(self): + os.mkdir(self.base) + os.mkdir(os.path.join(self.base, 'dirA')) + os.mkdir(os.path.join(self.base, 'dirB')) + os.mkdir(os.path.join(self.base, 'dirC')) + os.mkdir(os.path.join(self.base, 'dirC', 'dirD')) + os.mkdir(os.path.join(self.base, 'dirE')) + with open(os.path.join(self.base, 'fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(os.path.join(self.base, 'dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(os.path.join(self.base, 'dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(os.path.join(self.base, 'dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(os.path.join(self.base, 'dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + os.chmod(os.path.join(self.base, 'dirE'), 0) + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', os.path.join(self.base, 'linkA')) + os.symlink('non-existing', os.path.join(self.base, 'brokenLink')) + os.symlink('dirB', + os.path.join(self.base, 'linkB'), + target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirA', 'linkC'), + target_is_directory=True) + # This one goes upwards, creating a loop. + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirB', 'linkD'), + target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', os.path.join(self.base, 'brokenLinkLoop')) def tearDown(self): - os.chmod(self.parser.join(self.base, 'dirE'), 0o777) + os.chmod(os.path.join(self.base, 'dirE'), 0o777) os_helper.rmtree(self.base) def tempdir(self): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 9198a0cbc45cee..787fb0f82257e6 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1437,33 +1437,25 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() - parser = self.cls.parser - p = self.cls(self.base) - p.mkdir(parents=True) - p.joinpath('dirA').mkdir() - p.joinpath('dirB').mkdir() - p.joinpath('dirC').mkdir() - p.joinpath('dirC', 'dirD').mkdir() - p.joinpath('dirE').mkdir() - with p.joinpath('fileA').open('wb') as f: - f.write(b"this is file A\n") - with p.joinpath('dirB', 'fileB').open('wb') as f: - f.write(b"this is file B\n") - with p.joinpath('dirC', 'fileC').open('wb') as f: - f.write(b"this is file C\n") - with p.joinpath('dirC', 'novel.txt').open('wb') as f: - f.write(b"this is a novel\n") - with p.joinpath('dirC', 'dirD', 'fileD').open('wb') as f: - f.write(b"this is file D\n") - if self.can_symlink: - p.joinpath('linkA').symlink_to('fileA') - p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB', target_is_directory=True) - p.joinpath('dirA', 'linkC').symlink_to( - parser.join('..', 'dirB'), target_is_directory=True) - p.joinpath('dirB', 'linkD').symlink_to( - parser.join('..', 'dirB'), target_is_directory=True) - p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') + self.createTestHierarchy() + + def createTestHierarchy(self): + cls = self.cls + cls._files = { + f'{self.base}/fileA': b'this is file A\n', + f'{self.base}/dirB/fileB': b'this is file B\n', + f'{self.base}/dirC/fileC': b'this is file C\n', + f'{self.base}/dirC/dirD/fileD': b'this is file D\n', + f'{self.base}/dirC/novel.txt': b'this is a novel\n', + } + cls._directories = { + f'{self.base}': {'fileA', 'dirA', 'dirB', 'dirC', 'dirE'}, + f'{self.base}/dirA': set(), + f'{self.base}/dirB': {'fileB'}, + f'{self.base}/dirC': {'fileC', 'dirD', 'novel.txt'}, + f'{self.base}/dirC/dirD': {'fileD'}, + f'{self.base}/dirE': set(), + } def tearDown(self): cls = self.cls From 30efede33ca1fe32debbae93cc40b0e7e0b133b3 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Mon, 23 Dec 2024 22:17:47 +0100 Subject: [PATCH 09/44] gh-128195: Add `_REPLACE_WITH_TRUE` to the tier2 optimizer (GH-128203) Add `_REPLACE_WITH_TRUE` to the tier2 optimizer --- Python/optimizer_bytecodes.c | 4 ++++ Python/optimizer_cases.c.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e60c0d38425bfe..a14d119b7a1dec 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -899,6 +899,10 @@ dummy_func(void) { (void)version; } + op(_REPLACE_WITH_TRUE, (value -- res)) { + res = sym_new_const(ctx, Py_True); + } + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b46079ec8a1992..0fcf5e18ed5808 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -211,7 +211,7 @@ case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; } From 3f6a618e49b1c8c12a7bc0c26e846735e108dc97 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 24 Dec 2024 09:43:31 +0000 Subject: [PATCH 10/44] gh-127949: fix `DeprecationWarning` in test_inspect.py (#128215) --- Lib/test/test_inspect/test_inspect.py | 28 +++++++++++++-------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 2c950e46b3ed8a..d536d04d2e7d88 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1,5 +1,4 @@ from annotationlib import Format, ForwardRef -import asyncio import builtins import collections import copy @@ -73,11 +72,6 @@ def revise(filename, *args): git = mod.StupidGit() -def tearDownModule(): - if support.has_socket_support: - asyncio._set_event_loop_policy(None) - - def signatures_with_lexicographic_keyword_only_parameters(): """ Yields a whole bunch of functions with only keyword-only parameters, @@ -205,7 +199,7 @@ def test_excluding_predicates(self): self.assertFalse(inspect.ismethodwrapper(type("AnyClass", (), {}))) def test_ispackage(self): - self.istest(inspect.ispackage, 'asyncio') + self.istest(inspect.ispackage, 'unittest') self.istest(inspect.ispackage, 'importlib') self.assertFalse(inspect.ispackage(inspect)) self.assertFalse(inspect.ispackage(mod)) @@ -1166,16 +1160,20 @@ def f(self): # This is necessary when the test is run multiple times. sys.modules.pop("inspect_actual") - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) def test_nested_class_definition_inside_async_function(self): - import asyncio - self.addCleanup(asyncio.set_event_loop_policy, None) - self.assertSourceEqual(asyncio.run(mod2.func225()), 226, 227) + def run(coro): + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise RuntimeError("coroutine did not complete synchronously!") + finally: + coro.close() + + self.assertSourceEqual(run(mod2.func225()), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) - self.assertSourceEqual(asyncio.run(mod2.cls226().func232()), 233, 234) + self.assertSourceEqual(run(mod2.cls226().func232()), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) From a391d80f4bf5a3cf5aa95340ca848b9a0294778d Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 24 Dec 2024 17:30:26 +0530 Subject: [PATCH 11/44] gh-127949: deprecate asyncio policy classes (#128216) --- Doc/library/asyncio-policy.rst | 16 +++++++++ Lib/asyncio/__init__.py | 16 +++++++++ Lib/asyncio/events.py | 14 ++++---- Lib/asyncio/unix_events.py | 6 ++-- Lib/asyncio/windows_events.py | 10 +++--- Lib/test/test_asyncio/test_events.py | 34 +++++++++++++------- Lib/test/test_asyncio/test_runners.py | 2 +- Lib/test/test_asyncio/test_windows_events.py | 18 +++++++---- 8 files changed, 82 insertions(+), 34 deletions(-) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 2d05c3a9f7f157..ea7fe957da7c40 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -87,6 +87,10 @@ The abstract event loop policy base class is defined as follows: This method should never return ``None``. + .. deprecated:: next + The :class:`AbstractEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. _asyncio-policy-builtin: @@ -109,6 +113,10 @@ asyncio ships with the following built-in policies: The :meth:`get_event_loop` method of the default asyncio policy now raises a :exc:`RuntimeError` if there is no set event loop. + .. deprecated:: next + The :class:`DefaultEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. class:: WindowsSelectorEventLoopPolicy @@ -117,6 +125,10 @@ asyncio ships with the following built-in policies: .. availability:: Windows. + .. deprecated:: next + The :class:`WindowsSelectorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. class:: WindowsProactorEventLoopPolicy @@ -125,6 +137,10 @@ asyncio ships with the following built-in policies: .. availability:: Windows. + .. deprecated:: next + The :class:`WindowsProactorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. _asyncio-custom-policies: diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb7d2..edb615b1b6b1c6 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -45,3 +45,19 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ + +def __getattr__(name: str): + import warnings + + deprecated = { + "AbstractEventLoopPolicy", + "DefaultEventLoopPolicy", + "WindowsSelectorEventLoopPolicy", + "WindowsProactorEventLoopPolicy", + } + if name in deprecated: + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + # deprecated things have underscores in front of them + return globals()["_" + name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 1449245edc7c7e..3ade7747149b32 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,7 +5,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - 'AbstractEventLoopPolicy', + '_AbstractEventLoopPolicy', 'AbstractEventLoop', 'AbstractServer', 'Handle', 'TimerHandle', '_get_event_loop_policy', @@ -632,7 +632,7 @@ def set_debug(self, enabled): raise NotImplementedError -class AbstractEventLoopPolicy: +class _AbstractEventLoopPolicy: """Abstract policy for accessing the event loop.""" def get_event_loop(self): @@ -655,7 +655,7 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): +class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. In this policy, each thread has its own event loop. However, we @@ -758,8 +758,8 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() + from . import _DefaultEventLoopPolicy + _event_loop_policy = _DefaultEventLoopPolicy() def _get_event_loop_policy(): @@ -777,7 +777,7 @@ def _set_event_loop_policy(policy): If policy is None, the default policy is restored.""" global _event_loop_policy - if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy @@ -838,7 +838,7 @@ def new_event_loop(): def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: - _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 0227eb506c6016..f69c6a64c39ae6 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,7 +28,7 @@ __all__ = ( 'SelectorEventLoop', - 'DefaultEventLoopPolicy', + '_DefaultEventLoopPolicy', 'EventLoop', ) @@ -963,11 +963,11 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index bf99bc271c7acd..5f75b17d8ca649 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -29,8 +29,8 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy', - 'WindowsProactorEventLoopPolicy', 'EventLoop', + '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', + '_WindowsProactorEventLoopPolicy', 'EventLoop', ) @@ -891,13 +891,13 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop -class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = SelectorEventLoop -class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = ProactorEventLoop -DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy +_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy EventLoop = ProactorEventLoop diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index d43f66c13d2f96..c626670f72a084 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2695,14 +2695,26 @@ async def inner(): class PolicyTests(unittest.TestCase): + def test_abstract_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): + policy = asyncio.AbstractEventLoopPolicy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + + def test_default_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.DefaultEventLoopPolicy' is deprecated"): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) + def test_event_loop_policy(self): - policy = asyncio.AbstractEventLoopPolicy() + policy = asyncio._AbstractEventLoopPolicy() self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) def test_get_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) with self.assertRaises(RuntimeError): @@ -2710,7 +2722,7 @@ def test_get_event_loop(self): self.assertIsNone(policy._local._loop) def test_get_event_loop_does_not_call_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() with mock.patch.object( policy, "set_event_loop", @@ -2722,7 +2734,7 @@ def test_get_event_loop_does_not_call_set_event_loop(self): m_set_event_loop.assert_not_called() def test_get_event_loop_after_set_none(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() policy.set_event_loop(None) self.assertRaises(RuntimeError, policy.get_event_loop) @@ -2730,7 +2742,7 @@ def test_get_event_loop_after_set_none(self): def test_get_event_loop_thread(self, m_current_thread): def f(): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() self.assertRaises(RuntimeError, policy.get_event_loop) th = threading.Thread(target=f) @@ -2738,14 +2750,14 @@ def f(): th.join() def test_new_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() loop = policy.new_event_loop() self.assertIsInstance(loop, asyncio.AbstractEventLoop) loop.close() def test_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() old_loop = policy.new_event_loop() policy.set_event_loop(old_loop) @@ -2762,7 +2774,7 @@ def test_get_event_loop_policy(self): with self.assertWarnsRegex( DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): policy = asyncio.get_event_loop_policy() - self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + self.assertIsInstance(policy, asyncio._AbstractEventLoopPolicy) self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): @@ -2775,7 +2787,7 @@ def test_set_event_loop_policy(self): DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): old_policy = asyncio.get_event_loop_policy() - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() with self.assertWarnsRegex( DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): asyncio.set_event_loop_policy(policy) @@ -2862,7 +2874,7 @@ def test_get_event_loop_returns_running_loop(self): class TestError(Exception): pass - class Policy(asyncio.DefaultEventLoopPolicy): + class Policy(asyncio._DefaultEventLoopPolicy): def get_event_loop(self): raise TestError @@ -2908,7 +2920,7 @@ async def func(): def test_get_event_loop_returns_running_loop2(self): old_policy = asyncio._get_event_loop_policy() try: - asyncio._set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) + asyncio._set_event_loop_policy(asyncio._DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() self.addCleanup(loop.close) diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index e1f82f7f7bec0c..21f277bc2d8d5f 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -19,7 +19,7 @@ def interrupt_self(): _thread.interrupt_main() -class TestPolicy(asyncio.AbstractEventLoopPolicy): +class TestPolicy(asyncio._AbstractEventLoopPolicy): def __init__(self, loop_factory): self.loop_factory = loop_factory diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 28b05d24dc25a1..69e9905205eee0 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -328,14 +328,15 @@ class WinPolicyTests(WindowsEventsTestCase): def test_selector_win_policy(self): async def main(): - self.assertIsInstance( - asyncio.get_running_loop(), - asyncio.SelectorEventLoop) + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) old_policy = asyncio._get_event_loop_policy() try: - asyncio._set_event_loop_policy( - asyncio.WindowsSelectorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", + ): + asyncio._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: asyncio._set_event_loop_policy(old_policy) @@ -348,8 +349,11 @@ async def main(): old_policy = asyncio._get_event_loop_policy() try: - asyncio._set_event_loop_policy( - asyncio.WindowsProactorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", + ): + asyncio._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: asyncio._set_event_loop_policy(old_policy) From bd3e200cce6601684108e23e7621bbe05b53a323 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Tue, 24 Dec 2024 08:01:21 -0500 Subject: [PATCH 12/44] Add Windows version comments to the python manifest. (GH-127439) --- PC/python.manifest | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/PC/python.manifest b/PC/python.manifest index 8e1bc022adfb4f..19c9fc1b80a3ec 100644 --- a/PC/python.manifest +++ b/PC/python.manifest @@ -9,10 +9,15 @@ + + + + + From 3ddd70ceaaf67b111ee4251817e150396d6d10a9 Mon Sep 17 00:00:00 2001 From: Sergey Muraviov Date: Tue, 24 Dec 2024 16:06:41 +0300 Subject: [PATCH 13/44] gh-128217: Validate the normalized_environment variable instead of the similarly named function (GH-128220) --- Modules/_winapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 4ce689fe30e6df..260cab48091c16 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1048,7 +1048,7 @@ getenvironment(PyObject* environment) } normalized_environment = normalize_environment(environment); - if (normalize_environment == NULL) { + if (normalized_environment == NULL) { return NULL; } From 9fce90682553e2cfe93e98e2ae5948bf9c7c4456 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 24 Dec 2024 19:24:28 +0530 Subject: [PATCH 14/44] gh-127949: deprecate `asyncio.set_event_loop` (#128218) Deprecate `asyncio.set_event_loop` to be removed in Python 3.16. --- Doc/library/asyncio-eventloop.rst | 4 +++ Lib/asyncio/__main__.py | 2 +- Lib/asyncio/events.py | 32 +++++++++++++++-------- Lib/asyncio/runners.py | 4 +-- Lib/test/test_asyncgen.py | 2 +- Lib/test/test_asyncio/functional.py | 4 +-- Lib/test/test_asyncio/test_base_events.py | 8 +++--- Lib/test/test_asyncio/test_events.py | 26 +++++++++++------- Lib/test/test_asyncio/test_futures.py | 8 +++--- Lib/test/test_asyncio/test_streams.py | 10 +++---- Lib/test/test_asyncio/test_tasks.py | 18 ++++++------- Lib/test/test_asyncio/test_unix_events.py | 4 +-- Lib/test/test_asyncio/utils.py | 4 +-- Lib/test/test_coroutines.py | 2 +- Lib/test/test_type_params.py | 2 +- Lib/test/test_unittest/test_async_case.py | 2 +- 16 files changed, 77 insertions(+), 55 deletions(-) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 9f1aec148f8750..29f843123f8560 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -66,6 +66,10 @@ an event loop: Set *loop* as the current event loop for the current OS thread. + .. deprecated:: next + The :func:`set_event_loop` function is deprecated and will be removed + in Python 3.16. + .. function:: new_event_loop() Create and return a new event loop object. diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 95c636f9e02866..662ba649aa08be 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -149,7 +149,7 @@ def interrupt(self) -> None: return_code = 0 loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) repl_locals = {'asyncio': asyncio} for key in {'__name__', '__package__', diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 3ade7747149b32..6e291d28ec81ae 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,16 +5,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - '_AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - '_get_event_loop_policy', - 'get_event_loop_policy', - '_set_event_loop_policy', - 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - '_set_running_loop', 'get_running_loop', - '_get_running_loop', + "_AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "_get_event_loop_policy", + "get_event_loop_policy", + "_set_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "_set_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", ) import contextvars @@ -801,9 +807,13 @@ def get_event_loop(): return _get_event_loop_policy().get_event_loop() +def _set_event_loop(loop): + _get_event_loop_policy().set_event_loop(loop) + def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - _get_event_loop_policy().set_event_loop(loop) + warnings._deprecated('asyncio.set_event_loop', remove=(3,16)) + _set_event_loop(loop) def new_event_loop(): diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 0e63c34f60f4d9..b9adf291d4817f 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -74,7 +74,7 @@ def close(self): loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT)) finally: if self._set_event_loop: - events.set_event_loop(None) + events._set_event_loop(None) loop.close() self._loop = None self._state = _State.CLOSED @@ -147,7 +147,7 @@ def _lazy_init(self): if not self._set_event_loop: # Call set_event_loop only once to avoid calling # attach_loop multiple times on child watchers - events.set_event_loop(self._loop) + events._set_event_loop(self._loop) self._set_event_loop = True else: self._loop = self._loop_factory() diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 4bce6d5c1b1d2f..5bfd789185c675 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -624,7 +624,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) def tearDown(self): self.loop.close() diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py index d19c7a612ccf86..2934325b6dfbc7 100644 --- a/Lib/test/test_asyncio/functional.py +++ b/Lib/test/test_asyncio/functional.py @@ -24,7 +24,7 @@ def loop_exception_handler(self, loop, context): def setUp(self): self.loop = self.new_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) self.loop.set_exception_handler(self.loop_exception_handler) self.__unhandled_exceptions = [] @@ -39,7 +39,7 @@ def tearDown(self): self.fail('unexpected calls to loop.call_exception_handler()') finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) self.loop = None def tcp_server(self, server_prog, *, diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 08e38b047d519b..1e063c1352ecb9 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -331,10 +331,10 @@ def check_in_thread(loop, event, debug, create_loop, fut): if create_loop: loop2 = base_events.BaseEventLoop() try: - asyncio.set_event_loop(loop2) + asyncio._set_event_loop(loop2) self.check_thread(loop, debug) finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) loop2.close() else: self.check_thread(loop, debug) @@ -690,7 +690,7 @@ def default_exception_handler(self, context): loop = Loop() self.addCleanup(loop.close) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) def run_loop(): def zero_error(): @@ -1983,7 +1983,7 @@ def stop_loop_cb(loop): async def stop_loop_coro(loop): loop.stop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) self.loop.set_debug(True) self.loop.slow_callback_duration = 0.0 diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index c626670f72a084..c8439c9af5e6ba 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -58,7 +58,7 @@ async def doit(): return 'hello' loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) return loop.run_until_complete(doit()) @@ -2695,6 +2695,14 @@ async def inner(): class PolicyTests(unittest.TestCase): + def test_asyncio_set_event_loop_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop' is deprecated"): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self.assertIs(loop, asyncio.get_event_loop()) + loop.close() + def test_abstract_event_loop_policy_deprecation(self): with self.assertWarnsRegex( DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): @@ -2824,14 +2832,14 @@ def setUp(self): super().setUp() self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) def tearDown(self): try: super().tearDown() finally: self.loop.close() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) events._get_running_loop = self._get_running_loop_saved events._set_running_loop = self._set_running_loop_saved @@ -2885,7 +2893,7 @@ def get_event_loop(self): with self.assertRaises(TestError): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaises(TestError): asyncio.get_event_loop() @@ -2900,10 +2908,10 @@ async def func(): loop.run_until_complete(func()) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) with self.assertRaises(TestError): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaises(TestError): asyncio.get_event_loop() @@ -2927,7 +2935,7 @@ def test_get_event_loop_returns_running_loop2(self): with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() @@ -2938,10 +2946,10 @@ async def func(): loop.run_until_complete(func()) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) self.assertIs(asyncio.get_event_loop(), loop) - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 7db70a4c81d483..84b44011b9a844 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -178,8 +178,8 @@ async def test(): def test_constructor_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -566,8 +566,8 @@ async def test(): def test_wrap_future_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index c3ba90b309e49f..047ada8c5d23df 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -71,7 +71,7 @@ def _basetest_open_connection_no_loop_ssl(self, open_connection_fut): try: reader, writer = self.loop.run_until_complete(open_connection_fut) finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) writer.write(b'GET / HTTP/1.0\r\n\r\n') f = reader.read() data = self.loop.run_until_complete(f) @@ -839,8 +839,8 @@ def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set # Deprecated in 3.10, undeprecated in 3.12 - self.addCleanup(asyncio.set_event_loop, None) - asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) + asyncio._set_event_loop(self.loop) reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) @@ -863,8 +863,8 @@ def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set # Deprecated in 3.10, undeprecated in 3.12 - self.addCleanup(asyncio.set_event_loop, None) - asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) + asyncio._set_event_loop(self.loop) reader = mock.Mock() protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 7d6d0564a9a9db..b5363226ad79f4 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -212,8 +212,8 @@ async def test(): self.assertEqual(t.result(), 'ok') # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) @@ -2202,8 +2202,8 @@ def test_shield_coroutine_use_global_loop(self): async def coro(): return 42 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) @@ -2273,7 +2273,7 @@ async def kill_me(loop): self.assertEqual(self.all_tasks(loop=self.loop), {task}) - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) # execute the task so it waits for future self.loop._run_once() @@ -3278,8 +3278,8 @@ async def gather(): def test_constructor_empty_sequence_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.one_loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.one_loop) + self.addCleanup(asyncio._set_event_loop, None) fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) @@ -3386,8 +3386,8 @@ def test_constructor_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 async def coro(): return 'abc' - asyncio.set_event_loop(self.other_loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.other_loop) + self.addCleanup(asyncio._set_event_loop, None) gen1 = coro() gen2 = coro() fut = asyncio.gather(gen1, gen2) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index e9ee9702248015..ebb4cc0f7b64fd 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1116,11 +1116,11 @@ class TestFunctional(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) def tearDown(self): self.loop.close() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) def test_add_reader_invalid_argument(self): def assert_raises(): diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index b8dbe7feaac3f4..35ce13896da08f 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -541,7 +541,7 @@ def set_event_loop(self, loop, *, cleanup=True): if loop is None: raise AssertionError('loop is None') # ensure that the event loop is passed explicitly in asyncio - events.set_event_loop(None) + events._set_event_loop(None) if cleanup: self.addCleanup(self.close_loop, loop) @@ -554,7 +554,7 @@ def setUp(self): self._thread_cleanup = threading_helper.threading_setup() def tearDown(self): - events.set_event_loop(None) + events._set_event_loop(None) # Detect CPython bug #23353: ensure that yield/yield-from is not used # in an except block of a generator diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index a72c43f9b47947..840043d5271224 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2287,7 +2287,7 @@ async def f(): buffer.append('unreachable') loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) try: loop.run_until_complete(f()) except MyException: diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 433b19593bdd04..89f836cf722966 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1060,7 +1060,7 @@ async def coroutine[B](): co = get_coroutine() - self.addCleanup(asyncio.set_event_loop_policy, None) + self.addCleanup(asyncio._set_event_loop_policy, None) a, b = asyncio.run(co()) self.assertIsInstance(a, TypeVar) diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 993e6bf013cfbf..fc996b42149dcb 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -476,7 +476,7 @@ async def cleanup(self, fut): def test_setup_get_event_loop(self): # See https://github.com/python/cpython/issues/95736 # Make sure the default event loop is not used - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): From 7ed6c5c6961d0849f163d4d449fb36bae312b6bc Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 24 Dec 2024 07:56:42 -0800 Subject: [PATCH 15/44] gh-127847: Fix position in the special-cased zipfile seek (#127856) --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Peter Bierma Co-authored-by: Jason R. Coombs --- Lib/test/test_zipfile/test_core.py | 12 ++++++++++++ Lib/zipfile/__init__.py | 5 ++++- .../2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index c36228c033a414..124e088fd15b80 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2333,6 +2333,18 @@ def test_read_after_seek(self): fp.seek(1, os.SEEK_CUR) self.assertEqual(fp.read(-1), b'men!') + def test_uncompressed_interleaved_seek_read(self): + # gh-127847: Make sure the position in the archive is correct + # in the special case of seeking in a ZIP_STORED entry. + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("a.txt", "123") + zipf.writestr("b.txt", "456") + with zipfile.ZipFile(TESTFN, "r") as zipf: + with zipf.open("a.txt", "r") as a, zipf.open("b.txt", "r") as b: + self.assertEqual(a.read(1), b"1") + self.assertEqual(b.seek(1), 1) + self.assertEqual(b.read(1), b"5") + @requires_bz2() def test_decompress_without_3rd_party_library(self): data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 6907ae6d5b7464..f4d396abb6e639 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -819,7 +819,10 @@ def seek(self, offset, whence=0): raise ValueError("Can't reposition in the ZIP file while " "there is an open writing handle on it. " "Close the writing handle before trying to read.") - self._file.seek(offset, whence) + if whence == os.SEEK_CUR: + self._file.seek(self._pos + offset) + else: + self._file.seek(offset, whence) self._pos = self._file.tell() return self._pos diff --git a/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst b/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst new file mode 100644 index 00000000000000..3d6e36fb538bca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst @@ -0,0 +1 @@ +Fix the position when doing interleaved seeks and reads in uncompressed, unencrypted zip files returned by :meth:`zipfile.ZipFile.open`. From 7985d460c731b2c48419a33fc1820f9512bb6f21 Mon Sep 17 00:00:00 2001 From: Bogdan Romanyuk <65823030+wrongnull@users.noreply.github.com> Date: Tue, 24 Dec 2024 21:00:24 +0300 Subject: [PATCH 16/44] gh-128227: Regenerate `Doc/requirements-oldest-sphinx.txt` (#128228) --- Doc/requirements-oldest-sphinx.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index 3483faea6b56cb..c8027a05706c21 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -13,15 +13,15 @@ python-docs-theme>=2022.1 # Sphinx 7.2.6 comes from ``needs_sphinx = '7.2.6'`` in ``Doc/conf.py``. alabaster==0.7.16 -Babel==2.16.0 -certifi==2024.8.30 +babel==2.16.0 +certifi==2024.12.14 charset-normalizer==3.4.0 docutils==0.20.1 idna==3.10 imagesize==1.4.1 -Jinja2==3.1.4 -MarkupSafe==3.0.1 -packaging==24.1 +Jinja2==3.1.5 +MarkupSafe==3.0.2 +packaging==24.2 Pygments==2.18.0 requests==2.32.3 snowballstemmer==2.2.0 @@ -32,4 +32,4 @@ sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 -urllib3==2.2.3 +urllib3==2.3.0 From 418114c139666f33abff937e40ccbbbdce15bc39 Mon Sep 17 00:00:00 2001 From: Will Childs-Klein Date: Tue, 24 Dec 2024 12:29:27 -0600 Subject: [PATCH 17/44] gh-128035: Add ssl.HAS_PHA to detect libssl PHA support (GH-128036) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add ssl.HAS_PHA to detect libssl Post-Handshake-Auth support Co-authored-by: Tomas R. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/ssl.rst | 6 ++++++ Doc/whatsnew/3.14.rst | 8 ++++++++ Lib/ssl.py | 2 +- Lib/test/test_httplib.py | 4 ++-- Lib/test/test_ssl.py | 3 ++- .../2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst | 1 + Modules/_ssl.c | 6 ++++++ 7 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index f07d151a885692..9d7b6aa66cd443 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -934,6 +934,12 @@ Constants .. versionadded:: 3.13 +.. data:: HAS_PHA + + Whether the OpenSSL library has built-in support for TLS-PHA. + + .. versionadded:: next + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 97a37a82f76b9b..0dcee56b7d233f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -584,6 +584,14 @@ pydoc (Contributed by Jelle Zijlstra in :gh:`101552`.) +ssl +--- + +* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports + TLSv1.3 post-handshake client authentication (PHA). + (Contributed by Will Childs-Klein in :gh:`128036`.) + + symtable -------- diff --git a/Lib/ssl.py b/Lib/ssl.py index c8703b046cfd4b..05df4ad7f0f05c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -116,7 +116,7 @@ from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK, HAS_PHA ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 9d853d254db7c6..89963dadeb152b 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2073,8 +2073,8 @@ def test_host_port(self): def test_tls13_pha(self): import ssl - if not ssl.HAS_TLSv1_3: - self.skipTest('TLS 1.3 support required') + if not ssl.HAS_TLSv1_3 or not ssl.HAS_PHA: + self.skipTest('TLS 1.3 PHA support required') # just check status of PHA flag h = client.HTTPSConnection('localhost', 443) self.assertTrue(h._context.post_handshake_auth) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 3f6f890bbdc658..c16ef3f96f9a21 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4494,7 +4494,8 @@ def server_callback(identity): s.connect((HOST, server.port)) -@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") +@unittest.skipUnless(has_tls_version('TLSv1_3') and ssl.HAS_PHA, + "Test needs TLS 1.3 PHA") class TestPostHandshakeAuth(unittest.TestCase): def test_pha_setter(self): protocols = [ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst new file mode 100644 index 00000000000000..27815d48425334 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst @@ -0,0 +1 @@ +Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports TLSv1.3 post-handshake client authentication (PHA). Patch by Will Childs-Klein. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index e7df132869fee6..74cf99957389e2 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6553,6 +6553,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_PSK", 1); #endif +#ifdef SSL_VERIFY_POST_HANDSHAKE + addbool(m, "HAS_PHA", 1); +#else + addbool(m, "HAS_PHA", 0); +#endif + #undef addbool #undef ADD_INT_CONST From d9ed42bc00c74b3150be2a0eb28da03e01dffcc7 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 25 Dec 2024 09:26:51 +0200 Subject: [PATCH 18/44] gh-128201: Fix ``DeprecationWarning`` in ``test_pdb`` (#128202) --- Lib/test/test_pdb.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 58295cff84310f..9b0806d8b2a9bd 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2065,10 +2065,7 @@ def test_pdb_next_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', @@ -2129,10 +2126,7 @@ def test_pdb_next_command_for_asyncgen(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio._set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', @@ -2250,10 +2244,7 @@ def test_pdb_return_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio._set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', @@ -2350,10 +2341,7 @@ def test_pdb_until_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio._set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', From 76f1785657fde0ebee082b0df54da8f7c911c369 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 25 Dec 2024 17:51:27 +0530 Subject: [PATCH 19/44] gh-128002: use internal llist implementation for asyncio tasks (#128256) --- Modules/_asynciomodule.c | 70 ++++++++++++---------------------------- 1 file changed, 20 insertions(+), 50 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 27c16364457336..603b77a70b15d4 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -6,9 +6,9 @@ #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION_MUT() #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_freelist.h" // _Py_FREELIST_POP() +#include "pycore_llist.h" // struct llist_node #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_object.h" // _Py_SetImmortalUntracked() #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -60,8 +60,7 @@ typedef struct TaskObj { PyObject *task_coro; PyObject *task_name; PyObject *task_context; - struct TaskObj *next; - struct TaskObj *prev; + struct llist_node task_node; } TaskObj; typedef struct { @@ -136,21 +135,11 @@ typedef struct { /* Counter for autogenerated Task names */ uint64_t task_name_counter; - /* Circular linked-list of all tasks which are instances of asyncio.Task or subclasses - of it. Third party tasks implementations which don't inherit from - asyncio.Task are tracked separately using the 'non_asyncio_tasks' WeakSet. - `first` is used as a sentinel to mark the end of the linked-list. It avoids one - branch in checking for empty list when adding a new task, the list is - initialized with `head`, `head->next` and `head->prev` pointing to `first` - to mark an empty list. - + /* Head of circular linked-list of all tasks which are instances of `asyncio.Task` + or subclasses of it. Third party tasks implementations which don't inherit from + `asyncio.Task` are tracked separately using the `non_asyncio_tasks` WeakSet. */ - - struct { - TaskObj first; - TaskObj *head; - } asyncio_tasks; - + struct llist_node asyncio_tasks_head; } asyncio_state; static inline asyncio_state * @@ -1896,19 +1885,12 @@ register_task(asyncio_state *state, TaskObj *task) { ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); - assert(task != &state->asyncio_tasks.first); - if (task->next != NULL) { + if (task->task_node.next != NULL) { // already registered + assert(task->task_node.prev != NULL); goto exit; } - assert(task->prev == NULL); - assert(state->asyncio_tasks.head != NULL); - - task->next = state->asyncio_tasks.head; - task->prev = state->asyncio_tasks.head->prev; - state->asyncio_tasks.head->prev->next = task; - state->asyncio_tasks.head->prev = task; - + llist_insert_tail(&state->asyncio_tasks_head, &task->task_node); exit: ASYNCIO_STATE_UNLOCK(state); } @@ -1924,18 +1906,12 @@ unregister_task(asyncio_state *state, TaskObj *task) { ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); - assert(task != &state->asyncio_tasks.first); - if (task->next == NULL) { + if (task->task_node.next == NULL) { // not registered - assert(task->prev == NULL); - assert(state->asyncio_tasks.head != task); + assert(task->task_node.prev == NULL); goto exit; } - task->next->prev = task->prev; - task->prev->next = task->next; - task->next = NULL; - task->prev = NULL; - assert(state->asyncio_tasks.head != task); + llist_remove(&task->task_node); exit: ASYNCIO_STATE_UNLOCK(state); } @@ -3625,20 +3601,18 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(eager_iter); int err = 0; ASYNCIO_STATE_LOCK(state); - TaskObj *first = &state->asyncio_tasks.first; - TaskObj *head = state->asyncio_tasks.head->next; - Py_INCREF(head); - while (head != first) - { - if (add_one_task(state, tasks, (PyObject *)head, loop) < 0) { + struct llist_node *node; + llist_for_each_safe(node, &state->asyncio_tasks_head) { + TaskObj *task = llist_data(node, TaskObj, task_node); + Py_INCREF(task); + if (add_one_task(state, tasks, (PyObject *)task, loop) < 0) { + Py_DECREF(task); Py_DECREF(tasks); Py_DECREF(loop); - Py_DECREF(head); err = 1; break; } - Py_INCREF(head->next); - Py_SETREF(head, head->next); + Py_DECREF(task); } ASYNCIO_STATE_UNLOCK(state); if (err) { @@ -3847,11 +3821,7 @@ module_exec(PyObject *mod) { asyncio_state *state = get_asyncio_state(mod); - Py_SET_TYPE(&state->asyncio_tasks.first, state->TaskType); - _Py_SetImmortalUntracked((PyObject *)&state->asyncio_tasks.first); - state->asyncio_tasks.head = &state->asyncio_tasks.first; - state->asyncio_tasks.head->next = &state->asyncio_tasks.first; - state->asyncio_tasks.head->prev = &state->asyncio_tasks.first; + llist_init(&state->asyncio_tasks_head); #define CREATE_TYPE(m, tp, spec, base) \ do { \ From 81636d3bbd7f126692326bf957707e8a88c91739 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 25 Dec 2024 13:23:44 +0000 Subject: [PATCH 20/44] gh-128234: support emscripten and wasi in async contextlib tests by removing asyncio from contextlib async tests (#95888) Co-authored-by: Kumar Aditya --- Lib/test/test_contextlib_async.py | 94 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 88dcdadd5e027d..d496aa611d1068 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -1,4 +1,4 @@ -import asyncio +import functools from contextlib import ( asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack, nullcontext, aclosing, contextmanager) @@ -8,14 +8,31 @@ from test.test_contextlib import TestBaseExitStack -support.requires_working_socket(module=True) -def tearDownModule(): - asyncio._set_event_loop_policy(None) +def _run_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise AssertionError("coroutine did not complete") + finally: + coro.close() -class TestAbstractAsyncContextManager(unittest.IsolatedAsyncioTestCase): +def _async_test(async_fn): + """Decorator to turn an async function into a synchronous function""" + @functools.wraps(async_fn) + def wrapper(*args, **kwargs): + return _run_async_fn(async_fn, *args, **kwargs) + return wrapper + + +class TestAbstractAsyncContextManager(unittest.TestCase): + + @_async_test async def test_enter(self): class DefaultEnter(AbstractAsyncContextManager): async def __aexit__(self, *args): @@ -27,6 +44,7 @@ async def __aexit__(self, *args): async with manager as context: self.assertIs(manager, context) + @_async_test async def test_slots(self): class DefaultAsyncContextManager(AbstractAsyncContextManager): __slots__ = () @@ -38,6 +56,7 @@ async def __aexit__(self, *args): manager = DefaultAsyncContextManager() manager.var = 42 + @_async_test async def test_async_gen_propagates_generator_exit(self): # A regression test for https://bugs.python.org/issue33786. @@ -88,8 +107,9 @@ class NoneAexit(ManagerFromScratch): self.assertFalse(issubclass(NoneAexit, AbstractAsyncContextManager)) -class AsyncContextManagerTestCase(unittest.IsolatedAsyncioTestCase): +class AsyncContextManagerTestCase(unittest.TestCase): + @_async_test async def test_contextmanager_plain(self): state = [] @asynccontextmanager @@ -103,6 +123,7 @@ async def woohoo(): state.append(x) self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_finally(self): state = [] @asynccontextmanager @@ -120,6 +141,7 @@ async def woohoo(): raise ZeroDivisionError() self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_traceback(self): @asynccontextmanager async def f(): @@ -175,6 +197,7 @@ class StopAsyncIterationSubclass(StopAsyncIteration): self.assertEqual(frames[0].name, 'test_contextmanager_traceback') self.assertEqual(frames[0].line, 'raise stop_exc') + @_async_test async def test_contextmanager_no_reraise(self): @asynccontextmanager async def whee(): @@ -184,6 +207,7 @@ async def whee(): # Calling __aexit__ should not result in an exception self.assertFalse(await ctx.__aexit__(TypeError, TypeError("foo"), None)) + @_async_test async def test_contextmanager_trap_yield_after_throw(self): @asynccontextmanager async def whoo(): @@ -199,6 +223,7 @@ async def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.ag_suspended) + @_async_test async def test_contextmanager_trap_no_yield(self): @asynccontextmanager async def whoo(): @@ -208,6 +233,7 @@ async def whoo(): with self.assertRaises(RuntimeError): await ctx.__aenter__() + @_async_test async def test_contextmanager_trap_second_yield(self): @asynccontextmanager async def whoo(): @@ -221,6 +247,7 @@ async def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.ag_suspended) + @_async_test async def test_contextmanager_non_normalised(self): @asynccontextmanager async def whoo(): @@ -234,6 +261,7 @@ async def whoo(): with self.assertRaises(SyntaxError): await ctx.__aexit__(RuntimeError, None, None) + @_async_test async def test_contextmanager_except(self): state = [] @asynccontextmanager @@ -251,6 +279,7 @@ async def woohoo(): raise ZeroDivisionError(999) self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_except_stopiter(self): @asynccontextmanager async def woohoo(): @@ -277,6 +306,7 @@ class StopAsyncIterationSubclass(StopAsyncIteration): else: self.fail(f'{stop_exc} was suppressed') + @_async_test async def test_contextmanager_wrap_runtimeerror(self): @asynccontextmanager async def woohoo(): @@ -321,12 +351,14 @@ def test_contextmanager_doc_attrib(self): self.assertEqual(baz.__doc__, "Whee!") @support.requires_docstrings + @_async_test async def test_instance_docstring_given_cm_docstring(self): baz = self._create_contextmanager_attribs()(None) self.assertEqual(baz.__doc__, "Whee!") async with baz: pass # suppress warning + @_async_test async def test_keywords(self): # Ensure no keyword arguments are inhibited @asynccontextmanager @@ -335,6 +367,7 @@ async def woohoo(self, func, args, kwds): async with woohoo(self=11, func=22, args=33, kwds=44) as target: self.assertEqual(target, (11, 22, 33, 44)) + @_async_test async def test_recursive(self): depth = 0 ncols = 0 @@ -361,6 +394,7 @@ async def recursive(): self.assertEqual(ncols, 10) self.assertEqual(depth, 0) + @_async_test async def test_decorator(self): entered = False @@ -379,6 +413,7 @@ async def test(): await test() self.assertFalse(entered) + @_async_test async def test_decorator_with_exception(self): entered = False @@ -401,6 +436,7 @@ async def test(): await test() self.assertFalse(entered) + @_async_test async def test_decorating_method(self): @asynccontextmanager @@ -435,7 +471,7 @@ async def method(self, a, b, c=None): self.assertEqual(test.b, 2) -class AclosingTestCase(unittest.IsolatedAsyncioTestCase): +class AclosingTestCase(unittest.TestCase): @support.requires_docstrings def test_instance_docs(self): @@ -443,6 +479,7 @@ def test_instance_docs(self): obj = aclosing(None) self.assertEqual(obj.__doc__, cm_docstring) + @_async_test async def test_aclosing(self): state = [] class C: @@ -454,6 +491,7 @@ async def aclose(self): self.assertEqual(x, y) self.assertEqual(state, [1]) + @_async_test async def test_aclosing_error(self): state = [] class C: @@ -467,6 +505,7 @@ async def aclose(self): 1 / 0 self.assertEqual(state, [1]) + @_async_test async def test_aclosing_bpo41229(self): state = [] @@ -492,45 +531,27 @@ async def agenfunc(): self.assertEqual(state, [1]) -class TestAsyncExitStack(TestBaseExitStack, unittest.IsolatedAsyncioTestCase): +class TestAsyncExitStack(TestBaseExitStack, unittest.TestCase): class SyncAsyncExitStack(AsyncExitStack): - @staticmethod - def run_coroutine(coro): - loop = asyncio.new_event_loop() - t = loop.create_task(coro) - t.add_done_callback(lambda f: loop.stop()) - loop.run_forever() - - exc = t.exception() - if not exc: - return t.result() - else: - context = exc.__context__ - - try: - raise exc - except: - exc.__context__ = context - raise exc def close(self): - return self.run_coroutine(self.aclose()) + return _run_async_fn(self.aclose) def __enter__(self): - return self.run_coroutine(self.__aenter__()) + return _run_async_fn(self.__aenter__) def __exit__(self, *exc_details): - return self.run_coroutine(self.__aexit__(*exc_details)) + return _run_async_fn(self.__aexit__, *exc_details) exit_stack = SyncAsyncExitStack callback_error_internal_frames = [ - ('__exit__', 'return self.run_coroutine(self.__aexit__(*exc_details))'), - ('run_coroutine', 'raise exc'), - ('run_coroutine', 'raise exc'), + ('__exit__', 'return _run_async_fn(self.__aexit__, *exc_details)'), + ('_run_async_fn', 'coro.send(None)'), ('__aexit__', 'raise exc'), ('__aexit__', 'cb_suppress = cb(*exc_details)'), ] + @_async_test async def test_async_callback(self): expected = [ ((), {}), @@ -573,6 +594,7 @@ async def _exit(*args, **kwds): stack.push_async_callback(callback=_exit, arg=3) self.assertEqual(result, []) + @_async_test async def test_async_push(self): exc_raised = ZeroDivisionError async def _expect_exc(exc_type, exc, exc_tb): @@ -608,6 +630,7 @@ async def __aexit__(self, *exc_details): self.assertIs(stack._exit_callbacks[-1][1], _expect_exc) 1/0 + @_async_test async def test_enter_async_context(self): class TestCM(object): async def __aenter__(self): @@ -629,6 +652,7 @@ async def _exit(): self.assertEqual(result, [1, 2, 3, 4]) + @_async_test async def test_enter_async_context_errors(self): class LacksEnterAndExit: pass @@ -648,6 +672,7 @@ async def __aenter__(self): await stack.enter_async_context(LacksExit()) self.assertFalse(stack._exit_callbacks) + @_async_test async def test_async_exit_exception_chaining(self): # Ensure exception chaining matches the reference behaviour async def raise_exc(exc): @@ -679,6 +704,7 @@ async def suppress_exc(*exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) + @_async_test async def test_async_exit_exception_explicit_none_context(self): # Ensure AsyncExitStack chaining matches actual nested `with` statements # regarding explicit __context__ = None. @@ -713,6 +739,7 @@ async def my_cm_with_exit_stack(): else: self.fail("Expected IndexError, but no exception was raised") + @_async_test async def test_instance_bypass_async(self): class Example(object): pass cm = Example() @@ -725,7 +752,8 @@ class Example(object): pass self.assertIs(stack._exit_callbacks[-1][1], cm) -class TestAsyncNullcontext(unittest.IsolatedAsyncioTestCase): +class TestAsyncNullcontext(unittest.TestCase): + @_async_test async def test_async_nullcontext(self): class C: pass From 5c814c83cdd3dc42bd9682106ffb7ade7ce6b5b3 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Wed, 25 Dec 2024 18:42:04 +0100 Subject: [PATCH 21/44] gh-128198: Add missing error checks for usages of PyIter_Next() (GH-128199) --- Modules/_asynciomodule.c | 13 +++++++++++++ Objects/frameobject.c | 4 ++++ Objects/namespaceobject.c | 4 ++++ 3 files changed, 21 insertions(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 603b77a70b15d4..74db4c74af905a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3599,6 +3599,13 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(item); } Py_DECREF(eager_iter); + + if (PyErr_Occurred()) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + int err = 0; ASYNCIO_STATE_LOCK(state); struct llist_node *node; @@ -3636,6 +3643,12 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) } Py_DECREF(scheduled_iter); Py_DECREF(loop); + + if (PyErr_Occurred()) { + Py_DECREF(tasks); + return NULL; + } + return tasks; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 10fd3a982c36f4..4f0040df4f3017 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -264,6 +264,10 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) Py_DECREF(iter); + if (PyErr_Occurred()) { + return -1; + } + return 0; } diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index 5b7547103a2b3f..4ef3bd92f5a569 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -141,6 +141,10 @@ namespace_repr(PyObject *ns) goto error; } + if (PyErr_Occurred()) { + goto error; + } + separator = PyUnicode_FromString(", "); if (separator == NULL) goto error; From 3eb746a7b9122c7c3cfc3fbd41d9d079d8b02845 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 26 Dec 2024 20:02:23 +0530 Subject: [PATCH 22/44] gh-127949: add docs for asyncio policy deprecation (#128269) --- Doc/library/asyncio-policy.rst | 8 ++++++++ Doc/whatsnew/3.14.rst | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index ea7fe957da7c40..9f86234ce941d1 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -7,6 +7,14 @@ Policies ======== +.. warning:: + + Policies are deprecated and will be removed in Python 3.16. + Users are encouraged to use the :func:`asyncio.run` function + or the :class:`asyncio.Runner` with *loop_factory* to use + the desired loop implementation. + + An event loop policy is a global object used to get and set the current :ref:`event loop `, as well as create new event loops. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 0dcee56b7d233f..935c61c474e889 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -691,10 +691,36 @@ Deprecated (Contributed by Serhiy Storchaka in :gh:`58032`.) * :mod:`asyncio`: - :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + + * :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16; + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + * :func:`asyncio.set_event_loop` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) * :mod:`builtins`: Passing a complex number as the *real* or *imag* argument in the From c6563f3f22e715780e8182481879b0c9110b71ac Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Thu, 26 Dec 2024 15:39:15 +0100 Subject: [PATCH 23/44] gh-119786: Fix typos in `InternalDocs/frames.md` (#128275) Fix typos in `InternalDocs/frames.md` --- InternalDocs/frames.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 2598873ca98479..1a909009eea610 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -33,7 +33,7 @@ Each activation record is laid out as: * Stack This seems to provide the best performance without excessive complexity. -The specials have a fixed size, so the offset of the locals is know. The +The specials have a fixed size, so the offset of the locals is known. The interpreter needs to hold two pointers, a frame pointer and a stack pointer. #### Alternative layout @@ -52,7 +52,7 @@ an extra pointer for the locals, which can hurt performance. ### Generators and Coroutines Generators and coroutines contain a `_PyInterpreterFrame` -The specials sections contains the following pointers: +The specials section contains the following pointers: * Globals dict * Builtins dict @@ -69,7 +69,7 @@ and builtins, than strong references to both globals and builtins. When creating a backtrace or when calling `sys._getframe()` the frame becomes visible to Python code. When this happens a new `PyFrameObject` is created -and a strong reference to it placed in the `frame_obj` field of the specials +and a strong reference to it is placed in the `frame_obj` field of the specials section. The `frame_obj` field is initially `NULL`. The `PyFrameObject` may outlive a stack-allocated `_PyInterpreterFrame`. @@ -128,7 +128,7 @@ The `return_offset` field determines where a `RETURN` should go in the caller, relative to `instr_ptr`. It is only meaningful to the callee, so it needs to be set in any instruction that implements a call (to a Python function), including CALL, SEND and BINARY_SUBSCR_GETITEM, among others. If there is no -callee, then return_offset is meaningless. It is necessary to have a separate +callee, then return_offset is meaningless. It is necessary to have a separate field for the return offset because (1) if we apply this offset to `instr_ptr` while executing the `RETURN`, this is too early and would lose us information about the previous instruction which we could need for introspecting and From 8a26c7b2af59b999ae41b8189295cc550f297f6e Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Thu, 26 Dec 2024 23:39:44 +0900 Subject: [PATCH 24/44] Docs: Fix comment out in `c-api/typeobj.rst` (#128266) --- Doc/c-api/typeobj.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ba58cc1c26c70b..69c518d8e64cbc 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1023,6 +1023,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` values. + .. XXX are most flag bits *really* inherited individually? **Default:** From 42f7a00ae8b6b3fa09115e24b9512216c6c8978e Mon Sep 17 00:00:00 2001 From: da-woods Date: Thu, 26 Dec 2024 14:40:48 +0000 Subject: [PATCH 25/44] Clean up redundant ifdef in list getitem (#128257) It's already inside a `Py_GIL_DISABLED` block so the `#else` clause is always unused. --- Objects/listobject.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index a877bad66be45f..bbd53e7de94a31 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -335,11 +335,7 @@ list_item_impl(PyListObject *self, Py_ssize_t idx) if (!valid_index(idx, size)) { goto exit; } -#ifdef Py_GIL_DISABLED item = _Py_NewRefWithLock(self->ob_item[idx]); -#else - item = Py_NewRef(self->ob_item[idx]); -#endif exit: Py_END_CRITICAL_SECTION(); return item; From 9ddc388527477afdae17252628d63138631072ba Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 26 Dec 2024 14:50:20 +0000 Subject: [PATCH 26/44] gh-124761: add `socket.SO_REUSEPORT_LB` (#124961) --- Doc/library/socket.rst | 8 ++++++++ .../2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst | 1 + Modules/socketmodule.c | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 58323ba6514eac..8ba2bd1dcce8cc 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -674,6 +674,14 @@ Constants .. availability:: Linux >= 3.9 +.. data:: SO_REUSEPORT_LB + + Constant to enable duplicate address and port bindings with load balancing. + + .. versionadded:: next + + .. availability:: FreeBSD >= 12.0 + .. data:: AF_HYPERV HV_PROTOCOL_RAW HVSOCKET_CONNECT_TIMEOUT diff --git a/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst b/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst new file mode 100644 index 00000000000000..797dd31b368548 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst @@ -0,0 +1 @@ +Add :data:`~socket.SO_REUSEPORT_LB` constant to :mod:`socket` for FreeBSD. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 9394f1c940bedf..1e95be9b1bc9f4 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7916,6 +7916,9 @@ socket_exec(PyObject *m) ADD_INT_MACRO(m, SO_REUSEPORT); #endif #endif +#ifdef SO_REUSEPORT_LB + ADD_INT_MACRO(m, SO_REUSEPORT_LB); +#endif #ifdef SO_SNDBUF ADD_INT_MACRO(m, SO_SNDBUF); #endif From fb0b94223d481ca58b7c77332348d1ab2c9ab272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 26 Dec 2024 16:03:57 +0100 Subject: [PATCH 27/44] gh-87138: convert blake2b/2s types to heap types (#127669) --- Modules/blake2module.c | 89 ++++++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 94cdfe7fd2e962..6723e7de4675a5 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -379,13 +379,13 @@ class _blake2.blake2s "Blake2Object *" "&PyBlake2_BLAKE2sType" static Blake2Object * new_Blake2Object(PyTypeObject *type) { - Blake2Object *self; - self = (Blake2Object *)type->tp_alloc(type, 0); + Blake2Object *self = PyObject_GC_New(Blake2Object, type); if (self == NULL) { return NULL; } HASHLIB_INIT_MUTEX(self); + PyObject_GC_Track(self); return self; } @@ -454,7 +454,28 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, } self->impl = type_to_impl(type); - + // Ensure that the states are NULL-initialized in case of an error. + // See: py_blake2_clear() for more details. + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + self->blake2b_256_state = NULL; + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + self->blake2s_128_state = NULL; + break; +#endif + case Blake2b: + self->blake2b_state = NULL; + break; + case Blake2s: + self->blake2s_state = NULL; + break; + default: + Py_UNREACHABLE(); + } // Using Blake2b because we statically know that these are greater than the // Blake2s sizes -- this avoids a VLA. uint8_t salt_[HACL_HASH_BLAKE2B_SALT_BYTES] = { 0 }; @@ -595,7 +616,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, return (PyObject *)self; error: - Py_XDECREF(self); + Py_XDECREF(self); return NULL; } @@ -875,46 +896,70 @@ static PyGetSetDef py_blake2b_getsetters[] = { {NULL} }; - -static void -py_blake2b_dealloc(Blake2Object *self) +static int +py_blake2_clear(PyObject *op) { + Blake2Object *self = (Blake2Object *)op; + // The initialization function uses PyObject_GC_New() but explicitly + // initializes the HACL* internal state to NULL before allocating + // it. If an error occurs in the constructor, we should only free + // states that were allocated (i.e. that are not NULL). switch (self->impl) { #if HACL_CAN_COMPILE_SIMD256 case Blake2b_256: - if (self->blake2b_256_state != NULL) + if (self->blake2b_256_state != NULL) { Hacl_Hash_Blake2b_Simd256_free(self->blake2b_256_state); + self->blake2b_256_state = NULL; + } break; #endif #if HACL_CAN_COMPILE_SIMD128 case Blake2s_128: - if (self->blake2s_128_state != NULL) + if (self->blake2s_128_state != NULL) { Hacl_Hash_Blake2s_Simd128_free(self->blake2s_128_state); + self->blake2s_128_state = NULL; + } break; #endif case Blake2b: - // This happens if we hit "goto error" in the middle of the - // initialization function. We leverage the fact that tp_alloc - // guarantees that the contents of the object are NULL-initialized - // (see documentation for PyType_GenericAlloc) to detect this case. - if (self->blake2b_state != NULL) + if (self->blake2b_state != NULL) { Hacl_Hash_Blake2b_free(self->blake2b_state); + self->blake2b_state = NULL; + } break; case Blake2s: - if (self->blake2s_state != NULL) + if (self->blake2s_state != NULL) { Hacl_Hash_Blake2s_free(self->blake2s_state); + self->blake2s_state = NULL; + } break; default: Py_UNREACHABLE(); } + return 0; +} +static void +py_blake2_dealloc(PyObject *self) +{ PyTypeObject *type = Py_TYPE(self); - PyObject_Free(self); + PyObject_GC_UnTrack(self); + (void)py_blake2_clear(self); + type->tp_free(self); Py_DECREF(type); } +static int +py_blake2_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + static PyType_Slot blake2b_type_slots[] = { - {Py_tp_dealloc, py_blake2b_dealloc}, + {Py_tp_clear, py_blake2_clear}, + {Py_tp_dealloc, py_blake2_dealloc}, + {Py_tp_traverse, py_blake2_traverse}, {Py_tp_doc, (char *)py_blake2b_new__doc__}, {Py_tp_methods, py_blake2b_methods}, {Py_tp_getset, py_blake2b_getsetters}, @@ -923,7 +968,9 @@ static PyType_Slot blake2b_type_slots[] = { }; static PyType_Slot blake2s_type_slots[] = { - {Py_tp_dealloc, py_blake2b_dealloc}, + {Py_tp_clear, py_blake2_clear}, + {Py_tp_dealloc, py_blake2_dealloc}, + {Py_tp_traverse, py_blake2_traverse}, {Py_tp_doc, (char *)py_blake2s_new__doc__}, {Py_tp_methods, py_blake2b_methods}, {Py_tp_getset, py_blake2b_getsetters}, @@ -936,13 +983,15 @@ static PyType_Slot blake2s_type_slots[] = { static PyType_Spec blake2b_type_spec = { .name = "_blake2.blake2b", .basicsize = sizeof(Blake2Object), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2b_type_slots }; static PyType_Spec blake2s_type_spec = { .name = "_blake2.blake2s", .basicsize = sizeof(Blake2Object), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2s_type_slots }; From 3bd7730bbda3e38db5920a7b8c95958ca90342bf Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 26 Dec 2024 16:17:22 +0100 Subject: [PATCH 28/44] gh-126868: Add freelist for compact ints to `_PyLong_New` (#128181) Co-authored-by: Kumar Aditya --- ...-12-22-15-47-44.gh-issue-126868.RpjKez.rst | 1 + Objects/longobject.c | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst new file mode 100644 index 00000000000000..ede383deb4ad31 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst @@ -0,0 +1 @@ +Increase usage of freelist for :class:`int` allocation. diff --git a/Objects/longobject.c b/Objects/longobject.c index bd7ff68d0899c6..d449a01cedf886 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -156,7 +156,7 @@ PyLongObject * _PyLong_New(Py_ssize_t size) { assert(size >= 0); - PyLongObject *result; + PyLongObject *result = NULL; if (size > (Py_ssize_t)MAX_LONG_DIGITS) { PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); @@ -165,19 +165,25 @@ _PyLong_New(Py_ssize_t size) /* Fast operations for single digit integers (including zero) * assume that there is always at least one digit present. */ Py_ssize_t ndigits = size ? size : 1; - /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + - sizeof(digit)*size. Previous incarnations of this code used - sizeof() instead of the offsetof, but this risks being - incorrect in the presence of padding between the header - and the digits. */ - result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + - ndigits*sizeof(digit)); - if (!result) { - PyErr_NoMemory(); - return NULL; + + if (ndigits == 1) { + result = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); + } + if (result == NULL) { + /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + + sizeof(digit)*size. Previous incarnations of this code used + sizeof() instead of the offsetof, but this risks being + incorrect in the presence of padding between the header + and the digits. */ + result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + + ndigits*sizeof(digit)); + if (!result) { + PyErr_NoMemory(); + return NULL; + } + _PyObject_Init((PyObject*)result, &PyLong_Type); } _PyLong_SetSignAndDigitCount(result, size != 0, size); - _PyObject_Init((PyObject*)result, &PyLong_Type); /* The digit has to be initialized explicitly to avoid * use-of-uninitialized-value. */ result->long_value.ob_digit[0] = 0; From ea2b53739f1128184b4140decbeffeac6cfe966f Mon Sep 17 00:00:00 2001 From: Moshe Kaplan Date: Thu, 26 Dec 2024 16:53:37 -0500 Subject: [PATCH 29/44] Remove incorrect imports rationale comment in `http.server` (#128278) Remove reference to gethostbyaddr(), because it's not actually used within this code. --- Lib/http/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index a6f7aecc78763f..a90c8d34c394db 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -99,7 +99,7 @@ import posixpath import select import shutil -import socket # For gethostbyaddr() +import socket import socketserver import sys import time From 401bba6b58497ce59e7b45ad33e43ae8c67abcb9 Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Fri, 27 Dec 2024 02:03:47 +0100 Subject: [PATCH 30/44] gh-127537: Add __class_getitem__ to the python implementation of functools.partial (#127537) --- Lib/functools.py | 3 +++ Lib/test/test_functools.py | 6 ++++++ .../Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst | 5 +++++ 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst diff --git a/Lib/functools.py b/Lib/functools.py index eff6540c7f606e..786b8aedfd77f5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -433,6 +433,9 @@ def __setstate__(self, state): self._phcount = phcount self._merger = merger + __class_getitem__ = classmethod(GenericAlias) + + try: from _functools import partial, Placeholder, _PlaceholderType except ImportError: diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ffd2adb8665b45..4a0252cb637a52 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -473,6 +473,12 @@ class A: self.assertEqual(a.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) self.assertEqual(a.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + def test_partial_genericalias(self): + alias = self.partial[int] + self.assertIs(alias.__origin__, self.partial) + self.assertEqual(alias.__args__, (int,)) + self.assertEqual(alias.__parameters__, ()) + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst b/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst new file mode 100644 index 00000000000000..5e39933047993c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst @@ -0,0 +1,5 @@ +Add missing ``__class_getitem__`` method to the Python implementation of +:func:`functools.partial`, to make it compatible with the C version. This is +mainly relevant for alternative Python implementations like PyPy and +GraalPy, because CPython will usually use the C-implementation of that +function. From 08a0728d6c32986d35edb26872b4512a71ae60f3 Mon Sep 17 00:00:00 2001 From: Damien <81557462+Damien-Chen@users.noreply.github.com> Date: Fri, 27 Dec 2024 09:57:55 +0800 Subject: [PATCH 31/44] gh-125887: Update PyObject_HasAttr exception behavior (#125907) Update PyObject_HasAttr exception behavior Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/c-api/object.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index f97ade01e67850..a137688fe07545 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -111,7 +111,8 @@ Object Protocol .. note:: Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods are silently ignored. + :meth:`~object.__getattribute__` methods aren't propagated, + but instead given to :func:`sys.unraisablehook`. For proper error handling, use :c:func:`PyObject_HasAttrWithError`, :c:func:`PyObject_GetOptionalAttr` or :c:func:`PyObject_GetAttr` instead. From 0b5f1fae573a2c658eb000433ad7b87e9c40c697 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 27 Dec 2024 14:58:35 +0100 Subject: [PATCH 32/44] Mention loop_factory argument in docstring for asyncio.run() (#128288) --- Lib/asyncio/runners.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index b9adf291d4817f..14397b4ad0c732 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -177,6 +177,7 @@ def run(main, *, debug=None, loop_factory=None): running in the same thread. If debug is True, the event loop will be run in debug mode. + If loop_factory is passed, it is used for new event loop creation. This function always creates a new event loop and closes it at the end. It should be used as a main entry point for asyncio programs, and should From 71de839ec987bb67b98fcfecfc687281841a713c Mon Sep 17 00:00:00 2001 From: donBarbos Date: Fri, 27 Dec 2024 18:12:25 +0400 Subject: [PATCH 33/44] gh-127089: Add missing description for codes in `http.HTTPStatus` (#127100) Co-authored-by: Ethan Furman Co-authored-by: Andrew Svetlov --- Lib/http/__init__.py | 41 +++++++++++++++++++++++++--------------- Lib/test/test_httplib.py | 41 +++++++++++++++++++++++++--------------- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py index d64741ec0dd29a..9f278289420713 100644 --- a/Lib/http/__init__.py +++ b/Lib/http/__init__.py @@ -54,8 +54,9 @@ def is_server_error(self): CONTINUE = 100, 'Continue', 'Request received, please continue' SWITCHING_PROTOCOLS = (101, 'Switching Protocols', 'Switching to new protocol; obey Upgrade header') - PROCESSING = 102, 'Processing' - EARLY_HINTS = 103, 'Early Hints' + PROCESSING = 102, 'Processing', 'Server is processing the request' + EARLY_HINTS = (103, 'Early Hints', + 'Headers sent to prepare for the response') # success OK = 200, 'OK', 'Request fulfilled, document follows' @@ -67,9 +68,11 @@ def is_server_error(self): NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' - MULTI_STATUS = 207, 'Multi-Status' - ALREADY_REPORTED = 208, 'Already Reported' - IM_USED = 226, 'IM Used' + MULTI_STATUS = (207, 'Multi-Status', + 'Response contains multiple statuses in the body') + ALREADY_REPORTED = (208, 'Already Reported', + 'Operation has already been reported') + IM_USED = 226, 'IM Used', 'Request completed using instance manipulations' # redirection MULTIPLE_CHOICES = (300, 'Multiple Choices', @@ -128,15 +131,19 @@ def is_server_error(self): EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') IM_A_TEAPOT = (418, 'I\'m a Teapot', - 'Server refuses to brew coffee because it is a teapot.') + 'Server refuses to brew coffee because it is a teapot') MISDIRECTED_REQUEST = (421, 'Misdirected Request', 'Server is not able to produce a response') - UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_CONTENT = (422, 'Unprocessable Content', + 'Server is not able to process the contained instructions') UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT - LOCKED = 423, 'Locked' - FAILED_DEPENDENCY = 424, 'Failed Dependency' - TOO_EARLY = 425, 'Too Early' - UPGRADE_REQUIRED = 426, 'Upgrade Required' + LOCKED = 423, 'Locked', 'Resource of a method is locked' + FAILED_DEPENDENCY = (424, 'Failed Dependency', + 'Dependent action of the request failed') + TOO_EARLY = (425, 'Too Early', + 'Server refuses to process a request that might be replayed') + UPGRADE_REQUIRED = (426, 'Upgrade Required', + 'Server refuses to perform the request using the current protocol') PRECONDITION_REQUIRED = (428, 'Precondition Required', 'The origin server requires the request to be conditional') TOO_MANY_REQUESTS = (429, 'Too Many Requests', @@ -164,10 +171,14 @@ def is_server_error(self): 'The gateway server did not receive a timely response') HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', 'Cannot fulfill request') - VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' - INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' - LOOP_DETECTED = 508, 'Loop Detected' - NOT_EXTENDED = 510, 'Not Extended' + VARIANT_ALSO_NEGOTIATES = (506, 'Variant Also Negotiates', + 'Server has an internal configuration error') + INSUFFICIENT_STORAGE = (507, 'Insufficient Storage', + 'Server is not able to store the representation') + LOOP_DETECTED = (508, 'Loop Detected', + 'Server encountered an infinite loop while processing a request') + NOT_EXTENDED = (510, 'Not Extended', + 'Request does not meet the resource access policy') NETWORK_AUTHENTICATION_REQUIRED = (511, 'Network Authentication Required', 'The client needs to authenticate to gain network access') diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 89963dadeb152b..7a7ec555a2dbbb 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -594,8 +594,9 @@ def is_server_error(self): CONTINUE = 100, 'Continue', 'Request received, please continue' SWITCHING_PROTOCOLS = (101, 'Switching Protocols', 'Switching to new protocol; obey Upgrade header') - PROCESSING = 102, 'Processing' - EARLY_HINTS = 103, 'Early Hints' + PROCESSING = 102, 'Processing', 'Server is processing the request' + EARLY_HINTS = (103, 'Early Hints', + 'Headers sent to prepare for the response') # success OK = 200, 'OK', 'Request fulfilled, document follows' CREATED = 201, 'Created', 'Document created, URL follows' @@ -606,9 +607,11 @@ def is_server_error(self): NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' - MULTI_STATUS = 207, 'Multi-Status' - ALREADY_REPORTED = 208, 'Already Reported' - IM_USED = 226, 'IM Used' + MULTI_STATUS = (207, 'Multi-Status', + 'Response contains multiple statuses in the body') + ALREADY_REPORTED = (208, 'Already Reported', + 'Operation has already been reported') + IM_USED = 226, 'IM Used', 'Request completed using instance manipulations' # redirection MULTIPLE_CHOICES = (300, 'Multiple Choices', 'Object has several resources -- see URI list') @@ -665,15 +668,19 @@ def is_server_error(self): EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') IM_A_TEAPOT = (418, 'I\'m a Teapot', - 'Server refuses to brew coffee because it is a teapot.') + 'Server refuses to brew coffee because it is a teapot') MISDIRECTED_REQUEST = (421, 'Misdirected Request', 'Server is not able to produce a response') - UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_CONTENT = (422, 'Unprocessable Content', + 'Server is not able to process the contained instructions') UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT - LOCKED = 423, 'Locked' - FAILED_DEPENDENCY = 424, 'Failed Dependency' - TOO_EARLY = 425, 'Too Early' - UPGRADE_REQUIRED = 426, 'Upgrade Required' + LOCKED = 423, 'Locked', 'Resource of a method is locked' + FAILED_DEPENDENCY = (424, 'Failed Dependency', + 'Dependent action of the request failed') + TOO_EARLY = (425, 'Too Early', + 'Server refuses to process a request that might be replayed') + UPGRADE_REQUIRED = (426, 'Upgrade Required', + 'Server refuses to perform the request using the current protocol') PRECONDITION_REQUIRED = (428, 'Precondition Required', 'The origin server requires the request to be conditional') TOO_MANY_REQUESTS = (429, 'Too Many Requests', @@ -700,10 +707,14 @@ def is_server_error(self): 'The gateway server did not receive a timely response') HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', 'Cannot fulfill request') - VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' - INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' - LOOP_DETECTED = 508, 'Loop Detected' - NOT_EXTENDED = 510, 'Not Extended' + VARIANT_ALSO_NEGOTIATES = (506, 'Variant Also Negotiates', + 'Server has an internal configuration error') + INSUFFICIENT_STORAGE = (507, 'Insufficient Storage', + 'Server is not able to store the representation') + LOOP_DETECTED = (508, 'Loop Detected', + 'Server encountered an infinite loop while processing a request') + NOT_EXTENDED = (510, 'Not Extended', + 'Request does not meet the resource access policy') NETWORK_AUTHENTICATION_REQUIRED = (511, 'Network Authentication Required', 'The client needs to authenticate to gain network access') From 64173cd6f2d8dc95c6f8b67912d0edd1c1b707d5 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 27 Dec 2024 20:43:41 +0530 Subject: [PATCH 34/44] gh-127949: make deprecation of policy system more prominent (#128290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/deprecations/pending-removal-in-3.16.rst | 27 +++++++++++++++++++- Doc/library/asyncio-eventloop.rst | 16 ++++++++---- Doc/library/asyncio-runner.rst | 6 +++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index 6f6954b783a1ae..f2b818f14c63cd 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -19,10 +19,35 @@ Pending removal in Python 3.16 * :mod:`asyncio`: * :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, + and will be removed in Python 3.16; use :func:`inspect.iscoroutinefunction` instead. (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + * :func:`asyncio.set_event_loop` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) + * :mod:`builtins`: * Bitwise inversion on boolean types, ``~True`` or ``~False`` diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 29f843123f8560..ccb362d8c31ddf 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -62,6 +62,13 @@ an event loop: .. versionchanged:: 3.14 Raises a :exc:`RuntimeError` if there is no current event loop. + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, this function will always return the + running event loop. + + .. function:: set_event_loop(loop) Set *loop* as the current event loop for the current OS thread. @@ -1781,12 +1788,11 @@ By default asyncio is configured to use :class:`EventLoop`. import asyncio import selectors - class MyPolicy(asyncio.DefaultEventLoopPolicy): - def new_event_loop(self): - selector = selectors.SelectSelector() - return asyncio.SelectorEventLoop(selector) + async def main(): + ... - asyncio.set_event_loop_policy(MyPolicy()) + loop_factory = lambda: asyncio.SelectorEventLoop(selectors.SelectSelector()) + asyncio.run(main(), loop_factory=loop_factory) .. availability:: Unix, Windows. diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 28d5aaf3692baa..48d78099fd3ce7 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -76,6 +76,12 @@ Running an asyncio Program *coro* can be any awaitable object. + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, an explicit *loop_factory* is needed + to configure the event loop. + Runner context manager ====================== From aeb9b65aa26444529e4adc7d6e5b0d3dd9889ec2 Mon Sep 17 00:00:00 2001 From: Stephen Hansen Date: Fri, 27 Dec 2024 17:09:01 -0500 Subject: [PATCH 35/44] gh-127586: multiprocessing.Pool does not properly restore blocked signals (try 2) (GH-128011) Correct pthread_sigmask in resource_tracker to restore old signals Using SIG_UNBLOCK to remove blocked "ignored signals" may accidentally cause side effects if the calling parent already had said signals blocked to begin with and did not intend to unblock them when creating a pool. Use SIG_SETMASK instead with the previous mask of blocked signals to restore the original blocked set. Co-authored-by: Peter Bierma Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/resource_tracker.py | 7 ++++--- Lib/test/_test_multiprocessing.py | 21 +++++++++++++++++++ ...-12-03-20-28-08.gh-issue-127586.zgotYF.rst | 3 +++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 20ddd9c50e3d88..90e036ae905afa 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -155,13 +155,14 @@ def ensure_running(self): # that can make the child die before it registers signal handlers # for SIGINT and SIGTERM. The mask is unregistered after spawning # the child. + prev_sigmask = None try: if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = util.spawnv_passfds(exe, args, fds_to_pass) finally: - if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) except: os.close(w) raise diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 80b08b8ac66899..38a03f3391d31d 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6045,6 +6045,27 @@ def test_resource_tracker_exit_code(self): cleanup=cleanup, ) + @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available") + def test_resource_tracker_blocked_signals(self): + # + # gh-127586: Check that resource_tracker does not override blocked signals of caller. + # + from multiprocessing.resource_tracker import ResourceTracker + orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) + signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1} + + try: + for sig in signals: + signal.pthread_sigmask(signal.SIG_SETMASK, {sig}) + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker = ResourceTracker() + tracker.ensure_running() + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker._stop() + finally: + # restore sigmask to what it was before executing test + signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask) + class TestSimpleQueue(unittest.TestCase): @classmethod diff --git a/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst new file mode 100644 index 00000000000000..80217bd4a10503 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst @@ -0,0 +1,3 @@ +:class:`multiprocessing.pool.Pool` now properly restores blocked signal handlers +of the parent thread when creating processes via either *spawn* or +*forkserver*. From aab51c3414bc815c4c31e8ef2a9003f4a546faa9 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 28 Dec 2024 14:59:49 +0000 Subject: [PATCH 36/44] gh-128265: Support WASI/Emscripten on PDB tests, by removing asyncio from pdb tests (#128264) A part of `Lib/test/test_pdb.py` was previously unable to run on WASI/Emscripten platforms because it lacked support for `asyncio`. In fact, these tests could be rewritten without the `asyncio` framework because `test_pdb` tests the behavior of coroutines, which are not part of `asyncio`. Now reliance on the availability of `asyncio` has been removed and part of `test_pdb` that deals with coroutines working on WASI/Emscripten platforms. --- Lib/test/support/__init__.py | 29 +++++++++++ Lib/test/test_contextlib_async.py | 15 +----- Lib/test/test_inspect/test_inspect.py | 15 ++---- Lib/test/test_pdb.py | 71 +++++++++++++-------------- 4 files changed, 70 insertions(+), 60 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 5c738ffaa27713..cf3077f2a4a409 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -62,6 +62,7 @@ "force_not_colorized", "BrokenIter", "in_systemd_nspawn_sync_suppressed", + "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", ] @@ -2940,3 +2941,31 @@ def in_systemd_nspawn_sync_suppressed() -> bool: os.close(fd) return False + +def run_no_yield_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise AssertionError("coroutine did not complete") + finally: + coro.close() + + +@types.coroutine +def async_yield(v): + return (yield v) + + +def run_yielding_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + while True: + try: + coro.send(None) + except StopIteration as e: + return e.value + finally: + coro.close() diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index d496aa611d1068..7750186e56a5cc 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -3,24 +3,13 @@ asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack, nullcontext, aclosing, contextmanager) from test import support +from test.support import run_no_yield_async_fn as _run_async_fn import unittest import traceback from test.test_contextlib import TestBaseExitStack -def _run_async_fn(async_fn, /, *args, **kwargs): - coro = async_fn(*args, **kwargs) - try: - coro.send(None) - except StopIteration as e: - return e.value - else: - raise AssertionError("coroutine did not complete") - finally: - coro.close() - - def _async_test(async_fn): """Decorator to turn an async function into a synchronous function""" @functools.wraps(async_fn) @@ -546,7 +535,7 @@ def __exit__(self, *exc_details): exit_stack = SyncAsyncExitStack callback_error_internal_frames = [ ('__exit__', 'return _run_async_fn(self.__aexit__, *exc_details)'), - ('_run_async_fn', 'coro.send(None)'), + ('run_no_yield_async_fn', 'coro.send(None)'), ('__aexit__', 'raise exc'), ('__aexit__', 'cb_suppress = cb(*exc_details)'), ] diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d536d04d2e7d88..345a57a5cfee2d 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -36,6 +36,7 @@ from test.support import cpython_only, import_helper from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ +from test.support import run_no_yield_async_fn from test.support.import_helper import DirsOnSysPath, ready_to_import from test.support.os_helper import TESTFN, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python @@ -1161,19 +1162,11 @@ def f(self): sys.modules.pop("inspect_actual") def test_nested_class_definition_inside_async_function(self): - def run(coro): - try: - coro.send(None) - except StopIteration as e: - return e.value - else: - raise RuntimeError("coroutine did not complete synchronously!") - finally: - coro.close() + run = run_no_yield_async_fn - self.assertSourceEqual(run(mod2.func225()), 226, 227) + self.assertSourceEqual(run(mod2.func225), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) - self.assertSourceEqual(run(mod2.cls226().func232()), 233, 234) + self.assertSourceEqual(run(mod2.cls226().func232), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 9b0806d8b2a9bd..c5ee8c5fb25350 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -20,8 +20,7 @@ from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch -# gh-114275: WASI fails to run asyncio tests, similar skip than test_asyncio. -SKIP_ASYNCIO_TESTS = (not support.has_socket_support) +SKIP_CORO_TESTS = False class PdbTestInput(object): @@ -1987,7 +1986,7 @@ def test_next_until_return_at_return_event(): """ def test_pdb_next_command_for_generator(): - """Testing skip unwindng stack on yield for generators for "next" command + """Testing skip unwinding stack on yield for generators for "next" command >>> def test_gen(): ... yield 0 @@ -2049,23 +2048,23 @@ def test_pdb_next_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_next_command_for_coroutine(): - """Testing skip unwindng stack on yield for coroutines for "next" command + """Testing skip unwinding stack on yield for coroutines for "next" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) + ... await async_yield(0) + ... await async_yield(0) + ... await async_yield(0) >>> async def test_main(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', @@ -2088,13 +2087,13 @@ def test_pdb_next_command_for_coroutine(): -> async def test_coro(): (Pdb) step > (2)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (3)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (4)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next Internal StopIteration > (3)test_main() @@ -2108,13 +2107,13 @@ def test_pdb_next_command_for_coroutine(): """ def test_pdb_next_command_for_asyncgen(): - """Testing skip unwindng stack on yield for coroutines for "next" command + """Testing skip unwinding stack on yield for coroutines for "next" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def agen(): ... yield 1 - ... await asyncio.sleep(0) + ... await async_yield(0) ... yield 2 >>> async def test_coro(): @@ -2126,7 +2125,7 @@ def test_pdb_next_command_for_asyncgen(): ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', @@ -2163,14 +2162,14 @@ def test_pdb_next_command_for_asyncgen(): -> yield 1 (Pdb) next > (3)agen() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) continue 2 finished """ def test_pdb_return_command_for_generator(): - """Testing no unwindng stack on yield for generators + """Testing no unwinding stack on yield for generators for "return" command >>> def test_gen(): @@ -2228,23 +2227,23 @@ def test_pdb_return_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_return_command_for_coroutine(): - """Testing no unwindng stack on yield for coroutines for "return" command + """Testing no unwinding stack on yield for coroutines for "return" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) + ... await async_yield(0) + ... await async_yield(0) + ... await async_yield(0) >>> async def test_main(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', @@ -2264,16 +2263,16 @@ def test_pdb_return_command_for_coroutine(): -> async def test_coro(): (Pdb) step > (2)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (3)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) continue finished """ def test_pdb_until_command_for_generator(): - """Testing no unwindng stack on yield for generators + """Testing no unwinding stack on yield for generators for "until" command if target breakpoint is not reached >>> def test_gen(): @@ -2320,20 +2319,20 @@ def test_pdb_until_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_until_command_for_coroutine(): - """Testing no unwindng stack for coroutines + """Testing no unwinding stack for coroutines for "until" command if target breakpoint is not reached - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): ... print(0) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(1) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(2) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(3) >>> async def test_main(): @@ -2341,7 +2340,7 @@ def test_pdb_until_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', From 2cf396c368a188e9142843e566ce6d8e6eb08999 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sat, 28 Dec 2024 16:05:00 +0100 Subject: [PATCH 37/44] gh-119786: Fix typos in `InternalDocs/parser.md` (#128314) --- InternalDocs/parser.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 445b866fc0cb96..be47efe24356d4 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -56,7 +56,7 @@ an input string as its argument, and yields one of the following results: Note that "failure" results do not imply that the program is incorrect, nor do they necessarily mean that the parsing has failed. Since the choice operator is -ordered, a failure very often merely indicates "try the following option". A +ordered, a failure very often merely indicates "try the following option". A direct implementation of a PEG parser as a recursive descent parser will present exponential time performance in the worst case, because PEG parsers have infinite lookahead (this means that they can consider an arbitrary number of @@ -253,7 +253,7 @@ inside curly-braces, which specifies the return value of the alternative: If the action is omitted, a default action is generated: - If there is a single name in the rule, it gets returned. -- If there multiple names in the rule, a collection with all parsed +- If there are multiple names in the rule, a collection with all parsed expressions gets returned (the type of the collection will be different in C and Python). @@ -447,7 +447,7 @@ parser (the one used by the interpreter) just execute: $ make regen-pegen ``` -using the `Makefile` in the main directory. If you are on Windows you can +using the `Makefile` in the main directory. If you are on Windows you can use the Visual Studio project files to regenerate the parser or to execute: ```dos @@ -539,7 +539,7 @@ memoization is used. The C parser used by Python is highly optimized and memoization can be expensive both in memory and time. Although the memory cost is obvious (the parser needs memory for storing previous results in the cache) the execution time cost comes -for continuously checking if the given rule has a cache hit or not. In many +from continuously checking if the given rule has a cache hit or not. In many situations, just parsing it again can be faster. Pegen **disables memoization by default** except for rules with the special marker `memo` after the rule name (and type, if present): @@ -605,7 +605,7 @@ File "", line 1 SyntaxError: invalid syntax ``` -While soft keywords don't have this limitation if used in a context other the +While soft keywords don't have this limitation if used in a context other than one where they are defined as keywords: ```pycon From 492b224b991cd9027f1bc6d9988d01e94f764992 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 28 Dec 2024 21:49:45 +0300 Subject: [PATCH 38/44] gh-128279: Enhance the NetBSD compatibility for thread naming (#128280) Enhance NetBSD compatibility for thread naming in _threadmodule.c. --- Modules/_threadmodule.c | 3 +++ configure | 1 + configure.ac | 1 + 3 files changed, 5 insertions(+) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 75b34a8df7622c..2cbdfeb09b95ae 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -2438,6 +2438,9 @@ _thread_set_name_impl(PyObject *module, PyObject *name_obj) const char *name = PyBytes_AS_STRING(name_encoded); #ifdef __APPLE__ int rc = pthread_setname_np(name); +#elif defined(__NetBSD__) + pthread_t thread = pthread_self(); + int rc = pthread_setname_np(thread, "%s", (void *)name); #else pthread_t thread = pthread_self(); int rc = pthread_setname_np(thread, name); diff --git a/configure b/configure index a697bc1f87b012..299eff6bc3bf70 100755 --- a/configure +++ b/configure @@ -29148,6 +29148,7 @@ CPPFLAGS=$save_CPPFLAGS case "$ac_sys_system" in Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android SunOS*) PYTHREAD_NAME_MAXLEN=31;; + NetBSD*) PYTHREAD_NAME_MAXLEN=31;; Darwin) PYTHREAD_NAME_MAXLEN=63;; iOS) PYTHREAD_NAME_MAXLEN=63;; FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; diff --git a/configure.ac b/configure.ac index ebc15503f069cc..badb19d55895de 100644 --- a/configure.ac +++ b/configure.ac @@ -7514,6 +7514,7 @@ _RESTORE_VAR([CPPFLAGS]) case "$ac_sys_system" in Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android SunOS*) PYTHREAD_NAME_MAXLEN=31;; + NetBSD*) PYTHREAD_NAME_MAXLEN=31;; Darwin) PYTHREAD_NAME_MAXLEN=63;; iOS) PYTHREAD_NAME_MAXLEN=63;; FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; From f9a5a3a3ef34e63dc197156e9a5f57842859ca04 Mon Sep 17 00:00:00 2001 From: Calvin Bui <3604363+calvinbui@users.noreply.github.com> Date: Sun, 29 Dec 2024 08:05:34 +1100 Subject: [PATCH 39/44] gh-128192: support HTTP sha-256 digest authentication as per RFC-7617 (GH-128193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit support sha-256 digest authentication Co-authored-by: Peter Bierma Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Gregory P. Smith --- Doc/library/urllib.request.rst | 3 +++ Doc/whatsnew/3.14.rst | 8 ++++++ Lib/test/test_urllib2.py | 25 ++++++++++++++++--- Lib/urllib/request.py | 7 ++++-- Misc/ACKS | 1 + ...-12-23-11-14-07.gh-issue-128192.02mEhD.rst | 2 ++ 6 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 3c07dc4adf434a..b3efde3f189566 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -411,6 +411,9 @@ The following classes are provided: :ref:`http-password-mgr` for information on the interface that must be supported. + .. versionchanged:: 3.14 + Added support for HTTP digest authentication algorithm ``SHA-256``. + .. class:: HTTPDigestAuthHandler(password_mgr=None) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 935c61c474e889..2767fd3ca48b29 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -646,6 +646,14 @@ unittest (Contributed by Jacob Walls in :gh:`80958`.) +urllib +------ + +* Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by + supporting SHA-256 digest authentication as specified in :rfc:`7616`. + (Contributed by Calvin Bui in :gh:`128193`.) + + uuid ---- diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 4a9e653515be5b..96d91c1f1c2f8a 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1962,10 +1962,29 @@ def test_parse_proxy(self): self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), - def test_unsupported_algorithm(self): - handler = AbstractDigestAuthHandler() + +class TestDigestAlgorithms(unittest.TestCase): + def setUp(self): + self.handler = AbstractDigestAuthHandler() + + def test_md5_algorithm(self): + H, KD = self.handler.get_algorithm_impls('MD5') + self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") + self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") + + def test_sha_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA') + self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") + + def test_sha256_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA-256') + self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + self.assertEqual(KD("foo", "bar"), "a765a8beaa9d561d4c5cbed29d8f4e30870297fdfa9cb7d6e9848a95fec9f937") + + def test_invalid_algorithm(self): with self.assertRaises(ValueError) as exc: - handler.get_algorithm_impls('invalid') + self.handler.get_algorithm_impls('invalid') self.assertEqual( str(exc.exception), "Unsupported digest authentication algorithm 'invalid'" diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index c5a6a18a32bba1..0d1b594b8cf20b 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1048,7 +1048,7 @@ def http_error_407(self, req, fp, code, msg, headers): class AbstractDigestAuthHandler: - # Digest authentication is specified in RFC 2617. + # Digest authentication is specified in RFC 2617/7616. # XXX The client does not inspect the Authentication-Info header # in a successful response. @@ -1176,11 +1176,14 @@ def get_authorization(self, req, chal): return base def get_algorithm_impls(self, algorithm): + # algorithm names taken from RFC 7616 Section 6.1 # lambdas assume digest modules are imported at the top level if algorithm == 'MD5': H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest() - elif algorithm == 'SHA': + elif algorithm == 'SHA': # non-standard, retained for compatibility. H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest() + elif algorithm == 'SHA-256': + H = lambda x: hashlib.sha256(x.encode("ascii")).hexdigest() # XXX MD5-sess else: raise ValueError("Unsupported digest authentication " diff --git a/Misc/ACKS b/Misc/ACKS index 086930666822ad..c6e53317b37d78 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -258,6 +258,7 @@ Colm Buckley Erik de Bueger Jan-Hein Bührman Marc Bürg +Calvin Bui Lars Buitinck Artem Bulgakov Dick Bulterman diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst new file mode 100644 index 00000000000000..b80ab715ffc7db --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst @@ -0,0 +1,2 @@ +Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by +supporting SHA-256 digest authentication as specified in :rfc:`7616`. From c9159b7436363f0cfbddc3deae1bcf925b5c2d4b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 29 Dec 2024 06:22:29 +0000 Subject: [PATCH 40/44] expand the `asyncio.run_coroutine_threadsafe` recipes (#127576) Co-authored-by: Kumar Aditya --- Doc/library/asyncio-task.rst | 61 +++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f27e858cf420f4..4541cf28de0605 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1067,14 +1067,59 @@ Scheduling From Other Threads This function is meant to be called from a different OS thread than the one where the event loop is running. Example:: - # Create a coroutine - coro = asyncio.sleep(1, result=3) - - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) - - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 + def in_thread(loop: asyncio.AbstractEventLoop) -> None: + # Run some blocking IO + pathlib.Path("example.txt").write_text("hello world", encoding="utf8") + + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 + + async def amain() -> None: + # Get the running loop + loop = asyncio.get_running_loop() + + # Run something in a thread + await asyncio.to_thread(in_thread, loop) + + It's also possible to run the other way around. Example:: + + @contextlib.contextmanager + def loop_in_thread() -> Generator[asyncio.AbstractEventLoop]: + loop_fut = concurrent.futures.Future[asyncio.AbstractEventLoop]() + stop_event = asyncio.Event() + + async def main() -> None: + loop_fut.set_result(asyncio.get_running_loop()) + await stop_event.wait() + + with concurrent.futures.ThreadPoolExecutor(1) as tpe: + complete_fut = tpe.submit(asyncio.run, main()) + for fut in concurrent.futures.as_completed((loop_fut, complete_fut)): + if fut is loop_fut: + loop = loop_fut.result() + try: + yield loop + finally: + loop.call_soon_threadsafe(stop_event.set) + else: + fut.result() + + # Create a loop in another thread + with loop_in_thread() as loop: + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 If an exception is raised in the coroutine, the returned Future will be notified. It can also be used to cancel the task in From ffece5590e95e89d3b5b6847e3beb443ff65c3db Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 28 Dec 2024 22:32:32 -0800 Subject: [PATCH 41/44] gh-128192: mark new tests with skips based on hashlib algorithm availability (gh-128324) Puts the _hashlib get_fips_mode logic check into test.support rather than spreading it out among other tests. --- Lib/test/support/__init__.py | 8 ++++++++ Lib/test/test_urllib2.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index cf3077f2a4a409..42e7b876594fa7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2969,3 +2969,11 @@ def run_yielding_async_fn(async_fn, /, *args, **kwargs): return e.value finally: coro.close() + + +def is_libssl_fips_mode(): + try: + from _hashlib import get_fips_mode # ask _hashopenssl.c + except ImportError: + return False # more of a maybe, unless we add this to the _ssl module. + return get_fips_mode() != 0 diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 96d91c1f1c2f8a..085b24c25b2daa 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -27,6 +27,7 @@ import urllib.error import http.client + support.requires_working_socket(module=True) # XXX @@ -1963,20 +1964,29 @@ def test_parse_proxy(self): self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), -class TestDigestAlgorithms(unittest.TestCase): +skip_libssl_fips_mode = unittest.skipIf( + support.is_libssl_fips_mode(), + "conservative skip due to OpenSSL FIPS mode possible algorithm nerfing", +) + + +class TestDigestAuthAlgorithms(unittest.TestCase): def setUp(self): self.handler = AbstractDigestAuthHandler() + @skip_libssl_fips_mode def test_md5_algorithm(self): H, KD = self.handler.get_algorithm_impls('MD5') self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") + @skip_libssl_fips_mode def test_sha_algorithm(self): H, KD = self.handler.get_algorithm_impls('SHA') self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") + @skip_libssl_fips_mode def test_sha256_algorithm(self): H, KD = self.handler.get_algorithm_impls('SHA-256') self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") From 7e819ce0f32068de7914cd1ba3b4b95e91ea9873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 29 Dec 2024 19:30:53 +0100 Subject: [PATCH 42/44] gh-123424: add `ZipInfo._for_archive` to set suitable default properties (#123429) --------- Co-authored-by: Jason R. Coombs --- Doc/library/zipfile.rst | 11 +++++++ Doc/whatsnew/3.14.rst | 8 +++++ Lib/test/test_zipfile/_path/test_path.py | 19 +---------- Lib/test/test_zipfile/test_core.py | 29 ++++++++++++++++ Lib/zipfile/__init__.py | 33 ++++++++++++------- ...-08-28-16-10-37.gh-issue-123424.u96_i6.rst | 1 + 6 files changed, 72 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5583c6b24be5c6..afe1cd5c75fcbb 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -84,6 +84,17 @@ The module defines the following items: formerly protected :attr:`!_compresslevel`. The older protected name continues to work as a property for backwards compatibility. + + .. method:: _for_archive(archive) + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :meth:`ZipFile.writestr`. + + Returns self for chaining. + + .. versionadded:: 3.14 + + .. function:: is_zipfile(filename) Returns ``True`` if *filename* is a valid ZIP file based on its magic number, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2767fd3ca48b29..53415bb09bf080 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -661,6 +661,14 @@ uuid in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) +zipinfo +------- + +* Added :func:`ZipInfo._for_archive ` + to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object + as used by :func:`ZipFile.writestr `. + + (Contributed by Bénédikt Tran in :gh:`123424`.) .. Add improved modules above alphabetically, not here at the end. diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index aba515536f0c1a..1ee45f5fc57104 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -634,7 +634,7 @@ def test_backslash_not_separator(self): """ data = io.BytesIO() zf = zipfile.ZipFile(data, "w") - zf.writestr(DirtyZipInfo.for_name("foo\\bar", zf), b"content") + zf.writestr(DirtyZipInfo("foo\\bar")._for_archive(zf), b"content") zf.filename = '' root = zipfile.Path(zf) (first,) = root.iterdir() @@ -657,20 +657,3 @@ class DirtyZipInfo(zipfile.ZipInfo): def __init__(self, filename, *args, **kwargs): super().__init__(filename, *args, **kwargs) self.filename = filename - - @classmethod - def for_name(cls, name, archive): - """ - Construct the same way that ZipFile.writestr does. - - TODO: extract this functionality and re-use - """ - self = cls(filename=name, date_time=time.localtime(time.time())[:6]) - self.compress_type = archive.compression - self.compress_level = archive.compresslevel - if self.filename.endswith('/'): # pragma: no cover - self.external_attr = 0o40775 << 16 # drwxrwxr-x - self.external_attr |= 0x10 # MS-DOS directory flag - else: - self.external_attr = 0o600 << 16 # ?rw------- - return self diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 124e088fd15b80..49f39b9337df85 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -5,6 +5,7 @@ import itertools import os import posixpath +import stat import struct import subprocess import sys @@ -2211,6 +2212,34 @@ def test_create_empty_zipinfo_repr(self): zi = zipfile.ZipInfo(filename="empty") self.assertEqual(repr(zi), "") + def test_for_archive(self): + base_filename = TESTFN2.rstrip('/') + + with zipfile.ZipFile(TESTFN, mode="w", compresslevel=1, + compression=zipfile.ZIP_STORED) as zf: + # no trailing forward slash + zi = zipfile.ZipInfo(base_filename)._for_archive(zf) + self.assertEqual(zi.compress_level, 1) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + # ?rw- --- --- + filemode = stat.S_IRUSR | stat.S_IWUSR + # filemode is stored as the highest 16 bits of external_attr + self.assertEqual(zi.external_attr >> 16, filemode) + self.assertEqual(zi.external_attr & 0xFF, 0) # no MS-DOS flag + + with zipfile.ZipFile(TESTFN, mode="w", compresslevel=1, + compression=zipfile.ZIP_STORED) as zf: + # with a trailing slash + zi = zipfile.ZipInfo(f'{base_filename}/')._for_archive(zf) + self.assertEqual(zi.compress_level, 1) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + # d rwx rwx r-x + filemode = stat.S_IFDIR + filemode |= stat.S_IRWXU | stat.S_IRWXG + filemode |= stat.S_IROTH | stat.S_IXOTH + self.assertEqual(zi.external_attr >> 16, filemode) + self.assertEqual(zi.external_attr & 0xFF, 0x10) # MS-DOS flag + def test_create_empty_zipinfo_default_attributes(self): """Ensure all required attributes are set.""" zi = zipfile.ZipInfo() diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index f4d396abb6e639..052ef47b8f6598 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -13,6 +13,7 @@ import sys import threading import time +from typing import Self try: import zlib # We may need its compression method @@ -605,6 +606,24 @@ def from_file(cls, filename, arcname=None, *, strict_timestamps=True): return zinfo + def _for_archive(self, archive: ZipFile) -> Self: + """Resolve suitable defaults from the archive. + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :method:`ZipFile.writestr`. + + Return self. + """ + self.date_time = time.localtime(time.time())[:6] + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self + def is_dir(self): """Return True if this archive member is a directory.""" if self.filename.endswith('/'): @@ -1908,18 +1927,10 @@ def writestr(self, zinfo_or_arcname, data, the name of the file in the archive.""" if isinstance(data, str): data = data.encode("utf-8") - if not isinstance(zinfo_or_arcname, ZipInfo): - zinfo = ZipInfo(filename=zinfo_or_arcname, - date_time=time.localtime(time.time())[:6]) - zinfo.compress_type = self.compression - zinfo.compress_level = self.compresslevel - if zinfo.filename.endswith('/'): - zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.external_attr = 0o600 << 16 # ?rw------- - else: + if isinstance(zinfo_or_arcname, ZipInfo): zinfo = zinfo_or_arcname + else: + zinfo = ZipInfo(zinfo_or_arcname)._for_archive(self) if not self.fp: raise ValueError( diff --git a/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst b/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst new file mode 100644 index 00000000000000..4df4bbf2ba2b73 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst @@ -0,0 +1 @@ +Add :meth:`zipfile.ZipInfo._for_archive` setting default properties on :class:`~zipfile.ZipInfo` objects. Patch by Bénédikt Tran and Jason R. Coombs. From c78729f2df7c0e220f1080edb2a0d0cdd49e22df Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 29 Dec 2024 21:42:07 +0000 Subject: [PATCH 43/44] GH-127381: pathlib ABCs: remove `PathBase.stat()` (#128334) Remove the `PathBase.stat()` method. Its use of the `os.stat_result` API, with its 10 mandatory fields and low-level types, makes it an awkward fit for virtual filesystems. We'll look to add a `PathBase.info` attribute later - see GH-125413. --- Lib/pathlib/_abc.py | 31 ++------- Lib/pathlib/_local.py | 12 +++- Lib/test/test_pathlib/test_pathlib.py | 25 ++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 77 +++++++---------------- 4 files changed, 62 insertions(+), 83 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 6acc29ebab2bc5..4d5f7980fd30d2 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -16,7 +16,6 @@ import posixpath from errno import EINVAL from glob import _GlobberBase, _no_recurse_symlinks -from stat import S_ISDIR, S_ISLNK, S_ISREG from pathlib._os import copyfileobj @@ -450,15 +449,6 @@ class PathBase(PurePathBase): """ __slots__ = () - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - raise NotImplementedError - - # Convenience functions for querying the stat results - def exists(self, *, follow_symlinks=True): """ Whether this path exists. @@ -466,39 +456,26 @@ def exists(self, *, follow_symlinks=True): This method normally follows symlinks; to check whether a symlink exists, add the argument follow_symlinks=False. """ - try: - self.stat(follow_symlinks=follow_symlinks) - except (OSError, ValueError): - return False - return True + raise NotImplementedError def is_dir(self, *, follow_symlinks=True): """ Whether this path is a directory. """ - try: - return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) - except (OSError, ValueError): - return False + raise NotImplementedError def is_file(self, *, follow_symlinks=True): """ Whether this path is a regular file (also True for symlinks pointing to regular files). """ - try: - return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) - except (OSError, ValueError): - return False + raise NotImplementedError def is_symlink(self): """ Whether this path is a symbolic link. """ - try: - return S_ISLNK(self.stat(follow_symlinks=False).st_mode) - except (OSError, ValueError): - return False + raise NotImplementedError def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 915402e6c65b29..4484d9553c164f 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -7,7 +7,7 @@ from errno import * from glob import _StringGlobber, _no_recurse_symlinks from itertools import chain -from stat import S_IMODE, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from stat import S_IMODE, S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence try: @@ -725,7 +725,10 @@ def is_dir(self, *, follow_symlinks=True): """ if follow_symlinks: return os.path.isdir(self) - return PathBase.is_dir(self, follow_symlinks=follow_symlinks) + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False def is_file(self, *, follow_symlinks=True): """ @@ -734,7 +737,10 @@ def is_file(self, *, follow_symlinks=True): """ if follow_symlinks: return os.path.isfile(self) - return PathBase.is_file(self, follow_symlinks=follow_symlinks) + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False def is_mount(self): """ diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index fac8cbdf65a122..0efa8270210aef 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1835,6 +1835,31 @@ def test_symlink_to_unsupported(self): with self.assertRaises(pathlib.UnsupportedOperation): q.symlink_to(p) + def test_stat(self): + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() + # st_mode: files are the same, directory differs. + self.assertIsInstance(statA.st_mode, int) + self.assertEqual(statA.st_mode, statB.st_mode) + self.assertNotEqual(statA.st_mode, statC.st_mode) + self.assertNotEqual(statB.st_mode, statC.st_mode) + # st_ino: all different, + self.assertIsInstance(statA.st_ino, int) + self.assertNotEqual(statA.st_ino, statB.st_ino) + self.assertNotEqual(statA.st_ino, statC.st_ino) + self.assertNotEqual(statB.st_ino, statC.st_ino) + # st_dev: all the same. + self.assertIsInstance(statA.st_dev, int) + self.assertEqual(statA.st_dev, statB.st_dev) + self.assertEqual(statA.st_dev, statC.st_dev) + # other attributes not used by pathlib. + + def test_stat_no_follow_symlinks_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.stat(follow_symlinks=False)) + @needs_symlinks def test_stat_no_follow_symlinks(self): p = self.cls(self.base) / 'linkA' diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 787fb0f82257e6..81d9d34506f04b 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -2,7 +2,6 @@ import io import os import errno -import stat import unittest from pathlib._abc import PurePathBase, PathBase @@ -1294,11 +1293,6 @@ def close(self): super().close() -DummyPathStatResult = collections.namedtuple( - 'DummyPathStatResult', - 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') - - class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in @@ -1331,15 +1325,17 @@ def __repr__(self): def with_segments(self, *pathsegments): return type(self)(*pathsegments) - def stat(self, *, follow_symlinks=True): - path = str(self).rstrip('/') - if path in self._files: - st_mode = stat.S_IFREG - elif path in self._directories: - st_mode = stat.S_IFDIR - else: - raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) - return DummyPathStatResult(st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0) + def exists(self, *, follow_symlinks=True): + return self.is_dir() or self.is_file() + + def is_dir(self, *, follow_symlinks=True): + return str(self).rstrip('/') in self._directories + + def is_file(self, *, follow_symlinks=True): + return str(self) in self._files + + def is_symlink(self): + return False def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): @@ -1958,31 +1954,6 @@ def test_rglob_windows(self): self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - def test_stat(self): - statA = self.cls(self.base).joinpath('fileA').stat() - statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() - statC = self.cls(self.base).joinpath('dirC').stat() - # st_mode: files are the same, directory differs. - self.assertIsInstance(statA.st_mode, int) - self.assertEqual(statA.st_mode, statB.st_mode) - self.assertNotEqual(statA.st_mode, statC.st_mode) - self.assertNotEqual(statB.st_mode, statC.st_mode) - # st_ino: all different, - self.assertIsInstance(statA.st_ino, int) - self.assertNotEqual(statA.st_ino, statB.st_ino) - self.assertNotEqual(statA.st_ino, statC.st_ino) - self.assertNotEqual(statB.st_ino, statC.st_ino) - # st_dev: all the same. - self.assertIsInstance(statA.st_dev, int) - self.assertEqual(statA.st_dev, statB.st_dev) - self.assertEqual(statA.st_dev, statC.st_dev) - # other attributes not used by pathlib. - - def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(self.base) / 'fileA' - st = p.stat() - self.assertEqual(st, p.stat(follow_symlinks=False)) - def test_is_dir(self): P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir()) @@ -2054,26 +2025,26 @@ def test_is_symlink(self): def test_delete_file(self): p = self.cls(self.base) / 'fileA' p._delete() - self.assertFileNotFound(p.stat) + self.assertFalse(p.exists()) self.assertFileNotFound(p._delete) def test_delete_dir(self): base = self.cls(self.base) base.joinpath('dirA')._delete() - self.assertRaises(FileNotFoundError, base.joinpath('dirA').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').stat, - follow_symlinks=False) + self.assertFalse(base.joinpath('dirA').exists()) + self.assertFalse(base.joinpath('dirA', 'linkC').exists( + follow_symlinks=False)) base.joinpath('dirB')._delete() - self.assertRaises(FileNotFoundError, base.joinpath('dirB').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'fileB').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').stat, - follow_symlinks=False) + self.assertFalse(base.joinpath('dirB').exists()) + self.assertFalse(base.joinpath('dirB', 'fileB').exists()) + self.assertFalse(base.joinpath('dirB', 'linkD').exists( + follow_symlinks=False)) base.joinpath('dirC')._delete() - self.assertRaises(FileNotFoundError, base.joinpath('dirC').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD', 'fileD').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'fileC').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'novel.txt').stat) + self.assertFalse(base.joinpath('dirC').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD', 'fileD').exists()) + self.assertFalse(base.joinpath('dirC', 'fileC').exists()) + self.assertFalse(base.joinpath('dirC', 'novel.txt').exists()) def test_delete_missing(self): tmp = self.cls(self.base, 'delete') From ef63cca494571f50906baae1d176469a3dcf8838 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 29 Dec 2024 22:07:12 +0000 Subject: [PATCH 44/44] GH-127381: pathlib ABCs: remove uncommon `PurePathBase` methods (#127853) Remove `PurePathBase.relative_to()` and `is_relative_to()` because they don't account for *other* being an entirely different kind of path, and they can't use `__eq__()` because it's not on the `PurePathBase` interface. Remove `PurePathBase.drive`, `root`, `is_absolute()` and `as_posix()`. These are all too specific to local filesystems. --- Lib/pathlib/_abc.py | 65 ---- Lib/pathlib/_local.py | 5 + Lib/pathlib/_types.py | 2 - Lib/test/test_pathlib/test_pathlib.py | 356 +++++++++++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 370 +--------------------- 5 files changed, 365 insertions(+), 433 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 4d5f7980fd30d2..e6ff3fe1187512 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -205,21 +205,6 @@ def __str__(self): passing to system calls.""" raise NotImplementedError - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - return str(self).replace(self.parser.sep, '/') - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - return self.parser.splitdrive(self.anchor)[0] - - @property - def root(self): - """The root of the path, if any.""" - return self.parser.splitdrive(self.anchor)[1] - @property def anchor(self): """The concatenation of the drive and root, or ''.""" @@ -291,51 +276,6 @@ def with_suffix(self, suffix): else: return self.with_name(stem + suffix) - def relative_to(self, other, *, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = _explode_path(self) - anchor1, parts1 = _explode_path(other) - if anchor0 != anchor1: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if not part or part == '.': - pass - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif part == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - parts0.append('..') - return self.with_segments(*reversed(parts0)) - - def is_relative_to(self, other): - """Return True if the path is relative to another path or False. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = _explode_path(self) - anchor1, parts1 = _explode_path(other) - if anchor0 != anchor1: - return False - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if part and part != '.': - return False - return True - @property def parts(self): """An object providing sequence-like access to the @@ -387,11 +327,6 @@ def parents(self): parent = split(path)[0] return tuple(parents) - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - return self.parser.isabs(str(self)) - def match(self, path_pattern, *, case_sensitive=None): """ Return True if this path matches the given pattern. If the pattern is diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 4484d9553c164f..c5721a69d00470 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -437,6 +437,11 @@ def _parse_pattern(cls, pattern): parts.append('') return parts + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + @property def _raw_path(self): paths = self._raw_paths diff --git a/Lib/pathlib/_types.py b/Lib/pathlib/_types.py index baa4a7e2af5b68..72dac2e276fce0 100644 --- a/Lib/pathlib/_types.py +++ b/Lib/pathlib/_types.py @@ -15,7 +15,5 @@ class Parser(Protocol): sep: str def split(self, path: str) -> tuple[str, str]: ... - def splitdrive(self, path: str) -> tuple[str, str]: ... def splitext(self, path: str) -> tuple[str, str]: ... def normcase(self, path: str) -> str: ... - def isabs(self, path: str) -> bool: ... diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 0efa8270210aef..d13daf8ac8cb07 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -270,6 +270,12 @@ def test_as_bytes_common(self): P = self.cls self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + def test_as_posix_common(self): + P = self.cls + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self.assertEqual(P(pathstr).as_posix(), pathstr) + # Other tests for as_posix() are in test_equivalences(). + def test_eq_common(self): P = self.cls self.assertEqual(P('a/b'), P('a/b')) @@ -349,6 +355,51 @@ def test_repr_roundtrips(self): self.assertEqual(q, p) self.assertEqual(repr(q), r) + def test_drive_common(self): + P = self.cls + self.assertEqual(P('a/b').drive, '') + self.assertEqual(P('/a/b').drive, '') + self.assertEqual(P('').drive, '') + + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + + + def test_root_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').root, '') + self.assertEqual(P('a/b').root, '') + self.assertEqual(P('/').root, sep) + self.assertEqual(P('/a/b').root, sep) + + @needs_posix + def test_root_posix(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + def test_name_empty(self): P = self.cls self.assertEqual(P('').name, '') @@ -547,6 +598,311 @@ def assertOrderedEqual(a, b): self.assertFalse(p < q) self.assertFalse(p > q) + @needs_posix + def test_is_absolute_posix(self): + P = self.cls + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + @needs_windows + def test_is_absolute_windows(self): + P = self.cls + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) + + def test_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P('')), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to('a/b'), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P('')) + self.assertEqual(p.relative_to('/a/b'), P('')) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P('')) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + + @needs_windows + def test_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) + self.assertEqual(p.relative_to('c:foO'), P('Bar')) + self.assertEqual(p.relative_to('c:foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR')), P()) + self.assertEqual(p.relative_to('c:foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) + self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) + self.assertRaises(ValueError, p.relative_to, '', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) + p = P('C:/Foo/Bar') + self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) + self.assertEqual(p.relative_to('c:/foO'), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) + self.assertEqual(p.relative_to('c:/foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) + self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) + self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo')) + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('d:/')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) + + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P(''))) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + @needs_windows + def test_is_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + class PurePosixPathTest(PurePathTest): cls = pathlib.PurePosixPath diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 81d9d34506f04b..d588442bd11785 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -71,7 +71,7 @@ def __hash__(self): return hash(str(self)) def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + return "{}({!r})".format(self.__class__.__name__, str(self)) def with_segments(self, *pathsegments): return type(self)(*pathsegments) @@ -107,12 +107,6 @@ def test_constructor_common(self): P('a/b/c') P('/a/b/c') - def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls('')) - - def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls('')) - def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object # from a str subclass instance, and it then gets converted to @@ -161,7 +155,6 @@ def with_segments(self, *pathsegments): self.assertEqual(42, p.with_stem('foo').session_id) self.assertEqual(42, p.with_suffix('.foo').session_id) self.assertEqual(42, p.with_segments('foo').session_id) - self.assertEqual(42, p.relative_to('foo').session_id) self.assertEqual(42, p.parent.session_id) for parent in p.parents: self.assertEqual(42, parent.session_id) @@ -303,12 +296,6 @@ def test_str_windows(self): p = self.cls('//a/b/c/d') self.assertEqual(str(p), '\\\\a\\b\\c\\d') - def test_as_posix_common(self): - P = self.cls - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self.assertEqual(P(pathstr).as_posix(), pathstr) - # Other tests for as_posix() are in test_equivalences(). - def test_match_empty(self): P = self.cls self.assertRaises(ValueError, P('a').match, '') @@ -615,50 +602,6 @@ def test_parents_windows(self): with self.assertRaises(IndexError): par[2] - def test_drive_common(self): - P = self.cls - self.assertEqual(P('a/b').drive, '') - self.assertEqual(P('/a/b').drive, '') - self.assertEqual(P('').drive, '') - - @needs_windows - def test_drive_windows(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') - - def test_root_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').root, '') - self.assertEqual(P('a/b').root, '') - self.assertEqual(P('/').root, sep) - self.assertEqual(P('/a/b').root, sep) - - @needs_posix - def test_root_posix(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - @needs_windows - def test_root_windows(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - def test_anchor_common(self): P = self.cls sep = self.sep @@ -967,311 +910,6 @@ def test_with_suffix_invalid(self): self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') self.assertRaises(TypeError, P('a/b').with_suffix, None) - def test_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P('')), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P('')) - self.assertEqual(p.relative_to('a/b'), P('')) - self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) - self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) - self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P('')) - self.assertEqual(p.relative_to('/a/b'), P('')) - self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) - self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P('')) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - - @needs_windows - def test_relative_to_windows(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) - self.assertEqual(p.relative_to('c:foO'), P('Bar')) - self.assertEqual(p.relative_to('c:foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR')), P()) - self.assertEqual(p.relative_to('c:foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) - self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) - self.assertRaises(ValueError, p.relative_to, '', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) - p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) - self.assertEqual(p.relative_to('c:/foO'), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) - self.assertEqual(p.relative_to('c:/foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) - self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, 'c:') - self.assertRaises(ValueError, p.relative_to, P('c:')) - self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo')) - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('d:/')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) - self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - - def test_is_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P(''))) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P(''))) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) - - @needs_windows - def test_is_relative_to_windows(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:'))) - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:foO'))) - self.assertTrue(p.is_relative_to('c:foO')) - self.assertTrue(p.is_relative_to('c:foO/')) - self.assertTrue(p.is_relative_to(P('c:foO/baR'))) - self.assertTrue(p.is_relative_to('c:foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('Foo'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('C:/Foo'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) - p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:/'))) - self.assertTrue(p.is_relative_to(P('c:/foO'))) - self.assertTrue(p.is_relative_to('c:/foO/')) - self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) - self.assertTrue(p.is_relative_to('c:/foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to('c:')) - self.assertFalse(p.is_relative_to(P('C:/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo'))) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('d:/'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('//C/Foo'))) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) - self.assertTrue(p.is_relative_to('//sErver/sHare')) - self.assertTrue(p.is_relative_to('//sErver/sHare/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - - @needs_posix - def test_is_absolute_posix(self): - P = self.cls - self.assertFalse(P('').is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - @needs_windows - def test_is_absolute_windows(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - self.assertTrue(P('//?/UNC/').is_absolute()) - self.assertTrue(P('//?/UNC/spam').is_absolute()) - # # Tests for the virtual classes. @@ -1320,7 +958,7 @@ def __hash__(self): return hash(str(self)) def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + return "{}({!r})".format(self.__class__.__name__, str(self)) def with_segments(self, *pathsegments): return type(self)(*pathsegments) @@ -1631,8 +1269,8 @@ def ordered_walk(path): source_walk = ordered_walk(source) target_walk = ordered_walk(target) for source_item, target_item in zip(source_walk, target_walk, strict=True): - self.assertEqual(source_item[0].relative_to(source), - target_item[0].relative_to(target)) # dirpath + self.assertEqual(source_item[0].parts[len(source.parts):], + target_item[0].parts[len(target.parts):]) # dirpath self.assertEqual(source_item[1], target_item[1]) # dirnames self.assertEqual(source_item[2], target_item[2]) # filenames # Compare files and symlinks