diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 8405fb1917e714..79db1891a1ac03 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -710,6 +710,11 @@ process and user. :func:`socket.gethostname` or even ``socket.gethostbyaddr(socket.gethostname())``. + On macOS, iOS and Android, this returns the *kernel* name and version (i.e., + ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname()` + can be used to get the user-facing operating system name and version on iOS and + Android. + .. availability:: recent flavors of Unix. .. versionchanged:: 3.3 diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index dc2d871b47d5ef..57dc421bb6c1a5 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -148,6 +148,9 @@ Cross Platform Returns the system/OS name, such as ``'Linux'``, ``'Darwin'``, ``'Java'``, ``'Windows'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this returns the user-facing OS name (i.e, ``'iOS``, + ``'iPadOS'`` or ``'Android'``). To obtain the kernel name (``'Darwin'`` or + ``'Linux'``), use :func:`os.uname()`. .. function:: system_alias(system, release, version) @@ -161,6 +164,8 @@ Cross Platform Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this is the user-facing OS version. To obtain the + Darwin or Linux kernel version, use :func:`os.uname()`. .. function:: uname() @@ -230,7 +235,6 @@ Windows Platform macOS Platform -------------- - .. function:: mac_ver(release='', versioninfo=('','',''), machine='') Get macOS version information and return it as tuple ``(release, versioninfo, @@ -240,6 +244,24 @@ macOS Platform Entries which cannot be determined are set to ``''``. All tuple entries are strings. +iOS Platform +------------ + +.. function:: ios_ver(system='', release='', model='', is_simulator=False) + + Get iOS version information and return it as a + :func:`~collections.namedtuple` with the following attributes: + + * ``system`` is the OS name; either ``'iOS'`` or ``'iPadOS'``. + * ``release`` is the iOS version number as a string (e.g., ``'17.2'``). + * ``model`` is the device model identifier; this will be a string like + ``'iPhone13,2'`` for a physical device, or ``'iPhone'`` on a simulator. + * ``is_simulator`` is a boolean describing if the app is running on a + simulator or a physical device. + + Entries which cannot be determined are set to the defaults given as + parameters. + Unix Platforms -------------- diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index c0990882e58fe9..e5cc007d2b85d8 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -33,6 +33,13 @@ allow the remote browser to maintain its own windows on the display. If remote browsers are not available on Unix, the controlling process will launch a new browser and wait. +On iOS, the :envvar:`BROWSER` environment variable, as well as any arguments +controlling autoraise, browser preference, and new tab/window creation will be +ignored. Web pages will *always* be opened in the user's preferred browser, in +a new tab, with the browser being brought to the foreground. The use of the +:mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If +:mod:`ctypes` isn't available, calls to :func:`.open` will fail. + The script :program:`webbrowser` can be used as a command-line interface for the module. It accepts a URL as the argument. It accepts the following optional parameters: ``-n`` opens the URL in a new browser window, if possible; @@ -155,6 +162,8 @@ for the controller classes, all defined in this module. +------------------------+-----------------------------------------+-------+ | ``'chromium-browser'`` | :class:`Chromium('chromium-browser')` | | +------------------------+-----------------------------------------+-------+ +| ``'iosbrowser'`` | ``IOSBrowser`` | \(4) | ++------------------------+-----------------------------------------+-------+ Notes: @@ -169,11 +178,18 @@ Notes: Only on Windows platforms. (3) - Only on macOS platform. + Only on macOS. + +(4) + Only on iOS. + .. versionadded:: 3.3 Support for Chrome/Chromium has been added. +.. versionchanged:: 3.13 + Support for iOS has been added. + Here are some simple examples:: url = 'https://docs.python.org/' diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py new file mode 100644 index 00000000000000..db3fe23e45bca0 --- /dev/null +++ b/Lib/_ios_support.py @@ -0,0 +1,71 @@ +import sys +try: + from ctypes import cdll, c_void_p, c_char_p, util +except ImportError: + # ctypes is an optional module. If it's not present, we're limited in what + # we can tell about the system, but we don't want to prevent the module + # from working. + print("ctypes isn't available; iOS system calls will not be available") + objc = None +else: + # ctypes is available. Load the ObjC library, and wrap the objc_getClass, + # sel_registerName methods + lib = util.find_library("objc") + if lib is None: + # Failed to load the objc library + raise RuntimeError("ObjC runtime library couldn't be loaded") + + objc = cdll.LoadLibrary(lib) + objc.objc_getClass.restype = c_void_p + objc.objc_getClass.argtypes = [c_char_p] + objc.sel_registerName.restype = c_void_p + objc.sel_registerName.argtypes = [c_char_p] + + +def get_platform_ios(): + # Determine if this is a simulator using the multiarch value + is_simulator = sys.implementation._multiarch.endswith("simulator") + + # We can't use ctypes; abort + if not objc: + return None + + # Most of the methods return ObjC objects + objc.objc_msgSend.restype = c_void_p + # All the methods used have no arguments. + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + + # Equivalent of: + # device = [UIDevice currentDevice] + UIDevice = objc.objc_getClass(b"UIDevice") + SEL_currentDevice = objc.sel_registerName(b"currentDevice") + device = objc.objc_msgSend(UIDevice, SEL_currentDevice) + + # Equivalent of: + # device_systemVersion = [device systemVersion] + SEL_systemVersion = objc.sel_registerName(b"systemVersion") + device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion) + + # Equivalent of: + # device_systemName = [device systemName] + SEL_systemName = objc.sel_registerName(b"systemName") + device_systemName = objc.objc_msgSend(device, SEL_systemName) + + # Equivalent of: + # device_model = [device model] + SEL_model = objc.sel_registerName(b"model") + device_model = objc.objc_msgSend(device, SEL_model) + + # UTF8String returns a const char*; + SEL_UTF8String = objc.sel_registerName(b"UTF8String") + objc.objc_msgSend.restype = c_char_p + + # Equivalent of: + # system = [device_systemName UTF8String] + # release = [device_systemVersion UTF8String] + # model = [device_model UTF8String] + system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode() + release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode() + model = objc.objc_msgSend(device_model, SEL_UTF8String).decode() + + return system, release, model, is_simulator diff --git a/Lib/distutils/tests/test_cygwinccompiler.py b/Lib/distutils/tests/test_cygwinccompiler.py index 9dc869de4c8ef0..646b24faa86473 100644 --- a/Lib/distutils/tests/test_cygwinccompiler.py +++ b/Lib/distutils/tests/test_cygwinccompiler.py @@ -5,6 +5,9 @@ from io import BytesIO from test.support import run_unittest +if sys.platform != 'win32': + raise unittest.SkipTest("Cygwin tests only needed on Windows") + from distutils import cygwinccompiler from distutils.cygwinccompiler import (check_config_h, CONFIG_H_OK, CONFIG_H_NOTOK, diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py index 59676b0e0b0ce2..ba7bb5b7693a01 100644 --- a/Lib/distutils/tests/test_sysconfig.py +++ b/Lib/distutils/tests/test_sysconfig.py @@ -10,7 +10,7 @@ from distutils import sysconfig from distutils.ccompiler import get_default_compiler from distutils.tests import support -from test.support import run_unittest, swap_item +from test.support import is_apple_mobile, requires_subprocess, run_unittest, swap_item from test.support.os_helper import TESTFN from test.support.warnings_helper import check_warnings @@ -32,6 +32,7 @@ def cleanup_testfn(self): elif os.path.isdir(TESTFN): shutil.rmtree(TESTFN) + @unittest.skipIf(is_apple_mobile, "Header files not distributed with Apple mobile") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -48,6 +49,7 @@ def test_get_config_vars(self): self.assertIsInstance(cvars, dict) self.assertTrue(cvars) + @unittest.skipIf(is_apple_mobile, "Header files not distributed with Apple mobile") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -247,6 +249,7 @@ def test_SO_in_vars(self): self.assertIsNotNone(vars['SO']) self.assertEqual(vars['SO'], vars['EXT_SUFFIX']) + @requires_subprocess() def test_customize_compiler_before_get_config_vars(self): # Issue #21923: test that a Distribution compiler # instance can be called without an explicit call to diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py index d00c48981eb6d6..5d12b4779db65d 100644 --- a/Lib/distutils/unixccompiler.py +++ b/Lib/distutils/unixccompiler.py @@ -270,9 +270,9 @@ def find_library_file(self, dirs, lib, debug=0): static_f = self.library_filename(lib, lib_type='static') if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) + # On macOS users can specify an alternate SDK using + # '-isysroot ' or --sysroot=, calculate the SDK root + # if it is specified (and use it further on) # # Note that, as of Xcode 7, Apple SDKs may contain textual stub # libraries with .tbd extensions rather than the normal .dylib @@ -291,12 +291,14 @@ def find_library_file(self, dirs, lib, debug=0): cflags = sysconfig.get_config_var('CFLAGS') m = re.search(r'-isysroot\s*(\S+)', cflags) if m is None: - sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) + m = re.search(r'--sysroot=(\S+)', cflags) + if m is None: + sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) + else: + sysroot = m.group(1) else: sysroot = m.group(1) - - for dir in dirs: shared = os.path.join(dir, shared_f) dylib = os.path.join(dir, dylib_f) diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index 2ce5c5b64d62fa..e927f4af938983 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -89,10 +89,25 @@ def get_host_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support, distutils.sysconfig - osname, release, machine = _osx_support.get_platform_osx( - distutils.sysconfig.get_config_vars(), - osname, release, machine) + import distutils.sysconfig + config_vars = distutils.sysconfig.get_config_vars() + if sys.platform == "ios": + release = config_vars.get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + elif sys.platform == "tvos": + release = config_vars.get("TVOS_DEPLOYMENT_TARGET", "9.0") + osname = sys.platform + machine = sys.implementation._multiarch + elif sys.platform == "watchos": + release = config_vars.get("WATCHOS_DEPLOYMENT_TARGET", "4.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + config_vars, + osname, release, machine) return "%s-%s-%s" % (osname, release, machine) @@ -170,7 +185,7 @@ def check_environ (): if _environ_checked: return - if os.name == 'posix' and 'HOME' not in os.environ: + if os.name == 'posix' and 'HOME' not in os.environ and sys.platform not in {"ios", "tvos", "watchos"}: try: import pwd os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 90a1b34b5fff85..2e07663de439ed 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -62,6 +62,7 @@ def test_load_grammar_from_pickle(self): shutil.rmtree(tmpdir) @unittest.skipIf(sys.executable is None, 'sys.executable required') + @test.support.requires_subprocess() def test_load_grammar_from_subprocess(self): tmpdir = tempfile.mkdtemp() tmpsubdir = os.path.join(tmpdir, 'subdir') diff --git a/Lib/platform.py b/Lib/platform.py index 6a820c90a1ae42..8b97482697c89c 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -449,6 +449,30 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''): # If that also doesn't work return the default values return release, versioninfo, machine + +# A namedtuple for iOS version information. +IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def ios_ver(system="", release="", model="", is_simulator=False): + """Get iOS version information, and return it as a namedtuple: + (system, release, model, is_simulator). + + If values can't be determined, they are set to values provided as + parameters. + """ + if sys.platform == "ios": + import _ios_support + result = _ios_support.get_platform_ios() + if result is not None: + return IOSVersionInfo(*result) + + return IOSVersionInfo(system, release, model, is_simulator) + + def _java_getprop(name, default): from java.lang import System @@ -564,7 +588,7 @@ def _platform(*args): if cleaned == platform: break platform = cleaned - while platform[-1] == '-': + while platform and platform[-1] == '-': platform = platform[:-1] return platform @@ -605,7 +629,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ - if sys.platform in ('dos', 'win32', 'win16'): + if sys.platform in {'dos', 'win32', 'win16', 'ios', 'tvos', 'watchos'}: # XXX Others too ? return default @@ -744,6 +768,14 @@ def get_OpenVMS(): csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) return 'Alpha' if cpu_number >= 128 else 'VAX' + # On the iOS simulator, os.uname returns the architecture as uname.machine. + # On device it returns the model name for some reason; but there's only one + # CPU architecture for iOS devices, so we know the right answer. + def get_ios(): + if sys.implementation._multiarch.endswith("simulator"): + return os.uname().machine + return 'arm64' + def from_subprocess(): """ Fall back to `uname -p` @@ -893,6 +925,10 @@ def uname(): system = 'Windows' release = 'Vista' + # Normalize responses on iOS + if sys.platform == 'ios': + system, release, _, _ = ios_ver() + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) @@ -1205,11 +1241,14 @@ def platform(aliased=0, terse=0): system, release, version = system_alias(system, release, version) if system == 'Darwin': - # macOS (darwin kernel) - macos_release = mac_ver()[0] - if macos_release: - system = 'macOS' - release = macos_release + # macOS and iOS both report as a "Darwin" kernel + if sys.platform == "ios": + system, release, _, _ = ios_ver() + else: + macos_release = mac_ver()[0] + if macos_release: + system = 'macOS' + release = macos_release if system == 'Windows': # MS platforms diff --git a/Lib/site.py b/Lib/site.py index 5302037e0bf2c1..35bf9b03d3538c 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -276,8 +276,8 @@ def _getuserbase(): if env_base: return env_base - # VxWorks has no home directories - if sys.platform == "vxworks": + # iOS, tvOS, VxWorks and watchOS have no home directories + if sys.platform in {"ios", "tvos", "vxworks", "watchos"}: return None def joinuser(*args): diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index a764c82b0d30fe..8b23cd3f73317e 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -26,7 +26,7 @@ import sqlite3 as sqlite import sys -from test.support import check_disallow_instantiation, SHORT_TIMEOUT +from test.support import check_disallow_instantiation, SHORT_TIMEOUT, is_apple, requires_subprocess from test.support.os_helper import TESTFN, unlink @@ -87,7 +87,7 @@ def test_not_supported_error(self): # sqlite3_enable_shared_cache() is deprecated on macOS and calling it may raise # OperationalError on some buildbots. - @unittest.skipIf(sys.platform == "darwin", "shared cache is deprecated on macOS") + @unittest.skipIf(is_apple, "shared cache is deprecated on Apple platforms") def test_shared_cache_deprecated(self): for enable in (True, False): with self.assertWarns(DeprecationWarning) as cm: @@ -976,6 +976,7 @@ class MultiprocessTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) + @requires_subprocess() def test_ctx_mgr_rollback_if_commit_failed(self): # bpo-27334: ctx manager does not rollback if commit fails SCRIPT = f"""if 1: diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index daf9f000060a35..4a2144d5390937 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -20,6 +20,7 @@ # Keys for get_config_var() that are never converted to Python integers. _ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', 'MACOSX_DEPLOYMENT_TARGET', } @@ -56,6 +57,46 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + + # Downstream distributors can overwrite the default install scheme. + # This is done to support downstream modifications where distributors change + # the installation layout (eg. different site-packages directory). + # So, distributors will change the default scheme to one that correctly + # represents their layout. + # This presents an issue for projects/people that need to bootstrap virtual + # environments, like virtualenv. As distributors might now be customizing + # the default install scheme, there is no guarantee that the information + # returned by sysconfig.get_default_scheme/get_paths is correct for + # a virtual environment, the only guarantee we have is that it is correct + # for the *current* environment. When bootstrapping a virtual environment, + # we need to know its layout, so that we can place the files in the + # correct locations. + # The "*_venv" install scheme is a scheme to bootstrap virtual environments, + # essentially identical to the default posix_prefix/nt schemes. + # Downstream distributors who patch posix_prefix/nt scheme are encouraged to + # leave the following schemes unchanged + 'posix_venv': { + 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', + 'purelib': '{base}/lib/python{py_version_short}/site-packages', + 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'include': + '{installed_base}/include/python{py_version_short}{abiflags}', + 'platinclude': + '{installed_platbase}/include/python{py_version_short}{abiflags}', + 'scripts': '{base}/bin', + 'data': '{base}', + }, + 'nt_venv': { + 'stdlib': '{installed_base}/Lib', + 'platstdlib': '{base}/Lib', + 'purelib': '{base}/Lib/site-packages', + 'platlib': '{base}/Lib/site-packages', + 'include': '{installed_base}/Include', + 'platinclude': '{installed_base}/Include', + 'scripts': '{base}/Scripts', + 'data': '{base}', + }, } @@ -66,8 +107,8 @@ def _getuserbase(): if env_base: return env_base - # VxWorks has no home directories - if sys.platform == "vxworks": + # iOS, tvOS, VxWorks and watchOS have no home directories + if sys.platform in {"ios", "tvos", "vxworks", "watchos"}: return None def joinuser(*args): @@ -237,6 +278,7 @@ def _get_preferred_schemes(): 'home': 'posix_home', 'user': 'osx_framework_user', } + return { 'prefix': 'posix_prefix', 'home': 'posix_home', @@ -740,10 +782,15 @@ def get_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "12.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) return f"{osname}-{release}-{machine}" diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f474a756b33a90..21e792a86421ee 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -5908,6 +5908,8 @@ def test_gaps(self): ldt = tz.fromutc(udt.replace(tzinfo=tz)) self.assertEqual(ldt.fold, 0) + @unittest.skipIf(support.is_apple_mobile, + "FIXME: Edge case in timezone handling") def test_system_transitions(self): if ('Riyadh8' in self.zonename or # From tzdata NEWS file: diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index eef34f08121811..221159487882fd 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -267,6 +267,7 @@ def format_groups(groups): "HOMEDRIVE", "HOMEPATH", "IDLESTARTUP", + "IPHONEOS_DEPLOYMENT_TARGET", "LANG", "LDFLAGS", "LDSHARED", diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 4d10c1908a78fa..5eb0f0d7eed14a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1444,6 +1444,7 @@ def __exit__(self, exc_type, exc_value, exc_tb): if verbose: print("failed to clean up {}: {}".format(link, ex)) + @requires_subprocess() def _call(self, python, args, env, returncode): import subprocess cmd = [python, *args] diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 0313db6abdf2a2..a92dfef32ba45b 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -4,12 +4,13 @@ import os import re import stat -import support import sys import time import unittest import warnings +from test import support + # Filename used for testing if os.name == 'java': diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 6d699c8486cd20..f8bc838020da29 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -19,6 +19,7 @@ __cached_interp_requires_environment = None +@support.requires_subprocess() def interpreter_requires_environment(): """ Returns True if our sys.executable interpreter requires environment @@ -87,6 +88,7 @@ def fail(self, cmd_line): # Executing the interpreter in a subprocess +@support.requires_subprocess() def run_python_until_end(*args, **env_vars): env_required = interpreter_requires_environment() cwd = env_vars.pop('__cwd', None) @@ -146,6 +148,7 @@ def _assert_python(expected_success, /, *args, **env_vars): return res +@support.requires_subprocess() def assert_python_ok(*args, **env_vars): """ Assert that running the interpreter with `args` and optional environment @@ -160,6 +163,7 @@ def assert_python_ok(*args, **env_vars): return _assert_python(True, *args, **env_vars) +@support.requires_subprocess() def assert_python_failure(*args, **env_vars): """ Assert that running the interpreter with `args` and optional environment @@ -171,6 +175,7 @@ def assert_python_failure(*args, **env_vars): return _assert_python(False, *args, **env_vars) +@support.requires_subprocess() def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): """Run a Python subprocess with the given arguments. @@ -273,6 +278,7 @@ def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, return zip_name, os.path.join(zip_name, script_name_in_zip) +@support.requires_subprocess() def run_test_script(script): # use -u to try to get the full output if the test hangs or crash if support.verbose: diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 38c499bf722e85..8de5530605367e 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -1,8 +1,10 @@ import contextlib import errno +import os.path import socket -import unittest +import tempfile import sys +import unittest from .. import support @@ -267,3 +269,14 @@ def filter_error(err): # __cause__ or __context__? finally: socket.setdefaulttimeout(old_timeout) + + +def create_unix_domain_name(): + """ + Create a UNIX domain name: socket.bind() argument of a AF_UNIX socket. + + Return a path relative to the current directory to get a short path + (around 27 ASCII characters). + """ + return tempfile.mktemp(prefix="test_python_", suffix='.sock', + dir=os.path.curdir) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 01c1214c7f7cc2..7e16e2a369d484 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -315,11 +315,15 @@ def test_create_unix_connection_pathlib(self): self.loop.run_until_complete(coro) def test_create_unix_server_existing_path_nonsock(self): - with tempfile.NamedTemporaryFile() as file: - coro = self.loop.create_unix_server(lambda: None, file.name) - with self.assertRaisesRegex(OSError, - 'Address.*is already in use'): - self.loop.run_until_complete(coro) + path = test_utils.gen_unix_socket_path() + self.addCleanup(os_helper.unlink, path) + # create the file + open(path, "wb").close() + + coro = self.loop.create_unix_server(lambda: None, path) + with self.assertRaisesRegex(OSError, + 'Address.*is already in use'): + self.loop.run_until_complete(coro) def test_create_unix_server_ssl_bool(self): coro = self.loop.create_unix_server(lambda: None, path='spam', diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 0b9cde6878f37a..3f4c90e2ba6815 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -11,7 +11,6 @@ import socket import socketserver import sys -import tempfile import threading import time import unittest @@ -34,7 +33,7 @@ from asyncio import tasks from asyncio.log import logger from test import support -from test.support import threading_helper +from test.support import threading_helper, socket_helper def data_file(filename): @@ -250,8 +249,7 @@ class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer): def gen_unix_socket_path(): - with tempfile.NamedTemporaryFile() as file: - return file.name + return socket_helper.create_unix_domain_name() @contextlib.contextmanager diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index ecd1e120ecb515..3616b9957fbd33 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -9,6 +9,7 @@ import threading from test import support +from test.support import is_apple_mobile from test.support import os_helper from test.support import socket_helper from test.support import threading_helper @@ -658,6 +659,7 @@ def writable(self): @unittest.skipIf(sys.platform.startswith("sunos"), "OOB support is broken on Solaris") + @unittest.skipIf(is_apple_mobile, "FIXME: edge case in removed test module") def test_handle_expt(self): # Make sure handle_expt is called on OOB data received. # Note: this might fail on some platforms as OOB data is diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index e0feef7c653606..eab88bc256b694 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -1,6 +1,5 @@ import atexit import os -import sys import textwrap import unittest from test import support diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 10a61c60b57e88..2eab0fd6cf18ba 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -18,6 +18,7 @@ class AuditTest(unittest.TestCase): maxDiff = None + @support.requires_subprocess() def do_test(self, *args): with subprocess.Popen( [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], @@ -31,6 +32,7 @@ def do_test(self, *args): if p.returncode: self.fail("".join(p.stderr)) + @support.requires_subprocess() def run_python(self, *args): events = [] with subprocess.Popen( diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 71f934756e26a1..ca976abb853ff1 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -403,6 +403,7 @@ def test_LC_ALL_set_to_C(self): expected_warnings=[LEGACY_LOCALE_WARNING], coercion_expected=False) + @support.requires_subprocess() def test_PYTHONCOERCECLOCALE_set_to_one(self): # skip the test if the LC_CTYPE locale is C or coerced old_loc = locale.setlocale(locale.LC_CTYPE, None) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 6eae8c9cce3854..0cac02e48c39e9 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -152,6 +152,7 @@ class ThreadPoolMixin(ExecutorMixin): executor_type = futures.ThreadPoolExecutor +@support.requires_subprocess() class ProcessPoolForkMixin(ExecutorMixin): executor_type = futures.ProcessPoolExecutor ctx = "fork" @@ -166,6 +167,7 @@ def get_context(self): return super().get_context() +@support.requires_subprocess() class ProcessPoolSpawnMixin(ExecutorMixin): executor_type = futures.ProcessPoolExecutor ctx = "spawn" @@ -178,6 +180,7 @@ def get_context(self): return super().get_context() +@support.requires_subprocess() class ProcessPoolForkserverMixin(ExecutorMixin): executor_type = futures.ProcessPoolExecutor ctx = "forkserver" diff --git a/Lib/test/test_distutils.py b/Lib/test/test_distutils.py index d82d2b6423433e..f82aca2a01eb91 100644 --- a/Lib/test/test_distutils.py +++ b/Lib/test/test_distutils.py @@ -14,6 +14,8 @@ import distutils.tests +if support.is_apple_mobile: + raise unittest.SkipTest("FIXME: Edge case of test loader") def load_tests(*_): # used by unittest diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index ebd4ad91924cfb..d20dbc457c4155 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -4,7 +4,6 @@ from test import support from test.support import import_helper -from test.support import os_helper import doctest import functools import os @@ -14,7 +13,6 @@ import importlib.util import unittest import tempfile -import shutil import types import contextlib @@ -431,8 +429,9 @@ def test_DocTest(): r""" """ -class test_DocTestFinder: - def basics(): r""" +if not support.is_apple_mobile: + class test_DocTestFinder: + def basics(): r""" Unit tests for the `DocTestFinder` class. DocTestFinder is used to extract DocTests from an object's docstring @@ -2759,7 +2758,8 @@ def test_hook(pathdir): hook.remove() -def test_lineendings(): r""" +if not support.is_apple_mobile: + def test_lineendings(): r""" *nix systems use \n line endings, while Windows systems use \r\n, and old Mac systems used \r, which Python still recognizes as a line ending. Python handles this using universal newline mode for reading files. Let's make @@ -2875,7 +2875,8 @@ def test_unicode(): """ TestResults(failed=1, attempted=1) """ -def test_CLI(): r""" +if not support.is_apple_mobile: + def test_CLI(): r""" The doctest module can be used to run doctests against an arbitrary file. These tests test this CLI functionality. diff --git a/Lib/test/test_doctest2.py b/Lib/test/test_doctest2.py index ab8a0696736e23..f7007919113d91 100644 --- a/Lib/test/test_doctest2.py +++ b/Lib/test/test_doctest2.py @@ -13,9 +13,12 @@ import sys import unittest +from test import support + if sys.flags.optimize >= 2: raise unittest.SkipTest("Cannot test docstrings with -O2") + class C(object): """Class C. diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 8a436ad123b80f..4fa7571fe433ee 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -6,7 +6,7 @@ import types import unittest -from test.support import findfile +from test.support import findfile, requires_subprocess def abspath(filename): @@ -60,6 +60,7 @@ def generate_trace_command(self, script_file, subcommand=None): command += ["-c", subcommand] return command + @requires_subprocess() def trace(self, script_file, subcommand=None): command = self.generate_trace_command(script_file, subcommand) stdout, _ = subprocess.Popen(command, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 8c343f37210850..f13b5a1745bfb8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -72,6 +72,7 @@ def setUp(self): def tearDown(self): os.chdir(self.oldcwd) + @support.requires_subprocess() def run_embedded_interpreter(self, *args, env=None, timeout=None, returncode=0, input=None, cwd=None): @@ -1421,6 +1422,7 @@ def test_get_argc_argv(self): class SetConfigTests(unittest.TestCase): + @support.requires_subprocess() def test_set_config(self): # bpo-42260: Test _PyInterpreterState_SetConfig() cmd = [sys.executable, '-I', '-m', 'test._test_embed_set_config'] diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index e0f09e821da0c9..4bbdbdd15194f2 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -401,6 +401,7 @@ def test_is_enabled(self): finally: sys.stderr = orig_stderr + @support.requires_subprocess() def test_disabled_by_default(self): # By default, the module should be disabled code = "import faulthandler; print(faulthandler.is_enabled())" @@ -409,6 +410,7 @@ def test_disabled_by_default(self): output = subprocess.check_output(args) self.assertEqual(output.rstrip(), b"False") + @support.requires_subprocess() def test_sys_xoptions(self): # Test python -X faulthandler code = "import faulthandler; print(faulthandler.is_enabled())" @@ -421,6 +423,7 @@ def test_sys_xoptions(self): output = subprocess.check_output(args, env=env) self.assertEqual(output.rstrip(), b"True") + @support.requires_subprocess() def test_env_var(self): # empty env var code = "import faulthandler; print(faulthandler.is_enabled())" diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 0ef8fb42d73f39..7681e829d8b8e7 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -6,7 +6,7 @@ import sys import unittest from multiprocessing import Process -from test.support import cpython_only, requires_subprocess, verbose +from test.support import cpython_only, is_apple, requires_subprocess, verbose from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -25,7 +25,7 @@ def get_lockdata(): start_len = "qq" if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) - or sys.platform == 'darwin'): + or is_apple): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' diff --git a/Lib/test/test_file_eintr.py b/Lib/test/test_file_eintr.py index 01408d838a83cc..0386d4d774d936 100644 --- a/Lib/test/test_file_eintr.py +++ b/Lib/test/test_file_eintr.py @@ -15,6 +15,7 @@ import sys import time import unittest +from test import support # Test import all of the things we're about to try testing up front. import _io @@ -69,6 +70,7 @@ def fail_with_process_info(self, why, stdout=b'', stderr=b'', self.fail('Error from IO process %s:\nSTDOUT:\n%sSTDERR:\n%s\n' % (why, stdout.decode(), stderr.decode())) + @support.requires_subprocess() def _test_reading(self, data_to_write, read_and_verify_code): """Generic buffered read method test harness to validate EINTR behavior. diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 6c28b2b677ceeb..76ea55f14ad684 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,1415 +1,270 @@ +import os import unittest -import unittest.mock -from test.support import (verbose, refcount_test, - cpython_only) -from test.support.import_helper import import_module -from test.support.os_helper import temp_dir, TESTFN, unlink -from test.support.script_helper import assert_python_ok, make_script +import random +from test import support from test.support import threading_helper - -import gc -import sys -import sysconfig -import textwrap -import threading +import _thread as thread import time import weakref -try: - from _testcapi import with_tp_del -except ImportError: - def with_tp_del(cls): - class C(object): - def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') - return C - -try: - from _testcapi import ContainerNoGC -except ImportError: - ContainerNoGC = None - -### Support code -############################################################################### - -# Bug 1055820 has several tests of longstanding bugs involving weakrefs and -# cyclic gc. - -# An instance of C1055820 has a self-loop, so becomes cyclic trash when -# unreachable. -class C1055820(object): - def __init__(self, i): - self.i = i - self.loop = self - -class GC_Detector(object): - # Create an instance I. Then gc hasn't happened again so long as - # I.gc_happened is false. - - def __init__(self): - self.gc_happened = False - - def it_happened(ignored): - self.gc_happened = True - - # Create a piece of cyclic trash that triggers it_happened when - # gc collects it. - self.wr = weakref.ref(C1055820(666), it_happened) - -@with_tp_del -class Uncollectable(object): - """Create a reference cycle with multiple __del__ methods. - - An object in a reference cycle will never have zero references, - and so must be garbage collected. If one or more objects in the - cycle have __del__ methods, the gc refuses to guess an order, - and leaves the cycle uncollected.""" - def __init__(self, partner=None): - if partner is None: - self.partner = Uncollectable(partner=self) - else: - self.partner = partner - def __tp_del__(self): - pass - -if sysconfig.get_config_vars().get('PY_CFLAGS', ''): - BUILD_WITH_NDEBUG = ('-DNDEBUG' in sysconfig.get_config_vars()['PY_CFLAGS']) -else: - # Usually, sys.gettotalrefcount() is only present if Python has been - # compiled in debug mode. If it's missing, expect that Python has - # been released in release mode: with NDEBUG defined. - BUILD_WITH_NDEBUG = (not hasattr(sys, 'gettotalrefcount')) - -### Tests -############################################################################### - -class GCTests(unittest.TestCase): - def test_list(self): - l = [] - l.append(l) - gc.collect() - del l - self.assertEqual(gc.collect(), 1) - - def test_dict(self): - d = {} - d[1] = d - gc.collect() - del d - self.assertEqual(gc.collect(), 1) - - def test_tuple(self): - # since tuples are immutable we close the loop with a list - l = [] - t = (l,) - l.append(t) - gc.collect() - del t - del l - self.assertEqual(gc.collect(), 2) - - def test_class(self): - class A: - pass - A.a = A - gc.collect() - del A - self.assertNotEqual(gc.collect(), 0) - - def test_newstyleclass(self): - class A(object): - pass - gc.collect() - del A - self.assertNotEqual(gc.collect(), 0) - - def test_instance(self): - class A: - pass - a = A() - a.a = a - gc.collect() - del a - self.assertNotEqual(gc.collect(), 0) - - def test_newinstance(self): - class A(object): - pass - a = A() - a.a = a - gc.collect() - del a - self.assertNotEqual(gc.collect(), 0) - class B(list): - pass - class C(B, A): - pass - a = C() - a.a = a - gc.collect() - del a - self.assertNotEqual(gc.collect(), 0) - del B, C - self.assertNotEqual(gc.collect(), 0) - A.a = A() - del A - self.assertNotEqual(gc.collect(), 0) - self.assertEqual(gc.collect(), 0) - - def test_method(self): - # Tricky: self.__init__ is a bound method, it references the instance. - class A: - def __init__(self): - self.init = self.__init__ - a = A() - gc.collect() - del a - self.assertNotEqual(gc.collect(), 0) - - @cpython_only - def test_legacy_finalizer(self): - # A() is uncollectable if it is part of a cycle, make sure it shows up - # in gc.garbage. - @with_tp_del - class A: - def __tp_del__(self): pass - class B: - pass - a = A() - a.a = a - id_a = id(a) - b = B() - b.b = b - gc.collect() - del a - del b - self.assertNotEqual(gc.collect(), 0) - for obj in gc.garbage: - if id(obj) == id_a: - del obj.a - break - else: - self.fail("didn't find obj in garbage (finalizer)") - gc.garbage.remove(obj) - - @cpython_only - def test_legacy_finalizer_newclass(self): - # A() is uncollectable if it is part of a cycle, make sure it shows up - # in gc.garbage. - @with_tp_del - class A(object): - def __tp_del__(self): pass - class B(object): - pass - a = A() - a.a = a - id_a = id(a) - b = B() - b.b = b - gc.collect() - del a - del b - self.assertNotEqual(gc.collect(), 0) - for obj in gc.garbage: - if id(obj) == id_a: - del obj.a - break - else: - self.fail("didn't find obj in garbage (finalizer)") - gc.garbage.remove(obj) - - def test_function(self): - # Tricky: f -> d -> f, code should call d.clear() after the exec to - # break the cycle. - d = {} - exec("def f(): pass\n", d) - gc.collect() - del d - self.assertEqual(gc.collect(), 2) - - @refcount_test - def test_frame(self): - def f(): - frame = sys._getframe() - gc.collect() - f() - self.assertEqual(gc.collect(), 1) - - def test_saveall(self): - # Verify that cyclic garbage like lists show up in gc.garbage if the - # SAVEALL option is enabled. - - # First make sure we don't save away other stuff that just happens to - # be waiting for collection. - gc.collect() - # if this fails, someone else created immortal trash - self.assertEqual(gc.garbage, []) - - L = [] - L.append(L) - id_L = id(L) +from test import lock_tests - debug = gc.get_debug() - gc.set_debug(debug | gc.DEBUG_SAVEALL) - del L - gc.collect() - gc.set_debug(debug) +NUMTASKS = 10 +NUMTRIPS = 3 +POLL_SLEEP = 0.010 # seconds = 10 ms - self.assertEqual(len(gc.garbage), 1) - obj = gc.garbage.pop() - self.assertEqual(id(obj), id_L) +_print_mutex = thread.allocate_lock() - def test_del(self): - # __del__ methods can trigger collection, make this to happen - thresholds = gc.get_threshold() - gc.enable() - gc.set_threshold(1) +def verbose_print(arg): + """Helper function for printing out debugging output.""" + if support.verbose: + with _print_mutex: + print(arg) - class A: - def __del__(self): - dir(self) - a = A() - del a - gc.disable() - gc.set_threshold(*thresholds) +class BasicThreadTest(unittest.TestCase): - def test_del_newclass(self): - # __del__ methods can trigger collection, make this to happen - thresholds = gc.get_threshold() - gc.enable() - gc.set_threshold(1) - - class A(object): - def __del__(self): - dir(self) - a = A() - del a - - gc.disable() - gc.set_threshold(*thresholds) - - # The following two tests are fragile: - # They precisely count the number of allocations, - # which is highly implementation-dependent. - # For example, disposed tuples are not freed, but reused. - # To minimize variations, though, we first store the get_count() results - # and check them at the end. - @refcount_test - def test_get_count(self): - gc.collect() - a, b, c = gc.get_count() - x = [] - d, e, f = gc.get_count() - self.assertEqual((b, c), (0, 0)) - self.assertEqual((e, f), (0, 0)) - # This is less fragile than asserting that a equals 0. - self.assertLess(a, 5) - # Between the two calls to get_count(), at least one object was - # created (the list). - self.assertGreater(d, a) - - @refcount_test - def test_collect_generations(self): - gc.collect() - # This object will "trickle" into generation N + 1 after - # each call to collect(N) - x = [] - gc.collect(0) - # x is now in gen 1 - a, b, c = gc.get_count() - gc.collect(1) - # x is now in gen 2 - d, e, f = gc.get_count() - gc.collect(2) - # x is now in gen 3 - g, h, i = gc.get_count() - # We don't check a, d, g since their exact values depends on - # internal implementation details of the interpreter. - self.assertEqual((b, c), (1, 0)) - self.assertEqual((e, f), (0, 1)) - self.assertEqual((h, i), (0, 0)) - - def test_trashcan(self): - class Ouch: - n = 0 - def __del__(self): - Ouch.n = Ouch.n + 1 - if Ouch.n % 17 == 0: - gc.collect() + def setUp(self): + self.done_mutex = thread.allocate_lock() + self.done_mutex.acquire() + self.running_mutex = thread.allocate_lock() + self.random_mutex = thread.allocate_lock() + self.created = 0 + self.running = 0 + self.next_ident = 0 + + key = threading_helper.threading_setup() + self.addCleanup(threading_helper.threading_cleanup, *key) + + +class ThreadRunningTests(BasicThreadTest): + + def newtask(self): + with self.running_mutex: + self.next_ident += 1 + verbose_print("creating task %s" % self.next_ident) + thread.start_new_thread(self.task, (self.next_ident,)) + self.created += 1 + self.running += 1 + + def task(self, ident): + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % (ident, round(delay*1e6))) + time.sleep(delay) + verbose_print("task %s done" % ident) + with self.running_mutex: + self.running -= 1 + if self.created == NUMTASKS and self.running == 0: + self.done_mutex.release() + + def test_starting_threads(self): + with threading_helper.wait_threads_exit(): + # Basic test for thread creation. + for i in range(NUMTASKS): + self.newtask() + verbose_print("waiting for tasks to complete...") + self.done_mutex.acquire() + verbose_print("all tasks done") + + def test_stack_size(self): + # Various stack size tests. + self.assertEqual(thread.stack_size(), 0, "initial stack size is not 0") + + thread.stack_size(0) + self.assertEqual(thread.stack_size(), 0, "stack_size not reset to default") + + @unittest.skipIf(os.name not in ("nt", "posix"), 'test meant for nt and posix') + def test_nt_and_posix_stack_size(self): + try: + thread.stack_size(4096) + except ValueError: + verbose_print("caught expected ValueError setting " + "stack_size(4096)") + except thread.error: + self.skipTest("platform does not support changing thread stack " + "size") + + fail_msg = "stack_size(%d) failed - should succeed" + for tss in (262144, 0x100000, 0): + thread.stack_size(tss) + self.assertEqual(thread.stack_size(), tss, fail_msg % tss) + verbose_print("successfully set stack_size(%d)" % tss) + + for tss in (262144, 0x100000): + verbose_print("trying stack_size = (%d)" % tss) + self.next_ident = 0 + self.created = 0 + with threading_helper.wait_threads_exit(): + for i in range(NUMTASKS): + self.newtask() + + verbose_print("waiting for all tasks to complete") + self.done_mutex.acquire() + verbose_print("all tasks done") + + thread.stack_size(0) + + def test__count(self): + # Test the _count() function. + orig = thread._count() + mut = thread.allocate_lock() + mut.acquire() + started = [] + + def task(): + started.append(None) + mut.acquire() + mut.release() + + with threading_helper.wait_threads_exit(): + thread.start_new_thread(task, ()) + while not started: + time.sleep(POLL_SLEEP) + self.assertEqual(thread._count(), orig + 1) + # Allow the task to finish. + mut.release() + # The only reliable way to be sure that the thread ended from the + # interpreter's point of view is to wait for the function object to be + # destroyed. + done = [] + wr = weakref.ref(task, lambda _: done.append(None)) + del task + while not done: + time.sleep(POLL_SLEEP) + support.gc_collect() # For PyPy or other GCs. + self.assertEqual(thread._count(), orig) + + def test_unraisable_exception(self): + def task(): + started.release() + raise ValueError("task failed") + + started = thread.allocate_lock() + with support.catch_unraisable_exception() as cm: + with threading_helper.wait_threads_exit(): + started.acquire() + thread.start_new_thread(task, ()) + started.acquire() + + self.assertEqual(str(cm.unraisable.exc_value), "task failed") + self.assertIs(cm.unraisable.object, task) + self.assertEqual(cm.unraisable.err_msg, + "Exception ignored in thread started by") + self.assertIsNotNone(cm.unraisable.exc_traceback) + + +class Barrier: + def __init__(self, num_threads): + self.num_threads = num_threads + self.waiting = 0 + self.checkin_mutex = thread.allocate_lock() + self.checkout_mutex = thread.allocate_lock() + self.checkout_mutex.acquire() + + def enter(self): + self.checkin_mutex.acquire() + self.waiting = self.waiting + 1 + if self.waiting == self.num_threads: + self.waiting = self.num_threads - 1 + self.checkout_mutex.release() + return + self.checkin_mutex.release() + + self.checkout_mutex.acquire() + self.waiting = self.waiting - 1 + if self.waiting == 0: + self.checkin_mutex.release() + return + self.checkout_mutex.release() + + +class BarrierTest(BasicThreadTest): + + def test_barrier(self): + with threading_helper.wait_threads_exit(): + self.bar = Barrier(NUMTASKS) + self.running = NUMTASKS + for i in range(NUMTASKS): + thread.start_new_thread(self.task2, (i,)) + verbose_print("waiting for tasks to end") + self.done_mutex.acquire() + verbose_print("tasks done") + + def task2(self, ident): + for i in range(NUMTRIPS): + if ident == 0: + # give it a good chance to enter the next + # barrier before the others are all out + # of the current one + delay = 0 + else: + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % + (ident, round(delay * 1e6))) + time.sleep(delay) + verbose_print("task %s entering %s" % (ident, i)) + self.bar.enter() + verbose_print("task %s leaving barrier" % ident) + with self.running_mutex: + self.running -= 1 + # Must release mutex before releasing done, else the main thread can + # exit and set mutex to None as part of global teardown; then + # mutex.release() raises AttributeError. + finished = self.running == 0 + if finished: + self.done_mutex.release() + +class LockTests(lock_tests.LockTests): + locktype = thread.allocate_lock + + +class TestForkInThread(unittest.TestCase): + def setUp(self): + self.read_fd, self.write_fd = os.pipe() - # "trashcan" is a hack to prevent stack overflow when deallocating - # very deeply nested tuples etc. It works in part by abusing the - # type pointer and refcount fields, and that can yield horrible - # problems when gc tries to traverse the structures. - # If this test fails (as it does in 2.0, 2.1 and 2.2), it will - # most likely die via segfault. + @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') + @support.requires_fork() + @threading_helper.reap_threads + def test_forkinthread(self): + pid = None - # Note: In 2.3 the possibility for compiling without cyclic gc was - # removed, and that in turn allows the trashcan mechanism to work - # via much simpler means (e.g., it never abuses the type pointer or - # refcount fields anymore). Since it's much less likely to cause a - # problem now, the various constants in this expensive (we force a lot - # of full collections) test are cut back from the 2.2 version. - gc.enable() - N = 150 - for count in range(2): - t = [] - for i in range(N): - t = [t, Ouch()] - u = [] - for i in range(N): - u = [u, Ouch()] - v = {} - for i in range(N): - v = {1: v, 2: Ouch()} - gc.disable() + def fork_thread(read_fd, write_fd): + nonlocal pid - def test_trashcan_threads(self): - # Issue #13992: trashcan mechanism should be thread-safe - NESTING = 60 - N_THREADS = 2 + # fork in a thread + pid = os.fork() + if pid: + # parent process + return - def sleeper_gen(): - """A generator that releases the GIL when closed or dealloc'ed.""" + # child process try: - yield + os.close(read_fd) + os.write(write_fd, b"OK") finally: - time.sleep(0.000001) - - class C(list): - # Appending to a list is atomic, which avoids the use of a lock. - inits = [] - dels = [] - def __init__(self, alist): - self[:] = alist - C.inits.append(None) - def __del__(self): - # This __del__ is called by subtype_dealloc(). - C.dels.append(None) - # `g` will release the GIL when garbage-collected. This - # helps assert subtype_dealloc's behaviour when threads - # switch in the middle of it. - g = sleeper_gen() - next(g) - # Now that __del__ is finished, subtype_dealloc will proceed - # to call list_dealloc, which also uses the trashcan mechanism. + os._exit(0) - def make_nested(): - """Create a sufficiently nested container object so that the - trashcan mechanism is invoked when deallocating it.""" - x = C([]) - for i in range(NESTING): - x = [C([x])] - del x + with threading_helper.wait_threads_exit(): + thread.start_new_thread(fork_thread, (self.read_fd, self.write_fd)) + self.assertEqual(os.read(self.read_fd, 2), b"OK") + os.close(self.write_fd) - def run_thread(): - """Exercise make_nested() in a loop.""" - while not exit: - make_nested() + self.assertIsNotNone(pid) + support.wait_process(pid, exitcode=0) - old_switchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-5) + def tearDown(self): try: - exit = [] - threads = [] - for i in range(N_THREADS): - t = threading.Thread(target=run_thread) - threads.append(t) - with threading_helper.start_threads(threads, lambda: exit.append(1)): - time.sleep(1.0) - finally: - sys.setswitchinterval(old_switchinterval) - gc.collect() - self.assertEqual(len(C.inits), len(C.dels)) - - def test_boom(self): - class Boom: - def __getattr__(self, someattribute): - del self.attr - raise AttributeError - - a = Boom() - b = Boom() - a.attr = b - b.attr = a - - gc.collect() - garbagelen = len(gc.garbage) - del a, b - # a<->b are in a trash cycle now. Collection will invoke - # Boom.__getattr__ (to see whether a and b have __del__ methods), and - # __getattr__ deletes the internal "attr" attributes as a side effect. - # That causes the trash cycle to get reclaimed via refcounts falling to - # 0, thus mutating the trash graph as a side effect of merely asking - # whether __del__ exists. This used to (before 2.3b1) crash Python. - # Now __getattr__ isn't called. - self.assertEqual(gc.collect(), 4) - self.assertEqual(len(gc.garbage), garbagelen) - - def test_boom2(self): - class Boom2: - def __init__(self): - self.x = 0 - - def __getattr__(self, someattribute): - self.x += 1 - if self.x > 1: - del self.attr - raise AttributeError - - a = Boom2() - b = Boom2() - a.attr = b - b.attr = a - - gc.collect() - garbagelen = len(gc.garbage) - del a, b - # Much like test_boom(), except that __getattr__ doesn't break the - # cycle until the second time gc checks for __del__. As of 2.3b1, - # there isn't a second time, so this simply cleans up the trash cycle. - # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get - # reclaimed this way. - self.assertEqual(gc.collect(), 4) - self.assertEqual(len(gc.garbage), garbagelen) - - def test_boom_new(self): - # boom__new and boom2_new are exactly like boom and boom2, except use - # new-style classes. - - class Boom_New(object): - def __getattr__(self, someattribute): - del self.attr - raise AttributeError - - a = Boom_New() - b = Boom_New() - a.attr = b - b.attr = a - - gc.collect() - garbagelen = len(gc.garbage) - del a, b - self.assertEqual(gc.collect(), 4) - self.assertEqual(len(gc.garbage), garbagelen) - - def test_boom2_new(self): - class Boom2_New(object): - def __init__(self): - self.x = 0 - - def __getattr__(self, someattribute): - self.x += 1 - if self.x > 1: - del self.attr - raise AttributeError - - a = Boom2_New() - b = Boom2_New() - a.attr = b - b.attr = a - - gc.collect() - garbagelen = len(gc.garbage) - del a, b - self.assertEqual(gc.collect(), 4) - self.assertEqual(len(gc.garbage), garbagelen) - - def test_get_referents(self): - alist = [1, 3, 5] - got = gc.get_referents(alist) - got.sort() - self.assertEqual(got, alist) - - atuple = tuple(alist) - got = gc.get_referents(atuple) - got.sort() - self.assertEqual(got, alist) - - adict = {1: 3, 5: 7} - expected = [1, 3, 5, 7] - got = gc.get_referents(adict) - got.sort() - self.assertEqual(got, expected) - - got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0)) - got.sort() - self.assertEqual(got, [0, 0] + list(range(5))) - - self.assertEqual(gc.get_referents(1, 'a', 4j), []) - - def test_is_tracked(self): - # Atomic built-in types are not tracked, user-defined objects and - # mutable containers are. - # NOTE: types with special optimizations (e.g. tuple) have tests - # in their own test files instead. - self.assertFalse(gc.is_tracked(None)) - self.assertFalse(gc.is_tracked(1)) - self.assertFalse(gc.is_tracked(1.0)) - self.assertFalse(gc.is_tracked(1.0 + 5.0j)) - self.assertFalse(gc.is_tracked(True)) - self.assertFalse(gc.is_tracked(False)) - self.assertFalse(gc.is_tracked(b"a")) - self.assertFalse(gc.is_tracked("a")) - self.assertFalse(gc.is_tracked(bytearray(b"a"))) - self.assertFalse(gc.is_tracked(type)) - self.assertFalse(gc.is_tracked(int)) - self.assertFalse(gc.is_tracked(object)) - self.assertFalse(gc.is_tracked(object())) - - class UserClass: - pass - - class UserInt(int): - pass - - # Base class is object; no extra fields. - class UserClassSlots: - __slots__ = () - - # Base class is fixed size larger than object; no extra fields. - class UserFloatSlots(float): - __slots__ = () - - # Base class is variable size; no extra fields. - class UserIntSlots(int): - __slots__ = () - - self.assertTrue(gc.is_tracked(gc)) - self.assertTrue(gc.is_tracked(UserClass)) - self.assertTrue(gc.is_tracked(UserClass())) - self.assertTrue(gc.is_tracked(UserInt())) - self.assertTrue(gc.is_tracked([])) - self.assertTrue(gc.is_tracked(set())) - self.assertTrue(gc.is_tracked(UserClassSlots())) - self.assertTrue(gc.is_tracked(UserFloatSlots())) - self.assertTrue(gc.is_tracked(UserIntSlots())) - - def test_is_finalized(self): - # Objects not tracked by the always gc return false - self.assertFalse(gc.is_finalized(3)) - - storage = [] - class Lazarus: - def __del__(self): - storage.append(self) - - lazarus = Lazarus() - self.assertFalse(gc.is_finalized(lazarus)) - - del lazarus - gc.collect() - - lazarus = storage.pop() - self.assertTrue(gc.is_finalized(lazarus)) - - def test_bug1055820b(self): - # Corresponds to temp2b.py in the bug report. - - ouch = [] - def callback(ignored): - ouch[:] = [wr() for wr in WRs] - - Cs = [C1055820(i) for i in range(2)] - WRs = [weakref.ref(c, callback) for c in Cs] - c = None - - gc.collect() - self.assertEqual(len(ouch), 0) - # Make the two instances trash, and collect again. The bug was that - # the callback materialized a strong reference to an instance, but gc - # cleared the instance's dict anyway. - Cs = None - gc.collect() - self.assertEqual(len(ouch), 2) # else the callbacks didn't run - for x in ouch: - # If the callback resurrected one of these guys, the instance - # would be damaged, with an empty __dict__. - self.assertEqual(x, None) - - def test_bug21435(self): - # This is a poor test - its only virtue is that it happened to - # segfault on Tim's Windows box before the patch for 21435 was - # applied. That's a nasty bug relying on specific pieces of cyclic - # trash appearing in exactly the right order in finalize_garbage()'s - # input list. - # But there's no reliable way to force that order from Python code, - # so over time chances are good this test won't really be testing much - # of anything anymore. Still, if it blows up, there's _some_ - # problem ;-) - gc.collect() - - class A: + os.close(self.read_fd) + except OSError: pass - class B: - def __init__(self, x): - self.x = x - - def __del__(self): - self.attr = None - - def do_work(): - a = A() - b = B(A()) - - a.attr = b - b.attr = a - - do_work() - gc.collect() # this blows up (bad C pointer) when it fails - - @cpython_only - def test_garbage_at_shutdown(self): - import subprocess - code = """if 1: - import gc - import _testcapi - @_testcapi.with_tp_del - class X: - def __init__(self, name): - self.name = name - def __repr__(self): - return "" %% self.name - def __tp_del__(self): - pass - - x = X('first') - x.x = x - x.y = X('second') - del x - gc.set_debug(%s) - """ - def run_command(code): - p = subprocess.Popen([sys.executable, "-Wd", "-c", code], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - p.stdout.close() - p.stderr.close() - self.assertEqual(p.returncode, 0) - self.assertEqual(stdout, b"") - return stderr - - stderr = run_command(code % "0") - self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " - b"shutdown; use", stderr) - self.assertNotIn(b"", stderr) - # With DEBUG_UNCOLLECTABLE, the garbage list gets printed - stderr = run_command(code % "gc.DEBUG_UNCOLLECTABLE") - self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " - b"shutdown", stderr) - self.assertTrue( - (b"[, ]" in stderr) or - (b"[, ]" in stderr), stderr) - # With DEBUG_SAVEALL, no additional message should get printed - # (because gc.garbage also contains normally reclaimable cyclic - # references, and its elements get printed at runtime anyway). - stderr = run_command(code % "gc.DEBUG_SAVEALL") - self.assertNotIn(b"uncollectable objects at shutdown", stderr) - - def test_gc_main_module_at_shutdown(self): - # Create a reference cycle through the __main__ module and check - # it gets collected at interpreter shutdown. - code = """if 1: - class C: - def __del__(self): - print('__del__ called') - l = [C()] - l.append(l) - """ - rc, out, err = assert_python_ok('-c', code) - self.assertEqual(out.strip(), b'__del__ called') - - def test_gc_ordinary_module_at_shutdown(self): - # Same as above, but with a non-__main__ module. - with temp_dir() as script_dir: - module = """if 1: - class C: - def __del__(self): - print('__del__ called') - l = [C()] - l.append(l) - """ - code = """if 1: - import sys - sys.path.insert(0, %r) - import gctest - """ % (script_dir,) - make_script(script_dir, 'gctest', module) - rc, out, err = assert_python_ok('-c', code) - self.assertEqual(out.strip(), b'__del__ called') - - def test_global_del_SystemExit(self): - code = """if 1: - class ClassWithDel: - def __del__(self): - print('__del__ called') - a = ClassWithDel() - a.link = a - raise SystemExit(0)""" - self.addCleanup(unlink, TESTFN) - with open(TESTFN, 'w', encoding="utf-8") as script: - script.write(code) - rc, out, err = assert_python_ok(TESTFN) - self.assertEqual(out.strip(), b'__del__ called') - - def test_get_stats(self): - stats = gc.get_stats() - self.assertEqual(len(stats), 3) - for st in stats: - self.assertIsInstance(st, dict) - self.assertEqual(set(st), - {"collected", "collections", "uncollectable"}) - self.assertGreaterEqual(st["collected"], 0) - self.assertGreaterEqual(st["collections"], 0) - self.assertGreaterEqual(st["uncollectable"], 0) - # Check that collection counts are incremented correctly - if gc.isenabled(): - self.addCleanup(gc.enable) - gc.disable() - old = gc.get_stats() - gc.collect(0) - new = gc.get_stats() - self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) - self.assertEqual(new[1]["collections"], old[1]["collections"]) - self.assertEqual(new[2]["collections"], old[2]["collections"]) - gc.collect(2) - new = gc.get_stats() - self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) - self.assertEqual(new[1]["collections"], old[1]["collections"]) - self.assertEqual(new[2]["collections"], old[2]["collections"] + 1) - - def test_freeze(self): - gc.freeze() - self.assertGreater(gc.get_freeze_count(), 0) - gc.unfreeze() - self.assertEqual(gc.get_freeze_count(), 0) - - def test_get_objects(self): - gc.collect() - l = [] - l.append(l) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=0) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=1) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=2) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) - del l - gc.collect() - - def test_get_objects_arguments(self): - gc.collect() - self.assertEqual(len(gc.get_objects()), - len(gc.get_objects(generation=None))) - - self.assertRaises(ValueError, gc.get_objects, 1000) - self.assertRaises(ValueError, gc.get_objects, -1000) - self.assertRaises(TypeError, gc.get_objects, "1") - self.assertRaises(TypeError, gc.get_objects, 1.234) - - def test_resurrection_only_happens_once_per_object(self): - class A: # simple self-loop - def __init__(self): - self.me = self - - class Lazarus(A): - resurrected = 0 - resurrected_instances = [] - - def __del__(self): - Lazarus.resurrected += 1 - Lazarus.resurrected_instances.append(self) - - gc.collect() - gc.disable() - - # We start with 0 resurrections - laz = Lazarus() - self.assertEqual(Lazarus.resurrected, 0) - - # Deleting the instance and triggering a collection - # resurrects the object - del laz - gc.collect() - self.assertEqual(Lazarus.resurrected, 1) - self.assertEqual(len(Lazarus.resurrected_instances), 1) - - # Clearing the references and forcing a collection - # should not resurrect the object again. - Lazarus.resurrected_instances.clear() - self.assertEqual(Lazarus.resurrected, 1) - gc.collect() - self.assertEqual(Lazarus.resurrected, 1) - - gc.enable() - - def test_resurrection_is_transitive(self): - class Cargo: - def __init__(self): - self.me = self - - class Lazarus: - resurrected_instances = [] - - def __del__(self): - Lazarus.resurrected_instances.append(self) - - gc.collect() - gc.disable() - - laz = Lazarus() - cargo = Cargo() - cargo_id = id(cargo) - - # Create a cycle between cargo and laz - laz.cargo = cargo - cargo.laz = laz - - # Drop the references, force a collection and check that - # everything was resurrected. - del laz, cargo - gc.collect() - self.assertEqual(len(Lazarus.resurrected_instances), 1) - instance = Lazarus.resurrected_instances.pop() - self.assertTrue(hasattr(instance, "cargo")) - self.assertEqual(id(instance.cargo), cargo_id) - - gc.collect() - gc.enable() - - def test_resurrection_does_not_block_cleanup_of_other_objects(self): - - # When a finalizer resurrects objects, stats were reporting them as - # having been collected. This affected both collect()'s return - # value and the dicts returned by get_stats(). - N = 100 - - class A: # simple self-loop - def __init__(self): - self.me = self - - class Z(A): # resurrecting __del__ - def __del__(self): - zs.append(self) - - zs = [] - - def getstats(): - d = gc.get_stats()[-1] - return d['collected'], d['uncollectable'] - - gc.collect() - gc.disable() - - # No problems if just collecting A() instances. - oldc, oldnc = getstats() - for i in range(N): - A() - t = gc.collect() - c, nc = getstats() - self.assertEqual(t, 2*N) # instance object & its dict - self.assertEqual(c - oldc, 2*N) - self.assertEqual(nc - oldnc, 0) - - # But Z() is not actually collected. - oldc, oldnc = c, nc - Z() - # Nothing is collected - Z() is merely resurrected. - t = gc.collect() - c, nc = getstats() - self.assertEqual(t, 0) - self.assertEqual(c - oldc, 0) - self.assertEqual(nc - oldnc, 0) - - # Z() should not prevent anything else from being collected. - oldc, oldnc = c, nc - for i in range(N): - A() - Z() - t = gc.collect() - c, nc = getstats() - self.assertEqual(t, 2*N) - self.assertEqual(c - oldc, 2*N) - self.assertEqual(nc - oldnc, 0) - - # The A() trash should have been reclaimed already but the - # 2 copies of Z are still in zs (and the associated dicts). - oldc, oldnc = c, nc - zs.clear() - t = gc.collect() - c, nc = getstats() - self.assertEqual(t, 4) - self.assertEqual(c - oldc, 4) - self.assertEqual(nc - oldnc, 0) - - gc.enable() - - @unittest.skipIf(ContainerNoGC is None, - 'requires ContainerNoGC extension type') - def test_trash_weakref_clear(self): - # Test that trash weakrefs are properly cleared (bpo-38006). - # - # Structure we are creating: - # - # Z <- Y <- A--+--> WZ -> C - # ^ | - # +--+ - # where: - # WZ is a weakref to Z with callback C - # Y doesn't implement tp_traverse - # A contains a reference to itself, Y and WZ - # - # A, Y, Z, WZ are all trash. The GC doesn't know that Z is trash - # because Y does not implement tp_traverse. To show the bug, WZ needs - # to live long enough so that Z is deallocated before it. Then, if - # gcmodule is buggy, when Z is being deallocated, C will run. - # - # To ensure WZ lives long enough, we put it in a second reference - # cycle. That trick only works due to the ordering of the GC prev/next - # linked lists. So, this test is a bit fragile. - # - # The bug reported in bpo-38006 is caused because the GC did not - # clear WZ before starting the process of calling tp_clear on the - # trash. Normally, handle_weakrefs() would find the weakref via Z and - # clear it. However, since the GC cannot find Z, WR is not cleared and - # it can execute during delete_garbage(). That can lead to disaster - # since the callback might tinker with objects that have already had - # tp_clear called on them (leaving them in possibly invalid states). - - callback = unittest.mock.Mock() - - class A: - __slots__ = ['a', 'y', 'wz'] - - class Z: + try: + os.close(self.write_fd) + except OSError: pass - # setup required object graph, as described above - a = A() - a.a = a - a.y = ContainerNoGC(Z()) - a.wz = weakref.ref(a.y.value, callback) - # create second cycle to keep WZ alive longer - wr_cycle = [a.wz] - wr_cycle.append(wr_cycle) - # ensure trash unrelated to this test is gone - gc.collect() - gc.disable() - # release references and create trash - del a, wr_cycle - gc.collect() - # if called, it means there is a bug in the GC. The weakref should be - # cleared before Z dies. - callback.assert_not_called() - gc.enable() - - -class GCCallbackTests(unittest.TestCase): - def setUp(self): - # Save gc state and disable it. - self.enabled = gc.isenabled() - gc.disable() - self.debug = gc.get_debug() - gc.set_debug(0) - gc.callbacks.append(self.cb1) - gc.callbacks.append(self.cb2) - self.othergarbage = [] - - def tearDown(self): - # Restore gc state - del self.visit - gc.callbacks.remove(self.cb1) - gc.callbacks.remove(self.cb2) - gc.set_debug(self.debug) - if self.enabled: - gc.enable() - # destroy any uncollectables - gc.collect() - for obj in gc.garbage: - if isinstance(obj, Uncollectable): - obj.partner = None - del gc.garbage[:] - del self.othergarbage - gc.collect() - - def preclean(self): - # Remove all fluff from the system. Invoke this function - # manually rather than through self.setUp() for maximum - # safety. - self.visit = [] - gc.collect() - garbage, gc.garbage[:] = gc.garbage[:], [] - self.othergarbage.append(garbage) - self.visit = [] - - def cb1(self, phase, info): - self.visit.append((1, phase, dict(info))) - - def cb2(self, phase, info): - self.visit.append((2, phase, dict(info))) - if phase == "stop" and hasattr(self, "cleanup"): - # Clean Uncollectable from garbage - uc = [e for e in gc.garbage if isinstance(e, Uncollectable)] - gc.garbage[:] = [e for e in gc.garbage - if not isinstance(e, Uncollectable)] - for e in uc: - e.partner = None - - def test_collect(self): - self.preclean() - gc.collect() - # Algorithmically verify the contents of self.visit - # because it is long and tortuous. - - # Count the number of visits to each callback - n = [v[0] for v in self.visit] - n1 = [i for i in n if i == 1] - n2 = [i for i in n if i == 2] - self.assertEqual(n1, [1]*2) - self.assertEqual(n2, [2]*2) - - # Count that we got the right number of start and stop callbacks. - n = [v[1] for v in self.visit] - n1 = [i for i in n if i == "start"] - n2 = [i for i in n if i == "stop"] - self.assertEqual(n1, ["start"]*2) - self.assertEqual(n2, ["stop"]*2) - - # Check that we got the right info dict for all callbacks - for v in self.visit: - info = v[2] - self.assertTrue("generation" in info) - self.assertTrue("collected" in info) - self.assertTrue("uncollectable" in info) - - def test_collect_generation(self): - self.preclean() - gc.collect(2) - for v in self.visit: - info = v[2] - self.assertEqual(info["generation"], 2) - - @cpython_only - def test_collect_garbage(self): - self.preclean() - # Each of these cause four objects to be garbage: Two - # Uncollectables and their instance dicts. - Uncollectable() - Uncollectable() - C1055820(666) - gc.collect() - for v in self.visit: - if v[1] != "stop": - continue - info = v[2] - self.assertEqual(info["collected"], 2) - self.assertEqual(info["uncollectable"], 8) - - # We should now have the Uncollectables in gc.garbage - self.assertEqual(len(gc.garbage), 4) - for e in gc.garbage: - self.assertIsInstance(e, Uncollectable) - - # Now, let our callback handle the Uncollectable instances - self.cleanup=True - self.visit = [] - gc.garbage[:] = [] - gc.collect() - for v in self.visit: - if v[1] != "stop": - continue - info = v[2] - self.assertEqual(info["collected"], 0) - self.assertEqual(info["uncollectable"], 4) - - # Uncollectables should be gone - self.assertEqual(len(gc.garbage), 0) - - - @unittest.skipIf(BUILD_WITH_NDEBUG, - 'built with -NDEBUG') - def test_refcount_errors(self): - self.preclean() - # Verify the "handling" of objects with broken refcounts - - # Skip the test if ctypes is not available - import_module("ctypes") - - import subprocess - code = textwrap.dedent(''' - from test.support import gc_collect, SuppressCrashReport - - a = [1, 2, 3] - b = [a] - - # Avoid coredump when Py_FatalError() calls abort() - SuppressCrashReport().__enter__() - - # Simulate the refcount of "a" being too low (compared to the - # references held on it by live data), but keeping it above zero - # (to avoid deallocating it): - import ctypes - ctypes.pythonapi.Py_DecRef(ctypes.py_object(a)) - - # The garbage collector should now have a fatal error - # when it reaches the broken object - gc_collect() - ''') - p = subprocess.Popen([sys.executable, "-c", code], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - p.stdout.close() - p.stderr.close() - # Verify that stderr has a useful error message: - self.assertRegex(stderr, - br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.') - self.assertRegex(stderr, - br'refcount is too small') - # "address : 0x7fb5062efc18" - # "address : 7FB5062EFC18" - address_regex = br'[0-9a-fA-Fx]+' - self.assertRegex(stderr, - br'object address : ' + address_regex) - self.assertRegex(stderr, - br'object refcount : 1') - self.assertRegex(stderr, - br'object type : ' + address_regex) - self.assertRegex(stderr, - br'object type name: list') - self.assertRegex(stderr, - br'object repr : \[1, 2, 3\]') - - -class GCTogglingTests(unittest.TestCase): - def setUp(self): - gc.enable() - - def tearDown(self): - gc.disable() - - def test_bug1055820c(self): - # Corresponds to temp2c.py in the bug report. This is pretty - # elaborate. - - c0 = C1055820(0) - # Move c0 into generation 2. - gc.collect() - - c1 = C1055820(1) - c1.keep_c0_alive = c0 - del c0.loop # now only c1 keeps c0 alive - - c2 = C1055820(2) - c2wr = weakref.ref(c2) # no callback! - - ouch = [] - def callback(ignored): - ouch[:] = [c2wr()] - - # The callback gets associated with a wr on an object in generation 2. - c0wr = weakref.ref(c0, callback) - - c0 = c1 = c2 = None - - # What we've set up: c0, c1, and c2 are all trash now. c0 is in - # generation 2. The only thing keeping it alive is that c1 points to - # it. c1 and c2 are in generation 0, and are in self-loops. There's a - # global weakref to c2 (c2wr), but that weakref has no callback. - # There's also a global weakref to c0 (c0wr), and that does have a - # callback, and that callback references c2 via c2wr(). - # - # c0 has a wr with callback, which references c2wr - # ^ - # | - # | Generation 2 above dots - #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . - # | Generation 0 below dots - # | - # | - # ^->c1 ^->c2 has a wr but no callback - # | | | | - # <--v <--v - # - # So this is the nightmare: when generation 0 gets collected, we see - # that c2 has a callback-free weakref, and c1 doesn't even have a - # weakref. Collecting generation 0 doesn't see c0 at all, and c0 is - # the only object that has a weakref with a callback. gc clears c1 - # and c2. Clearing c1 has the side effect of dropping the refcount on - # c0 to 0, so c0 goes away (despite that it's in an older generation) - # and c0's wr callback triggers. That in turn materializes a reference - # to c2 via c2wr(), but c2 gets cleared anyway by gc. - - # We want to let gc happen "naturally", to preserve the distinction - # between generations. - junk = [] - i = 0 - detector = GC_Detector() - while not detector.gc_happened: - i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") - self.assertEqual(len(ouch), 0) - junk.append([]) # this will eventually trigger gc - - self.assertEqual(len(ouch), 1) # else the callback wasn't invoked - for x in ouch: - # If the callback resurrected c2, the instance would be damaged, - # with an empty __dict__. - self.assertEqual(x, None) - - def test_bug1055820d(self): - # Corresponds to temp2d.py in the bug report. This is very much like - # test_bug1055820c, but uses a __del__ method instead of a weakref - # callback to sneak in a resurrection of cyclic trash. - - ouch = [] - class D(C1055820): - def __del__(self): - ouch[:] = [c2wr()] - - d0 = D(0) - # Move all the above into generation 2. - gc.collect() - - c1 = C1055820(1) - c1.keep_d0_alive = d0 - del d0.loop # now only c1 keeps d0 alive - - c2 = C1055820(2) - c2wr = weakref.ref(c2) # no callback! - - d0 = c1 = c2 = None - - # What we've set up: d0, c1, and c2 are all trash now. d0 is in - # generation 2. The only thing keeping it alive is that c1 points to - # it. c1 and c2 are in generation 0, and are in self-loops. There's - # a global weakref to c2 (c2wr), but that weakref has no callback. - # There are no other weakrefs. - # - # d0 has a __del__ method that references c2wr - # ^ - # | - # | Generation 2 above dots - #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . - # | Generation 0 below dots - # | - # | - # ^->c1 ^->c2 has a wr but no callback - # | | | | - # <--v <--v - # - # So this is the nightmare: when generation 0 gets collected, we see - # that c2 has a callback-free weakref, and c1 doesn't even have a - # weakref. Collecting generation 0 doesn't see d0 at all. gc clears - # c1 and c2. Clearing c1 has the side effect of dropping the refcount - # on d0 to 0, so d0 goes away (despite that it's in an older - # generation) and d0's __del__ triggers. That in turn materializes - # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc. - - # We want to let gc happen "naturally", to preserve the distinction - # between generations. - detector = GC_Detector() - junk = [] - i = 0 - while not detector.gc_happened: - i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") - self.assertEqual(len(ouch), 0) - junk.append([]) # this will eventually trigger gc - - self.assertEqual(len(ouch), 1) # else __del__ wasn't invoked - for x in ouch: - # If __del__ resurrected c2, the instance would be damaged, with an - # empty __dict__. - self.assertEqual(x, None) - - -class PythonFinalizationTests(unittest.TestCase): - def test_ast_fini(self): - # bpo-44184: Regression test for subtype_dealloc() when deallocating - # an AST instance also destroy its AST type: subtype_dealloc() must - # not access the type memory after deallocating the instance, since - # the type memory can be freed as well. The test is also related to - # _PyAST_Fini() which clears references to AST types. - code = textwrap.dedent(""" - import ast - import codecs - - # Small AST tree to keep their AST types alive - tree = ast.parse("def f(x, y): return 2*x-y") - x = [tree] - x.append(x) - - # Put the cycle somewhere to survive until the last GC collection. - # Codec search functions are only cleared at the end of - # interpreter_clear(). - def search_func(encoding): - return None - search_func.a = x - codecs.register(search_func) - """) - assert_python_ok("-c", code) - - -def setUpModule(): - global enabled, debug - enabled = gc.isenabled() - gc.disable() - assert not gc.isenabled() - debug = gc.get_debug() - gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak - gc.collect() # Delete 2nd generation garbage - - -def tearDownModule(): - gc.set_debug(debug) - # test gc.enable() even if GC is disabled by default - if verbose: - print("restoring automatic collection") - # make sure to always test gc.enable() - gc.enable() - assert gc.isenabled() - if not enabled: - gc.disable() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index 3452e46213a76c..3519f2f9837300 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -22,6 +22,8 @@ def test_username_takes_username_from_env(self, environ): environ.get.return_value = expected_name self.assertEqual(expected_name, getpass.getuser()) + @unittest.skipIf(support.is_apple_mobile, + "FIXME: getpwuid implementation not complete on simulator") def test_username_priorities_of_env_values(self, environ): environ.get.return_value = None try: diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index f86e767ac0e59c..14434981e9504c 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -12,6 +12,7 @@ from subprocess import PIPE, Popen from test.support import import_helper from test.support import os_helper +from test.support import requires_subprocess from test.support import _4G, bigmemtest from test.support.script_helper import assert_python_ok, assert_python_failure @@ -752,6 +753,7 @@ def wrapper(*args, **kwargs): class TestCommandLine(unittest.TestCase): data = b'This is a simple test with gzip' + @requires_subprocess() def test_decompress_stdin_stdout(self): with io.BytesIO() as bytes_io: with gzip.GzipFile(fileobj=bytes_io, mode='wb') as gzip_file: @@ -787,6 +789,7 @@ def test_decompress_infile_outfile_error(self): self.assertEqual(rc, 1) self.assertEqual(out, b'') + @requires_subprocess() @create_and_remove_directory(TEMPDIR) def test_compress_stdin_outfile(self): args = sys.executable, '-m', 'gzip' diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 21fec2ad960e3a..c1cecb42f99447 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -9,6 +9,7 @@ from test.support import import_helper from test.support import os_helper from test.support import script_helper +from test.support import is_apple_mobile import unittest import warnings with warnings.catch_warnings(): @@ -225,6 +226,7 @@ def test_load_from_source(self): self.assertIsNot(orig_getenv, new_os.getenv) @requires_load_dynamic + @unittest.skipIf(is_apple_mobile, "FIXME: edge case of module loader") def test_issue15828_load_extensions(self): # Issue 15828 picked up that the adapter between the old imp API # and importlib couldn't handle C extensions @@ -237,6 +239,7 @@ def test_issue15828_load_extensions(self): self.assertEqual(mod.__name__, example) @requires_load_dynamic + @unittest.skipIf(is_apple_mobile, "FIXME: edge case of module loader") def test_issue16421_multiple_modules_in_one_dll(self): # Issue 16421: loading several modules from the same compiled file fails m = '_testimportmultiple' @@ -264,6 +267,7 @@ def test_load_dynamic_ImportError_path(self): self.assertEqual(name, err.exception.name) @requires_load_dynamic + @unittest.skipIf(is_apple_mobile, "FIXME: edge case of module loader") def test_load_module_extension_file_is_None(self): # When loading an extension module and the file is None, open one # on the behalf of imp.load_dynamic(). @@ -277,6 +281,7 @@ def test_load_module_extension_file_is_None(self): imp.load_module(name, None, *found[1:]) @requires_load_dynamic + @unittest.skipIf(is_apple_mobile, "FIXME: edge case of module loader") def test_issue24748_load_module_skips_sys_modules_check(self): name = 'test.imp_dummy' try: diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 97447619b90d2d..9abe0e0b7c0df5 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -19,7 +19,7 @@ from unittest import mock from test.support import os_helper -from test.support import (is_jython, swap_attr, swap_item, cpython_only) +from test.support import (is_apple_mobile, is_jython, swap_attr, swap_item, cpython_only) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath) from test.support.os_helper import ( @@ -93,6 +93,8 @@ def test_from_import_missing_attr_has_name_and_path(self): self.assertRegex(str(cm.exception), r"cannot import name 'i_dont_exist' from 'os' \(.*os.py\)") @cpython_only + @unittest.skipIf(is_apple_mobile, + "FIXME: edge case in loader") def test_from_import_missing_attr_has_name_and_so_path(self): import _testcapi with self.assertRaises(ImportError) as cm: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 5e430dad061b01..ad0728408df2fd 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -24,7 +24,7 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only +from test.support import cpython_only, is_apple_mobile from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath from test.support.os_helper import TESTFN @@ -727,6 +727,7 @@ def test_method_in_dynamic_class(self): @unittest.skipIf(not hasattr(unicodedata, '__file__') or unicodedata.__file__.endswith('.py'), "unicodedata is not an external binary module") + @unittest.skipIf(is_apple_mobile, "FIXME: Edge case of module loader") def test_findsource_binary(self): self.assertRaises(OSError, inspect.getsource, unicodedata) self.assertRaises(OSError, inspect.findsource, unicodedata) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 1d7fca6efb1cc7..0715cc75b661df 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -85,6 +85,7 @@ class TestTool(unittest.TestCase): } """) + @support.requires_subprocess() def test_stdin_stdout(self): args = sys.executable, '-m', 'json.tool' process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True) @@ -140,6 +141,7 @@ def test_writing_in_place(self): self.assertEqual(out, b'') self.assertEqual(err, b'') + @support.requires_subprocess() def test_jsonlines(self): args = sys.executable, '-m', 'json.tool', '--json-lines' process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True) @@ -160,6 +162,7 @@ def test_sort_keys_flag(self): self.expect_without_sort_keys.encode().splitlines()) self.assertEqual(err, b'') + @support.requires_subprocess() def test_indent(self): input_ = '[1, 2]' expect = textwrap.dedent('''\ @@ -173,6 +176,7 @@ def test_indent(self): self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @support.requires_subprocess() def test_no_indent(self): input_ = '[1,\n2]' expect = '[1, 2]\n' @@ -181,6 +185,7 @@ def test_no_indent(self): self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @support.requires_subprocess() def test_tab(self): input_ = '[1, 2]' expect = '[\n\t1,\n\t2\n]\n' @@ -189,6 +194,7 @@ def test_tab(self): self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @support.requires_subprocess() def test_compact(self): input_ = '[ 1 ,\n 2]' expect = '[1,2]\n' @@ -197,6 +203,7 @@ def test_compact(self): self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @unittest.skipIf(support.is_apple_mobile, "FIXME: Edge case in encoding handling") def test_no_ensure_ascii_flag(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' @@ -208,6 +215,7 @@ def test_no_ensure_ascii_flag(self): expected = [b'{', b' "key": "\xf0\x9f\x92\xa9"', b"}"] self.assertEqual(lines, expected) + @unittest.skipIf(support.is_apple_mobile, "FIXME: Edge case in encoding handling") def test_ensure_ascii_default(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' @@ -220,6 +228,7 @@ def test_ensure_ascii_default(self): self.assertEqual(lines, expected) @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows") + @support.requires_subprocess() def test_broken_pipe_error(self): cmd = [sys.executable, '-m', 'json.tool'] proc = subprocess.Popen(cmd, diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 74e950450cfb20..b3dc348d2bcadf 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -43,6 +43,7 @@ import tempfile from test.support.script_helper import assert_python_ok, assert_python_failure from test import support +from test.support import is_apple_mobile from test.support import os_helper from test.support import socket_helper from test.support import threading_helper diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py index 32f07ab290f1ef..3ca7616feef4dc 100644 --- a/Lib/test/test_mailcap.py +++ b/Lib/test/test_mailcap.py @@ -2,7 +2,7 @@ import os import copy import test.support -from test.support import os_helper +from test.support import os_helper, requires_subprocess import unittest import sys @@ -220,6 +220,7 @@ def test_findmatch(self): @unittest.skipUnless(os.name == "posix", "Requires 'test' command on system") @unittest.skipIf(sys.platform == "vxworks", "'test' command is not supported on VxWorks") + @requires_subprocess() def test_test(self): # findmatch() will automatically check any "test" conditions and skip # the entry if the check fails. diff --git a/Lib/test/test_multiprocessing_fork.py b/Lib/test/test_multiprocessing_fork.py index 5000edb7c5c299..5c2f1e08cc9044 100644 --- a/Lib/test/test_multiprocessing_fork.py +++ b/Lib/test/test_multiprocessing_fork.py @@ -13,6 +13,9 @@ if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") +if support.is_apple_mobile: + raise unittest.SkipTest("Can't use fork on Apple mobile") + test._test_multiprocessing.install_tests_in_module_dict(globals(), 'fork') if __name__ == '__main__': diff --git a/Lib/test/test_multiprocessing_forkserver.py b/Lib/test/test_multiprocessing_forkserver.py index 6ad5faf9e8a329..0c0f470b6d3fdc 100644 --- a/Lib/test/test_multiprocessing_forkserver.py +++ b/Lib/test/test_multiprocessing_forkserver.py @@ -10,6 +10,9 @@ if sys.platform == "win32": raise unittest.SkipTest("forkserver is not available on Windows") +if support.is_apple_mobile: + raise unittest.SkipTest("Can't use fork on Apple mobile") + test._test_multiprocessing.install_tests_in_module_dict(globals(), 'forkserver') if __name__ == '__main__': diff --git a/Lib/test/test_multiprocessing_spawn.py b/Lib/test/test_multiprocessing_spawn.py index 6558952308f25c..9c3901a63f3a94 100644 --- a/Lib/test/test_multiprocessing_spawn.py +++ b/Lib/test/test_multiprocessing_spawn.py @@ -6,6 +6,9 @@ if support.PGO: raise unittest.SkipTest("test is not helpful for PGO") +if support.is_apple_mobile: + raise unittest.SkipTest("Can't use fork on Apple mobile") + test._test_multiprocessing.install_tests_in_module_dict(globals(), 'spawn') if __name__ == '__main__': diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 0dcd996eadfc3f..c064bc7afa363c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -33,6 +33,8 @@ from test import support from test.support import import_helper from test.support import os_helper +from test.support import requires_fork +from test.support import requires_subprocess from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper @@ -1095,6 +1097,7 @@ def test_environb(self): value_str = value.decode(sys.getfilesystemencoding(), 'surrogateescape') self.assertEqual(os.environ['bytes'], value_str) + @requires_subprocess() def test_putenv_unsetenv(self): name = "PYTHONTESTVAR" value = "testvalue" @@ -2306,6 +2309,7 @@ def test_setreuid(self): self.assertRaises(OverflowError, os.setreuid, 0, self.UID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') + @requires_subprocess() def test_setreuid_neg1(self): # Needs to accept -1. We run this in a subprocess to avoid # altering the test runner's process state (issue8045). @@ -2323,6 +2327,7 @@ def test_setregid(self): self.assertRaises(OverflowError, os.setregid, 0, self.GID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()') + @requires_subprocess() def test_setregid_neg1(self): # Needs to accept -1. We run this in a subprocess to avoid # altering the test runner's process state (issue8045). @@ -2995,6 +3000,7 @@ def test_device_encoding(self): class PidTests(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid") + @requires_subprocess() def test_getppid(self): p = subprocess.Popen([sys.executable, '-c', 'import os; print(os.getppid())'], @@ -3003,6 +3009,7 @@ def test_getppid(self): # We are the parent of our subprocess self.assertEqual(int(stdout), os.getpid()) + @requires_fork() def check_waitpid(self, code, exitcode, callback=None): if sys.platform == 'win32': # On Windows, os.spawnv() simply joins arguments with spaces: @@ -3065,6 +3072,7 @@ def kill_process(pid): self.check_waitpid(code, exitcode=-signum, callback=kill_process) +@requires_fork() class SpawnTests(unittest.TestCase): def create_args(self, *, with_env=False, use_bytes=False): self.exitcode = 17 @@ -3147,6 +3155,7 @@ def test_spawnvpe(self): self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnv') + @requires_fork() def test_nowait(self): args = self.create_args() pid = os.spawnv(os.P_NOWAIT, args[0], args) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index bf3fc5fb249d21..3da409e99177fb 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -12,6 +12,7 @@ from unittest import mock from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper from test.support.os_helper import TESTFN, FakePath @@ -1468,6 +1469,7 @@ def _test_home(self, p): self.assertIs(type(p), type(q)) self.assertTrue(p.is_absolute()) + @unittest.skipIf(is_apple_mobile, "No home folder on Apple mobile") def test_home(self): with os_helper.EnvironmentVarGuard() as env: self._test_home(self.cls.home()) @@ -2546,6 +2548,7 @@ def test_rglob(self): 'pwd module does not expose getpwall()') @unittest.skipIf(sys.platform == "vxworks", "no home directory on VxWorks") + @unittest.skipIf(is_apple_mobile, "No home folder on Apple mobile") def test_expanduser(self): P = self.cls import_helper.import_module('pwd') diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 77f68084482d9c..94cde4008ea912 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -13,7 +13,7 @@ from contextlib import ExitStack, redirect_stdout from io import StringIO -from test.support import os_helper +from test.support import os_helper, requires_subprocess # This little helper class is essential for testing pdb under doctest. from test.test_doctest import _FakeInput from unittest.mock import patch @@ -1531,6 +1531,7 @@ def test_pdb_issue_gh_101673(): """ +@requires_subprocess() class PdbTestCase(unittest.TestCase): def tearDown(self): os_helper.unlink(os_helper.TESTFN) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index fff1b2452d2266..964e8fa3154b46 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -10,6 +10,14 @@ from test import support from test.support import os_helper +try: + # Some of the iOS tests need ctypes to operate. + # Confirm that the ctypes module is available + # is available. + import _ctypes +except ImportError: + _ctypes = None + FEDORA_OS_RELEASE = """\ NAME=Fedora VERSION="32 (Thirty Two)" @@ -228,6 +236,29 @@ def test_uname(self): self.assertEqual(res[-1], res.processor) self.assertEqual(len(res), 6) + if os.name == "posix": + uname = os.uname() + self.assertEqual(res.node, uname.nodename) + self.assertEqual(res.version, uname.version) + self.assertEqual(res.machine, uname.machine) + + if sys.platform == "android": + self.assertEqual(res.system, "Android") + self.assertEqual(res.release, platform.android_ver().release) + elif sys.platform == "ios": + # Platform module needs ctypes for full operation. If ctypes + # isn't available, there's no ObjC module, and dummy values are + # returned. + if _ctypes: + self.assertIn(res.system, {"iOS", "iPadOS"}) + self.assertEqual(res.release, platform.ios_ver().release) + else: + self.assertEqual(res.system, "") + self.assertEqual(res.release, "") + else: + self.assertEqual(res.system, uname.sysname) + self.assertEqual(res.release, uname.release) + def test_uname_cast_to_tuple(self): res = platform.uname() expected = ( @@ -277,6 +308,7 @@ def test_uname_asdict(self): self.assertIn('processor', res) @unittest.skipIf(sys.platform in ['win32', 'OpenVMS'], "uname -p not used") + @support.requires_subprocess() def test_uname_processor(self): """ On some systems, the processor must match the output @@ -318,6 +350,7 @@ def test_java_ver(self): def test_win32_ver(self): res = platform.win32_ver() + @support.requires_subprocess() def test_mac_ver(self): res = platform.mac_ver() @@ -370,6 +403,56 @@ def test_mac_ver_with_fork(self): # parent support.wait_process(pid, exitcode=0) + def test_ios_ver(self): + result = platform.ios_ver() + + # ios_ver is only fully available on iOS where ctypes is available. + if sys.platform == "ios" and _ctypes: + system, release, model, is_simulator = result + # Result is a namedtuple + self.assertEqual(result.system, system) + self.assertEqual(result.release, release) + self.assertEqual(result.model, model) + self.assertEqual(result.is_simulator, is_simulator) + + # We can't assert specific values without reproducing the logic of + # ios_ver(), so we check that the values are broadly what we expect. + + # System is either iOS or iPadOS, depending on the test device + self.assertIn(system, {"iOS", "iPadOS"}) + + # Release is a numeric version specifier with at least 2 parts + parts = release.split(".") + self.assertGreaterEqual(len(parts), 2) + self.assertTrue(all(part.isdigit() for part in parts)) + + # If this is a simulator, we get a high level device descriptor + # with no identifying model number. If this is a physical device, + # we get a model descriptor like "iPhone13,1" + if is_simulator: + self.assertIn(model, {"iPhone", "iPad"}) + else: + self.assertTrue( + (model.startswith("iPhone") or model.startswith("iPad")) + and "," in model + ) + + self.assertEqual(type(is_simulator), bool) + else: + # On non-iOS platforms, calling ios_ver doesn't fail; you get + # default values + self.assertEqual(result.system, "") + self.assertEqual(result.release, "") + self.assertEqual(result.model, "") + self.assertFalse(result.is_simulator) + + # Check the fallback values can be overridden by arguments + override = platform.ios_ver("Foo", "Bar", "Whiz", True) + self.assertEqual(override.system, "Foo") + self.assertEqual(override.release, "Bar") + self.assertEqual(override.model, "Whiz") + self.assertTrue(override.is_simulator) + def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception if os.path.isdir(sys.executable) and \ diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 82bbb3af9f1b39..2a09eed55b6db4 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -8,6 +8,7 @@ import time import unittest from test.support import cpython_only +from test.support import requires_subprocess from test.support import threading_helper from test.support.os_helper import TESTFN @@ -120,6 +121,7 @@ def fileno(self): # Another test case for poll(). This is copied from the test case for # select(), modified to use poll() instead. + @requires_subprocess() def test_poll2(self): cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, diff --git a/Lib/test/test_popen.py b/Lib/test/test_popen.py index cac2f6177f3257..50ceffc81b853e 100644 --- a/Lib/test/test_popen.py +++ b/Lib/test/test_popen.py @@ -7,7 +7,7 @@ from test import support import os, sys -if not hasattr(os, 'popen'): +if not hasattr(os, 'popen') or support.is_apple_mobile: raise unittest.SkipTest("need os.popen()") # Test that command-lines get down as we expect. diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index e8bb3ce2f21536..bc56b0a7a90eed 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,7 +1,7 @@ "Test posix functions" from test import support -from test.support import is_apple +from test.support import import_helper from test.support import os_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok @@ -1049,6 +1049,7 @@ def _create_and_do_getcwd(dirname, current_path_length = 0): @unittest.skipUnless(hasattr(posix, 'getgrouplist'), "test needs posix.getgrouplist()") @unittest.skipUnless(hasattr(pwd, 'getpwuid'), "test needs pwd.getpwuid()") @unittest.skipUnless(hasattr(os, 'getuid'), "test needs os.getuid()") + @unittest.skipIf(support.is_apple_mobile, "FIXME: edge case of getpwuid() on simulator") def test_getgrouplist(self): user = pwd.getpwuid(os.getuid())[0] group = pwd.getpwuid(os.getuid())[3] @@ -1057,6 +1058,7 @@ def test_getgrouplist(self): @unittest.skipUnless(hasattr(os, 'getegid'), "test needs os.getegid()") @unittest.skipUnless(hasattr(os, 'popen'), "test needs os.popen()") + @support.requires_subprocess() def test_getgroups(self): with os.popen('id -G 2>/dev/null') as idg: groups = idg.read().strip() @@ -1115,7 +1117,7 @@ def test_sched_priority(self): self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) # Apple plaforms return 15 without checking the argument. - if not is_apple: + if not support.is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 50cba256cef039..c0f99fddd51daf 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -5,6 +5,7 @@ from posixpath import realpath, abspath, dirname, basename from test import test_genericpath from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper from test.support.os_helper import FakePath from unittest import mock @@ -267,6 +268,7 @@ def test_expanduser_home_envvar(self): @unittest.skipIf(sys.platform == "vxworks", "no home directory on VxWorks") + @unittest.skipIf(is_apple_mobile, "FIXME: Edge case of getpwuid handling") def test_expanduser_pwd(self): pwd = import_helper.import_module('pwd') diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 5ed98dbff1737e..794d6436b61ab0 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -230,6 +230,7 @@ def setUp(self): def tearDown(self): os_helper.rmtree(self.directory) + @support.requires_subprocess() def pycompilecmd(self, *args, **kwargs): # assert_python_* helpers don't return proc object. We'll just use # subprocess.run() instead of spawn_python() and its friends to test diff --git a/Lib/test/test_quopri.py b/Lib/test/test_quopri.py index 715544c8a9669d..152d1858dcdd24 100644 --- a/Lib/test/test_quopri.py +++ b/Lib/test/test_quopri.py @@ -3,6 +3,7 @@ import sys, io, subprocess import quopri +from test import support ENCSAMPLE = b"""\ @@ -180,6 +181,7 @@ def test_decode_header(self): for p, e in self.HSTRINGS: self.assertEqual(quopri.decodestring(e, header=True), p) + @support.requires_subprocess() def test_scriptencode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri"], @@ -196,6 +198,7 @@ def test_scriptencode(self): self.assertEqual(cout[i], e[i]) self.assertEqual(cout, e) + @support.requires_subprocess() def test_scriptdecode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri", "-d"], diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 62e6c28280e61f..76ae976e69aeae 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -513,6 +513,7 @@ def parse_random_seed(self, output): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed + @support.requires_subprocess() def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 03bf8d8b5483fb..9775c3e89afe21 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -5,9 +5,10 @@ import unittest import subprocess from textwrap import dedent -from test.support import cpython_only, SuppressCrashReport +from test.support import cpython_only, SuppressCrashReport, requires_subprocess from test.support.script_helper import kill_python +@requires_subprocess() def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): """Run the Python REPL with the given arguments. diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index e6d36825f15a11..93e1ab556ae929 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,7 +12,7 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose +from test.support import no_tracing, verbose, requires_subprocess from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir from test.support.script_helper import make_script, make_zip_script @@ -780,6 +780,7 @@ def run(self, *args, **kwargs): ) super().run(*args, **kwargs) + @requires_subprocess() def assertSigInt(self, *args, **kwargs): proc = subprocess.run(*args, **kwargs, text=True, stderr=subprocess.PIPE) self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n")) diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py index cf32cf2f6a6f8b..810c67aa2b32f3 100644 --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -47,6 +47,7 @@ def test_returned_list_identity(self): self.assertIsNot(w, x) @unittest.skipUnless(hasattr(os, 'popen'), "need os.popen()") + @support.requires_subprocess() def test_select(self): code = textwrap.dedent(''' import time diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index f124709993b279..26d51621647656 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -89,6 +89,7 @@ def test_valid_signals(self): self.assertLess(len(s), signal.NSIG) @unittest.skipUnless(sys.executable, "sys.executable required.") + @support.requires_subprocess() def test_keyboard_interrupt_exit_code(self): """KeyboardInterrupt triggers exit via SIGINT.""" process = subprocess.run( @@ -139,6 +140,7 @@ def test_issue9324(self): signal.signal(7, handler) @unittest.skipUnless(sys.executable, "sys.executable required.") + @support.requires_subprocess() def test_keyboard_interrupt_exit_code(self): """KeyboardInterrupt triggers an exit using STATUS_CONTROL_C_EXIT.""" # We don't test via os.kill(os.getpid(), signal.CTRL_C_EVENT) here @@ -612,6 +614,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(signal, 'siginterrupt'), "needs signal.siginterrupt()") class SiginterruptTest(unittest.TestCase): + @support.requires_fork() def readpipe_interrupted(self, interrupt): """Perform a read during which a signal will arrive. Return True if the read is interrupted by the signal and raises an exception. Return False diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index c70e1fa9ae1041..18086ac55b92bc 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -221,6 +221,7 @@ def test_addsitedir_hidden_flags(self): pth_file.cleanup() @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows') + @support.requires_subprocess() def test_addsitedir_hidden_file_attribute(self): pth_file = PthFile() pth_file.cleanup(prep=True) @@ -249,6 +250,7 @@ def test_get_path(self): @unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 " "user-site (site.ENABLE_USER_SITE)") + @support.requires_subprocess() def test_s_option(self): # (ncoghlan) Change this to use script_helper... usersite = os.path.normpath(site.USER_SITE) @@ -534,6 +536,7 @@ def test_license_exists_at_url(self): class StartupImportTests(unittest.TestCase): + @support.requires_subprocess() def test_startup_imports(self): # Get sys.path in isolated mode (python3 -I) popen = subprocess.Popen([sys.executable, '-I', '-c', @@ -582,17 +585,20 @@ def test_startup_imports(self): }.difference(sys.builtin_module_names) self.assertFalse(modules.intersection(collection_mods), stderr) + @support.requires_subprocess() def test_startup_interactivehook(self): r = subprocess.Popen([sys.executable, '-c', 'import sys; sys.exit(hasattr(sys, "__interactivehook__"))']).wait() self.assertTrue(r, "'__interactivehook__' not added by site") + @support.requires_subprocess() def test_startup_interactivehook_isolated(self): # issue28192 readline is not automatically enabled in isolated mode r = subprocess.Popen([sys.executable, '-I', '-c', 'import sys; sys.exit(hasattr(sys, "__interactivehook__"))']).wait() self.assertFalse(r, "'__interactivehook__' added in isolated mode") + @support.requires_subprocess() def test_startup_interactivehook_isolated_explicit(self): # issue28192 readline can be explicitly enabled in isolated mode r = subprocess.Popen([sys.executable, '-I', '-c', @@ -631,6 +637,7 @@ def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): sys_path.append(abs_path) return sys_path + @support.requires_subprocess() def test_underpth_nosite_file(self): libpath = os.path.dirname(os.path.dirname(encodings.__file__)) exe_prefix = os.path.dirname(sys.executable) @@ -659,6 +666,7 @@ def test_underpth_nosite_file(self): "sys.path is incorrect" ) + @support.requires_subprocess() def test_underpth_file(self): libpath = os.path.dirname(os.path.dirname(encodings.__file__)) exe_prefix = os.path.dirname(sys.executable) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 3b4f5e8969ca6f..99d267a663e5bb 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -701,7 +701,7 @@ def setUp(self): super().setUp() def bindSock(self, sock): - path = tempfile.mktemp(dir=self.dir_path) + path = socket_helper.create_unix_domain_name() socket_helper.bind_unix_socket(sock, path) self.addCleanup(os_helper.unlink, path) @@ -1925,12 +1925,13 @@ def test_socket_fileno(self): self._test_socket_fileno(s, socket.AF_INET6, socket.SOCK_STREAM) if hasattr(socket, "AF_UNIX"): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) + unix_name = socket_helper.create_unix_domain_name() + self.addCleanup(os_helper.unlink, unix_name) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.addCleanup(s.close) try: - s.bind(os.path.join(tmpdir, 'socket')) + s.bind(unix_name) except PermissionError: pass else: diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 211321f37617e9..faa038816a7b5a 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -96,8 +96,7 @@ def pickaddr(self, proto): else: # XXX: We need a way to tell AF_UNIX to pick its own name # like AF_INET provides port==0. - dir = None - fn = tempfile.mktemp(prefix='unix_socket.', dir=dir) + fn = socket_helper.create_unix_domain_name() self.test_files.append(fn) return fn diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 219c25cd2f6c4a..d2ad941c051ccb 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -1,7 +1,7 @@ # -*- coding: koi8-r -*- import unittest -from test.support import script_helper, captured_stdout +from test.support import script_helper, captured_stdout, requires_subprocess, is_apple_mobile from test.support.os_helper import TESTFN, unlink, rmtree from test.support.import_helper import unload import importlib @@ -12,13 +12,14 @@ class MiscSourceEncodingTest(unittest.TestCase): + @unittest.skipIf(is_apple_mobile, "FIXME: Edge case of encoding") def test_pep263(self): self.assertEqual( - "ðÉÔÏÎ".encode("utf-8"), + "�����".encode("utf-8"), b'\xd0\x9f\xd0\xb8\xd1\x82\xd0\xbe\xd0\xbd' ) self.assertEqual( - "\ð".encode("utf-8"), + "\�".encode("utf-8"), b'\\\xd0\x9f' ) @@ -65,6 +66,7 @@ def test_issue7820(self): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') + @requires_subprocess() def test_20731(self): sub = subprocess.Popen([sys.executable, os.path.join(os.path.dirname(__file__), diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 0d3afe612a3afe..3fc38f8a754f32 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -48,6 +48,8 @@ if support.PGO: raise unittest.SkipTest("test is not helpful for PGO") +if not support.has_subprocess_support: + raise unittest.SkipTest("test requires subprocess support") mswindows = (sys.platform == "win32") diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index 007d68817c677e..e64733e71f2a0c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -2,6 +2,7 @@ import importlib import platform import sys +import sys from test import support from test.support import import_helper from test.support import warnings_helper @@ -22,7 +23,8 @@ def test_untested_modules_can_be_imported(self): import distutils.bcppcompiler import distutils.ccompiler - import distutils.cygwinccompiler + if sys.platform.startswith("win"): + import distutils.cygwinccompiler import distutils.filelist import distutils.text_file import distutils.unixccompiler diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 79290986c494d5..86c2e6ff817a90 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -489,6 +489,7 @@ def test_reap_children(self): # pending child process support.reap_children() + @support.requires_subprocess() def check_options(self, args, func, expected=None): code = f'from test.support import {func}; print(repr({func}()))' cmd = [sys.executable, *args, '-c', code] diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1094d40849d8be..61f6c6b2840f3a 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -633,6 +633,7 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() + @support.requires_subprocess() def test_ioencoding(self): env = dict(os.environ) @@ -680,6 +681,7 @@ def test_ioencoding(self): 'requires OS support of non-ASCII encodings') @unittest.skipUnless(sys.getfilesystemencoding() == locale.getpreferredencoding(False), 'requires FS encoding to match locale') + @support.requires_subprocess() def test_ioencoding_nonascii(self): env = dict(os.environ) @@ -692,6 +694,7 @@ def test_ioencoding_nonascii(self): @unittest.skipIf(sys.base_prefix != sys.prefix, 'Test is not venv-compatible') + @support.requires_subprocess() def test_executable(self): # sys.executable should be absolute self.assertEqual(os.path.abspath(sys.executable), sys.executable) @@ -726,6 +729,7 @@ def test_getfilesystemencoding(self): expected = None self.check_fsencoding(fs_encoding, expected) + @support.requires_subprocess() def c_locale_get_error_handler(self, locale, isolated=False, encoding=None): # Force the POSIX locale env = os.environ.copy() @@ -933,6 +937,7 @@ def test_getandroidapilevel(self): self.assertIsInstance(level, int) self.assertGreater(level, 0) + @support.requires_subprocess() def test_sys_tracebacklimit(self): code = """if 1: import sys @@ -979,6 +984,7 @@ def test__enablelegacywindowsfsencoding(self): out = out.decode('ascii', 'replace').rstrip() self.assertEqual(out, 'mbcs replace') + @support.requires_subprocess() def test_orig_argv(self): code = textwrap.dedent(''' import sys diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 5ee9839c0487d1..e4b37afab4e96c 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -5,7 +5,7 @@ import shutil from copy import copy -from test.support import (captured_stdout, PythonSymlink) +from test.support import (is_apple_mobile, captured_stdout, PythonSymlink) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) @@ -258,12 +258,14 @@ def test_get_platform(self): # XXX more platforms to tests here + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) def test_get_scheme_names(self): - wanted = ['nt', 'posix_home', 'posix_prefix'] + wanted = ['nt', 'nt_venv', 'posix_home', 'posix_prefix', 'posix_venv'] if HAS_USER_BASE: wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) self.assertEqual(get_scheme_names(), tuple(sorted(wanted))) @@ -364,6 +366,8 @@ def test_platform_in_subprocess(self): self.assertEqual(status, 0) self.assertEqual(my_platform, test_platform) + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -440,6 +444,8 @@ class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 6e06bfc7f2f78d..e1fb2a39750988 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -201,6 +201,7 @@ def supports_iter(self): @unittest.skipUnless(hasattr(os, 'fork'), "os.fork is required for this test") + @support.requires_fork() def test_process_awareness(self): # ensure that the random source differs between # child and parent. @@ -462,6 +463,7 @@ def test_file_mode(self): self.assertEqual(mode, expected) @unittest.skipUnless(has_spawnl, 'os.spawnl not available') + @support.requires_fork() def test_noinherit(self): # _mkstemp_inner file handles are not inherited by child processes diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 4ae8a833b990d7..76ea55f14ad684 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -225,6 +225,7 @@ def setUp(self): self.read_fd, self.write_fd = os.pipe() @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') + @support.requires_fork() @threading_helper.reap_threads def test_forkinthread(self): pid = None diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index c54806e5946b68..1a01b06e7223d4 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1258,6 +1258,7 @@ def test_releasing_unacquired_lock(self): lock = threading.Lock() self.assertRaises(RuntimeError, lock.release) + @support.requires_subprocess() def test_recursion_limit(self): # Issue 9670 # test that excessive recursion within a non-main thread causes diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 18cd4aba24af26..1c65268411ece6 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -130,6 +130,7 @@ def __str__(self): str_name = '.'.join([X.__module__, X.__qualname__]) self.assertEqual(err[0], "%s: %s\n" % (str_name, str_value)) + @support.requires_subprocess() def test_encoded_file(self): # Test that tracebacks are correctly printed for encoded source files: # - correct line number (Issue2384) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index c54e11380230f1..ebbd71aabe7bea 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -18,7 +18,7 @@ import shlex from test.support import (captured_stdout, captured_stderr, requires_zlib, skip_if_broken_multiprocessing_synchronize, - requires_subprocess) + requires_subprocess, is_apple_mobile) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py index aa166baa400bc7..964fd82e38f3d3 100644 --- a/Lib/test/test_wait3.py +++ b/Lib/test/test_wait3.py @@ -15,6 +15,9 @@ if not hasattr(os, 'wait3'): raise unittest.SkipTest("os.wait3 not defined") +if support.is_apple_mobile: + raise unittest.SkipTest("os.wait3 doesn't work on Apple mobile") + class Wait3Test(ForkWait): def wait_impl(self, cpid, *, exitcode): # This many iterations can be required, since some previously run diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 673cc995d3f5a4..547fcf40ac7bbc 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -5,8 +5,10 @@ import subprocess from unittest import mock from test import support +from test.support import is_apple_mobile from test.support import import_helper from test.support import os_helper +from test.support import requires_subprocess URL = 'http://www.example.com' @@ -22,6 +24,7 @@ def wait(self, seconds=None): return 0 +@requires_subprocess() class CommandTestMixin: def _test(self, meth, *, args=[URL], kw={}, options, arguments): @@ -217,6 +220,73 @@ def test_open_new_tab(self): arguments=['openURL({},new-tab)'.format(URL)]) +@unittest.skipUnless(sys.platform == "ios", "Test only applicable to iOS") +class IOSBrowserTest(unittest.TestCase): + def _obj_ref(self, *args): + # Construct a string representation of the arguments that can be used + # as a proxy for object instance references + return "|".join(str(a) for a in args) + + @unittest.skipIf(getattr(webbrowser, "objc", None) is None, + "iOS Webbrowser tests require ctypes") + def setUp(self): + # Intercept the the objc library. Wrap the calls to get the + # references to classes and selectors to return strings, and + # wrap msgSend to return stringified object references + self.orig_objc = webbrowser.objc + + webbrowser.objc = mock.Mock() + webbrowser.objc.objc_getClass = lambda cls: f"C#{cls.decode()}" + webbrowser.objc.sel_registerName = lambda sel: f"S#{sel.decode()}" + webbrowser.objc.objc_msgSend.side_effect = self._obj_ref + + def tearDown(self): + webbrowser.objc = self.orig_objc + + def _test(self, meth, **kwargs): + # The browser always gets focus, there's no concept of separate browser + # windows, and there's no API-level control over creating a new tab. + # Therefore, all calls to webbrowser are effectively the same. + getattr(webbrowser, meth)(URL, **kwargs) + + # The ObjC String version of the URL is created with UTF-8 encoding + url_string_args = [ + "C#NSString", + "S#stringWithCString:encoding:", + b'http://www.example.com', + 4, + ] + # The NSURL version of the URL is created from that string + url_obj_args = [ + "C#NSURL", + "S#URLWithString:", + self._obj_ref(*url_string_args), + ] + # The openURL call is invoked on the shared application + shared_app_args = ["C#UIApplication", "S#sharedApplication"] + + # Verify that the last call is the one that opens the URL. + webbrowser.objc.objc_msgSend.assert_called_with( + self._obj_ref(*shared_app_args), + "S#openURL:options:completionHandler:", + self._obj_ref(*url_obj_args), + None, + None + ) + + def test_open(self): + self._test('open') + + def test_open_with_autoraise_false(self): + self._test('open', autoraise=False) + + def test_open_new(self): + self._test('open_new') + + def test_open_new_tab(self): + self._test('open_new_tab') + + class BrowserRegistrationTest(unittest.TestCase): def setUp(self): @@ -300,6 +370,10 @@ def test_synthesize(self): webbrowser.register(name, None, webbrowser.GenericBrowser(name)) webbrowser.get(sys.executable) + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: @@ -311,6 +385,10 @@ def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') webbrowser.get() + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment_preferred(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index b4f71a8b3f64df..50bf8f4690d1e8 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -21,7 +21,7 @@ from tempfile import TemporaryFile from random import randint, random, randbytes -from test.support import script_helper +from test.support import script_helper, requires_subprocess from test.support import (findfile, requires_zlib, requires_bz2, requires_lzma, captured_stdout) from test.support.os_helper import TESTFN, unlink, rmtree, temp_dir, temp_cwd @@ -2839,6 +2839,7 @@ def test_read_zip64_with_exe_prepended(self): @unittest.skipUnless(sys.executable, 'sys.executable required.') @unittest.skipUnless(os.access('/bin/bash', os.X_OK), 'Test relies on #!/bin/bash working.') + @requires_subprocess() def test_execute_zip2(self): output = subprocess.check_output([self.exe_zip, sys.executable]) self.assertIn(b'number in executable: 5', output) @@ -2846,6 +2847,7 @@ def test_execute_zip2(self): @unittest.skipUnless(sys.executable, 'sys.executable required.') @unittest.skipUnless(os.access('/bin/bash', os.X_OK), 'Test relies on #!/bin/bash working.') + @requires_subprocess() def test_execute_zip64(self): output = subprocess.check_output([self.exe_zip64, sys.executable]) self.assertIn(b'number in executable: 5', output) diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 7bf50a33728e53..d172895be51e19 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -13,7 +13,7 @@ import inspect import linecache import unittest -from test.support import os_helper +from test.support import os_helper, is_apple_mobile from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) @@ -92,6 +92,7 @@ def test_inspect_getsource_issue4223(self): finally: del sys.modules["zip_pkg"] + @unittest.skipIf(is_apple_mobile, "FIXME: Edge case of doctest importer") def test_doctest_issue4197(self): # To avoid having to keep two copies of the doctest module's # unit tests in sync, this test works by taking the source of diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py index b7fbbc1e7baddd..a544819d516a33 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/unittest/test/test_program.py @@ -1,5 +1,3 @@ -import io - import os import sys import subprocess @@ -427,6 +425,7 @@ def testParseArgsSelectedTestNames(self): self.assertEqual(program.testNamePatterns, ['*foo*', '*bar*', '*pat*']) + @support.requires_subprocess() def testSelectedTestNamesFunctionalTest(self): def run_unittest(args): p = subprocess.Popen([sys.executable, '-m', 'unittest'] + args, diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index 0082d394dc954a..c6ef4851745669 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -9,6 +9,7 @@ from unittest.test.support import (LoggingResult, ResultWithNoStartTestRunStopTestRun) +from test import support def resultFactory(*_): @@ -1176,6 +1177,7 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @support.requires_subprocess() def test_warnings(self): """ Check that warnings argument of TextTestRunner correctly affects the diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index ec3cece48c9587..34a772ab3c41db 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -532,6 +532,9 @@ def register_standard_browsers(): # OS X can use below Unix support (but we prefer using the OS X # specific stuff) + if sys.platform == "ios": + register("iosbrowser", None, IOSBrowser(), preferred=True) + if sys.platform == "serenityos": # SerenityOS webbrowser, simply called "Browser". register("Browser", None, BackgroundBrowser("Browser")) @@ -688,6 +691,70 @@ def open(self, url, new=0, autoraise=True): rc = osapipe.close() return not rc +# +# Platform support for iOS +# +if sys.platform == "ios": + from _ios_support import objc + if objc: + # If objc exists, we know ctypes is also importable. + from ctypes import c_void_p, c_char_p, c_ulong + + class IOSBrowser(BaseBrowser): + def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + # If ctypes isn't available, we can't open a browser + if objc is None: + return False + + # All the messages in this call return object references. + objc.objc_msgSend.restype = c_void_p + + # This is the equivalent of: + # NSString url_string = + # [NSString stringWithCString:url.encode("utf-8") + # encoding:NSUTF8StringEncoding]; + NSString = objc.objc_getClass(b"NSString") + constructor = objc.sel_registerName(b"stringWithCString:encoding:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_char_p, c_ulong] + url_string = objc.objc_msgSend( + NSString, + constructor, + url.encode("utf-8"), + 4, # NSUTF8StringEncoding = 4 + ) + + # Create an NSURL object representing the URL + # This is the equivalent of: + # NSURL *nsurl = [NSURL URLWithString:url]; + NSURL = objc.objc_getClass(b"NSURL") + urlWithString_ = objc.sel_registerName(b"URLWithString:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_void_p] + ns_url = objc.objc_msgSend(NSURL, urlWithString_, url_string) + + # Get the shared UIApplication instance + # This code is the equivalent of: + # UIApplication shared_app = [UIApplication sharedApplication] + UIApplication = objc.objc_getClass(b"UIApplication") + sharedApplication = objc.sel_registerName(b"sharedApplication") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + shared_app = objc.objc_msgSend(UIApplication, sharedApplication) + + # Open the URL on the shared application + # This code is the equivalent of: + # [shared_app openURL:ns_url + # options:NIL + # completionHandler:NIL]; + openURL_ = objc.sel_registerName(b"openURL:options:completionHandler:") + objc.objc_msgSend.argtypes = [ + c_void_p, c_void_p, c_void_p, c_void_p, c_void_p + ] + # Method returns void + objc.objc_msgSend.restype = None + objc.objc_msgSend(shared_app, openURL_, ns_url, None, None) + + return True + def main(): import getopt diff --git a/Makefile.pre.in b/Makefile.pre.in index dbac7e91964a42..29110df6e7d2c4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -178,12 +178,18 @@ PYTHONFRAMEWORKPREFIX= @PYTHONFRAMEWORKPREFIX@ PYTHONFRAMEWORKINSTALLDIR= @PYTHONFRAMEWORKINSTALLDIR@ PYTHONFRAMEWORKINSTALLNAMEPREFIX= @PYTHONFRAMEWORKINSTALLNAMEPREFIX@ RESSRCDIR= @RESSRCDIR@ -# Deployment target selected during configure, to be checked +# macOS deployment target selected during configure, to be checked # by distutils. The export statement is needed to ensure that the # deployment target is active during build. MACOSX_DEPLOYMENT_TARGET=@CONFIGURE_MACOSX_DEPLOYMENT_TARGET@ @EXPORT_MACOSX_DEPLOYMENT_TARGET@export MACOSX_DEPLOYMENT_TARGET +# iOS Deployment target selected during configure. Unlike macOS, the iOS +# deployment target is controlled using `-mios-version-min` arguments added to +# CFLAGS and LDFLAGS by the configure script. This variable is not used during +# the build, and is only listed here so it will be included in sysconfigdata. +IPHONEOS_DEPLOYMENT_TARGET=@IPHONEOS_DEPLOYMENT_TARGET@ + # Option to install to strip binaries STRIPFLAG=-s @@ -1282,11 +1288,23 @@ testios: cp -r $(srcdir)/iOS/testbed $(XCFOLDER) # Copy the framework from the install location to the testbed project. cp -r $(PYTHONFRAMEWORKPREFIX)/* $(XCFOLDER)/Python.xcframework/ios-arm64_x86_64-simulator + # Run the test suite for the Xcode project, targeting the iOS simulator. - # If the suite fails, extract and print the console output, then re-raise the failure + # If the suite fails, touch a file in the test folder as a marker if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) ; then \ - xcrun xcresulttool get --path $(XCRESULT) --id $$(xcrun xcresulttool get --path $(XCRESULT) --format json | $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])"); \ - echo ; \ + touch $(XCFOLDER)/failed; \ + fi + + # Regardless of success or failure, extract and print the test output + xcrun xcresulttool get --path $(XCRESULT) \ + --id $$( \ + xcrun xcresulttool get --path $(XCRESULT) --format json | \ + $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])" \ + ) \ + --format json | \ + $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['subsections']['_values'][1]['subsections']['_values'][0]['emittedOutput']['_value'])" + + @if test -e $(XCFOLDER)/failed ; then \ exit 1; \ fi @@ -1919,8 +1937,8 @@ frameworkinstallmobileheaders: frameworkinstallunversionedstructure inclinstall echo "Removing old framework headers"; \ rm -rf $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; \ fi - mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" - $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" + mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" + $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status diff --git a/Misc/NEWS.d/next/Library/2024-03-19-14-35-57.gh-issue-114099.siNSpK.rst b/Misc/NEWS.d/next/Library/2024-03-19-14-35-57.gh-issue-114099.siNSpK.rst new file mode 100644 index 00000000000000..9b57cbb812db4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-19-14-35-57.gh-issue-114099.siNSpK.rst @@ -0,0 +1 @@ +Modify standard library to allow for iOS platform differences. diff --git a/Modules/getpath.c b/Modules/getpath.c index ef6dd59a084d8d..e8f91831286e5e 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -10,6 +10,7 @@ #include #ifdef __APPLE__ +# include "TargetConditionals.h" # include #endif @@ -1090,7 +1091,7 @@ resolve_symlinks(wchar_t **path_p) #endif /* HAVE_READLINK */ -#ifdef WITH_NEXT_FRAMEWORK +#if defined(WITH_NEXT_FRAMEWORK) && !defined(TARGET_OS_IPHONE) static PyStatus calculate_argv0_path_framework(PyCalculatePath *calculate, _PyPathConfig *pathconfig) { @@ -1184,7 +1185,7 @@ calculate_argv0_path(PyCalculatePath *calculate, return _PyStatus_NO_MEMORY(); } -#ifdef WITH_NEXT_FRAMEWORK +#if defined(WITH_NEXT_FRAMEWORK) && !defined(TARGET_OS_IPHONE) status = calculate_argv0_path_framework(calculate, pathconfig); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index feffa43cfd0de4..50397b35207581 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -332,8 +332,6 @@ corresponding Unix manual entries for more information on calls."); # else /* Unix functions that the configure script doesn't check for */ # ifndef __VXWORKS__ -# define HAVE_EXECV 1 -# define HAVE_FORK 1 # if defined(__USLC__) && defined(__SCO_VERSION__) /* SCO UDK Compiler */ # define HAVE_FORK1 1 # endif @@ -346,7 +344,6 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_KILL 1 # define HAVE_OPENDIR 1 # define HAVE_PIPE 1 -# define HAVE_SYSTEM 1 # define HAVE_WAIT 1 # define HAVE_TTYNAME 1 # endif /* _MSC_VER */ diff --git a/Python/marshal.c b/Python/marshal.c index 67540e08ed9922..062e3047fb85cd 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -14,6 +14,10 @@ #include "marshal.h" #include "pycore_hashtable.h" +#ifdef __APPLE__ +# include "TargetConditionals.h" +#endif /* __APPLE__ */ + /*[clinic input] module marshal [clinic start generated code]*/ @@ -33,9 +37,12 @@ module marshal * #if defined(MS_WINDOWS) && defined(_DEBUG) */ #if defined(MS_WINDOWS) -#define MAX_MARSHAL_STACK_DEPTH 1000 +# define MAX_MARSHAL_STACK_DEPTH 1000 +// TARGET_OS_IPHONE covers any non-macOS Apple platform. +#elif defined(__APPLE__) && TARGET_OS_IPHONE +# define MAX_MARSHAL_STACK_DEPTH 1500 #else -#define MAX_MARSHAL_STACK_DEPTH 2000 +# define MAX_MARSHAL_STACK_DEPTH 2000 #endif #define TYPE_NULL '0' diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 50cf340e543b40..bd2c036bb53b04 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -40,6 +40,7 @@ static const char* _Py_stdlib_module_names[] = { "_heapq", "_imp", "_io", +"_ios_support", "_json", "_locale", "_lsprof", diff --git a/configure b/configure index 405f419a5bb78d..042a2fd4b84b72 100755 --- a/configure +++ b/configure @@ -738,7 +738,7 @@ CPPFLAGS LDFLAGS CFLAGS CC -IOS_DEPLOYMENT_TARGET +IPHONEOS_DEPLOYMENT_TARGET EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET _PYTHON_HOST_PLATFORM @@ -3001,7 +3001,7 @@ if test "$cross_compiling" = yes; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for python interpreter for cross build" >&5 $as_echo_n "checking for python interpreter for cross build... " >&6; } if test -z "$PYTHON_FOR_BUILD"; then - for interp in python$PACKAGE_VERSION python3 python; do + for interp in $PYTHON_FOR_REGEN python$PACKAGE_VERSION python3 python; do which $interp >/dev/null 2>&1 || continue if $interp -c "import sys;sys.exit(not '.'.join(str(n) for n in sys.version_info[:2]) == '$PACKAGE_VERSION')"; then break @@ -3019,6 +3019,8 @@ elif test "$cross_compiling" = maybe; then as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5 else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $interp" >&5 +$as_echo "$interp" >&6; } fi @@ -3577,15 +3579,16 @@ if test "$cross_compiling" = yes; then _host_device=`echo $host | cut -d '-' -f4` _host_device=${_host_device:=os} - IOS_DEPLOYMENT_TARGET=${_host_os:3} - IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=12.0} case "$host_cpu" in aarch64) - _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} ;; *) - _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} ;; esac ;; @@ -3729,6 +3732,9 @@ fi CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. + + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -3754,9 +3760,8 @@ fi case $ac_sys_system in #( iOS) : - as_fn_append CFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" - as_fn_append LDFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" - + as_fn_append CFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" + as_fn_append LDFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" ;; #( *) : ;; @@ -12271,7 +12276,7 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - for ac_func in getentropy getgroups + for ac_func in getentropy getgroups system do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -17674,7 +17679,15 @@ $as_echo_n "checking for ensurepip... " >&6; } if test "${with_ensurepip+set}" = set; then : withval=$with_ensurepip; else - with_ensurepip=upgrade + + case $ac_sys_system in #( + iOS) : + with_ensurepip=no ;; #( + *) : + with_ensurepip=upgrade + ;; +esac + fi case $with_ensurepip in #( diff --git a/configure.ac b/configure.ac index ddb9c49aa6f13e..85294a21d03762 100644 --- a/configure.ac +++ b/configure.ac @@ -71,7 +71,7 @@ AC_SUBST(PYTHON_FOR_REGEN) if test "$cross_compiling" = yes; then AC_MSG_CHECKING([for python interpreter for cross build]) if test -z "$PYTHON_FOR_BUILD"; then - for interp in python$PACKAGE_VERSION python3 python; do + for interp in $PYTHON_FOR_REGEN python$PACKAGE_VERSION python3 python; do which $interp >/dev/null 2>&1 || continue if $interp -c "import sys;sys.exit(not '.'.join(str(n) for n in sys.version_info@<:@:2@:>@) == '$PACKAGE_VERSION')"; then break @@ -88,6 +88,7 @@ elif test "$cross_compiling" = maybe; then AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH]) else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' + AC_MSG_RESULT($interp) fi AC_SUBST(PYTHON_FOR_BUILD) @@ -629,16 +630,16 @@ if test "$cross_compiling" = yes; then _host_device=`echo $host | cut -d '-' -f4` _host_device=${_host_device:=os} - dnl IOS_DEPLOYMENT_TARGET is the minimum supported iOS version - IOS_DEPLOYMENT_TARGET=${_host_os:3} - IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=12.0} case "$host_cpu" in aarch64) - _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} ;; *) - _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} ;; esac ;; @@ -773,6 +774,9 @@ AC_SUBST(EXPORT_MACOSX_DEPLOYMENT_TARGET) CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. +AC_SUBST([IPHONEOS_DEPLOYMENT_TARGET]) + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -798,9 +802,8 @@ fi dnl Add the compiler flag for the iOS minimum supported OS version. AS_CASE([$ac_sys_system], [iOS], [ - AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) - AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) - AC_SUBST([IOS_DEPLOYMENT_TARGET]) + AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) + AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) ], ) @@ -4011,7 +4014,7 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - AC_CHECK_FUNCS([getentropy getgroups]) + AC_CHECK_FUNCS([ getentropy getgroups system ]) fi AC_CHECK_DECL(dirfd, @@ -6001,7 +6004,12 @@ AC_ARG_WITH(ensurepip, [AS_HELP_STRING([--with-ensurepip@<:@=install|upgrade|no@:>@], ["install" or "upgrade" using bundled pip (default is upgrade)])], [], - [with_ensurepip=upgrade]) + [ + AS_CASE([$ac_sys_system], + [iOS], [with_ensurepip=no], + [with_ensurepip=upgrade] + ) + ]) AS_CASE($with_ensurepip, [yes|upgrade],[ENSUREPIP=upgrade], [install],[ENSUREPIP=install], diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in index 52c0a6e7fd7a55..c3e261ecd9eff7 100644 --- a/iOS/Resources/Info.plist.in +++ b/iOS/Resources/Info.plist.in @@ -29,6 +29,6 @@ iPhoneOS MinimumOSVersion - @IOS_DEPLOYMENT_TARGET@ + @IPHONEOS_DEPLOYMENT_TARGET@ diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index 4389c08ac1960d..d57cfc3dbe0304 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -441,7 +441,7 @@ CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = 3HEZE76D99; + DEVELOPMENT_TEAM = ""; ENABLE_USER_SCRIPT_SANDBOXING = NO; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; INFOPLIST_FILE = "iOSTestbed/iOSTestbed-Info.plist"; @@ -471,7 +471,7 @@ CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = 3HEZE76D99; + DEVELOPMENT_TEAM = ""; ENABLE_TESTABILITY = YES; ENABLE_USER_SCRIPT_SANDBOXING = NO; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; diff --git a/pyconfig.h.in b/pyconfig.h.in index 57c84e5f8fac01..b40023b5f61bb2 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1122,6 +1122,9 @@ /* Define to 1 if you have the `sysconf' function. */ #undef HAVE_SYSCONF +/* Define to 1 if you have the `system' function. */ +#undef HAVE_SYSTEM + /* Define to 1 if you have the header file. */ #undef HAVE_SYSEXITS_H @@ -1699,4 +1702,3 @@ #endif #endif /*Py_PYCONFIG_H*/ - diff --git a/setup.py b/setup.py index 2f6d174bcb7d6b..09ea06965e92aa 100644 --- a/setup.py +++ b/setup.py @@ -1042,8 +1042,10 @@ def detect_simple_extensions(self): self.add(Extension('_csv', ['_csv.c'])) # POSIX subprocess module helper. - self.add(Extension('_posixsubprocess', ['_posixsubprocess.c'], - extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) + # iOS/tvOS/watchOS doesn't have this. + if not (IOS or TVOS or WATCHOS): + self.add(Extension('_posixsubprocess', ['_posixsubprocess.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) def detect_test_extensions(self): # Python C API test module @@ -1533,6 +1535,15 @@ class db_found(Exception): pass ], libraries=dblibs) break + + # iOS, tvOS and watchOS have DBM in the system libraries. + if IOS or TVOS or WATCHOS: + dbmext = Extension('_dbm', ['_dbmmodule.c'], + define_macros=[ + ('HAVE_NDBM_H',None), + ], + libraries=['dbm']) + if dbmext is not None: self.add(dbmext) else: @@ -1612,7 +1623,8 @@ def detect_sqlite(self): if sqlite_libfile: sqlite_libdir = [os.path.abspath(os.path.dirname(sqlite_libfile))] - if sqlite_incdir and sqlite_libdir: + # iOS, tvOS and watchOS provide SQLITE as part of the system libraries. + if (sqlite_incdir and sqlite_libdir) or IOS or TVOS or WATCHOS: sqlite_srcs = ['_sqlite/cache.c', '_sqlite/connection.c', '_sqlite/cursor.c', @@ -1645,7 +1657,7 @@ def detect_sqlite(self): # Only include the directory where sqlite was found if it does # not already exist in set include directories, otherwise you # can end up with a bad search path order. - if sqlite_incdir not in self.compiler.include_dirs: + if sqlite_incdir and sqlite_incdir not in self.compiler.include_dirs: include_dirs.append(sqlite_incdir) # avoid a runtime library path for a system library dir if sqlite_libdir and sqlite_libdir[0] in self.lib_dirs: @@ -1728,6 +1740,9 @@ def detect_compress_exts(self): self.missing.append('zlib') else: self.missing.append('zlib') + elif IOS or TVOS or WATCHOS: + # iOS/tvOS/watchOS include zlib in the system libraries. + self.add(Extension('zlib', ['zlibmodule.c'], libraries=['z'])) else: self.missing.append('zlib')