diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 35b6a822a0dfffd..3fd23b9313c4536 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -56,7 +56,11 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; assert(PyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); +#else return mp->ma_used; +#endif } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index de2013323d2c720..5433ba48eefc692 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -8,7 +8,7 @@ extern "C" { #endif -#define COMMON_FIELDS(PREFIX) \ +#define _Py_COMMON_FIELDS(PREFIX) \ PyObject *PREFIX ## globals; \ PyObject *PREFIX ## builtins; \ PyObject *PREFIX ## name; \ @@ -19,7 +19,7 @@ extern "C" { PyObject *PREFIX ## closure; /* NULL or a tuple of cell objects */ typedef struct { - COMMON_FIELDS(fc_) + _Py_COMMON_FIELDS(fc_) } PyFrameConstructor; /* Function objects and code objects should not be confused with each other: @@ -35,7 +35,7 @@ typedef struct { typedef struct { PyObject_HEAD - COMMON_FIELDS(func_) + _Py_COMMON_FIELDS(func_) PyObject *func_doc; /* The __doc__ attribute, can be anything */ PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ PyObject *func_weakreflist; /* List of weak references */ @@ -60,6 +60,8 @@ typedef struct { */ } PyFunctionObject; +#undef _Py_COMMON_FIELDS + PyAPI_DATA(PyTypeObject) PyFunction_Type; #define PyFunction_Check(op) Py_IS_TYPE((op), &PyFunction_Type) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 946f82ae3c20e38..8d88b5c1d15cb84 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -182,7 +182,7 @@ static inline void _Py_LeaveRecursiveCall(void) { extern struct _PyInterpreterFrame* _PyEval_GetFrame(void); -extern PyObject* _Py_MakeCoro(PyFunctionObject *func); +PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 5c55282fa39e6f6..bc8100b58e8ea32 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -290,6 +290,8 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ +extern void _Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsize, Py_ssize_t *rootsize); + // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. // Usage: diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 74d9e4cac72c0e9..f913928f38bd058 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -110,7 +110,17 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) return code->co_framesize - FRAME_SPECIALS_SIZE; } -void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); +static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) +{ + assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); + *dest = *src; + for (int i = 1; i < src->stacktop; i++) { + dest->localsplus[i] = src->localsplus[i]; + } + // Don't leave a dangling pointer to the old frame when creating generators + // and coroutines: + dest->previous = NULL; +} /* Consumes reference to func and locals. Does not initialize frame->previous, which happens @@ -256,7 +266,7 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) extern _PyInterpreterFrame * _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); -void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); /* Pushes a frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 88b052f4544b155..7df8003196d8cc0 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -688,7 +688,7 @@ static inline PyDictObject * _PyObject_GetManagedDict(PyObject *obj) { PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj); - return (PyDictObject *)FT_ATOMIC_LOAD_PTR_RELAXED(dorv->dict); + return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict); } static inline PyDictValues * diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5636debbf4a7f2c..400d7c334db8e73 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -805,7 +805,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case RETURN_CONST: return 0; case RETURN_GENERATOR: - return 0; + return 1; case RETURN_VALUE: return 0; case SEND: @@ -1310,6 +1310,7 @@ _PyOpcode_macro_expansion[256] = { [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, 0, 0 } } }, [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, 0, 0 } } }, [RETURN_CONST] = { .nuops = 2, .uops = { { _LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, 0, 0 } } }, [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, 0, 0 } } }, diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 09c4501c38c935c..7e533bd138469ba 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -164,6 +164,7 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef2(PyTypeObject *, PyTypeObject *, PyModuleDef *); // PyType_Ready() must be called if _PyType_IsReady() is false. // See also the Py_TPFLAGS_READY flag. diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index f0558743b32f5e7..bb49d6e77d25622 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -231,6 +231,7 @@ extern "C" { #define _PUSH_NULL PUSH_NULL #define _REPLACE_WITH_TRUE 424 #define _RESUME_CHECK RESUME_CHECK +#define _RETURN_GENERATOR RETURN_GENERATOR #define _SAVE_RETURN_OFFSET 425 #define _SEND 426 #define _SEND_GEN SEND_GEN diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 2da4c4d4e21e939..b8cdfae83914603 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -219,6 +219,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -445,6 +446,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_NULL] = "_PUSH_NULL", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RETURN_GENERATOR] = "_RETURN_GENERATOR", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SET_ADD] = "_SET_ADD", @@ -894,6 +896,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _SET_FUNCTION_ATTRIBUTE: return 2; + case _RETURN_GENERATOR: + return 0; case _BUILD_SLICE: return 2 + ((oparg == 3) ? 1 : 0); case _CONVERT_VALUE: diff --git a/Include/object.h b/Include/object.h index ffcacf1a3ef4ed9..5aaf11c5194f0e3 100644 --- a/Include/object.h +++ b/Include/object.h @@ -349,7 +349,8 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) { #if defined(Py_GIL_DISABLED) - return (op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL); + return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == + _Py_IMMORTAL_REFCNT_LOCAL); #elif SIZEOF_VOID_P > 4 return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); #else diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 22cdfc93d8ad32d..8e4d49c859534d8 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -2142,6 +2142,9 @@ def is_loopback(self): RFC 2373 2.5.3. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_loopback return self._ip == 1 @property @@ -2258,7 +2261,7 @@ def is_unspecified(self): @property def is_loopback(self): - return self._ip == 1 and self.network.is_loopback + return super().is_loopback and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 7e5516eeccbe0bd..56b08f4c7234aa5 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -167,56 +167,76 @@ def splitdrive(p): return drive, root + tail -def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: - - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - empty = b'' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' - empty = '' - normp = p.replace(altsep, sep) - if normp[:1] == sep: - if normp[1:2] == sep: - # UNC drives, e.g. \\server\share or \\?\UNC\server\share - # Device drives, e.g. \\.\device or \\?\device - start = 8 if normp[:8].upper() == unc_prefix else 2 - index = normp.find(sep, start) - if index == -1: - return p, empty, empty - index2 = normp.find(sep, index + 1) - if index2 == -1: - return p, empty, empty - return p[:index2], p[index2:index2 + 1], p[index2 + 1:] +try: + from nt import _path_splitroot_ex +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. The drive is defined + exactly as in splitdrive(). On Windows, the root may be a single path + separator or an empty string. The tail contains anything after the root. + For example: + + splitroot('//server/share/') == ('//server/share', '/', '') + splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') + splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') + splitroot('Windows/notepad') == ('', '', 'Windows/notepad') + """ + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'\\' + altsep = b'/' + colon = b':' + unc_prefix = b'\\\\?\\UNC\\' + empty = b'' else: - # Relative path with root, e.g. \Windows - return empty, p[:1], p[1:] - elif normp[1:2] == colon: - if normp[2:3] == sep: - # Absolute drive-letter path, e.g. X:\Windows - return p[:2], p[2:3], p[3:] + sep = '\\' + altsep = '/' + colon = ':' + unc_prefix = '\\\\?\\UNC\\' + empty = '' + normp = p.replace(altsep, sep) + if normp[:1] == sep: + if normp[1:2] == sep: + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + start = 8 if normp[:8].upper() == unc_prefix else 2 + index = normp.find(sep, start) + if index == -1: + return p, empty, empty + index2 = normp.find(sep, index + 1) + if index2 == -1: + return p, empty, empty + return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + else: + # Relative path with root, e.g. \Windows + return empty, p[:1], p[1:] + elif normp[1:2] == colon: + if normp[2:3] == sep: + # Absolute drive-letter path, e.g. X:\Windows + return p[:2], p[2:3], p[3:] + else: + # Relative path with drive, e.g. X:Windows + return p[:2], empty, p[2:] else: - # Relative path with drive, e.g. X:Windows - return p[:2], empty, p[2:] - else: - # Relative path, e.g. Windows - return empty, empty, p + # Relative path, e.g. Windows + return empty, empty, p +else: + def splitroot(p): + """Split a pathname into drive, root and tail. The drive is defined + exactly as in splitdrive(). On Windows, the root may be a single path + separator or an empty string. The tail contains anything after the root. + For example: + + splitroot('//server/share/') == ('//server/share', '/', '') + splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') + splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') + splitroot('Windows/notepad') == ('', '', 'Windows/notepad') + """ + p = os.fspath(p) + if isinstance(p, bytes): + drive, root, tail = _path_splitroot_ex(os.fsdecode(p)) + return os.fsencode(drive), os.fsencode(root), os.fsencode(tail) + return _path_splitroot_ex(p) # Split a path in head (everything up to the last '/') and tail (the diff --git a/Lib/posixpath.py b/Lib/posixpath.py index f1960ddb88e5905..56b7915826daf4b 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -134,33 +134,53 @@ def splitdrive(p): return p[:0], p -def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: - - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'/' - empty = b'' - else: - sep = '/' - empty = '' - if p[:1] != sep: - # Relative path, e.g.: 'foo' - return empty, empty, p - elif p[1:2] != sep or p[2:3] == sep: - # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. - return empty, sep, p[1:] - else: - # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see - # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - return empty, p[:2], p[2:] +try: + from posix import _path_splitroot_ex +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. On Posix, drive is always + empty; the root may be empty, a single slash, or two slashes. The tail + contains anything after the root. For example: + + splitroot('foo/bar') == ('', '', 'foo/bar') + splitroot('/foo/bar') == ('', '/', 'foo/bar') + splitroot('//foo/bar') == ('', '//', 'foo/bar') + splitroot('///foo/bar') == ('', '/', '//foo/bar') + """ + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + empty = b'' + else: + sep = '/' + empty = '' + if p[:1] != sep: + # Relative path, e.g.: 'foo' + return empty, empty, p + elif p[1:2] != sep or p[2:3] == sep: + # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + return empty, sep, p[1:] + else: + # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + return empty, p[:2], p[2:] +else: + def splitroot(p): + """Split a pathname into drive, root and tail. On Posix, drive is always + empty; the root may be empty, a single slash, or two slashes. The tail + contains anything after the root. For example: + + splitroot('foo/bar') == ('', '', 'foo/bar') + splitroot('/foo/bar') == ('', '/', 'foo/bar') + splitroot('//foo/bar') == ('', '//', 'foo/bar') + splitroot('///foo/bar') == ('', '/', '//foo/bar') + """ + p = os.fspath(p) + if isinstance(p, bytes): + # Optimisation: the drive is always empty + _, root, tail = _path_splitroot_ex(os.fsdecode(p)) + return b'', os.fsencode(root), os.fsencode(tail) + return _path_splitroot_ex(p) # Return the tail (basename) part of a path, same as split(path)[1]. diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 9dcce7dc76ced48..57e6a3b4f1e6eba 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -26,6 +26,7 @@ def _iterdump(connection, *, filter=None): writeable_schema = False cu = connection.cursor() + cu.row_factory = None # Make sure we get predictable results. # Disable foreign key constraints, if there is any foreign key violation. violations = cu.execute("PRAGMA foreign_key_check").fetchall() if violations: diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 296601e8ee4f5f7..6ab7b685c2e18bd 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -153,7 +153,7 @@ class C(): pass # free-threading requires mimalloc (not malloc) -@support.requires_gil_enabled +@support.requires_gil_enabled() class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index c004f4637700190..e2e772a52d764e5 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1286,5 +1286,17 @@ def testfunc(n): self.assertEqual(res, 32 * 32) self.assertIsNone(ex) + def test_return_generator(self): + def gen(): + yield None + def testfunc(n): + for i in range(n): + gen() + return i + res, ex = self._run_with_optimizer(testfunc, 20) + self.assertEqual(res, 19) + self.assertIsNotNone(ex) + self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py index fe67bf9ecc88803..714853c7b4732d9 100644 --- a/Lib/test/test_gdb/test_backtrace.py +++ b/Lib/test/test_gdb/test_backtrace.py @@ -49,7 +49,7 @@ def test_bt_full(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") - @support.requires_gil_enabled + @support.requires_gil_enabled() @support.requires_resource('cpu') def test_threads(self): 'Verify that "py-bt" indicates threads that are waiting for the GIL' diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index f1519df673747a9..c3ecf2a742941a6 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2446,6 +2446,22 @@ def testIpv4MappedPrivateCheck(self): self.assertEqual( False, ipaddress.ip_address('::ffff:172.32.0.0').is_private) + def testIpv4MappedLoopbackCheck(self): + # test networks + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.100.200.254/128').is_loopback) + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.42.0.0/112').is_loopback) + self.assertEqual(False, ipaddress.ip_network( + '::ffff:128.0.0.0').is_loopback) + # test addresses + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.100.200.254').is_loopback) + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.42.0.0').is_loopback) + self.assertEqual(False, ipaddress.ip_address( + '::ffff:128.0.0.0').is_loopback) + def testAddrExclude(self): addr1 = ipaddress.ip_network('10.1.1.0/24') addr2 = ipaddress.ip_network('10.1.1.0/26') diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 31156130fcc7474..7f91bf1c2b837a4 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -374,6 +374,7 @@ def test_normpath(self): tester("ntpath.normpath('\\\\foo\\')", '\\\\foo\\') tester("ntpath.normpath('\\\\foo')", '\\\\foo') tester("ntpath.normpath('\\\\')", '\\\\') + tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 7261b7f0dc93d0a..d508f238f84fb5d 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -190,6 +190,21 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + def test_dump_custom_row_factory(self): + # gh-118221: iterdump should be able to cope with custom row factories. + def dict_factory(cu, row): + fields = [col[0] for col in cu.description] + return dict(zip(fields, row)) + + self.cx.row_factory = dict_factory + CREATE_TABLE = "CREATE TABLE test(t);" + expected = ["BEGIN TRANSACTION;", CREATE_TABLE, "COMMIT;"] + + self.cu.execute(CREATE_TABLE) + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + self.assertEqual(self.cx.row_factory, dict_factory) + def test_dump_virtual_tables(self): # gh-64662 expected = [ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-19-08-50-48.gh-issue-102511.qDEB66.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-19-08-50-48.gh-issue-102511.qDEB66.rst new file mode 100644 index 000000000000000..dfdf250710778e5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-19-08-50-48.gh-issue-102511.qDEB66.rst @@ -0,0 +1 @@ +Speed up :func:`os.path.splitroot` with a native implementation. diff --git a/Misc/NEWS.d/next/Library/2024-04-05-15-51-01.gh-issue-117566.54nABf.rst b/Misc/NEWS.d/next/Library/2024-04-05-15-51-01.gh-issue-117566.54nABf.rst new file mode 100644 index 000000000000000..56c2fb0e25d494c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-05-15-51-01.gh-issue-117566.54nABf.rst @@ -0,0 +1,3 @@ +:meth:`ipaddress.IPv6Address.is_loopback` will now return ``True`` for +IPv4-mapped loopback addresses, i.e. addresses in the +``::ffff:127.0.0.0/104`` address space. diff --git a/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst b/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst new file mode 100644 index 000000000000000..9b0ea9978a195eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst @@ -0,0 +1,2 @@ +Fix a bug where :func:`sqlite3.iterdump` could fail if a custom :attr:`row +factory ` was used. Patch by Erlend Aasland. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 2481455ac0d1437..fa425f4f740d312 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -32,6 +32,7 @@ #include #include "pycore_long.h" // _PyLong_IsZero() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_typeobject.h" #include "complexobject.h" #include "mpdecimal.h" @@ -120,11 +121,8 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = PyType_GetModuleByDef(Py_TYPE(left), &_decimal_module); - if (mod == NULL) { - PyErr_Clear(); - mod = PyType_GetModuleByDef(Py_TYPE(right), &_decimal_module); - } + PyObject *mod = _PyType_GetModuleByDef2(Py_TYPE(left), Py_TYPE(right), + &_decimal_module); assert(mod != NULL); return get_module_state(mod); } diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c index fa3251c4b5b3100..4fd98b82d762ff4 100644 --- a/Modules/_testcapi/run.c +++ b/Modules/_testcapi/run.c @@ -74,8 +74,10 @@ run_fileexflags(PyObject *mod, PyObject *pos_args) result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); -#if !defined(__wasi__) - /* The behavior of fileno() after fclose() is undefined. */ +#if defined(__linux__) || defined(MS_WINDOWS) || defined(__APPLE__) + /* The behavior of fileno() after fclose() is undefined, but it is + * the only practical way to check whether the file was closed. + * Only test this on the known platforms. */ if (closeit && result && fileno(fp) >= 0) { PyErr_SetString(PyExc_AssertionError, "File was not closed after excution"); Py_DECREF(result); diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 0398629e3c10ce9..a0d1f3238a6733b 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2248,6 +2248,64 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__path_splitroot_ex__doc__, +"_path_splitroot_ex($module, /, path)\n" +"--\n" +"\n"); + +#define OS__PATH_SPLITROOT_EX_METHODDEF \ + {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, + +static PyObject * +os__path_splitroot_ex_impl(PyObject *module, PyObject *path); + +static PyObject * +os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_splitroot_ex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("_path_splitroot_ex", "argument 'path'", "str", args[0]); + goto exit; + } + path = args[0]; + return_value = os__path_splitroot_ex_impl(module, path); + +exit: + return return_value; +} + PyDoc_STRVAR(os__path_normpath__doc__, "_path_normpath($module, /, path)\n" "--\n" @@ -12602,4 +12660,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=511f0788a6b90db0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c4698b47007cd6eb input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5e54cf64cd563ec..c9d67ccbb8c9089 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5467,6 +5467,49 @@ os__path_islink_impl(PyObject *module, PyObject *path) #endif /* MS_WINDOWS */ +/*[clinic input] +os._path_splitroot_ex + + path: unicode + +[clinic start generated code]*/ + +static PyObject * +os__path_splitroot_ex_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=de97403d3dfebc40 input=f1470e12d899f9ac]*/ +{ + Py_ssize_t len, drvsize, rootsize; + PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; + + wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); + if (!buffer) { + goto exit; + } + + _Py_skiproot(buffer, len, &drvsize, &rootsize); + drv = PyUnicode_FromWideChar(buffer, drvsize); + if (drv == NULL) { + goto exit; + } + root = PyUnicode_FromWideChar(&buffer[drvsize], rootsize); + if (root == NULL) { + goto exit; + } + tail = PyUnicode_FromWideChar(&buffer[drvsize + rootsize], + len - drvsize - rootsize); + if (tail == NULL) { + goto exit; + } + result = Py_BuildValue("(OOO)", drv, root, tail); +exit: + PyMem_Free(buffer); + Py_XDECREF(drv); + Py_XDECREF(root); + Py_XDECREF(tail); + return result; +} + + /*[clinic input] os._path_normpath @@ -16799,6 +16842,7 @@ static PyMethodDef posix_methods[] = { OS__FINDFIRSTFILE_METHODDEF OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF + OS__PATH_SPLITROOT_EX_METHODDEF OS__PATH_NORMPATH_METHODDEF OS_GETLOADAVG_METHODDEF OS_URANDOM_METHODDEF diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 2644516bc307706..afcf535f8c0a78a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1097,10 +1097,11 @@ compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + PyObject *ep_key = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key); + assert(ep_key != NULL); + assert(PyUnicode_CheckExact(ep_key)); + if (ep_key == key || + (unicode_get_hash(ep_key) == hash && unicode_eq(ep_key, key))) { return 1; } return 0; @@ -1761,10 +1762,12 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, else { assert(old_value != NULL); if (DK_IS_UNICODE(mp->ma_keys)) { - DK_UNICODE_ENTRIES(mp->ma_keys)[ix].me_value = value; + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } else { - DK_ENTRIES(mp->ma_keys)[ix].me_value = value; + PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } } mp->ma_version_tag = new_version; @@ -1810,15 +1813,15 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, if (unicode) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys); ep->me_key = key; - ep->me_value = value; + STORE_VALUE(ep, value); } else { PyDictKeyEntry *ep = DK_ENTRIES(newkeys); ep->me_key = key; ep->me_hash = hash; - ep->me_value = value; + STORE_VALUE(ep, value); } - mp->ma_used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, FT_ATOMIC_LOAD_SSIZE_RELAXED(mp->ma_used) + 1); mp->ma_version_tag = new_version; newkeys->dk_usable--; newkeys->dk_nentries++; @@ -2510,7 +2513,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); - mp->ma_used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, FT_ATOMIC_LOAD_SSIZE(mp->ma_used) - 1); mp->ma_version_tag = new_version; if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); @@ -6895,7 +6898,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr } #ifdef Py_GIL_DISABLED - PyObject *value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]); if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) { *attr = value; return true; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d55c246d80dd6a7..07b7ef3df46a5c2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -304,11 +304,6 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[i] = UNINITIALIZED; } stacks[0] = EMPTY_STACK; - if (code_obj->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) - { - // Generators get sent None while starting: - stacks[0] = push_value(stacks[0], Object); - } int todo = 1; while (todo) { todo = 0; diff --git a/Objects/setobject.c b/Objects/setobject.c index 0d88f4ff922d249..19975e3d4d18e25 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2333,6 +2333,13 @@ set_init(PySetObject *self, PyObject *args, PyObject *kwds) if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; + if (Py_REFCNT(self) == 1 && self->fill == 0) { + self->hash = -1; + if (iterable == NULL) { + return 0; + } + return set_update_local(self, iterable); + } Py_BEGIN_CRITICAL_SECTION(self); if (self->fill) set_clear_internal(self); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 808e11fcbaf1ff9..07e0a5a02da87ff 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4825,11 +4825,24 @@ PyType_GetModuleState(PyTypeObject *type) /* Get the module of the first superclass where the module has the * given PyModuleDef. */ -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +static inline PyObject * +get_module_by_def(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + // type_ready_mro() ensures that no heap type is + // contained in a static type MRO. + return NULL; + } + else { + PyHeapTypeObject *ht = (PyHeapTypeObject*)type; + PyObject *module = ht->ht_module; + if (module && _PyModule_GetDef(module) == def) { + return module; + } + } + PyObject *res = NULL; BEGIN_TYPE_LOCK() @@ -4837,12 +4850,14 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) // The type must be ready assert(mro != NULL); assert(PyTuple_Check(mro)); - // mro_invoke() ensures that the type MRO cannot be empty, so we don't have - // to check i < PyTuple_GET_SIZE(mro) at the first loop iteration. + // mro_invoke() ensures that the type MRO cannot be empty. assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which + // we already checked above. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); Py_ssize_t n = PyTuple_GET_SIZE(mro); - for (Py_ssize_t i = 0; i < n; i++) { + for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped @@ -4857,14 +4872,37 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) } } END_TYPE_LOCK() + return res; +} - if (res == NULL) { +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + PyObject *module = get_module_by_def(type, def); + if (module == NULL) { PyErr_Format( PyExc_TypeError, "PyType_GetModuleByDef: No superclass of '%s' has the given module", type->tp_name); } - return res; + return module; +} + +PyObject * +_PyType_GetModuleByDef2(PyTypeObject *left, PyTypeObject *right, + PyModuleDef *def) +{ + PyObject *module = get_module_by_def(left, def); + if (module == NULL) { + module = get_module_by_def(right, def); + if (module == NULL) { + PyErr_Format( + PyExc_TypeError, + "PyType_GetModuleByDef: No superclass of '%s' nor '%s' has " + "the given module", left->tp_name, right->tp_name); + } + } + return module; } void * diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c31617d35b02f5c..485504914912f91 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -837,12 +837,7 @@ dummy_func( _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } -#endif + LLTRACE_RESUME_FRAME(); } macro(RETURN_VALUE) = @@ -3186,12 +3181,7 @@ dummy_func( tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } -#endif + LLTRACE_RESUME_FRAME(); } macro(CALL_BOUND_METHOD_EXACT_ARGS) = @@ -3877,7 +3867,7 @@ dummy_func( } } - tier1 inst(RETURN_GENERATOR, (--)) { + inst(RETURN_GENERATOR, (-- res)) { assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -3887,19 +3877,19 @@ dummy_func( assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; + frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); + res = (PyObject *)gen; _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); LOAD_IP(frame->return_offset); - goto resume_frame; + LOAD_SP(); + LLTRACE_RESUME_FRAME(); } inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 224cd1da7d4a0e9..871d1747e2bb8d2 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -86,6 +86,18 @@ #define PRE_DISPATCH_GOTO() ((void)0) #endif +#if LLTRACE +#define LLTRACE_RESUME_FRAME() \ +do { \ + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); \ + if (lltrace < 0) { \ + goto exit_unwind; \ + } \ +} while (0) +#else +#define LLTRACE_RESUME_FRAME() ((void)0) +#endif + #ifdef Py_GIL_DISABLED #define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) #else diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7403d6fdaf0e2b2..1eb3da9b70002c4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -988,12 +988,7 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); break; } @@ -3213,12 +3208,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); break; } @@ -3833,6 +3823,35 @@ break; } + case _RETURN_GENERATOR: { + PyObject *res; + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + JUMP_TO_ERROR(); + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + res = (PyObject *)gen; + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + LOAD_SP(); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + case _BUILD_SLICE: { PyObject *step = NULL; PyObject *stop; diff --git a/Python/fileutils.c b/Python/fileutils.c index 882d3299575cf3b..54853ba2f75d9d0 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2295,6 +2295,99 @@ PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ +void +_Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsize, + Py_ssize_t *rootsize) +{ + assert(drvsize); + assert(rootsize); +#ifndef MS_WINDOWS +#define IS_SEP(x) (*(x) == SEP) + *drvsize = 0; + if (!IS_SEP(&path[0])) { + // Relative path, e.g.: 'foo' + *rootsize = 0; + } + else if (!IS_SEP(&path[1]) || IS_SEP(&path[2])) { + // Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + *rootsize = 1; + } + else { + // Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + *rootsize = 2; + } +#undef IS_SEP +#else + const wchar_t *pEnd = size >= 0 ? &path[size] : NULL; +#define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) +#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) + if (IS_SEP(&path[0])) { + if (IS_SEP(&path[1])) { + // Device drives, e.g. \\.\device or \\?\device + // UNC drives, e.g. \\server\share or \\?\UNC\server\share + Py_ssize_t idx; + if (path[2] == L'?' && IS_SEP(&path[3]) && + (path[4] == L'U' || path[4] == L'u') && + (path[5] == L'N' || path[5] == L'n') && + (path[6] == L'C' || path[6] == L'c') && + IS_SEP(&path[7])) + { + idx = 8; + } + else { + idx = 2; + } + while (!SEP_OR_END(&path[idx])) { + idx++; + } + if (IS_END(&path[idx])) { + *drvsize = idx; + *rootsize = 0; + } + else { + idx++; + while (!SEP_OR_END(&path[idx])) { + idx++; + } + *drvsize = idx; + if (IS_END(&path[idx])) { + *rootsize = 0; + } + else { + *rootsize = 1; + } + } + } + else { + // Relative path with root, e.g. \Windows + *drvsize = 0; + *rootsize = 1; + } + } + else if (!IS_END(&path[0]) && path[1] == L':') { + *drvsize = 2; + if (IS_SEP(&path[2])) { + // Absolute drive-letter path, e.g. X:\Windows + *rootsize = 1; + } + else { + // Relative path with drive, e.g. X:Windows + *rootsize = 0; + } + } + else { + // Relative path, e.g. Windows + *drvsize = 0; + *rootsize = 0; + } +#undef SEP_OR_END +#undef IS_SEP +#undef IS_END +#endif +} + // The caller must ensure "buffer" is big enough. static int join_relfile(wchar_t *buffer, size_t bufsize, @@ -2411,49 +2504,39 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) - // Skip leading '.\' if (p1[0] == L'.' && IS_SEP(&p1[1])) { + // Skip leading '.\' path = &path[2]; - while (IS_SEP(path) && !IS_END(path)) { + while (IS_SEP(path)) { path++; } p1 = p2 = minP2 = path; lastC = SEP; } + else { + Py_ssize_t drvsize, rootsize; + _Py_skiproot(path, size, &drvsize, &rootsize); + if (drvsize || rootsize) { + // Skip past root and update minP2 + p1 = &path[drvsize + rootsize]; +#ifndef ALTSEP + p2 = p1; +#else + for (; p2 < p1; ++p2) { + if (*p2 == ALTSEP) { + *p2 = SEP; + } + } +#endif + minP2 = p2 - 1; + lastC = *minP2; #ifdef MS_WINDOWS - // Skip past drive segment and update minP2 - else if (p1[0] && p1[1] == L':') { - *p2++ = *p1++; - *p2++ = *p1++; - minP2 = p2; - lastC = L':'; - } - // Skip past all \\-prefixed paths, including \\?\, \\.\, - // and network paths, including the first segment. - else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { - int sepCount = 2; - *p2++ = SEP; - *p2++ = SEP; - p1 += 2; - for (; !IS_END(p1) && sepCount; ++p1) { - if (IS_SEP(p1)) { - --sepCount; - *p2++ = lastC = SEP; - } else { - *p2++ = lastC = *p1; + if (lastC != SEP) { + minP2++; } +#endif } - minP2 = p2 - 1; - } -#else - // Skip past two leading SEPs - else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) { - *p2++ = *p1++; - *p2++ = *p1++; - minP2 = p2 - 1; // Absolute path has SEP at minP2 - lastC = SEP; } -#endif /* MS_WINDOWS */ /* if pEnd is specified, check that. Else, check for null terminator */ for (; !IS_END(p1); ++p1) { diff --git a/Python/frame.c b/Python/frame.c index f88a8f0d73d3f87..db9d13359a23ca2 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -53,18 +53,6 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) return f; } -void -_PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) -{ - assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); - Py_ssize_t size = ((char*)&src->localsplus[src->stacktop]) - (char *)src; - memcpy(dest, src, size); - // Don't leave a dangling pointer to the old frame when creating generators - // and coroutines: - dest->previous = NULL; -} - - static void take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 058cac8bedd9179..0c58f3f87d40412 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -997,12 +997,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); } DISPATCH(); } @@ -1786,12 +1781,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); } DISPATCH(); } @@ -4992,12 +4982,7 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); } DISPATCH(); } @@ -5006,6 +4991,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); + PyObject *res; assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -5015,19 +5001,22 @@ assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; + frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); + res = (PyObject *)gen; _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); LOAD_IP(frame->return_offset); - goto resume_frame; + LOAD_SP(); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); } TARGET(RETURN_VALUE) { @@ -5050,12 +5039,7 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index b17c2998e2504bf..e5c70f72f9c3247 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -697,7 +697,8 @@ translate_bytecode_to_trace( // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) int nuops = expansion->nuops; RESERVE(nuops + 1); /* One extra for exit */ - if (expansion->uops[nuops-1].uop == _POP_FRAME) { + int16_t last_op = expansion->uops[nuops-1].uop; + if (last_op == _POP_FRAME || last_op == _RETURN_GENERATOR) { // Check for trace stack underflow now: // We can't bail e.g. in the middle of // LOAD_CONST + _POP_FRAME. @@ -756,7 +757,7 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } - if (uop == _POP_FRAME) { + if (uop == _POP_FRAME || uop == _RETURN_GENERATOR) { TRACE_STACK_POP(); /* Set the operand to the function or code object returned to, * to assist optimization passes. (See _PUSH_FRAME below.) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a76edd62c94c131..9315d7228b57323 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -369,7 +369,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) static PyCodeObject * get_code(_PyUOpInstruction *op) { - assert(op->opcode == _PUSH_FRAME || op->opcode == _POP_FRAME); + assert(op->opcode == _PUSH_FRAME || op->opcode == _POP_FRAME || op->opcode == _RETURN_GENERATOR); PyCodeObject *co = NULL; uint64_t operand = op->operand; if (operand == 0) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 481fb8387af4167..8bc563427747905 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -651,6 +651,28 @@ dummy_func(void) { } } + op(_RETURN_GENERATOR, ( -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + goto done; + } + } + op(_CHECK_STACK_SPACE, ( --)) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0a7d96d30ad3e89..4f0941a3cc3e095 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1840,6 +1840,29 @@ break; } + case _RETURN_GENERATOR: { + _Py_UopsSymbol *res; + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + goto done; + } + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + case _BUILD_SLICE: { _Py_UopsSymbol *slice; slice = sym_new_not_null(ctx); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 1408103ba80f96f..e4ca32bebc5a22b 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -14,7 +14,6 @@ race:set_allocator_unlocked race:_add_to_weak_set race:_in_weak_set race:_mi_heap_delayed_free_partial -race:_Py_IsImmortal race:_Py_IsOwnedByCurrentThread race:_PyEval_EvalFrameDefault race:_PyFunction_SetVersion @@ -25,11 +24,10 @@ race:_PyInterpreterState_IsRunningMain race:_PyObject_GC_IS_SHARED race:_PyObject_GC_SET_SHARED race:_PyObject_GC_TRACK +# https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 +race:_PyParkingLot_Park race:_PyType_HasFeature race:assign_version_tag -race:compare_unicode_unicode -race:delitem_common -race:dictresize race:gc_collect_main race:gc_restore_tid race:initialize_new_array @@ -44,3 +42,6 @@ race:set_inheritable race:start_the_world race:tstate_set_detached race:unicode_hash + +# https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 +thread:pthread_create