From 36aab34fab3d05f254fe80ca542c38c5f9ae11fe Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 14 Nov 2023 00:31:02 +0000 Subject: [PATCH 01/49] gh-107149: make new opcode util functions private rather than public and unstable (#112042) --- Include/cpython/compile.h | 12 ----------- Include/internal/pycore_compile.h | 14 +++++++++++++ Modules/_opcode.c | 35 ++++++++++++++++--------------- Python/compile.c | 16 +++++++------- Python/intrinsics.c | 5 +++-- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index 265f5397b45116..0d587505ef7f85 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -68,15 +68,3 @@ typedef struct { #define PY_INVALID_STACK_EFFECT INT_MAX PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump); - -PyAPI_FUNC(int) PyUnstable_OpcodeIsValid(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode); - -PyAPI_FUNC(PyObject*) PyUnstable_GetUnaryIntrinsicName(int index); -PyAPI_FUNC(PyObject*) PyUnstable_GetBinaryIntrinsicName(int index); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index a5a7146f5ee917..e5870759ba74f1 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -102,6 +102,20 @@ int _PyCompile_EnsureArrayLargeEnough( int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); + +// Export for '_opcode' extention module +PyAPI_FUNC(int) _PyCompile_OpcodeIsValid(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasArg(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasConst(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasName(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasJump(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasFree(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasLocal(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasExc(int opcode); + +PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index); +PyAPI_FUNC(PyObject*) _PyCompile_GetBinaryIntrinsicName(int index); + /* Access compiler internals for unit testing */ // Export for '_testinternalcapi' shared extension diff --git a/Modules/_opcode.c b/Modules/_opcode.c index f0b547795b847a..93c71377f03a76 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -6,6 +6,7 @@ #include "compile.h" #include "opcode.h" #include "internal/pycore_code.h" +#include "internal/pycore_compile.h" #include "internal/pycore_intrinsics.h" /*[clinic input] @@ -78,7 +79,7 @@ static int _opcode_is_valid_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b0d918ea1d073f65 input=fe23e0aa194ddae0]*/ { - return PyUnstable_OpcodeIsValid(opcode); + return _PyCompile_OpcodeIsValid(opcode); } /*[clinic input] @@ -94,8 +95,8 @@ static int _opcode_has_arg_impl(PyObject *module, int opcode) /*[clinic end generated code: output=7a062d3b2dcc0815 input=93d878ba6361db5f]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasArg(opcode); + return _PyCompile_OpcodeIsValid(opcode) && + _PyCompile_OpcodeHasArg(opcode); } /*[clinic input] @@ -111,8 +112,8 @@ static int _opcode_has_const_impl(PyObject *module, int opcode) /*[clinic end generated code: output=c646d5027c634120 input=a6999e4cf13f9410]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasConst(opcode); + return _PyCompile_OpcodeIsValid(opcode) && + _PyCompile_OpcodeHasConst(opcode); } /*[clinic input] @@ -128,8 +129,8 @@ static int _opcode_has_name_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasName(opcode); + return _PyCompile_OpcodeIsValid(opcode) && + _PyCompile_OpcodeHasName(opcode); } /*[clinic input] @@ -145,8 +146,8 @@ static int _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic end generated code: output=e9c583c669f1c46a input=35f711274357a0c3]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasJump(opcode); + return _PyCompile_OpcodeIsValid(opcode) && + _PyCompile_OpcodeHasJump(opcode); } @@ -168,8 +169,8 @@ static int _opcode_has_free_impl(PyObject *module, int opcode) /*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasFree(opcode); + return _PyCompile_OpcodeIsValid(opcode) && + _PyCompile_OpcodeHasFree(opcode); } @@ -186,8 +187,8 @@ static int _opcode_has_local_impl(PyObject *module, int opcode) /*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasLocal(opcode); + return _PyCompile_OpcodeIsValid(opcode) && + _PyCompile_OpcodeHasLocal(opcode); } /*[clinic input] @@ -203,8 +204,8 @@ static int _opcode_has_exc_impl(PyObject *module, int opcode) /*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasExc(opcode); + return _PyCompile_OpcodeIsValid(opcode) && + _PyCompile_OpcodeHasExc(opcode); } /*[clinic input] @@ -309,7 +310,7 @@ _opcode_get_intrinsic1_descs_impl(PyObject *module) return NULL; } for (int i=0; i <= MAX_INTRINSIC_1; i++) { - PyObject *name = PyUnstable_GetUnaryIntrinsicName(i); + PyObject *name = _PyCompile_GetUnaryIntrinsicName(i); if (name == NULL) { Py_DECREF(list); return NULL; @@ -336,7 +337,7 @@ _opcode_get_intrinsic2_descs_impl(PyObject *module) return NULL; } for (int i=0; i <= MAX_INTRINSIC_2; i++) { - PyObject *name = PyUnstable_GetBinaryIntrinsicName(i); + PyObject *name = _PyCompile_GetBinaryIntrinsicName(i); if (name == NULL) { Py_DECREF(list); return NULL; diff --git a/Python/compile.c b/Python/compile.c index 6c64b405b2b286..8b1eef79a79eae 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -883,49 +883,49 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) } int -PyUnstable_OpcodeIsValid(int opcode) +_PyCompile_OpcodeIsValid(int opcode) { return IS_VALID_OPCODE(opcode); } int -PyUnstable_OpcodeHasArg(int opcode) +_PyCompile_OpcodeHasArg(int opcode) { return OPCODE_HAS_ARG(opcode); } int -PyUnstable_OpcodeHasConst(int opcode) +_PyCompile_OpcodeHasConst(int opcode) { return OPCODE_HAS_CONST(opcode); } int -PyUnstable_OpcodeHasName(int opcode) +_PyCompile_OpcodeHasName(int opcode) { return OPCODE_HAS_NAME(opcode); } int -PyUnstable_OpcodeHasJump(int opcode) +_PyCompile_OpcodeHasJump(int opcode) { return OPCODE_HAS_JUMP(opcode); } int -PyUnstable_OpcodeHasFree(int opcode) +_PyCompile_OpcodeHasFree(int opcode) { return OPCODE_HAS_FREE(opcode); } int -PyUnstable_OpcodeHasLocal(int opcode) +_PyCompile_OpcodeHasLocal(int opcode) { return OPCODE_HAS_LOCAL(opcode); } int -PyUnstable_OpcodeHasExc(int opcode) +_PyCompile_OpcodeHasExc(int opcode) { return IS_BLOCK_PUSH_OPCODE(opcode); } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index bbd79ec473f470..d3146973b75178 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -5,6 +5,7 @@ #include "pycore_frame.h" #include "pycore_function.h" #include "pycore_global_objects.h" +#include "pycore_compile.h" // _PyCompile_GetUnaryIntrinsicName, etc #include "pycore_intrinsics.h" // INTRINSIC_PRINT #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_runtime.h" // _Py_ID() @@ -269,7 +270,7 @@ _PyIntrinsics_BinaryFunctions[] = { #undef INTRINSIC_FUNC_ENTRY PyObject* -PyUnstable_GetUnaryIntrinsicName(int index) +_PyCompile_GetUnaryIntrinsicName(int index) { if (index < 0 || index > MAX_INTRINSIC_1) { return NULL; @@ -278,7 +279,7 @@ PyUnstable_GetUnaryIntrinsicName(int index) } PyObject* -PyUnstable_GetBinaryIntrinsicName(int index) +_PyCompile_GetBinaryIntrinsicName(int index) { if (index < 0 || index > MAX_INTRINSIC_2) { return NULL; From 31ad7e061ebebc484e00ed3ad5ff61061341c35e Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 13 Nov 2023 17:27:19 -0800 Subject: [PATCH 02/49] GH-111520: Add back the operand local (GH-111813) --- Python/ceval.c | 6 +--- Python/executor_cases.c.h | 44 +++++++++++++-------------- Tools/cases_generator/instructions.py | 2 +- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 42100393f60ff7..584b64a6e6333d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -992,9 +992,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int OPT_STAT_INC(traces_executed); _PyUOpInstruction *next_uop = current_executor->trace; -#ifdef Py_DEBUG - uint64_t operand; // Used by several DPRINTF() calls -#endif + uint64_t operand; #ifdef Py_STATS uint64_t trace_uop_execution_counter = 0; #endif @@ -1002,9 +1000,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int for (;;) { opcode = next_uop->opcode; oparg = next_uop->oparg; -#ifdef Py_DEBUG operand = next_uop->operand; -#endif DPRINTF(3, "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", (int)(next_uop - current_executor->trace), diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2f4d55b5ddf7d5..877328a4ab67ab 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -204,7 +204,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - uint32_t version = (uint32_t)next_uop[-1].operand; + uint32_t version = (uint32_t)operand; // This one is a bit weird, because we expect *some* failures: assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); @@ -1128,7 +1128,7 @@ } case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)next_uop[-1].operand; + uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); @@ -1137,7 +1137,7 @@ } case _GUARD_BUILTINS_VERSION: { - uint16_t version = (uint16_t)next_uop[-1].operand; + uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); @@ -1148,7 +1148,7 @@ case _LOAD_GLOBAL_MODULE: { PyObject *res; PyObject *null = NULL; - uint16_t index = (uint16_t)next_uop[-1].operand; + uint16_t index = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -1166,7 +1166,7 @@ case _LOAD_GLOBAL_BUILTINS: { PyObject *res; PyObject *null = NULL; - uint16_t index = (uint16_t)next_uop[-1].operand; + uint16_t index = (uint16_t)operand; PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -1615,7 +1615,7 @@ case _GUARD_TYPE_VERSION: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)next_uop[-1].operand; + uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); @@ -1637,7 +1637,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)next_uop[-1].operand; + uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); @@ -1654,7 +1654,7 @@ case _CHECK_ATTR_MODULE: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)next_uop[-1].operand; + uint32_t type_version = (uint32_t)operand; DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -1667,7 +1667,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)next_uop[-1].operand; + uint16_t index = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); @@ -1701,7 +1701,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)next_uop[-1].operand; + uint16_t hint = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); @@ -1732,7 +1732,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)next_uop[-1].operand; + uint16_t index = (uint16_t)operand; char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); @@ -1749,7 +1749,7 @@ case _CHECK_ATTR_CLASS: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)next_uop[-1].operand; + uint32_t type_version = (uint32_t)operand; DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); @@ -1761,7 +1761,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)next_uop[-1].operand; + PyObject *descr = (PyObject *)operand; STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -1787,7 +1787,7 @@ PyObject *value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)next_uop[-1].operand; + uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); PyDictValues *values = _PyDictOrValues_GetValues(dorv); @@ -1809,7 +1809,7 @@ PyObject *value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)next_uop[-1].operand; + uint16_t index = (uint16_t)operand; char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -2383,7 +2383,7 @@ case _GUARD_KEYS_VERSION: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)next_uop[-1].operand; + uint32_t keys_version = (uint32_t)operand; PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); @@ -2395,7 +2395,7 @@ PyObject *attr; PyObject *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)next_uop[-1].operand; + PyObject *descr = (PyObject *)operand; assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -2414,7 +2414,7 @@ PyObject *attr; PyObject *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)next_uop[-1].operand; + PyObject *descr = (PyObject *)operand; assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2432,7 +2432,7 @@ PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)next_uop[-1].operand; + PyObject *descr = (PyObject *)operand; assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2446,7 +2446,7 @@ PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)next_uop[-1].operand; + PyObject *descr = (PyObject *)operand; assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2473,7 +2473,7 @@ PyObject *attr; PyObject *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)next_uop[-1].operand; + PyObject *descr = (PyObject *)operand; assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2522,7 +2522,7 @@ PyObject *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)next_uop[-1].operand; + uint32_t func_version = (uint32_t)operand; DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 181d8badf7bf22..9039ac5c6f127e 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -166,7 +166,7 @@ def write_body( f"{func}(&this_instr[{active.offset + 1}].cache);" ) else: - out.emit(f"{typ}{ceffect.name} = ({typ.strip()})next_uop[-1].operand;") + out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") # Write the body, substituting a goto for ERROR_IF() and other stuff assert dedent <= 0 From e31d65e0b7bb6d6fee4e8df54e10976b4cfab1de Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 14 Nov 2023 02:41:20 -0500 Subject: [PATCH 03/49] gh-111622: Fix doc for items views (#112051) They are set-like even when some values are not hashable, but work even better when all are. --- Doc/library/stdtypes.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8160740c7a05bd..f204b287b565eb 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4755,14 +4755,17 @@ support membership tests: .. versionadded:: 3.10 -Keys views are set-like since their entries are unique and :term:`hashable`. If all -values are hashable, so that ``(key, value)`` pairs are unique and hashable, -then the items view is also set-like. (Values views are not treated as set-like +Keys views are set-like since their entries are unique and :term:`hashable`. +Items views also have set-like operations since the (key, value) pairs +are unique and the keys are hashable. +If all values in an items view are hashable as well, +then the items view can interoperate with other sets. +(Values views are not treated as set-like since the entries are generally not unique.) For set-like views, all of the operations defined for the abstract base class :class:`collections.abc.Set` are available (for example, ``==``, ``<``, or ``^``). While using set operators, -set-like views accept any iterable as the other operand, unlike sets which only -accept sets as the input. +set-like views accept any iterable as the other operand, +unlike sets which only accept sets as the input. An example of dictionary view usage:: From 18203a6bc9ccf3a9ba901574dfc772474b027e2a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Nov 2023 11:25:39 +0200 Subject: [PATCH 04/49] gh-111789: Use PyDict_GetItemRef() in Objects/ (GH-111827) --- Objects/abstract.c | 7 +-- Objects/funcobject.c | 9 ++-- Objects/moduleobject.c | 29 +++++----- Objects/object.c | 31 ++++------- Objects/typeobject.c | 120 ++++++++++++++++------------------------- 5 files changed, 76 insertions(+), 120 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index b6762893b8fd5d..43842fbdd6aedd 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -204,12 +204,7 @@ int PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) { if (PyDict_CheckExact(obj)) { - *result = PyDict_GetItemWithError(obj, key); /* borrowed */ - if (*result) { - Py_INCREF(*result); - return 1; - } - return PyErr_Occurred() ? -1 : 0; + return PyDict_GetItemRef(obj, key, result); } *result = PyObject_GetItem(obj, key); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index e2fd6f99e451b3..4d88dd2229295d 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -92,8 +92,8 @@ PyFunction_ClearWatcher(int watcher_id) PyFunctionObject * _PyFunction_FromConstructor(PyFrameConstructor *constr) { - PyObject *module = Py_XNewRef(PyDict_GetItemWithError(constr->fc_globals, &_Py_ID(__name__))); - if (!module && PyErr_Occurred()) { + PyObject *module; + if (PyDict_GetItemRef(constr->fc_globals, &_Py_ID(__name__), &module) < 0) { return NULL; } @@ -158,12 +158,11 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname Py_INCREF(doc); // __module__: Use globals['__name__'] if it exists, or NULL. - PyObject *module = PyDict_GetItemWithError(globals, &_Py_ID(__name__)); + PyObject *module; PyObject *builtins = NULL; - if (module == NULL && _PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &module) < 0) { goto error; } - Py_XINCREF(module); builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref if (builtins == NULL) { diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index a3d55b76aa1012..bba77ce8ab7e7b 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -522,6 +522,7 @@ PyModule_GetNameObject(PyObject *mod) } PyObject *name; if (PyDict_GetItemRef(dict, &_Py_ID(__name__), &name) <= 0) { + // error or not found goto error; } if (!PyUnicode_Check(name)) { @@ -562,6 +563,7 @@ PyModule_GetFilenameObject(PyObject *mod) } PyObject *fileobj; if (PyDict_GetItemRef(dict, &_Py_ID(__file__), &fileobj) <= 0) { + // error or not found goto error; } if (!PyUnicode_Check(fileobj)) { @@ -816,28 +818,28 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyErr_Clear(); } assert(m->md_dict != NULL); - getattr = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__getattr__)); + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + return NULL; + } if (getattr) { PyObject *result = PyObject_CallOneArg(getattr, name); if (result == NULL && suppress == 1 && PyErr_ExceptionMatches(PyExc_AttributeError)) { // suppress AttributeError PyErr_Clear(); } + Py_DECREF(getattr); return result; } - if (PyErr_Occurred()) { + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name) < 0) { return NULL; } - mod_name = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__name__)); if (mod_name && PyUnicode_Check(mod_name)) { - Py_INCREF(mod_name); - PyObject *spec = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__spec__)); - if (spec == NULL && PyErr_Occurred()) { + PyObject *spec; + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) { Py_DECREF(mod_name); return NULL; } if (suppress != 1) { - Py_XINCREF(spec); if (_PyModuleSpec_IsInitializing(spec)) { PyErr_Format(PyExc_AttributeError, "partially initialized " @@ -856,14 +858,12 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) "module '%U' has no attribute '%U'", mod_name, name); } - Py_XDECREF(spec); } + Py_XDECREF(spec); Py_DECREF(mod_name); return NULL; } - else if (PyErr_Occurred()) { - return NULL; - } + Py_XDECREF(mod_name); if (suppress != 1) { PyErr_Format(PyExc_AttributeError, "module has no attribute '%U'", name); @@ -957,11 +957,8 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) return NULL; } - PyObject *annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); - if (annotations) { - Py_INCREF(annotations); - } - else if (!PyErr_Occurred()) { + PyObject *annotations; + if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) { annotations = PyDict_New(); if (annotations) { int result = PyDict_SetItem( diff --git a/Objects/object.c b/Objects/object.c index 1003029f0ed722..1f5b2b4f79e5fa 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1490,19 +1490,14 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } if (dict != NULL) { Py_INCREF(dict); - PyObject *attr = PyDict_GetItemWithError(dict, name); - if (attr != NULL) { - *method = Py_NewRef(attr); + if (PyDict_GetItemRef(dict, name, method) != 0) { + // found or error Py_DECREF(dict); Py_XDECREF(descr); return 0; } + // not found Py_DECREF(dict); - - if (PyErr_Occurred()) { - Py_XDECREF(descr); - return 0; - } } if (meth_found) { @@ -1607,21 +1602,17 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } if (dict != NULL) { Py_INCREF(dict); - res = PyDict_GetItemWithError(dict, name); + int rc = PyDict_GetItemRef(dict, name, &res); + Py_DECREF(dict); if (res != NULL) { - Py_INCREF(res); - Py_DECREF(dict); goto done; } - else { - Py_DECREF(dict); - if (PyErr_Occurred()) { - if (suppress && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - } - else { - goto done; - } + else if (rc < 0) { + if (suppress && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + goto done; } } } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index c1a4cb9aaeacfd..4464b5af8cd15b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1092,14 +1092,9 @@ type_module(PyTypeObject *type, void *context) if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { PyObject *dict = lookup_tp_dict(type); - mod = PyDict_GetItemWithError(dict, &_Py_ID(__module__)); - if (mod == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_AttributeError, "__module__"); - } - return NULL; + if (PyDict_GetItemRef(dict, &_Py_ID(__module__), &mod) == 0) { + PyErr_Format(PyExc_AttributeError, "__module__"); } - Py_INCREF(mod); } else { const char *s = strrchr(type->tp_name, '.'); @@ -1134,17 +1129,16 @@ type_abstractmethods(PyTypeObject *type, void *context) PyObject *mod = NULL; /* type itself has an __abstractmethods__ descriptor (this). Don't return that. */ - if (type != &PyType_Type) { - PyObject *dict = lookup_tp_dict(type); - mod = PyDict_GetItemWithError(dict, &_Py_ID(__abstractmethods__)); + if (type == &PyType_Type) { + PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); } - if (!mod) { - if (!PyErr_Occurred()) { + else { + PyObject *dict = lookup_tp_dict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__abstractmethods__), &mod) == 0) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); } - return NULL; } - return Py_NewRef(mod); + return mod; } static int @@ -1435,18 +1429,14 @@ type_get_doc(PyTypeObject *type, void *context) return _PyType_GetDocFromInternalDoc(type->tp_name, type->tp_doc); } PyObject *dict = lookup_tp_dict(type); - result = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); - if (result == NULL) { - if (!PyErr_Occurred()) { - result = Py_NewRef(Py_None); - } - } - else if (Py_TYPE(result)->tp_descr_get) { - result = Py_TYPE(result)->tp_descr_get(result, NULL, - (PyObject *)type); + if (PyDict_GetItemRef(dict, &_Py_ID(__doc__), &result) == 0) { + result = Py_NewRef(Py_None); } - else { - Py_INCREF(result); + else if (result) { + descrgetfunc descr_get = Py_TYPE(result)->tp_descr_get; + if (descr_get) { + Py_SETREF(result, descr_get(result, NULL, (PyObject *)type)); + } } return result; } @@ -1477,16 +1467,16 @@ type_get_annotations(PyTypeObject *type, void *context) PyObject *annotations; PyObject *dict = lookup_tp_dict(type); - annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + return NULL; + } if (annotations) { - if (Py_TYPE(annotations)->tp_descr_get) { - annotations = Py_TYPE(annotations)->tp_descr_get( - annotations, NULL, (PyObject *)type); - } else { - Py_INCREF(annotations); + descrgetfunc get = Py_TYPE(annotations)->tp_descr_get; + if (get) { + Py_SETREF(annotations, get(annotations, NULL, (PyObject *)type)); } } - else if (!PyErr_Occurred()) { + else { annotations = PyDict_New(); if (annotations) { int result = PyDict_SetItem( @@ -1533,16 +1523,11 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_type_params(PyTypeObject *type, void *context) { - PyObject *params = PyDict_GetItemWithError(lookup_tp_dict(type), &_Py_ID(__type_params__)); - - if (params) { - return Py_NewRef(params); - } - if (PyErr_Occurred()) { - return NULL; + PyObject *params; + if (PyDict_GetItemRef(lookup_tp_dict(type), &_Py_ID(__type_params__), ¶ms) == 0) { + return PyTuple_New(0); } - - return PyTuple_New(0); + return params; } static int @@ -3436,18 +3421,13 @@ type_new_set_module(PyTypeObject *type) return 0; } - PyObject *module = PyDict_GetItemWithError(globals, &_Py_ID(__name__)); - if (module == NULL) { - if (PyErr_Occurred()) { - return -1; - } - return 0; - } - - if (PyDict_SetItem(dict, &_Py_ID(__module__), module) < 0) { - return -1; + PyObject *module; + r = PyDict_GetItemRef(globals, &_Py_ID(__name__), &module); + if (module) { + r = PyDict_SetItem(dict, &_Py_ID(__module__), module); + Py_DECREF(module); } - return 0; + return r; } @@ -3458,23 +3438,24 @@ type_new_set_ht_name(PyTypeObject *type) { PyHeapTypeObject *et = (PyHeapTypeObject *)type; PyObject *dict = lookup_tp_dict(type); - PyObject *qualname = PyDict_GetItemWithError(dict, &_Py_ID(__qualname__)); + PyObject *qualname; + if (PyDict_GetItemRef(dict, &_Py_ID(__qualname__), &qualname) < 0) { + return -1; + } if (qualname != NULL) { if (!PyUnicode_Check(qualname)) { PyErr_Format(PyExc_TypeError, "type __qualname__ must be a str, not %s", Py_TYPE(qualname)->tp_name); + Py_DECREF(qualname); return -1; } - et->ht_qualname = Py_NewRef(qualname); + et->ht_qualname = qualname; if (PyDict_DelItem(dict, &_Py_ID(__qualname__)) < 0) { return -1; } } else { - if (PyErr_Occurred()) { - return -1; - } et->ht_qualname = Py_NewRef(et->ht_name); } return 0; @@ -5888,24 +5869,22 @@ _PyType_GetSlotNames(PyTypeObject *cls) /* Get the slot names from the cache in the class if possible. */ PyObject *dict = lookup_tp_dict(cls); - slotnames = PyDict_GetItemWithError(dict, &_Py_ID(__slotnames__)); + if (PyDict_GetItemRef(dict, &_Py_ID(__slotnames__), &slotnames) < 0) { + return NULL; + } if (slotnames != NULL) { if (slotnames != Py_None && !PyList_Check(slotnames)) { PyErr_Format(PyExc_TypeError, "%.200s.__slotnames__ should be a list or None, " "not %.200s", cls->tp_name, Py_TYPE(slotnames)->tp_name); + Py_DECREF(slotnames); return NULL; } - return Py_NewRef(slotnames); - } - else { - if (PyErr_Occurred()) { - return NULL; - } - /* The class does not have the slot names cached yet. */ + return slotnames; } + /* The class does not have the slot names cached yet. */ copyreg = import_copyreg(); if (copyreg == NULL) return NULL; @@ -10264,23 +10243,18 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * return NULL; /* keep a strong reference to mro because su_obj_type->tp_mro can be - replaced during PyDict_GetItemWithError(dict, name) */ + replaced during PyDict_GetItemRef(dict, name, &res) */ Py_INCREF(mro); do { PyObject *obj = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(obj)); assert(dict != NULL && PyDict_Check(dict)); - res = PyDict_GetItemWithError(dict, name); - if (res != NULL) { - Py_INCREF(res); + if (PyDict_GetItemRef(dict, name, &res) != 0) { + // found or error Py_DECREF(mro); return res; } - else if (PyErr_Occurred()) { - Py_DECREF(mro); - return NULL; - } i++; } while (i < n); From c98600bed47b0be4d9d601c78252154118e7366f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Nov 2023 11:28:34 +0200 Subject: [PATCH 05/49] gh-111789: Use PyDict_GetItemRef() in _ctypes (GH-111828) --- Modules/_ctypes/_ctypes.c | 104 ++++++++++++++++--------------------- Modules/_ctypes/callproc.c | 35 ++++++------- 2 files changed, 61 insertions(+), 78 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6e0709cc1e4a4d..f909a9496b6526 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -243,26 +243,13 @@ PyDict_SetItemProxy(PyObject *dict, PyObject *key, PyObject *item) static int _PyDict_GetItemProxy(PyObject *dict, PyObject *key, PyObject **presult) { - PyObject *item = PyDict_GetItemWithError(dict, key); - if (item == NULL) { - if (PyErr_Occurred()) { - return -1; - } - *presult = NULL; - return 0; + int rc = PyDict_GetItemRef(dict, key, presult); + PyObject *item = *presult; + if (item && PyWeakref_CheckProxy(item)) { + rc = PyWeakref_GetRef(item, presult); + Py_DECREF(item); } - - if (!PyWeakref_CheckProxy(item)) { - *presult = Py_NewRef(item); - return 0; - } - PyObject *ref; - if (PyWeakref_GetRef(item, &ref) < 0) { - return -1; - } - // ref is NULL if the referenced object was destroyed - *presult = ref; - return 0; + return rc; } /******************************************************************/ @@ -565,18 +552,19 @@ StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isSt dict->paramfunc = StructUnionType_paramfunc; - fields = PyDict_GetItemWithError((PyObject *)dict, &_Py_ID(_fields_)); + if (PyDict_GetItemRef((PyObject *)dict, &_Py_ID(_fields_), &fields) < 0) { + Py_DECREF(result); + return NULL; + } if (fields) { if (PyObject_SetAttr((PyObject *)result, &_Py_ID(_fields_), fields) < 0) { Py_DECREF(result); + Py_DECREF(fields); return NULL; } + Py_DECREF(fields); return (PyObject *)result; } - else if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } else { StgDictObject *basedict = PyType_stgdict((PyObject *)result->tp_base); @@ -1110,11 +1098,15 @@ PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) stgdict->paramfunc = PyCPointerType_paramfunc; stgdict->flags |= TYPEFLAG_ISPOINTER; - proto = PyDict_GetItemWithError(typedict, &_Py_ID(_type_)); /* Borrowed ref */ + if (PyDict_GetItemRef(typedict, &_Py_ID(_type_), &proto) < 0) { + Py_DECREF((PyObject *)stgdict); + return NULL; + } if (proto) { StgDictObject *itemdict; const char *current_format; if (-1 == PyCPointerType_SetProto(stgdict, proto)) { + Py_DECREF(proto); Py_DECREF((PyObject *)stgdict); return NULL; } @@ -1134,15 +1126,12 @@ PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } else { stgdict->format = _ctypes_alloc_format_string("&", current_format); } + Py_DECREF(proto); if (stgdict->format == NULL) { Py_DECREF((PyObject *)stgdict); return NULL; } } - else if (PyErr_Occurred()) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } /* create the new instance (which is a class, since we are a metatype!) */ @@ -2461,58 +2450,61 @@ make_funcptrtype_dict(StgDictObject *stgdict) stgdict->getfunc = NULL; stgdict->ffi_type_pointer = ffi_type_pointer; - ob = PyDict_GetItemWithError((PyObject *)stgdict, &_Py_ID(_flags_)); + if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_flags_), &ob) < 0) { + return -1; + } if (!ob || !PyLong_Check(ob)) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, + PyErr_SetString(PyExc_TypeError, "class must define _flags_ which must be an integer"); - } + Py_XDECREF(ob); return -1; } stgdict->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; + Py_DECREF(ob); /* _argtypes_ is optional... */ - ob = PyDict_GetItemWithError((PyObject *)stgdict, &_Py_ID(_argtypes_)); + if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_argtypes_), &ob) < 0) { + return -1; + } if (ob) { converters = converters_from_argtypes(ob); - if (!converters) + if (!converters) { + Py_DECREF(ob); return -1; - stgdict->argtypes = Py_NewRef(ob); + } + stgdict->argtypes = ob; stgdict->converters = converters; } - else if (PyErr_Occurred()) { + + if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_restype_), &ob) < 0) { return -1; } - - ob = PyDict_GetItemWithError((PyObject *)stgdict, &_Py_ID(_restype_)); if (ob) { if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "_restype_ must be a type, a callable, or None"); + Py_DECREF(ob); return -1; } - stgdict->restype = Py_NewRef(ob); + stgdict->restype = ob; if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), &stgdict->checker) < 0) { return -1; } } - else if (PyErr_Occurred()) { +/* XXX later, maybe. + if (PyDict_GetItemRef((PyObject *)stgdict, &_Py _ID(_errcheck_), &ob) < 0) { return -1; } -/* XXX later, maybe. - ob = _PyDict_GetItemIdWithError((PyObject *)stgdict, &PyId__errcheck_); if (ob) { if (!PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "_errcheck_ must be callable"); + Py_DECREF(ob); return -1; } - stgdict->errcheck = Py_NewRef(ob); - } - else if (PyErr_Occurred()) { - return -1; + stgdict->errcheck = ob; } */ return 0; @@ -3812,13 +3804,12 @@ _get_arg(int *pindex, PyObject *name, PyObject *defval, PyObject *inargs, PyObje return Py_NewRef(v); } if (kwds && name) { - v = PyDict_GetItemWithError(kwds, name); + if (PyDict_GetItemRef(kwds, name, &v) < 0) { + return NULL; + } if (v) { ++*pindex; - return Py_NewRef(v); - } - else if (PyErr_Occurred()) { - return NULL; + return v; } } if (defval) { @@ -4870,15 +4861,12 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) return NULL; PyObject *result; - if (_PyDict_GetItemProxy(cache, key, &result) < 0) { - Py_DECREF(key); - return NULL; - } - if (result) { + if (_PyDict_GetItemProxy(cache, key, &result) != 0) { + // found or error Py_DECREF(key); return result; } - + // not found if (!PyType_Check(itemtype)) { PyErr_SetString(PyExc_TypeError, "Expected a type object"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 2a0c26a0fec1b6..3b11cd7f58ce4b 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -168,16 +168,18 @@ _ctypes_get_errobj(int **pspace) if (error_object_name == NULL) return NULL; } - errobj = PyDict_GetItemWithError(dict, error_object_name); + if (PyDict_GetItemRef(dict, error_object_name, &errobj) < 0) { + return NULL; + } if (errobj) { if (!PyCapsule_IsValid(errobj, CTYPES_CAPSULE_NAME_PYMEM)) { PyErr_SetString(PyExc_RuntimeError, "ctypes.error_object is an invalid capsule"); + Py_DECREF(errobj); return NULL; } - Py_INCREF(errobj); } - else if (!PyErr_Occurred()) { + else { void *space = PyMem_Calloc(2, sizeof(int)); if (space == NULL) return NULL; @@ -192,9 +194,6 @@ _ctypes_get_errobj(int **pspace) return NULL; } } - else { - return NULL; - } *pspace = (int *)PyCapsule_GetPointer(errobj, CTYPES_CAPSULE_NAME_PYMEM); return errobj; } @@ -1922,13 +1921,11 @@ create_pointer_type(PyObject *module, PyObject *cls) PyTypeObject *typ; PyObject *key; - result = PyDict_GetItemWithError(_ctypes_ptrtype_cache, cls); - if (result) { - return Py_NewRef(result); - } - else if (PyErr_Occurred()) { - return NULL; + if (PyDict_GetItemRef(_ctypes_ptrtype_cache, cls, &result) != 0) { + // found or error + return result; } + // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); result = PyObject_CallFunction((PyObject *)Py_TYPE(&PyCPointer_Type), @@ -1986,16 +1983,14 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *result; PyObject *typ; - typ = PyDict_GetItemWithError(_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg)); - if (typ) { - return PyObject_CallOneArg(typ, arg); - } - else if (PyErr_Occurred()) { + if (PyDict_GetItemRef(_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { return NULL; } - typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); - if (typ == NULL) - return NULL; + if (typ == NULL) { + typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); + if (typ == NULL) + return NULL; + } result = PyObject_CallOneArg(typ, arg); Py_DECREF(typ); return result; From 95365625f4aa8615c94fbfa10f6a0264ca3054db Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Nov 2023 11:29:20 +0200 Subject: [PATCH 06/49] gh-111789: Simplify import.c by using PyDict_GetItemRef() (GH-111979) --- Python/import.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/Python/import.c b/Python/import.c index b6ffba5c5746e2..12f586abc2eed8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2372,11 +2372,11 @@ get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, if (nhooks < 0) return NULL; /* Shouldn't happen */ - importer = PyDict_GetItemWithError(path_importer_cache, p); - if (importer != NULL || _PyErr_Occurred(tstate)) { - return Py_XNewRef(importer); + if (PyDict_GetItemRef(path_importer_cache, p, &importer) != 0) { + // found or error + return importer; } - + // not found /* set path_importer_cache[p] to None to avoid recursion */ if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) return NULL; @@ -2565,7 +2565,7 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level { PyObject *abs_name; PyObject *package = NULL; - PyObject *spec; + PyObject *spec = NULL; Py_ssize_t last_dot; PyObject *base; int level_up; @@ -2578,20 +2578,18 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level _PyErr_SetString(tstate, PyExc_TypeError, "globals must be a dict"); goto error; } - package = PyDict_GetItemWithError(globals, &_Py_ID(__package__)); + if (PyDict_GetItemRef(globals, &_Py_ID(__package__), &package) < 0) { + goto error; + } if (package == Py_None) { + Py_DECREF(package); package = NULL; } - else if (package == NULL && _PyErr_Occurred(tstate)) { - goto error; - } - spec = PyDict_GetItemWithError(globals, &_Py_ID(__spec__)); - if (spec == NULL && _PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(globals, &_Py_ID(__spec__), &spec) < 0) { goto error; } if (package != NULL) { - Py_INCREF(package); if (!PyUnicode_Check(package)) { _PyErr_SetString(tstate, PyExc_TypeError, "package must be a string"); @@ -2635,16 +2633,15 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level goto error; } - package = PyDict_GetItemWithError(globals, &_Py_ID(__name__)); + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &package) < 0) { + goto error; + } if (package == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_KeyError, - "'__name__' not in globals"); - } + _PyErr_SetString(tstate, PyExc_KeyError, + "'__name__' not in globals"); goto error; } - Py_INCREF(package); if (!PyUnicode_Check(package)) { _PyErr_SetString(tstate, PyExc_TypeError, "__name__ must be a string"); @@ -2692,6 +2689,7 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level } } + Py_XDECREF(spec); base = PyUnicode_Substring(package, 0, last_dot); Py_DECREF(package); if (base == NULL || PyUnicode_GET_LENGTH(name) == 0) { @@ -2708,6 +2706,7 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level "with no known parent package"); error: + Py_XDECREF(spec); Py_XDECREF(package); return NULL; } From 16055c160412544e2a49794aaf3aa70c584f843a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Nov 2023 11:29:49 +0200 Subject: [PATCH 07/49] gh-111789: Simplify ceval.c by using PyDict_GetItemRef() (GH-111980) --- Python/ceval.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 584b64a6e6333d..fe1bf310bdc5ea 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1592,14 +1592,14 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, continue; PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); if (func->func_kwdefaults != NULL) { - PyObject *def = PyDict_GetItemWithError(func->func_kwdefaults, varname); + PyObject *def; + if (PyDict_GetItemRef(func->func_kwdefaults, varname, &def) < 0) { + goto fail_post_args; + } if (def) { - localsplus[i] = Py_NewRef(def); + localsplus[i] = def; continue; } - else if (_PyErr_Occurred(tstate)) { - goto fail_post_args; - } } missing++; } @@ -2401,13 +2401,9 @@ PyEval_GetBuiltins(void) PyObject * _PyEval_GetBuiltin(PyObject *name) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *attr = PyDict_GetItemWithError(PyEval_GetBuiltins(), name); - if (attr) { - Py_INCREF(attr); - } - else if (!_PyErr_Occurred(tstate)) { - _PyErr_SetObject(tstate, PyExc_AttributeError, name); + PyObject *attr; + if (PyDict_GetItemRef(PyEval_GetBuiltins(), name, &attr) == 0) { + PyErr_SetObject(PyExc_AttributeError, name); } return attr; } @@ -2558,12 +2554,12 @@ static PyObject * import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name, PyObject *fromlist, PyObject *level) { - PyObject *import_func = _PyDict_GetItemWithError(frame->f_builtins, - &_Py_ID(__import__)); + PyObject *import_func; + if (PyDict_GetItemRef(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { + return NULL; + } if (import_func == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found"); - } + _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found"); return NULL; } @@ -2574,6 +2570,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, /* Fast path for not overloaded __import__. */ if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { + Py_DECREF(import_func); int ilevel = PyLong_AsInt(level); if (ilevel == -1 && _PyErr_Occurred(tstate)) { return NULL; @@ -2587,7 +2584,6 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, } PyObject* args[5] = {name, frame->f_globals, locals, fromlist, level}; - Py_INCREF(import_func); PyObject *res = PyObject_Vectorcall(import_func, args, 5, NULL); Py_DECREF(import_func); return res; From 324531df909721978446d504186738a33ab03fd5 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 14 Nov 2023 05:47:46 -0500 Subject: [PATCH 08/49] gh-111903: Add `@critical_section` directive to Argument Clinic. (#111904) The `@critical_section` directive instructs Argument Clinic to generate calls to `Py_BEGIN_CRITICAL_SECTION()` and `Py_END_CRITICAL_SECTION()` around the bound function. In `--disable-gil` builds, these calls will lock and unlock the `self` object. They are no-ops in the default build. This is used in one place (`_io._Buffered.close`) as a demonstration. Subsequent PRs will use it more widely in the `_io.Buffered` bindings. --- Lib/test/clinic.test.c | 75 +++++++++++++++++++ ...-11-09-13-04-29.gh-issue-111903.7Prryr.rst | 4 + Modules/_io/bufferedio.c | 4 +- Modules/_io/clinic/bufferedio.c.h | 10 ++- Tools/clinic/clinic.py | 32 ++++++-- 5 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-11-09-13-04-29.gh-issue-111903.7Prryr.rst diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 2ef7a6e8eb2795..da730c5b2495c8 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5467,3 +5467,78 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) /*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ + + +/*[clinic input] +@critical_section +test_critical_section +[clinic start generated code]*/ + +PyDoc_STRVAR(test_critical_section__doc__, +"test_critical_section($module, /)\n" +"--\n" +"\n"); + +#define TEST_CRITICAL_SECTION_METHODDEF \ + {"test_critical_section", (PyCFunction)test_critical_section, METH_NOARGS, test_critical_section__doc__}, + +static PyObject * +test_critical_section_impl(PyObject *module); + +static PyObject * +test_critical_section(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = test_critical_section_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +static PyObject * +test_critical_section_impl(PyObject *module) +/*[clinic end generated code: output=9d5a87bb28aa3f0c input=8c58956d6ff00f80]*/ + + +/*[clinic input] +@critical_section +test_critical_section_meth_o + a: object(subclass_of="&PyUnicode_Type") + / +[clinic start generated code]*/ + +PyDoc_STRVAR(test_critical_section_meth_o__doc__, +"test_critical_section_meth_o($module, a, /)\n" +"--\n" +"\n"); + +#define TEST_CRITICAL_SECTION_METH_O_METHODDEF \ + {"test_critical_section_meth_o", (PyCFunction)test_critical_section_meth_o, METH_O, test_critical_section_meth_o__doc__}, + +static PyObject * +test_critical_section_meth_o_impl(PyObject *module, PyObject *a); + +static PyObject * +test_critical_section_meth_o(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("test_critical_section_meth_o", "argument", "str", arg); + goto exit; + } + a = arg; + Py_BEGIN_CRITICAL_SECTION(module); + return_value = test_critical_section_meth_o_impl(module, a); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +static PyObject * +test_critical_section_meth_o_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=7a9d7420802d1202 input=376533f51eceb6c3]*/ diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-11-09-13-04-29.gh-issue-111903.7Prryr.rst b/Misc/NEWS.d/next/Tools-Demos/2023-11-09-13-04-29.gh-issue-111903.7Prryr.rst new file mode 100644 index 00000000000000..41601f7511797a --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-11-09-13-04-29.gh-issue-111903.7Prryr.rst @@ -0,0 +1,4 @@ +Argument Clinic now supports the ``@critical_section`` directive that +instructs Argument Clinic to generate a critical section around the function +call, which locks the ``self`` object in ``--disable-gil`` builds. Patch by +Sam Gross. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index e8caf9f0df6dbf..169b2b3d105669 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytes_Join() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() @@ -521,12 +522,13 @@ buffered_closed_get(buffered *self, void *context) } /*[clinic input] +@critical_section _io._Buffered.close [clinic start generated code]*/ static PyObject * _io__Buffered_close_impl(buffered *self) -/*[clinic end generated code: output=7280b7b42033be0c input=d20b83d1ddd7d805]*/ +/*[clinic end generated code: output=7280b7b42033be0c input=56d95935b03fd326]*/ { PyObject *res = NULL; int r; diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index f6ac2699135917..3fe03506b6cfa5 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -324,7 +324,13 @@ _io__Buffered_close_impl(buffered *self); static PyObject * _io__Buffered_close(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_detach__doc__, @@ -1075,4 +1081,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=090e70253e35fc22 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b23847480eba3d9b input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2ea93e610b086b..a205a5182ee99f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -470,6 +470,10 @@ def __init__(self) -> None: # The C statements required to clean up after the impl call. self.cleanup: list[str] = [] + # The C statements to generate critical sections (per-object locking). + self.lock: list[str] = [] + self.unlock: list[str] = [] + class FormatCounterFormatter(string.Formatter): """ @@ -1109,7 +1113,8 @@ def output_templates( condition=include.condition) has_option_groups = parameters and (parameters[0].group or parameters[-1].group) - default_return_converter = f.return_converter.type == 'PyObject *' + simple_return = (f.return_converter.type == 'PyObject *' + and not f.critical_section) new_or_init = f.kind.new_or_init vararg: int | str = NO_VARARG @@ -1183,7 +1188,9 @@ def parser_body( """) + "\n" finale = normalize_snippet(""" {modifications} + {lock} {return_value} = {c_basename}_impl({impl_arguments}); + {unlock} {return_conversion} {post_parsing} @@ -1219,7 +1226,7 @@ def parser_body( flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - return_error = ('return NULL;' if default_return_converter + return_error = ('return NULL;' if simple_return else 'goto exit;') parser_code = [normalize_snippet(""" if (nargs) {{ @@ -1228,7 +1235,7 @@ def parser_body( }} """ % return_error, indent=4)] - if default_return_converter: + if simple_return: parser_definition = '\n'.join([ parser_prototype, '{{', @@ -1245,7 +1252,7 @@ def parser_body( converters[0].format_unit == 'O'): meth_o_prototype = self.METH_O_PROTOTYPE - if default_return_converter: + if simple_return: # maps perfectly to METH_O, doesn't need a return converter. # so we skip making a parse function # and call directly into the impl function. @@ -1858,6 +1865,10 @@ def render_function( selfless = parameters[1:] assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" + if f.critical_section: + data.lock.append('Py_BEGIN_CRITICAL_SECTION({self_name});') + data.unlock.append('Py_END_CRITICAL_SECTION();') + last_group = 0 first_optional = len(selfless) positional = selfless and selfless[-1].is_positional_only() @@ -1937,6 +1948,8 @@ def render_function( template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) template_dict['cleanup'] = format_escape("".join(data.cleanup)) template_dict['return_value'] = data.return_value + template_dict['lock'] = "\n".join(data.lock) + template_dict['unlock'] = "\n".join(data.unlock) # used by unpack tuple code generator unpack_min = first_optional @@ -1961,6 +1974,8 @@ def render_function( modifications=template_dict['modifications'], post_parsing=template_dict['post_parsing'], cleanup=template_dict['cleanup'], + lock=template_dict['lock'], + unlock=template_dict['unlock'], ) # Only generate the "exit:" label @@ -2954,6 +2969,7 @@ class Function: # functions with optional groups because we can't represent # those accurately with inspect.Signature in 3.4. docstring_only: bool = False + critical_section: bool = False def __post_init__(self) -> None: self.parent = self.cls or self.module @@ -5108,6 +5124,7 @@ class DSLParser: coexist: bool parameter_continuation: str preserve_output: bool + critical_section: bool from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') def __init__(self, clinic: Clinic) -> None: @@ -5142,6 +5159,7 @@ def reset(self) -> None: self.forced_text_signature: str | None = None self.parameter_continuation = '' self.preserve_output = False + self.critical_section = False def directive_version(self, required: str) -> None: global version @@ -5270,6 +5288,9 @@ def at_classmethod(self) -> None: fail("Can't set @classmethod, function is not a normal callable") self.kind = CLASS_METHOD + def at_critical_section(self) -> None: + self.critical_section = True + def at_staticmethod(self) -> None: if self.kind is not CALLABLE: fail("Can't set @staticmethod, function is not a normal callable") @@ -5492,7 +5513,8 @@ def state_modulename_name(self, line: str) -> None: return_converter = CReturnConverter() self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, - return_converter=return_converter, kind=self.kind, coexist=self.coexist) + return_converter=return_converter, kind=self.kind, coexist=self.coexist, + critical_section=self.critical_section) self.block.signatures.append(self.function) # insert a self converter automatically From f44d6ff6e0c9eeb0bb246a3dd8f99d40b7050054 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 14 Nov 2023 04:22:25 -0800 Subject: [PATCH 09/49] gh-110944: Make pdb completion work for alias and convenience vars (GH-110945) --- Lib/pdb.py | 18 +++++++++++++++- Lib/test/test_pdb.py | 21 +++++++++++++++++++ ...-10-16-18-41-51.gh-issue-110944.CmUKXo.rst | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-16-18-41-51.gh-issue-110944.CmUKXo.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index a4b02e010a6be6..ed78d749a47fa8 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -238,7 +238,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, try: import readline # remove some common file name delimiters - readline.set_completer_delims(' \t\n`@#$%^&*()=+[{]}\\|;:\'",<>?') + readline.set_completer_delims(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') except ImportError: pass self.allow_kbdint = False @@ -686,6 +686,18 @@ def set_convenience_variable(self, frame, name, value): # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. + def completenames(self, text, line, begidx, endidx): + # Overwrite completenames() of cmd so for the command completion, + # if no current command matches, check for expressions as well + commands = super().completenames(text, line, begidx, endidx) + for alias in self.aliases: + if alias.startswith(text): + commands.append(alias) + if commands: + return commands + else: + return self._complete_expression(text, line, begidx, endidx) + def _complete_location(self, text, line, begidx, endidx): # Complete a file/module/function location for break/tbreak/clear. if line.strip().endswith((':', ',')): @@ -720,6 +732,10 @@ def _complete_expression(self, text, line, begidx, endidx): # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. ns = {**self.curframe.f_globals, **self.curframe_locals} + if text.startswith("$"): + # Complete convenience variables + conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) + return [f"${name}" for name in conv_vars if name.startswith(text[1:])] if '.' in text: # Walk an attribute chain up to the last part, similar to what # rlcompleter does. This will bail if any of the parts are not diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2ec4e2ee0e0360..67a4053a2ac8bc 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3289,6 +3289,27 @@ def test_basic_completion(self): self.assertIn(b'continue', output) self.assertIn(b'hello!', output) + def test_expression_completion(self): + script = textwrap.dedent(""" + value = "speci" + import pdb; pdb.Pdb().set_trace() + """) + + # Complete: value + 'al' + input = b"val\t + 'al'\n" + # Complete: p value + 'es' + input += b"p val\t + 'es'\n" + # Complete: $_frame + input += b"$_fra\t\n" + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'special', output) + self.assertIn(b'species', output) + self.assertIn(b'$_frame', output) + def load_tests(loader, tests, pattern): from test import test_pdb diff --git a/Misc/NEWS.d/next/Library/2023-10-16-18-41-51.gh-issue-110944.CmUKXo.rst b/Misc/NEWS.d/next/Library/2023-10-16-18-41-51.gh-issue-110944.CmUKXo.rst new file mode 100644 index 00000000000000..ec9ca5a11f1ac8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-16-18-41-51.gh-issue-110944.CmUKXo.rst @@ -0,0 +1 @@ +Support alias and convenience vars for :mod:`pdb` completion From 4f04172c9287c507f1426e02ddfc432f1f3ade54 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Nov 2023 13:51:00 +0100 Subject: [PATCH 10/49] gh-111262: Add PyDict_Pop() function (#112028) _PyDict_Pop_KnownHash(): remove the default value and the return type becomes an int. Co-authored-by: Stefan Behnel Co-authored-by: Antoine Pitrou --- Doc/c-api/dict.rst | 27 ++++ Doc/whatsnew/3.13.rst | 6 + Include/cpython/dictobject.h | 2 + Include/internal/pycore_dict.h | 6 +- Lib/test/test_capi/test_dict.py | 87 +++++++++++++ ...-11-10-10-21-38.gh-issue-111262.2utB5m.rst | 4 + Modules/_functoolsmodule.c | 28 +++-- Modules/_testcapi/dict.c | 87 ++++++++++++- Modules/_threadmodule.c | 7 +- Modules/socketmodule.c | 10 +- Objects/dictobject.c | 119 +++++++++++++----- Objects/odictobject.c | 5 +- Objects/structseq.c | 12 +- Python/import.c | 4 +- Python/sysmodule.c | 4 +- 15 files changed, 335 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-11-10-10-21-38.gh-issue-111262.2utB5m.rst diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 8ee351918006e4..8471c98d044872 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -173,6 +173,33 @@ Dictionary Objects .. versionadded:: 3.4 + +.. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) + + Remove *key* from dictionary *p* and optionally return the removed value. + Do not raise :exc:`KeyError` if the key missing. + + - If the key is present, set *\*result* to a new reference to the removed + value if *result* is not ``NULL``, and return ``1``. + - If the key is missing, set *\*result* to ``NULL`` if *result* is not + ``NULL``, and return ``0``. + - On error, raise an exception and return ``-1``. + + This is similar to :meth:`dict.pop`, but without the default value and + not raising :exc:`KeyError` if the key missing. + + .. versionadded:: 3.13 + + +.. c:function:: int PyDict_PopString(PyObject *p, const char *key, PyObject **result) + + Similar to :c:func:`PyDict_Pop`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyDict_Items(PyObject *p) Return a :c:type:`PyListObject` containing all the items from the dictionary. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 81e133bb5454ec..136fe901ce39fb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1175,6 +1175,12 @@ New Features Python ``list.extend()`` and ``list.clear()`` methods. (Contributed by Victor Stinner in :gh:`111138`.) +* Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a + key from a dictionary and optionally return the removed value. This is + similar to :meth:`dict.pop`, but without the default value and not raising + :exc:`KeyError` if the key missing. + (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 64a40425ba7714..64753849d32061 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -46,6 +46,8 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); +PyAPI_FUNC(int) PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result); +PyAPI_FUNC(int) PyDict_PopString(PyObject *dict, const char *key, PyObject **result); PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value); /* Dictionary watchers */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index d01ef55de51f5d..89f30a452c0c64 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -116,7 +116,11 @@ extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); -extern PyObject *_PyDict_Pop_KnownHash(PyObject *, PyObject *, Py_hash_t, PyObject *); +extern int _PyDict_Pop_KnownHash( + PyDictObject *dict, + PyObject *key, + Py_hash_t hash, + PyObject **result); #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 67f12a56313b6f..57a7238588eae0 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -432,6 +432,93 @@ def test_dict_mergefromseq2(self): # CRASHES mergefromseq2({}, NULL, 0) # CRASHES mergefromseq2(NULL, {}, 0) + def test_dict_pop(self): + # Test PyDict_Pop() + dict_pop = _testcapi.dict_pop + dict_pop_null = _testcapi.dict_pop_null + + # key present, get removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_pop(mydict, "key"), (1, "value")) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_pop(mydict, "key2"), (1, "value2")) + self.assertEqual(mydict, {}) + + # key present, ignore removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_pop_null(mydict, "key"), 1) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_pop_null(mydict, "key2"), 1) + self.assertEqual(mydict, {}) + + # key missing, expect removed value; empty dict has a fast path + self.assertEqual(dict_pop({}, "key"), (0, NULL)) + self.assertEqual(dict_pop({"a": 1}, "key"), (0, NULL)) + + # key missing, ignored removed value; empty dict has a fast path + self.assertEqual(dict_pop_null({}, "key"), 0) + self.assertEqual(dict_pop_null({"a": 1}, "key"), 0) + + # dict error + not_dict = UserDict({1: 2}) + self.assertRaises(SystemError, dict_pop, not_dict, "key") + self.assertRaises(SystemError, dict_pop_null, not_dict, "key") + + # key error; don't hash key if dict is empty + not_hashable_key = ["list"] + self.assertEqual(dict_pop({}, not_hashable_key), (0, NULL)) + with self.assertRaises(TypeError): + dict_pop({'key': 1}, not_hashable_key) + dict_pop({}, NULL) # key is not checked if dict is empty + + # CRASHES dict_pop(NULL, "key") + # CRASHES dict_pop({"a": 1}, NULL) + + def test_dict_popstring(self): + # Test PyDict_PopString() + dict_popstring = _testcapi.dict_popstring + dict_popstring_null = _testcapi.dict_popstring_null + + # key present, get removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_popstring(mydict, "key"), (1, "value")) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_popstring(mydict, "key2"), (1, "value2")) + self.assertEqual(mydict, {}) + + # key present, ignore removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_popstring_null(mydict, "key"), 1) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_popstring_null(mydict, "key2"), 1) + self.assertEqual(mydict, {}) + + # key missing; empty dict has a fast path + self.assertEqual(dict_popstring({}, "key"), (0, NULL)) + self.assertEqual(dict_popstring_null({}, "key"), 0) + self.assertEqual(dict_popstring({"a": 1}, "key"), (0, NULL)) + self.assertEqual(dict_popstring_null({"a": 1}, "key"), 0) + + # non-ASCII key + non_ascii = '\U0001f40d' + dct = {'\U0001f40d': 123} + self.assertEqual(dict_popstring(dct, '\U0001f40d'.encode()), (1, 123)) + dct = {'\U0001f40d': 123} + self.assertEqual(dict_popstring_null(dct, '\U0001f40d'.encode()), 1) + + # dict error + not_dict = UserDict({1: 2}) + self.assertRaises(SystemError, dict_popstring, not_dict, "key") + self.assertRaises(SystemError, dict_popstring_null, not_dict, "key") + + # key error + self.assertRaises(UnicodeDecodeError, dict_popstring, {1: 2}, INVALID_UTF8) + self.assertRaises(UnicodeDecodeError, dict_popstring_null, {1: 2}, INVALID_UTF8) + + # CRASHES dict_popstring(NULL, "key") + # CRASHES dict_popstring({}, NULL) + # CRASHES dict_popstring({"a": 1}, NULL) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C API/2023-11-10-10-21-38.gh-issue-111262.2utB5m.rst b/Misc/NEWS.d/next/C API/2023-11-10-10-21-38.gh-issue-111262.2utB5m.rst new file mode 100644 index 00000000000000..d432b7ebd3a565 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-10-10-21-38.gh-issue-111262.2utB5m.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a key +from a dictionary and optionally return the removed value. This is similar to +:meth:`dict.pop`, but without the default value and not raising :exc:`KeyError` +if the key missing. Patch by Stefan Behnel and Victor Stinner. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 8ea493ad9ab278..ca440e4c70c0d9 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1087,19 +1087,9 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds The cache dict holds one reference to the link. We created one other reference when the link was created. The linked list only has borrowed references. */ - popresult = _PyDict_Pop_KnownHash(self->cache, link->key, - link->hash, Py_None); - if (popresult == Py_None) { - /* Getting here means that the user function call or another - thread has already removed the old key from the dictionary. - This link is now an orphan. Since we don't want to leave the - cache in an inconsistent state, we don't restore the link. */ - Py_DECREF(popresult); - Py_DECREF(link); - Py_DECREF(key); - return result; - } - if (popresult == NULL) { + int res = _PyDict_Pop_KnownHash((PyDictObject*)self->cache, link->key, + link->hash, &popresult); + if (res < 0) { /* An error arose while trying to remove the oldest key (the one being evicted) from the cache. We restore the link to its original position as the oldest link. Then we allow the @@ -1110,10 +1100,22 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds Py_DECREF(result); return NULL; } + if (res == 0) { + /* Getting here means that the user function call or another + thread has already removed the old key from the dictionary. + This link is now an orphan. Since we don't want to leave the + cache in an inconsistent state, we don't restore the link. */ + assert(popresult == NULL); + Py_DECREF(link); + Py_DECREF(key); + return result; + } + /* Keep a reference to the old key and old result to prevent their ref counts from going to zero during the update. That will prevent potentially arbitrary object clean-up code (i.e. __del__) from running while we're still adjusting the links. */ + assert(popresult != NULL); oldkey = link->key; oldresult = link->result; diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 5f6a1a037dcba2..42e056b7d07a31 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -331,6 +331,88 @@ dict_mergefromseq2(PyObject *self, PyObject *args) } +static PyObject * +dict_pop(PyObject *self, PyObject *args) +{ + // Test PyDict_Pop(dict, key, &value) + PyObject *dict, *key; + if (!PyArg_ParseTuple(args, "OO", &dict, &key)) { + return NULL; + } + NULLABLE(dict); + NULLABLE(key); + PyObject *result = UNINITIALIZED_PTR; + int res = PyDict_Pop(dict, key, &result); + if (res < 0) { + assert(result == NULL); + return NULL; + } + if (res == 0) { + assert(result == NULL); + result = Py_NewRef(Py_None); + } + else { + assert(result != NULL); + } + return Py_BuildValue("iN", res, result); +} + + +static PyObject * +dict_pop_null(PyObject *self, PyObject *args) +{ + // Test PyDict_Pop(dict, key, NULL) + PyObject *dict, *key; + if (!PyArg_ParseTuple(args, "OO", &dict, &key)) { + return NULL; + } + NULLABLE(dict); + NULLABLE(key); + RETURN_INT(PyDict_Pop(dict, key, NULL)); +} + + +static PyObject * +dict_popstring(PyObject *self, PyObject *args) +{ + PyObject *dict; + const char *key; + Py_ssize_t key_size; + if (!PyArg_ParseTuple(args, "Oz#", &dict, &key, &key_size)) { + return NULL; + } + NULLABLE(dict); + PyObject *result = UNINITIALIZED_PTR; + int res = PyDict_PopString(dict, key, &result); + if (res < 0) { + assert(result == NULL); + return NULL; + } + if (res == 0) { + assert(result == NULL); + result = Py_NewRef(Py_None); + } + else { + assert(result != NULL); + } + return Py_BuildValue("iN", res, result); +} + + +static PyObject * +dict_popstring_null(PyObject *self, PyObject *args) +{ + PyObject *dict; + const char *key; + Py_ssize_t key_size; + if (!PyArg_ParseTuple(args, "Oz#", &dict, &key, &key_size)) { + return NULL; + } + NULLABLE(dict); + RETURN_INT(PyDict_PopString(dict, key, NULL)); +} + + static PyMethodDef test_methods[] = { {"dict_check", dict_check, METH_O}, {"dict_checkexact", dict_checkexact, METH_O}, @@ -358,7 +440,10 @@ static PyMethodDef test_methods[] = { {"dict_merge", dict_merge, METH_VARARGS}, {"dict_update", dict_update, METH_VARARGS}, {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, - + {"dict_pop", dict_pop, METH_VARARGS}, + {"dict_pop_null", dict_pop_null, METH_VARARGS}, + {"dict_popstring", dict_popstring, METH_VARARGS}, + {"dict_popstring_null", dict_popstring_null, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 88ca9032b5e679..c608789b5fbd5c 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -967,11 +967,8 @@ local_clear(localobject *self) HEAD_UNLOCK(runtime); while (tstate) { if (tstate->dict) { - PyObject *v = _PyDict_Pop(tstate->dict, self->key, Py_None); - if (v != NULL) { - Py_DECREF(v); - } - else { + if (PyDict_Pop(tstate->dict, self->key, NULL) < 0) { + // Silently ignore error PyErr_Clear(); } } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 97f4d174666750..2e1a0974ceb75d 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -392,16 +392,10 @@ remove_unusable_flags(PyObject *m) break; } else { - PyObject *flag_name = PyUnicode_FromString(win_runtime_flags[i].flag_name); - if (flag_name == NULL) { + if (PyDict_PopString(dict, win_runtime_flags[i].flag_name, + NULL) < 0) { return -1; } - PyObject *v = _PyDict_Pop(dict, flag_name, Py_None); - Py_DECREF(flag_name); - if (v == NULL) { - return -1; - } - Py_DECREF(v); } } return 0; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 719d438897ca6c..d3d16c59f0ea7c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2226,64 +2226,119 @@ PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) return _PyDict_Next(op, ppos, pkey, pvalue, NULL); } + /* Internal version of dict.pop(). */ -PyObject * -_PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *deflt) +int +_PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, + PyObject **result) { - Py_ssize_t ix; - PyObject *old_value; - PyDictObject *mp; - PyInterpreterState *interp = _PyInterpreterState_GET(); - - assert(PyDict_Check(dict)); - mp = (PyDictObject *)dict; + assert(PyDict_Check(mp)); if (mp->ma_used == 0) { - if (deflt) { - return Py_NewRef(deflt); + if (result) { + *result = NULL; } - _PyErr_SetKeyError(key); - return NULL; + return 0; } - ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) - return NULL; + + PyObject *old_value; + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; + } + if (ix == DKIX_EMPTY || old_value == NULL) { - if (deflt) { - return Py_NewRef(deflt); + if (result) { + *result = NULL; } - _PyErr_SetKeyError(key); - return NULL; + return 0; } + assert(old_value != NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, mp, key, NULL); delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); ASSERT_CONSISTENT(mp); - return old_value; + if (result) { + *result = old_value; + } + else { + Py_DECREF(old_value); + } + return 1; } -PyObject * -_PyDict_Pop(PyObject *dict, PyObject *key, PyObject *deflt) + +int +PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) { - Py_hash_t hash; + if (!PyDict_Check(op)) { + if (result) { + *result = NULL; + } + PyErr_BadInternalCall(); + return -1; + } + PyDictObject *dict = (PyDictObject *)op; - if (((PyDictObject *)dict)->ma_used == 0) { - if (deflt) { - return Py_NewRef(deflt); + if (dict->ma_used == 0) { + if (result) { + *result = NULL; } - _PyErr_SetKeyError(key); - return NULL; + return 0; } + + Py_hash_t hash; if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + if (hash == -1) { + if (result) { + *result = NULL; + } + return -1; + } + } + return _PyDict_Pop_KnownHash(dict, key, hash, result); +} + + +int +PyDict_PopString(PyObject *op, const char *key, PyObject **result) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + if (result != NULL) { + *result = NULL; + } + return -1; } - return _PyDict_Pop_KnownHash(dict, key, hash, deflt); + + int res = PyDict_Pop(op, key_obj, result); + Py_DECREF(key_obj); + return res; } + +PyObject * +_PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) +{ + PyObject *result; + if (PyDict_Pop(dict, key, &result) == 0) { + if (default_value != NULL) { + return Py_NewRef(default_value); + } + _PyErr_SetKeyError(key); + return NULL; + } + return result; +} + + /* Internal version of dict.from_keys(). It is subclass-friendly. */ PyObject * _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) diff --git a/Objects/odictobject.c b/Objects/odictobject.c index b99896319e0136..b5280c39e1be54 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1049,7 +1049,10 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, return NULL; } /* Now delete the value from the dict. */ - value = _PyDict_Pop_KnownHash(od, key, hash, failobj); + if (_PyDict_Pop_KnownHash((PyDictObject *)od, key, hash, + &value) == 0) { + value = Py_NewRef(failobj); + } } else if (value == NULL && !PyErr_Occurred()) { /* Apply the fallback value, if necessary. */ diff --git a/Objects/structseq.c b/Objects/structseq.c index db4aebebdd8404..581d6ad240885a 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -8,7 +8,6 @@ */ #include "Python.h" -#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -417,14 +416,13 @@ structseq_replace(PyStructSequence *self, PyObject *args, PyObject *kwargs) // We do not support types with unnamed fields, so we can iterate over // i >= n_visible_fields case without slicing with (i - n_unnamed_fields). for (i = 0; i < n_fields; ++i) { - PyObject *key = PyUnicode_FromString(Py_TYPE(self)->tp_members[i].name); - if (!key) { + PyObject *ob; + if (PyDict_PopString(kwargs, Py_TYPE(self)->tp_members[i].name, + &ob) < 0) { goto error; } - PyObject *ob = _PyDict_Pop(kwargs, key, self->ob_item[i]); - Py_DECREF(key); - if (!ob) { - goto error; + if (ob == NULL) { + ob = Py_NewRef(self->ob_item[i]); } result->ob_item[i] = ob; } diff --git a/Python/import.c b/Python/import.c index 12f586abc2eed8..f37393bbdc4269 100644 --- a/Python/import.c +++ b/Python/import.c @@ -395,8 +395,8 @@ remove_module(PyThreadState *tstate, PyObject *name) PyObject *modules = MODULES(tstate->interp); if (PyDict_CheckExact(modules)) { - PyObject *mod = _PyDict_Pop(modules, name, Py_None); - Py_XDECREF(mod); + // Error is reported to the caller + (void)PyDict_Pop(modules, name, NULL); } else if (PyMapping_DelItem(modules, name) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e28523284f1517..c17de44731b703 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -125,11 +125,9 @@ sys_set_object(PyInterpreterState *interp, PyObject *key, PyObject *v) } PyObject *sd = interp->sysdict; if (v == NULL) { - v = _PyDict_Pop(sd, key, Py_None); - if (v == NULL) { + if (PyDict_Pop(sd, key, NULL) < 0) { return -1; } - Py_DECREF(v); return 0; } else { From b11c443bb2ebfdd009e43ff208fa6324b658d15d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Nov 2023 15:38:49 +0200 Subject: [PATCH 11/49] gh-111789: Simplify bytecodes.c by using PyDict_GetItemRef() (GH-111978) --- Python/bytecodes.c | 63 ++++++++++--------------------------- Python/executor_cases.c.h | 64 ++++++++++---------------------------- Python/generated_cases.c.h | 64 ++++++++++---------------------------- 3 files changed, 51 insertions(+), 140 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d914f27ba9dcf0..bb8123ee0bdfe3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -643,16 +643,12 @@ dummy_func( inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetKeyError(sub); - } - DECREF_INPUTS(); - ERROR_IF(true, error); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); } - Py_INCREF(res); // Do this before DECREF'ing dict, sub DECREF_INPUTS(); + ERROR_IF(rc <= 0, error); // not found or error } inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { @@ -1349,14 +1345,10 @@ dummy_func( GOTO_ERROR(error); } if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { GOTO_ERROR(error); } - else { + if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { GOTO_ERROR(error); } @@ -1383,14 +1375,10 @@ dummy_func( GOTO_ERROR(error); } if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { GOTO_ERROR(error); } - else { + if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { GOTO_ERROR(error); } @@ -1663,34 +1651,17 @@ dummy_func( ERROR_IF(true, error); } /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(LOCALS())) { - ann_dict = _PyDict_GetItemWithError(LOCALS(), - &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - ERROR_IF(_PyErr_Occurred(tstate), error); - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - ERROR_IF(ann_dict == NULL, error); - err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - ERROR_IF(err, error); - } + ERROR_IF(PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0, error); + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + ERROR_IF(ann_dict == NULL, error); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + ERROR_IF(err, error); } else { - /* do the same if locals() is not a dict */ - ERROR_IF(PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0, error); - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - ERROR_IF(ann_dict == NULL, error); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - ERROR_IF(err, error); - } - else { - Py_DECREF(ann_dict); - } + Py_DECREF(ann_dict); } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 877328a4ab67ab..ac542a562ea6ca 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -512,18 +512,13 @@ dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetKeyError(sub); - } - Py_DECREF(dict); - Py_DECREF(sub); - if (true) goto pop_2_error_tier_two; + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); } - Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); Py_DECREF(sub); + if (rc <= 0) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -1022,14 +1017,10 @@ GOTO_ERROR(error); } if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { GOTO_ERROR(error); } - else { + if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { GOTO_ERROR(error); } @@ -1059,14 +1050,10 @@ GOTO_ERROR(error); } if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { GOTO_ERROR(error); } - else { + if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { GOTO_ERROR(error); } @@ -1404,34 +1391,17 @@ if (true) goto error_tier_two; } /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(LOCALS())) { - ann_dict = _PyDict_GetItemWithError(LOCALS(), - &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) goto error_tier_two; - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error_tier_two; - err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error_tier_two; - } + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error_tier_two; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error_tier_two; } else { - /* do the same if locals() is not a dict */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error_tier_two; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error_tier_two; - } - else { - Py_DECREF(ann_dict); - } + Py_DECREF(ann_dict); } break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b9a2b2275491e7..6cbc54c8e6201e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -896,18 +896,13 @@ dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetKeyError(sub); - } - Py_DECREF(dict); - Py_DECREF(sub); - if (true) goto pop_2_error; + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); } - Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -1950,14 +1945,10 @@ GOTO_ERROR(error); } if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { GOTO_ERROR(error); } - else { + if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { GOTO_ERROR(error); } @@ -1990,14 +1981,10 @@ GOTO_ERROR(error); } if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { GOTO_ERROR(error); } - else { + if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { GOTO_ERROR(error); } @@ -2422,34 +2409,17 @@ if (true) goto error; } /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(LOCALS())) { - ann_dict = _PyDict_GetItemWithError(LOCALS(), - &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) goto error; - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; } else { - /* do the same if locals() is not a dict */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } + Py_DECREF(ann_dict); } DISPATCH(); } From a519b87958da0b340caef48349d6e3c23c98e47e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Nov 2023 15:30:33 +0000 Subject: [PATCH 12/49] GH-111848: Convert remaining jumps to deopts into tier 2 code. (GH-112045) --- Include/internal/pycore_opcode_metadata.h | 142 +++++++++++++--------- Lib/test/test_capi/test_misc.py | 10 +- Python/abstract_interp_cases.c.h | 14 ++- Python/bytecodes.c | 35 ++++-- Python/executor_cases.c.h | 31 +++-- Python/generated_cases.c.h | 16 +-- Python/optimizer.c | 65 ++++------ 7 files changed, 184 insertions(+), 129 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 290564181b24aa..69ac38829f9632 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -76,46 +76,50 @@ #define _STORE_ATTR_SLOT 348 #define _SPECIALIZE_COMPARE_OP 349 #define _COMPARE_OP 350 -#define _IS_NONE 351 -#define _SPECIALIZE_FOR_ITER 352 -#define _FOR_ITER 353 -#define _ITER_CHECK_LIST 354 -#define _ITER_JUMP_LIST 355 -#define _GUARD_NOT_EXHAUSTED_LIST 356 -#define _ITER_NEXT_LIST 357 -#define _ITER_CHECK_TUPLE 358 -#define _ITER_JUMP_TUPLE 359 -#define _GUARD_NOT_EXHAUSTED_TUPLE 360 -#define _ITER_NEXT_TUPLE 361 -#define _ITER_CHECK_RANGE 362 -#define _ITER_JUMP_RANGE 363 -#define _GUARD_NOT_EXHAUSTED_RANGE 364 -#define _ITER_NEXT_RANGE 365 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 366 -#define _GUARD_KEYS_VERSION 367 -#define _LOAD_ATTR_METHOD_WITH_VALUES 368 -#define _LOAD_ATTR_METHOD_NO_DICT 369 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 370 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 371 -#define _CHECK_ATTR_METHOD_LAZY_DICT 372 -#define _LOAD_ATTR_METHOD_LAZY_DICT 373 -#define _SPECIALIZE_CALL 374 -#define _CALL 375 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 376 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 377 -#define _CHECK_PEP_523 378 -#define _CHECK_FUNCTION_EXACT_ARGS 379 -#define _CHECK_STACK_SPACE 380 -#define _INIT_CALL_PY_EXACT_ARGS 381 -#define _PUSH_FRAME 382 -#define _SPECIALIZE_BINARY_OP 383 -#define _BINARY_OP 384 -#define _POP_JUMP_IF_FALSE 385 -#define _POP_JUMP_IF_TRUE 386 -#define _JUMP_TO_TOP 387 -#define _SAVE_RETURN_OFFSET 388 -#define _INSERT 389 -#define _CHECK_VALIDITY 390 +#define _POP_JUMP_IF_FALSE 351 +#define _POP_JUMP_IF_TRUE 352 +#define _IS_NONE 353 +#define _SPECIALIZE_FOR_ITER 354 +#define _FOR_ITER 355 +#define _ITER_CHECK_LIST 356 +#define _ITER_JUMP_LIST 357 +#define _GUARD_NOT_EXHAUSTED_LIST 358 +#define _ITER_NEXT_LIST 359 +#define _ITER_CHECK_TUPLE 360 +#define _ITER_JUMP_TUPLE 361 +#define _GUARD_NOT_EXHAUSTED_TUPLE 362 +#define _ITER_NEXT_TUPLE 363 +#define _ITER_CHECK_RANGE 364 +#define _ITER_JUMP_RANGE 365 +#define _GUARD_NOT_EXHAUSTED_RANGE 366 +#define _ITER_NEXT_RANGE 367 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 368 +#define _GUARD_KEYS_VERSION 369 +#define _LOAD_ATTR_METHOD_WITH_VALUES 370 +#define _LOAD_ATTR_METHOD_NO_DICT 371 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 372 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 373 +#define _CHECK_ATTR_METHOD_LAZY_DICT 374 +#define _LOAD_ATTR_METHOD_LAZY_DICT 375 +#define _SPECIALIZE_CALL 376 +#define _CALL 377 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 378 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 379 +#define _CHECK_PEP_523 380 +#define _CHECK_FUNCTION_EXACT_ARGS 381 +#define _CHECK_STACK_SPACE 382 +#define _INIT_CALL_PY_EXACT_ARGS 383 +#define _PUSH_FRAME 384 +#define _SPECIALIZE_BINARY_OP 385 +#define _BINARY_OP 386 +#define _GUARD_IS_TRUE_POP 387 +#define _GUARD_IS_FALSE_POP 388 +#define _GUARD_IS_NONE_POP 389 +#define _GUARD_IS_NOT_NONE_POP 390 +#define _JUMP_TO_TOP 391 +#define _SAVE_RETURN_OFFSET 392 +#define _INSERT 393 +#define _CHECK_VALIDITY 394 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -505,12 +509,16 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case ENTER_EXECUTOR: return 0; - case POP_JUMP_IF_FALSE: + case _POP_JUMP_IF_FALSE: return 1; - case POP_JUMP_IF_TRUE: + case _POP_JUMP_IF_TRUE: return 1; case _IS_NONE: return 1; + case POP_JUMP_IF_TRUE: + return 1; + case POP_JUMP_IF_FALSE: + return 1; case POP_JUMP_IF_NONE: return 1; case POP_JUMP_IF_NOT_NONE: @@ -725,9 +733,13 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case RESERVED: return 0; - case _POP_JUMP_IF_FALSE: + case _GUARD_IS_TRUE_POP: return 1; - case _POP_JUMP_IF_TRUE: + case _GUARD_IS_FALSE_POP: + return 1; + case _GUARD_IS_NONE_POP: + return 1; + case _GUARD_IS_NOT_NONE_POP: return 1; case _JUMP_TO_TOP: return 0; @@ -1135,12 +1147,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case ENTER_EXECUTOR: return 0; - case POP_JUMP_IF_FALSE: + case _POP_JUMP_IF_FALSE: return 0; - case POP_JUMP_IF_TRUE: + case _POP_JUMP_IF_TRUE: return 0; case _IS_NONE: return 1; + case POP_JUMP_IF_TRUE: + return 0; + case POP_JUMP_IF_FALSE: + return 0; case POP_JUMP_IF_NONE: return 0; case POP_JUMP_IF_NOT_NONE: @@ -1355,9 +1371,13 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RESERVED: return 0; - case _POP_JUMP_IF_FALSE: + case _GUARD_IS_TRUE_POP: return 0; - case _POP_JUMP_IF_TRUE: + case _GUARD_IS_FALSE_POP: + return 0; + case _GUARD_IS_NONE_POP: + return 0; + case _GUARD_IS_NOT_NONE_POP: return 0; case _JUMP_TO_TOP: return 0; @@ -1639,9 +1659,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [JUMP] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_NO_INTERRUPT] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, - [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, + [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, + [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, [_IS_NONE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, + [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1749,8 +1771,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_IS_TRUE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_IS_FALSE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_IS_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_IS_NOT_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1871,6 +1895,10 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [CONTAINS_OP] = { .nuops = 1, .uops = { { CONTAINS_OP, 0, 0 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { CHECK_EXC_MATCH, 0, 0 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, 0, 0 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 0, 0 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 0, 0 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 0, 0 } } }, [GET_LEN] = { .nuops = 1, .uops = { { GET_LEN, 0, 0 } } }, [MATCH_CLASS] = { .nuops = 1, .uops = { { MATCH_CLASS, 0, 0 } } }, [MATCH_MAPPING] = { .nuops = 1, .uops = { { MATCH_MAPPING, 0, 0 } } }, @@ -1972,6 +2000,8 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", [_SPECIALIZE_COMPARE_OP] = "_SPECIALIZE_COMPARE_OP", [_COMPARE_OP] = "_COMPARE_OP", + [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", + [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [_IS_NONE] = "_IS_NONE", [_SPECIALIZE_FOR_ITER] = "_SPECIALIZE_FOR_ITER", [_FOR_ITER] = "_FOR_ITER", @@ -2006,8 +2036,10 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_PUSH_FRAME] = "_PUSH_FRAME", [_SPECIALIZE_BINARY_OP] = "_SPECIALIZE_BINARY_OP", [_BINARY_OP] = "_BINARY_OP", - [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", - [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", + [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_INSERT] = "_INSERT", @@ -2256,8 +2288,8 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_ATTR] = 9, [COMPARE_OP] = 1, [JUMP_BACKWARD] = 1, - [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_TRUE] = 1, + [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, [FOR_ITER] = 1, diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index d526bbf62dda0b..fe5c36c0c0dec9 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2610,7 +2610,7 @@ def testfunc(n): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_FALSE", uops) + self.assertIn("_GUARD_IS_TRUE_POP", uops) def test_pop_jump_if_none(self): def testfunc(a): @@ -2625,7 +2625,7 @@ def testfunc(a): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_TRUE", uops) + self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_not_none(self): def testfunc(a): @@ -2641,7 +2641,7 @@ def testfunc(a): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_FALSE", uops) + self.assertIn("_GUARD_IS_NONE_POP", uops) def test_pop_jump_if_true(self): def testfunc(n): @@ -2656,7 +2656,7 @@ def testfunc(n): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_TRUE", uops) + self.assertIn("_GUARD_IS_FALSE_POP", uops) def test_jump_backward(self): def testfunc(n): @@ -2806,7 +2806,7 @@ def testfunc(n): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_TRUE", uops) + self.assertIn("_GUARD_IS_FALSE_POP", uops) if __name__ == "__main__": diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 8892d266da00b7..a2f6aa8def8f69 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -914,12 +914,22 @@ break; } - case _POP_JUMP_IF_FALSE: { + case _GUARD_IS_TRUE_POP: { STACK_SHRINK(1); break; } - case _POP_JUMP_IF_TRUE: { + case _GUARD_IS_FALSE_POP: { + STACK_SHRINK(1); + break; + } + + case _GUARD_IS_NONE_POP: { + STACK_SHRINK(1); + break; + } + + case _GUARD_IS_NOT_NONE_POP: { STACK_SHRINK(1); break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bb8123ee0bdfe3..8a7dcb8416eb8c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2368,7 +2368,7 @@ dummy_func( goto enter_tier_one; } - inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) { + replaced op(_POP_JUMP_IF_FALSE, (unused/1, cond -- )) { assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION @@ -2377,7 +2377,7 @@ dummy_func( JUMPBY(oparg * flag); } - inst(POP_JUMP_IF_TRUE, (unused/1, cond -- )) { + replaced op(_POP_JUMP_IF_TRUE, (unused/1, cond -- )) { assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION @@ -2396,9 +2396,13 @@ dummy_func( } } - macro(POP_JUMP_IF_NONE) = _IS_NONE + POP_JUMP_IF_TRUE; + macro(POP_JUMP_IF_TRUE) = _POP_JUMP_IF_TRUE; - macro(POP_JUMP_IF_NOT_NONE) = _IS_NONE + POP_JUMP_IF_FALSE; + macro(POP_JUMP_IF_FALSE) = _POP_JUMP_IF_FALSE; + + macro(POP_JUMP_IF_NONE) = _IS_NONE + _POP_JUMP_IF_TRUE; + + macro(POP_JUMP_IF_NOT_NONE) = _IS_NONE + _POP_JUMP_IF_FALSE; inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. @@ -3963,16 +3967,23 @@ dummy_func( ///////// Tier-2 only opcodes ///////// - op(_POP_JUMP_IF_FALSE, (flag -- )) { - if (Py_IsFalse(flag)) { - next_uop = current_executor->trace + oparg; - } + op (_GUARD_IS_TRUE_POP, (flag -- )) { + DEOPT_IF(Py_IsFalse(flag)); + assert(Py_IsTrue(flag)); } - op(_POP_JUMP_IF_TRUE, (flag -- )) { - if (Py_IsTrue(flag)) { - next_uop = current_executor->trace + oparg; - } + op (_GUARD_IS_FALSE_POP, (flag -- )) { + DEOPT_IF(Py_IsTrue(flag)); + assert(Py_IsFalse(flag)); + } + + op (_GUARD_IS_NONE_POP, (val -- )) { + DEOPT_IF(!Py_IsNone(val)); + } + + op (_GUARD_IS_NOT_NONE_POP, (val -- )) { + DEOPT_IF(Py_IsNone(val)); + Py_DECREF(val); } op(_JUMP_TO_TOP, (--)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ac542a562ea6ca..4e29fb9f0fa93d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3185,22 +3185,37 @@ break; } - case _POP_JUMP_IF_FALSE: { + case _GUARD_IS_TRUE_POP: { PyObject *flag; flag = stack_pointer[-1]; - if (Py_IsFalse(flag)) { - next_uop = current_executor->trace + oparg; - } + DEOPT_IF(Py_IsFalse(flag), _GUARD_IS_TRUE_POP); + assert(Py_IsTrue(flag)); STACK_SHRINK(1); break; } - case _POP_JUMP_IF_TRUE: { + case _GUARD_IS_FALSE_POP: { PyObject *flag; flag = stack_pointer[-1]; - if (Py_IsTrue(flag)) { - next_uop = current_executor->trace + oparg; - } + DEOPT_IF(Py_IsTrue(flag), _GUARD_IS_FALSE_POP); + assert(Py_IsFalse(flag)); + STACK_SHRINK(1); + break; + } + + case _GUARD_IS_NONE_POP: { + PyObject *val; + val = stack_pointer[-1]; + DEOPT_IF(!Py_IsNone(val), _GUARD_IS_NONE_POP); + STACK_SHRINK(1); + break; + } + + case _GUARD_IS_NOT_NONE_POP: { + PyObject *val; + val = stack_pointer[-1]; + DEOPT_IF(Py_IsNone(val), _GUARD_IS_NOT_NONE_POP); + Py_DECREF(val); STACK_SHRINK(1); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6cbc54c8e6201e..fe0cbfe6330e51 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3435,14 +3435,14 @@ goto enter_tier_one; } - TARGET(POP_JUMP_IF_FALSE) { + TARGET(POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; cond = stack_pointer[-1]; assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -3451,14 +3451,14 @@ DISPATCH(); } - TARGET(POP_JUMP_IF_TRUE) { + TARGET(POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; cond = stack_pointer[-1]; assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -3485,7 +3485,7 @@ Py_DECREF(value); } } - // POP_JUMP_IF_TRUE + // _POP_JUMP_IF_TRUE cond = b; { assert(PyBool_Check(cond)); @@ -3517,7 +3517,7 @@ Py_DECREF(value); } } - // POP_JUMP_IF_FALSE + // _POP_JUMP_IF_FALSE cond = b; { assert(PyBool_Check(cond)); diff --git a/Python/optimizer.c b/Python/optimizer.c index e142bd0488a69b..bc518d0088b373 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -385,7 +385,7 @@ PyTypeObject _PyUOpExecutor_Type = { .tp_methods = executor_methods, }; -/* TO DO -- Generate this table */ +/* TO DO -- Generate these tables */ static const uint16_t _PyUop_Replacements[OPCODE_METADATA_SIZE] = { [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, @@ -393,6 +393,18 @@ _PyUop_Replacements[OPCODE_METADATA_SIZE] = { [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, }; +static const uint16_t +BRANCH_TO_GUARD[4][2] = { + [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP, + [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_FALSE_POP, + [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_FALSE_POP, + [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_TRUE_POP, + [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NOT_NONE_POP, + [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NONE_POP, + [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NONE_POP, + [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, +}; + #define TRACE_STACK_SIZE 5 /* Returns 1 on success, @@ -528,45 +540,23 @@ translate_bytecode_to_trace( } switch (opcode) { - case POP_JUMP_IF_NONE: - { - RESERVE(2, 2); - ADD_TO_TRACE(_IS_NONE, 0, 0); - opcode = POP_JUMP_IF_TRUE; - goto pop_jump_if_bool; - } - case POP_JUMP_IF_NOT_NONE: - { - RESERVE(2, 2); - ADD_TO_TRACE(_IS_NONE, 0, 0); - opcode = POP_JUMP_IF_FALSE; - goto pop_jump_if_bool; - } - case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { -pop_jump_if_bool: - RESERVE(1, 2); - max_length -= 2; // Really the start of the stubs + RESERVE(1, 0); int counter = instr[1].cache; int bitcount = _Py_popcount32(counter); - bool jump_likely = bitcount > 8; - bool jump_sense = opcode == POP_JUMP_IF_TRUE; - uint32_t uopcode = jump_sense ^ jump_likely ? - _POP_JUMP_IF_TRUE : _POP_JUMP_IF_FALSE; + int jump_likely = bitcount > 8; + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *target_instr = next_instr + oparg; - _Py_CODEUNIT *stub_target = jump_likely ? next_instr : target_instr; - DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, sense=%d, uopcode=%s\n", + DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, uopcode=%s\n", uop_name(opcode), oparg, - counter, bitcount, jump_likely, jump_sense, uop_name(uopcode)); + counter, bitcount, jump_likely, uop_name(uopcode)); ADD_TO_TRACE(uopcode, max_length, 0); - ADD_TO_STUB(max_length, _SET_IP, INSTR_IP(stub_target, code), 0); - ADD_TO_STUB(max_length + 1, _EXIT_TRACE, 0, 0); if (jump_likely) { + _Py_CODEUNIT *target_instr = next_instr + oparg; DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; @@ -797,16 +787,13 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) } /* All other micro-ops fall through, so i+1 is reachable */ SET_BIT(used, i+1); - switch(opcode) { - case NOP: - /* Don't count NOPs as used */ - count--; - UNSET_BIT(used, i); - break; - case _POP_JUMP_IF_FALSE: - case _POP_JUMP_IF_TRUE: - /* Mark target as reachable */ - SET_BIT(used, buffer[i].oparg); + if (OPCODE_HAS_JUMP(opcode)) { + /* Mark target as reachable */ + SET_BIT(used, buffer[i].oparg); + } + if (opcode == NOP) { + count--; + UNSET_BIT(used, i); } } return count; From ee06fffd38cb51ce1c045da9d8336d9ce13c318a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Nov 2023 17:37:56 +0200 Subject: [PATCH 13/49] gh-111942: Fix crashes in TextIOWrapper.reconfigure() (GH-111976) * Fix crash when encoding is not string or None. * Fix crash when both line_buffering and write_through raise exception when converted ti int. * Add a number of tests for constructor and reconfigure() method with invalid arguments. --- Lib/test/test_io.py | 86 ++++++++++++++++++- ...-11-10-22-08-28.gh-issue-111942.MDFm6v.rst | 2 + Modules/_io/textio.c | 39 ++++++++- 3 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-10-22-08-28.gh-issue-111942.MDFm6v.rst diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 3c3be870fac09e..fe622e836ddac2 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -77,6 +77,10 @@ def _default_chunk_size(): ) +class BadIndex: + def __index__(self): + 1/0 + class MockRawIOWithoutRead: """A RawIO implementation without read(), so as to exercise the default RawIO.read() which calls readinto().""" @@ -2709,8 +2713,31 @@ def test_constructor(self): self.assertEqual(t.encoding, "utf-8") self.assertEqual(t.line_buffering, True) self.assertEqual("\xe9\n", t.readline()) - self.assertRaises(TypeError, t.__init__, b, encoding="utf-8", newline=42) - self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') + invalid_type = TypeError if self.is_C else ValueError + with self.assertRaises(invalid_type): + t.__init__(b, encoding=42) + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding='utf-8\0') + with self.assertRaises(invalid_type): + t.__init__(b, encoding="utf-8", errors=42) + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding="utf-8", errors='\udcfe') + if support.Py_DEBUG or sys.flags.dev_mode: + # TODO: If encoded to UTF-8, should also be checked for + # embedded null characters. + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", errors='replace\0') + with self.assertRaises(TypeError): + t.__init__(b, encoding="utf-8", newline=42) + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\n\0') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='xyzzy') def test_uninitialized(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) @@ -3756,6 +3783,59 @@ def test_reconfigure_defaults(self): self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n') + def test_reconfigure_errors(self): + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r') + with self.assertRaises(TypeError): # there was a crash + txt.reconfigure(encoding=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(encoding='\udcfe') + with self.assertRaises(LookupError): + txt.reconfigure(encoding='locale\0') + # TODO: txt.reconfigure(encoding='utf-8\0') + # TODO: txt.reconfigure(encoding='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(errors=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(errors='\udcfe') + # TODO: txt.reconfigure(errors='ignore\0') + # TODO: txt.reconfigure(errors='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(newline=42) + with self.assertRaises(ValueError): + txt.reconfigure(newline='\udcfe') + with self.assertRaises(ValueError): + txt.reconfigure(newline='xyz') + if not self.is_C: + # TODO: Should fail in C too. + with self.assertRaises(ValueError): + txt.reconfigure(newline='\n\0') + if self.is_C: + # TODO: Use __bool__(), not __index__(). + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(line_buffering=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(line_buffering=2**1000) + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(write_through=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(write_through=2**1000) + with self.assertRaises(ZeroDivisionError): # there was a crash + txt.reconfigure(line_buffering=BadIndex(), + write_through=BadIndex()) + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + self.assertIs(txt.line_buffering, False) + self.assertIs(txt.write_through, False) + + txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n', + line_buffering=True, write_through=True) + self.assertEqual(txt.encoding, 'latin1') + self.assertEqual(txt.errors, 'ignore') + self.assertIs(txt.line_buffering, True) + self.assertIs(txt.write_through, True) + def test_reconfigure_newline(self): raw = self.BytesIO(b'CR\rEOF') txt = self.TextIOWrapper(raw, 'ascii', newline='\n') @@ -4781,9 +4861,11 @@ def load_tests(loader, tests, pattern): if test.__name__.startswith("C"): for name, obj in c_io_ns.items(): setattr(test, name, obj) + test.is_C = True elif test.__name__.startswith("Py"): for name, obj in py_io_ns.items(): setattr(test, name, obj) + test.is_C = False suite = loader.suiteClass() for test in tests: diff --git a/Misc/NEWS.d/next/Library/2023-11-10-22-08-28.gh-issue-111942.MDFm6v.rst b/Misc/NEWS.d/next/Library/2023-11-10-22-08-28.gh-issue-111942.MDFm6v.rst new file mode 100644 index 00000000000000..4fc505c8f257a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-10-22-08-28.gh-issue-111942.MDFm6v.rst @@ -0,0 +1,2 @@ +Fix crashes in :meth:`io.TextIOWrapper.reconfigure` when pass invalid +arguments, e.g. non-string encoding. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index e6a971e2250d63..5c37e99067f886 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1284,30 +1284,40 @@ textiowrapper_change_encoding(textio *self, PyObject *encoding, errors = &_Py_ID(strict); } } + Py_INCREF(errors); + const char *c_encoding = PyUnicode_AsUTF8(encoding); + if (c_encoding == NULL) { + Py_DECREF(encoding); + Py_DECREF(errors); + return -1; + } const char *c_errors = PyUnicode_AsUTF8(errors); if (c_errors == NULL) { Py_DECREF(encoding); + Py_DECREF(errors); return -1; } // Create new encoder & decoder PyObject *codec_info = _PyCodec_LookupTextEncoding( - PyUnicode_AsUTF8(encoding), "codecs.open()"); + c_encoding, "codecs.open()"); if (codec_info == NULL) { Py_DECREF(encoding); + Py_DECREF(errors); return -1; } if (_textiowrapper_set_decoder(self, codec_info, c_errors) != 0 || _textiowrapper_set_encoder(self, codec_info, c_errors) != 0) { Py_DECREF(codec_info); Py_DECREF(encoding); + Py_DECREF(errors); return -1; } Py_DECREF(codec_info); Py_SETREF(self->encoding, encoding); - Py_SETREF(self->errors, Py_NewRef(errors)); + Py_SETREF(self->errors, errors); return _textiowrapper_fix_encoder_state(self); } @@ -1338,6 +1348,26 @@ _io_TextIOWrapper_reconfigure_impl(textio *self, PyObject *encoding, int write_through; const char *newline = NULL; + if (encoding != Py_None && !PyUnicode_Check(encoding)) { + PyErr_Format(PyExc_TypeError, + "reconfigure() argument 'encoding' must be str or None, not %s", + Py_TYPE(encoding)->tp_name); + return NULL; + } + if (errors != Py_None && !PyUnicode_Check(errors)) { + PyErr_Format(PyExc_TypeError, + "reconfigure() argument 'errors' must be str or None, not %s", + Py_TYPE(errors)->tp_name); + return NULL; + } + if (newline_obj != NULL && newline_obj != Py_None && + !PyUnicode_Check(newline_obj)) + { + PyErr_Format(PyExc_TypeError, + "reconfigure() argument 'newline' must be str or None, not %s", + Py_TYPE(newline_obj)->tp_name); + return NULL; + } /* Check if something is in the read buffer */ if (self->decoded_chars != NULL) { if (encoding != Py_None || errors != Py_None || newline_obj != NULL) { @@ -1357,9 +1387,12 @@ _io_TextIOWrapper_reconfigure_impl(textio *self, PyObject *encoding, line_buffering = convert_optional_bool(line_buffering_obj, self->line_buffering); + if (line_buffering < 0) { + return NULL; + } write_through = convert_optional_bool(write_through_obj, self->write_through); - if (line_buffering < 0 || write_through < 0) { + if (write_through < 0) { return NULL; } From 0ff6368519ed7542ad8b443de01108690102420a Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 14 Nov 2023 19:07:46 +0300 Subject: [PATCH 14/49] gh-111906: Fix warnings during mimalloc build on FreeBSD (#111907) Fix `unused function` warnings during mimalloc build on FreeBSD. --- Objects/mimalloc/prim/unix/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index 790d7418eaba03..cffbb2d0b4d7b2 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -50,7 +50,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__FreeBSD__) #define MI_HAS_SYSCALL_H #include #endif @@ -76,7 +76,7 @@ static int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__APPLE__) && !defined(_AIX) // avoid unused warnings +#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__FreeBSD__) // avoid unused warnings static int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); From 9302f05f9af07332c414b3c19003efd1b1763cf3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Nov 2023 22:02:28 +0200 Subject: [PATCH 15/49] gh-111942: Fix SystemError in the TextIOWrapper constructor (#112061) In non-debug more the check for the "errors" argument is skipped, and then PyUnicode_AsUTF8() can fail, but its result was not checked. Co-authored-by: Victor Stinner --- Lib/test/test_io.py | 4 +--- .../2023-11-14-18-43-55.gh-issue-111942.x1pnrj.rst | 2 ++ Modules/_io/textio.c | 8 ++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-14-18-43-55.gh-issue-111942.x1pnrj.rst diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index fe622e836ddac2..ab3389215e93d0 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2725,9 +2725,7 @@ def test_constructor(self): if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: with self.assertRaises(UnicodeEncodeError): t.__init__(b, encoding="utf-8", errors='\udcfe') - if support.Py_DEBUG or sys.flags.dev_mode: - # TODO: If encoded to UTF-8, should also be checked for - # embedded null characters. + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: with self.assertRaises(ValueError): t.__init__(b, encoding="utf-8", errors='replace\0') with self.assertRaises(TypeError): diff --git a/Misc/NEWS.d/next/Library/2023-11-14-18-43-55.gh-issue-111942.x1pnrj.rst b/Misc/NEWS.d/next/Library/2023-11-14-18-43-55.gh-issue-111942.x1pnrj.rst new file mode 100644 index 00000000000000..ca58a6fa5d6ae1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-14-18-43-55.gh-issue-111942.x1pnrj.rst @@ -0,0 +1,2 @@ +Fix SystemError in the TextIOWrapper constructor with non-encodable "errors" +argument in non-debug mode. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 5c37e99067f886..8d19198502fe12 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1112,6 +1112,10 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, else if (io_check_errors(errors)) { return -1; } + const char *errors_str = _PyUnicode_AsUTF8NoNUL(errors); + if (errors_str == NULL) { + return -1; + } if (validate_newline(newline) < 0) { return -1; @@ -1184,11 +1188,11 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, /* Build the decoder object */ _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); self->state = state; - if (_textiowrapper_set_decoder(self, codec_info, PyUnicode_AsUTF8(errors)) != 0) + if (_textiowrapper_set_decoder(self, codec_info, errors_str) != 0) goto error; /* Build the encoder object */ - if (_textiowrapper_set_encoder(self, codec_info, PyUnicode_AsUTF8(errors)) != 0) + if (_textiowrapper_set_encoder(self, codec_info, errors_str) != 0) goto error; /* Finished sorting out the codec details */ From 62802b6228f001e1a4af6ac668a21d2dcec0ce57 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 01:19:20 +0100 Subject: [PATCH 16/49] gh-111545: Add Include/cpython/pyhash.h header file (#112063) Move non-limited C API to a new Include/cpython/pyhash.h header file. --- Include/cpython/pyhash.h | 13 +++++++++++++ Include/pyhash.h | 19 ++++++------------- Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 5 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 Include/cpython/pyhash.h diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h new file mode 100644 index 00000000000000..62ae6084bbcf53 --- /dev/null +++ b/Include/cpython/pyhash.h @@ -0,0 +1,13 @@ +#ifndef Py_CPYTHON_HASH_H +# error "this header file must not be included directly" +#endif + +/* hash function definition */ +typedef struct { + Py_hash_t (*const hash)(const void *, Py_ssize_t); + const char *name; + const int hash_bits; + const int seed_bits; +} PyHash_FuncDef; + +PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); diff --git a/Include/pyhash.h b/Include/pyhash.h index 6e969f86fa2625..3e23e2758808d7 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -4,19 +4,6 @@ extern "C" { #endif -#ifndef Py_LIMITED_API -/* hash function definition */ -typedef struct { - Py_hash_t (*const hash)(const void *, Py_ssize_t); - const char *name; - const int hash_bits; - const int seed_bits; -} PyHash_FuncDef; - -PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); -#endif - - /* Cutoff for small string DJBX33A optimization in range [1, cutoff). * * About 50% of the strings in a typical Python application are smaller than @@ -60,6 +47,12 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); # endif /* uint64_t && uint32_t && aligned */ #endif /* Py_HASH_ALGORITHM */ +#ifndef Py_LIMITED_API +# define Py_CPYTHON_HASH_H +# include "cpython/pyhash.h" +# undef Py_CPYTHON_HASH_H +#endif + #ifdef __cplusplus } #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index a74cb7ebc706d9..4fa0337fcc7777 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1768,6 +1768,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pyerrors.h \ $(srcdir)/Include/cpython/pyfpe.h \ $(srcdir)/Include/cpython/pyframe.h \ + $(srcdir)/Include/cpython/pyhash.h \ $(srcdir)/Include/cpython/pylifecycle.h \ $(srcdir)/Include/cpython/pymem.h \ $(srcdir)/Include/cpython/pystate.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f6fbd0fccc3f9b..be1b98dba02fc5 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -172,6 +172,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index eb7ba0417c8146..4f0da8f35998b7 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -468,6 +468,9 @@ Include\cpython + + Include\cpython + Include\cpython From fe9db901b2446b047e537447ea5bad3d470b0f78 Mon Sep 17 00:00:00 2001 From: adder32 Date: Wed, 15 Nov 2023 02:00:24 +0100 Subject: [PATCH 17/49] gh-111995: Add getnameinfo extension flag (#111994) Add getnameinfo extension NI_IDN flag. --- .../Library/2023-11-14-16-31-59.gh-issue-111995.OoX8JJ.rst | 2 ++ Modules/socketmodule.c | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-11-14-16-31-59.gh-issue-111995.OoX8JJ.rst diff --git a/Misc/NEWS.d/next/Library/2023-11-14-16-31-59.gh-issue-111995.OoX8JJ.rst b/Misc/NEWS.d/next/Library/2023-11-14-16-31-59.gh-issue-111995.OoX8JJ.rst new file mode 100644 index 00000000000000..773f620c97b608 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-14-16-31-59.gh-issue-111995.OoX8JJ.rst @@ -0,0 +1,2 @@ +Added the ``NI_IDN`` constant to the :mod:`socket` module when present in C +at build time for use with :func:`socket.getnameinfo`. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 2e1a0974ceb75d..9ac2001c0132d3 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8803,6 +8803,9 @@ socket_exec(PyObject *m) #ifdef NI_DGRAM ADD_INT_MACRO(m, NI_DGRAM); #endif +#ifdef NI_IDN + ADD_INT_MACRO(m, NI_IDN); +#endif /* shutdown() parameters */ #ifdef SHUT_RD From e0f512797596282bff63260f8102592aad37cdf1 Mon Sep 17 00:00:00 2001 From: DPR Date: Wed, 15 Nov 2023 09:17:51 +0800 Subject: [PATCH 18/49] gh-109538: Avoid RuntimeError when StreamWriter is deleted with closed loop (#111983) Issue a ResourceWarning instead. Co-authored-by: Hugo van Kemenade --- Lib/asyncio/streams.py | 8 ++-- Lib/test/test_asyncio/test_streams.py | 41 ++++++++++++++++++- ...-11-11-16-42-48.gh-issue-109538.cMG5ux.rst | 1 + 3 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-11-16-42-48.gh-issue-109538.cMG5ux.rst diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index f82b10c3803656..ffb48b9cd6cb70 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -406,9 +406,11 @@ async def start_tls(self, sslcontext, *, def __del__(self, warnings=warnings): if not self._transport.is_closing(): - self.close() - warnings.warn(f"unclosed {self!r}", ResourceWarning) - + if self._loop.is_closed(): + warnings.warn("loop is closed", ResourceWarning) + else: + self.close() + warnings.warn(f"unclosed {self!r}", ResourceWarning) class StreamReader: diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 3fea7b9f6911aa..ccb7dbf667c320 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1082,10 +1082,11 @@ async def inner(httpd): self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() self.assertTrue(data.endswith(b'\r\n\r\nTest message')) - with self.assertWarns(ResourceWarning): + with self.assertWarns(ResourceWarning) as cm: del wr gc.collect() - + self.assertEqual(len(cm.warnings), 1) + self.assertTrue(str(cm.warnings[0].message).startswith("unclosed None: port = socket_helper.find_unused_port() diff --git a/Misc/NEWS.d/next/Library/2023-11-11-16-42-48.gh-issue-109538.cMG5ux.rst b/Misc/NEWS.d/next/Library/2023-11-11-16-42-48.gh-issue-109538.cMG5ux.rst new file mode 100644 index 00000000000000..d1ee4c054a3f19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-11-16-42-48.gh-issue-109538.cMG5ux.rst @@ -0,0 +1 @@ +Issue warning message instead of having :class:`RuntimeError` be displayed when event loop has already been closed at :meth:`StreamWriter.__del__`. From 55f3cce821f8f18ddb485aa07bdf0190c358d081 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 03:41:29 +0100 Subject: [PATCH 19/49] gh-111545: Test PyHash_GetFuncDef() function (#112098) Add Modules/_testcapi/hash.c and Lib/test/test_capi/test_hash.py. --- Doc/c-api/hash.rst | 48 ++++++++++++++++++++++++++ Doc/c-api/utilities.rst | 1 + Lib/test/test_capi/test_hash.py | 33 ++++++++++++++++++ Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/hash.c | 56 +++++++++++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 ++ PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 ++ 9 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 Doc/c-api/hash.rst create mode 100644 Lib/test/test_capi/test_hash.py create mode 100644 Modules/_testcapi/hash.c diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst new file mode 100644 index 00000000000000..4dc121d7fbaa9b --- /dev/null +++ b/Doc/c-api/hash.rst @@ -0,0 +1,48 @@ +.. highlight:: c + +PyHash API +---------- + +See also the :c:member:`PyTypeObject.tp_hash` member. + +.. c:type:: Py_hash_t + + Hash value type: signed integer. + + .. versionadded:: 3.2 + +.. c:type:: Py_uhash_t + + Hash value type: unsigned integer. + + .. versionadded:: 3.2 + + +.. c:type:: PyHash_FuncDef + + Hash function definition used by :c:func:`PyHash_GetFuncDef`. + + .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + + Hash function. + + .. c:member:: const char *name + + Hash function name (UTF-8 encoded string). + + .. c:member:: const int hash_bits + + Internal size of the hash value in bits. + + .. c:member:: const int seed_bits + + Size of seed input in bits. + + .. versionadded:: 3.4 + + +.. c:function:: PyHash_FuncDef* PyHash_GetFuncDef(void) + + Get the hash function definition. + + .. versionadded:: 3.4 diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index ccbf14e1850f68..48ae54acebe887 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -17,6 +17,7 @@ and parsing function arguments and constructing Python values from C values. marshal.rst arg.rst conversion.rst + hash.rst reflection.rst codec.rst perfmaps.rst diff --git a/Lib/test/test_capi/test_hash.py b/Lib/test/test_capi/test_hash.py new file mode 100644 index 00000000000000..59dec15bc21445 --- /dev/null +++ b/Lib/test/test_capi/test_hash.py @@ -0,0 +1,33 @@ +import sys +import unittest +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') + + +SIZEOF_PY_HASH_T = _testcapi.SIZEOF_VOID_P + + +class CAPITest(unittest.TestCase): + def test_hash_getfuncdef(self): + # Test PyHash_GetFuncDef() + hash_getfuncdef = _testcapi.hash_getfuncdef + func_def = hash_getfuncdef() + + match func_def.name: + case "fnv": + self.assertEqual(func_def.hash_bits, 8 * SIZEOF_PY_HASH_T) + self.assertEqual(func_def.seed_bits, 16 * SIZEOF_PY_HASH_T) + case "siphash13": + self.assertEqual(func_def.hash_bits, 64) + self.assertEqual(func_def.seed_bits, 128) + case "siphash24": + self.assertEqual(func_def.hash_bits, 64) + self.assertEqual(func_def.seed_bits, 128) + case _: + self.fail(f"unknown function name: {func_def.name!r}") + + # compare with sys.hash_info + hash_info = sys.hash_info + self.assertEqual(func_def.name, hash_info.algorithm) + self.assertEqual(func_def.hash_bits, hash_info.hash_bits) + self.assertEqual(func_def.seed_bits, hash_info.seed_bits) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d144ad312ef560..54650ea9c1d4ac 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -159,7 +159,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/hash.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/hash.c b/Modules/_testcapi/hash.c new file mode 100644 index 00000000000000..d0b8127020c5c1 --- /dev/null +++ b/Modules/_testcapi/hash.c @@ -0,0 +1,56 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +hash_getfuncdef(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // bind PyHash_GetFuncDef() + PyHash_FuncDef *def = PyHash_GetFuncDef(); + + PyObject *types = PyImport_ImportModule("types"); + if (types == NULL) { + return NULL; + } + + PyObject *result = PyObject_CallMethod(types, "SimpleNamespace", NULL); + Py_DECREF(types); + if (result == NULL) { + return NULL; + } + + // ignore PyHash_FuncDef.hash + + PyObject *value = PyUnicode_FromString(def->name); + int res = PyObject_SetAttrString(result, "name", value); + Py_DECREF(value); + if (res < 0) { + return NULL; + } + + value = PyLong_FromLong(def->hash_bits); + res = PyObject_SetAttrString(result, "hash_bits", value); + Py_DECREF(value); + if (res < 0) { + return NULL; + } + + value = PyLong_FromLong(def->seed_bits); + res = PyObject_SetAttrString(result, "seed_bits", value); + Py_DECREF(value); + if (res < 0) { + return NULL; + } + + return result; +} + +static PyMethodDef test_methods[] = { + {"hash_getfuncdef", hash_getfuncdef, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Hash(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 9af414e10d0160..29817edd69b134 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -58,6 +58,7 @@ int _PyTestCapi_Init_Codec(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Sys(PyObject *module); +int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_VectorcallLimited(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 300025ce3705fe..999bd866f14814 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3995,6 +3995,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Hash(m) < 0) { + return NULL; + } PyState_AddModule(m, &_testcapimodule); return m; diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 2d8a652a84b92d..1c15541d3ec735 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -124,6 +124,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 573e54d56a2020..6059959bb9a040 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -102,6 +102,9 @@ Source Files + + Source Files + Source Files From 4744f59a5e86690e76c35d9b9d79971ebe9a87d7 Mon Sep 17 00:00:00 2001 From: Radislav Chugunov <52372310+chgnrdv@users.noreply.github.com> Date: Wed, 15 Nov 2023 07:00:34 +0300 Subject: [PATCH 20/49] gh-112062: Make `_struct` module thread-safe in `--disable-gil` builds (#112094) * gh-112062: Make `_struct` module thread-safe in --disable-gil builds --- Modules/_struct.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Modules/_struct.c b/Modules/_struct.c index ff1bf4e96c5f21..0116b03ea95115 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2250,12 +2250,6 @@ cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) return 1; } - if (state->cache == NULL) { - state->cache = PyDict_New(); - if (state->cache == NULL) - return 0; - } - s_object = PyDict_GetItemWithError(state->cache, fmt); if (s_object != NULL) { *ptr = (PyStructObject *)Py_NewRef(s_object); @@ -2288,7 +2282,7 @@ static PyObject * _clearcache_impl(PyObject *module) /*[clinic end generated code: output=ce4fb8a7bf7cb523 input=463eaae04bab3211]*/ { - Py_CLEAR(get_struct_state(module)->cache); + PyDict_Clear(get_struct_state(module)->cache); Py_RETURN_NONE; } @@ -2512,6 +2506,11 @@ _structmodule_exec(PyObject *m) { _structmodulestate *state = get_struct_state(m); + state->cache = PyDict_New(); + if (state->cache == NULL) { + return -1; + } + state->PyStructType = PyType_FromModuleAndSpec( m, &PyStructType_spec, NULL); if (state->PyStructType == NULL) { From 6c214dea7c503eb42bd130d43e8880f39bff0350 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 15 Nov 2023 04:04:28 +0000 Subject: [PATCH 21/49] Docs: Add the time to the HTML last updated format (#110091) --- Doc/conf.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index c92ea60ee07094..f1b411126c4e87 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -288,9 +288,8 @@ "pr_id": os.getenv("READTHEDOCS_VERSION") } -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +# This 'Last updated on:' timestamp is inserted at the bottom of every page. +html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime()) # Path to find HTML templates. templates_path = ['tools/templates'] From d4f83e1e3a19e2f881115f20d58ae6a019ddb48f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 15 Nov 2023 06:28:30 +0200 Subject: [PATCH 22/49] gh-111789: Use PyDict_GetItemRef() in Modules/_asynciomodule.c (GH-112072) --- Modules/_asynciomodule.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 6d06208dd1d70d..3a11cdc926f138 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3514,15 +3514,11 @@ _asyncio_current_task_impl(PyObject *module, PyObject *loop) Py_INCREF(loop); } - ret = PyDict_GetItemWithError(state->current_tasks, loop); + int rc = PyDict_GetItemRef(state->current_tasks, loop, &ret); Py_DECREF(loop); - if (ret == NULL && PyErr_Occurred()) { - return NULL; - } - else if (ret == NULL) { + if (rc == 0) { Py_RETURN_NONE; } - Py_INCREF(ret); return ret; } From 422c0f030170490c001575303d9e628a0298d457 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 15 Nov 2023 15:26:49 +0300 Subject: [PATCH 23/49] gh-108303: Delete `imp_dummy` test file (#112110) --- Lib/test/imp_dummy.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 Lib/test/imp_dummy.py diff --git a/Lib/test/imp_dummy.py b/Lib/test/imp_dummy.py deleted file mode 100644 index 2a4deb49547cf4..00000000000000 --- a/Lib/test/imp_dummy.py +++ /dev/null @@ -1,3 +0,0 @@ -# Fodder for test of issue24748 in test_imp - -dummy_name = True From 0cfdd6e3d17fee8c1c1f4b42b2146abcb43aa34b Mon Sep 17 00:00:00 2001 From: Ryuji Tsutsui Date: Thu, 16 Nov 2023 00:40:19 +0900 Subject: [PATCH 24/49] Fix typo in perf profiling docs (#112112) --- Doc/howto/perf_profiling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index af7b67d2042950..bb1c00e0aa51d5 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -97,7 +97,7 @@ Then we can use ``perf report`` to analyze the data: | | | | | |--2.97%--_PyObject_Malloc ... -As you can see, the Python functions are not shown in the output, only ``_Py_Eval_EvalFrameDefault`` +As you can see, the Python functions are not shown in the output, only ``_PyEval_EvalFrameDefault`` (the function that evaluates the Python bytecode) shows up. Unfortunately that's not very useful because all Python functions use the same C function to evaluate bytecode so we cannot know which Python function corresponds to which bytecode-evaluating function. From 4bbb367ba65e1df7307f7c6a33afd3c369592188 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 15 Nov 2023 15:48:58 +0000 Subject: [PATCH 25/49] GH-111848: Set the IP when de-optimizing (GH-112065) * Replace jumps with deopts in tier 2 * Fewer special cases of uop names * Add target field to uop IR * Remove more redundant SET_IP and _CHECK_VALIDITY micro-ops * Extend whitelist of non-escaping API functions. --- Include/internal/pycore_opcode_metadata.h | 104 +++++++++++----------- Include/internal/pycore_uops.h | 5 +- Python/ceval.c | 2 + Python/optimizer.c | 48 +++++----- Python/optimizer_analysis.c | 22 +++-- Tools/cases_generator/flags.py | 35 ++++++-- 6 files changed, 118 insertions(+), 98 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 69ac38829f9632..4d98b23df5d927 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1469,7 +1469,7 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SI const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_CLOSURE] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, @@ -1488,24 +1488,24 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [END_SEND] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, [_SPECIALIZE_TO_BOOL] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, [_TO_BOOL] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [_GUARD_BOTH_INT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [_GUARD_BOTH_FLOAT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, [_BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, @@ -1528,7 +1528,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [_SPECIALIZE_STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, [_STORE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1558,16 +1558,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [_SPECIALIZE_UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [_UNPACK_SEQUENCE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [_SPECIALIZE_STORE_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, [_STORE_ATTR] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1593,7 +1593,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1632,8 +1632,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_CHECK_ATTR_CLASS] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [_GUARD_DORV_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, @@ -1648,7 +1648,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1658,14 +1658,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_NO_INTERRUPT] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, - [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, - [_IS_NONE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, - [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, - [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, - [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, + [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_IS_NONE] = { true, INSTR_FMT_IX, 0 }, + [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1681,13 +1681,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [_GUARD_NOT_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, 0 }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [_ITER_CHECK_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_ITER_JUMP_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [_GUARD_NOT_EXHAUSTED_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, 0 }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [_ITER_CHECK_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_ITER_JUMP_RANGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [_GUARD_NOT_EXHAUSTED_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, @@ -1701,17 +1701,17 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [SETUP_CLEANUP] = { true, 0, 0 }, [SETUP_WITH] = { true, 0, 0 }, [POP_BLOCK] = { true, 0, 0 }, - [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_GUARD_KEYS_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, [_LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1720,16 +1720,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_CALL] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_CHECK_FUNCTION_EXACT_ARGS] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_PUSH_FRAME] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [_PUSH_FRAME] = { true, INSTR_FMT_IX, 0 }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1740,7 +1740,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1756,7 +1756,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_SPECIALIZE_BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [_BINARY_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1764,17 +1764,17 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [_GUARD_IS_TRUE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_IS_FALSE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_IS_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_IS_NOT_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_IS_TRUE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_IS_FALSE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_IS_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_IS_NOT_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index d8a7d978f1304e..0ecbd2dfd1af73 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -13,8 +13,9 @@ extern "C" { #define _Py_UOP_MAX_TRACE_LENGTH 128 typedef struct { - uint32_t opcode; - uint32_t oparg; + uint16_t opcode; + uint16_t oparg; + uint32_t target; uint64_t operand; // A cache entry } _PyUOpInstruction; diff --git a/Python/ceval.c b/Python/ceval.c index fe1bf310bdc5ea..d684c72cc9e302 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1067,6 +1067,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int UOP_STAT_INC(opcode, miss); frame->return_offset = 0; // Dispatch to frame->instr_ptr _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = next_uop[-1].target + _PyCode_CODE((PyCodeObject *)frame->f_executable); Py_DECREF(current_executor); // Fall through // Jump here from ENTER_EXECUTOR @@ -1077,6 +1078,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from _EXIT_TRACE exit_trace: _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = next_uop[-1].target + _PyCode_CODE((PyCodeObject *)frame->f_executable); Py_DECREF(current_executor); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); goto enter_tier_one; diff --git a/Python/optimizer.c b/Python/optimizer.c index bc518d0088b373..e14ad89bbe2921 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -446,7 +446,8 @@ translate_bytecode_to_trace( #define DPRINTF(level, ...) #endif -#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND) \ + +#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ DPRINTF(2, \ " ADD_TO_TRACE(%s, %d, %" PRIu64 ")\n", \ uop_name(OPCODE), \ @@ -458,23 +459,12 @@ translate_bytecode_to_trace( trace[trace_length].opcode = (OPCODE); \ trace[trace_length].oparg = (OPARG); \ trace[trace_length].operand = (OPERAND); \ + trace[trace_length].target = (TARGET); \ trace_length++; #define INSTR_IP(INSTR, CODE) \ ((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) -#define ADD_TO_STUB(INDEX, OPCODE, OPARG, OPERAND) \ - DPRINTF(2, " ADD_TO_STUB(%d, %s, %d, %" PRIu64 ")\n", \ - (INDEX), \ - uop_name(OPCODE), \ - (OPARG), \ - (uint64_t)(OPERAND)); \ - assert(reserved > 0); \ - reserved--; \ - trace[(INDEX)].opcode = (OPCODE); \ - trace[(INDEX)].oparg = (OPARG); \ - trace[(INDEX)].operand = (OPERAND); - // Reserve space for n uops #define RESERVE_RAW(n, opname) \ if (trace_length + (n) > max_length) { \ @@ -483,7 +473,7 @@ translate_bytecode_to_trace( OPT_STAT_INC(trace_too_long); \ goto done; \ } \ - reserved = (n); // Keep ADD_TO_TRACE / ADD_TO_STUB honest + reserved = (n); // Keep ADD_TO_TRACE honest // Reserve space for main+stub uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE #define RESERVE(main, stub) RESERVE_RAW((main) + (stub) + 3, uop_name(opcode)) @@ -493,7 +483,7 @@ translate_bytecode_to_trace( if (trace_stack_depth >= TRACE_STACK_SIZE) { \ DPRINTF(2, "Trace stack overflow\n"); \ OPT_STAT_INC(trace_stack_overflow); \ - ADD_TO_TRACE(_SET_IP, 0, 0); \ + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ goto done; \ } \ trace_stack[trace_stack_depth].code = code; \ @@ -513,22 +503,28 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); - + uint32_t target = 0; top: // Jump here after _PUSH_FRAME or likely branches for (;;) { + target = INSTR_IP(instr, code); RESERVE_RAW(3, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE - ADD_TO_TRACE(_SET_IP, INSTR_IP(instr, code), 0); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0); + ADD_TO_TRACE(_SET_IP, target, 0, target); + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; uint32_t extras = 0; - while (opcode == EXTENDED_ARG) { + + if (opcode == EXTENDED_ARG) { instr++; extras += 1; opcode = instr->op.code; oparg = (oparg << 8) | instr->op.arg; + if (opcode == EXTENDED_ARG) { + instr--; + goto done; + } } if (opcode == ENTER_EXECUTOR) { @@ -554,7 +550,7 @@ translate_bytecode_to_trace( DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, uopcode=%s\n", uop_name(opcode), oparg, counter, bitcount, jump_likely, uop_name(uopcode)); - ADD_TO_TRACE(uopcode, max_length, 0); + ADD_TO_TRACE(uopcode, max_length, 0, target); if (jump_likely) { _Py_CODEUNIT *target_instr = next_instr + oparg; DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", @@ -569,7 +565,7 @@ translate_bytecode_to_trace( { if (instr + 2 - oparg == initial_instr && code == initial_code) { RESERVE(1, 0); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0); + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); } else { OPT_STAT_INC(inner_loop); @@ -653,7 +649,7 @@ translate_bytecode_to_trace( expansion->uops[i].offset); Py_FatalError("garbled expansion"); } - ADD_TO_TRACE(uop, oparg, operand); + ADD_TO_TRACE(uop, oparg, operand, target); if (uop == _POP_FRAME) { TRACE_STACK_POP(); DPRINTF(2, @@ -682,7 +678,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(new_code->co_filename), new_code->co_firstlineno); OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(_SET_IP, 0, 0); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } if (new_code->co_version != func_version) { @@ -690,7 +686,7 @@ translate_bytecode_to_trace( // Perhaps it may happen again, so don't bother tracing. // TODO: Reason about this -- is it better to bail or not? DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(_SET_IP, 0, 0); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } // Increment IP to the return address @@ -707,7 +703,7 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } - ADD_TO_TRACE(_SET_IP, 0, 0); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } } @@ -732,7 +728,7 @@ translate_bytecode_to_trace( assert(code == initial_code); // Skip short traces like _SET_IP, LOAD_FAST, _SET_IP, _EXIT_TRACE if (trace_length > 4) { - ADD_TO_TRACE(_EXIT_TRACE, 0, 0); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); DPRINTF(1, "Created a trace for %s (%s:%d) at byte offset %d -- length %d+%d\n", PyUnicode_AsUTF8(code->co_qualname), diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 3c8596463fd2b4..0f9bc085f22f1c 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -17,21 +17,15 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { // Note that we don't enter stubs, those SET_IPs are needed. int last_set_ip = -1; - bool need_ip = true; bool maybe_invalid = false; for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; if (opcode == _SET_IP) { - if (!need_ip && last_set_ip >= 0) { - buffer[last_set_ip].opcode = NOP; - } - need_ip = false; + buffer[pc].opcode = NOP; last_set_ip = pc; } else if (opcode == _CHECK_VALIDITY) { if (maybe_invalid) { - /* Exiting the trace requires that IP is correct */ - need_ip = true; maybe_invalid = false; } else { @@ -42,12 +36,16 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) break; } else { - // If opcode has ERROR or DEOPT, set need_ip to true - if (_PyOpcode_opcode_metadata[opcode].flags & (HAS_ERROR_FLAG | HAS_DEOPT_FLAG) || opcode == _PUSH_FRAME) { - need_ip = true; - } - if (_PyOpcode_opcode_metadata[opcode].flags & HAS_ESCAPES_FLAG) { + if (OPCODE_HAS_ESCAPES(opcode)) { maybe_invalid = true; + if (last_set_ip >= 0) { + buffer[last_set_ip].opcode = _SET_IP; + } + } + if (OPCODE_HAS_ERROR(opcode) || opcode == _PUSH_FRAME) { + if (last_set_ip >= 0) { + buffer[last_set_ip].opcode = _SET_IP; + } } } } diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 42e1763ad4d484..0066c9e74512c3 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -5,7 +5,7 @@ import parsing from typing import AbstractSet -WHITELIST = ( +NON_ESCAPING_FUNCTIONS = ( "Py_INCREF", "_PyDictOrValues_IsValues", "_PyObject_DictOrValuesPointer", @@ -31,9 +31,29 @@ "_PyLong_IsNonNegativeCompact", "_PyLong_CompactValue", "_Py_NewRef", + "_Py_IsImmortal", + "_Py_STR", + "_PyLong_Add", + "_PyLong_Multiply", + "_PyLong_Subtract", + "Py_NewRef", + "_PyList_ITEMS", + "_PyTuple_ITEMS", + "_PyList_AppendTakeRef", + "_Py_atomic_load_uintptr_relaxed", + "_PyFrame_GetCode", + "_PyThreadState_HasStackSpace", ) -def makes_escaping_api_call(instr: parsing.Node) -> bool: +ESCAPING_FUNCTIONS = ( + "import_name", + "import_from", +) + + +def makes_escaping_api_call(instr: parsing.InstDef) -> bool: + if "CALL_INTRINSIC" in instr.name: + return True; tkns = iter(instr.tokens) for tkn in tkns: if tkn.kind != lx.IDENTIFIER: @@ -44,13 +64,17 @@ def makes_escaping_api_call(instr: parsing.Node) -> bool: return False if next_tkn.kind != lx.LPAREN: continue + if tkn.text in ESCAPING_FUNCTIONS: + return True if not tkn.text.startswith("Py") and not tkn.text.startswith("_Py"): continue if tkn.text.endswith("Check"): continue + if tkn.text.startswith("Py_Is"): + continue if tkn.text.endswith("CheckExact"): continue - if tkn.text in WHITELIST: + if tkn.text in NON_ESCAPING_FUNCTIONS: continue return True return False @@ -74,7 +98,7 @@ def __post_init__(self) -> None: self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} @staticmethod - def fromInstruction(instr: parsing.Node) -> "InstructionFlags": + def fromInstruction(instr: parsing.InstDef) -> "InstructionFlags": has_free = ( variable_used(instr, "PyCell_New") or variable_used(instr, "PyCell_GET") @@ -101,8 +125,7 @@ def fromInstruction(instr: parsing.Node) -> "InstructionFlags": or variable_used(instr, "resume_with_error") ), HAS_ESCAPES_FLAG=( - variable_used(instr, "tstate") - or makes_escaping_api_call(instr) + makes_escaping_api_call(instr) ), ) From b338ffa4bc078fd363e8b0078eef4e0d6a071546 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 16:59:32 +0100 Subject: [PATCH 26/49] gh-112026: Add again include in Python.h (#112046) Add again and includes in Python.h, but don't include them in the limited C API version 3.13 and newer. --- Doc/whatsnew/3.13.rst | 18 ------------------ Include/Python.h | 15 ++++++++++++--- ...3-11-13-17-57-11.gh-issue-112026.WJLJcI.rst | 3 +++ 3 files changed, 15 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-11-13-17-57-11.gh-issue-112026.WJLJcI.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 136fe901ce39fb..b64cfc51f75701 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1191,17 +1191,6 @@ Porting to Python 3.13 also the ``HAVE_IEEEFP_H`` macro. (Contributed by Victor Stinner in :gh:`108765`.) -* ``Python.h`` no longer includes the ```` standard header file. If - needed, it should now be included explicitly. For example, it provides the - functions: ``read()``, ``write()``, ``close()``, ``isatty()``, ``lseek()``, - ``getpid()``, ``getcwd()``, ``sysconf()``, ``getpagesize()``, ``alarm()`` and - ``pause()``. - As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no - longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H`` - macros defined by ``Python.h`` can be used to decide if ```` and - ```` header files can be included. - (Contributed by Victor Stinner in :gh:`108765`.) - * ``Python.h`` no longer includes these standard header files: ````, ```` and ````. If needed, they should now be included explicitly. For example, ```` provides the ``clock()`` and @@ -1210,13 +1199,6 @@ Porting to Python 3.13 and ``setitimer()`` functions. (Contributed by Victor Stinner in :gh:`108765`.) -* ``Python.h`` no longer includes the ```` standard header file. If - needed, it should now be included explicitly. For example, it provides - ``isalpha()`` and ``tolower()`` functions which are locale dependent. Python - provides locale independent functions, like :c:func:`!Py_ISALPHA` and - :c:func:`!Py_TOLOWER`. - (Contributed by Victor Stinner in :gh:`108765`.) - * If the :c:macro:`Py_LIMITED_API` macro is defined, :c:macro:`!Py_BUILD_CORE`, :c:macro:`!Py_BUILD_CORE_BUILTIN` and :c:macro:`!Py_BUILD_CORE_MODULE` macros are now undefined by ````. diff --git a/Include/Python.h b/Include/Python.h index a1f26afbb12256..196751c3201e62 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -26,15 +26,24 @@ # include // ssize_t #endif -// errno.h, stdio.h, stdlib.h and string.h headers are no longer used by -// Python, but kept for backward compatibility (avoid compiler warnings). -// They are no longer included by limited C API version 3.11 and newer. +// , , and headers are no longer used +// by Python, but kept for the backward compatibility of existing third party C +// extensions. They are not included by limited C API version 3.11 and newer. +// +// The and headers are not included by limited C API +// version 3.13 and newer. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # include // errno # include // FILE* # include // getenv() # include // memcpy() #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030d0000 +# include // tolower() +# ifndef MS_WINDOWS +# include // close() +# endif +#endif // Include Python header files diff --git a/Misc/NEWS.d/next/C API/2023-11-13-17-57-11.gh-issue-112026.WJLJcI.rst b/Misc/NEWS.d/next/C API/2023-11-13-17-57-11.gh-issue-112026.WJLJcI.rst new file mode 100644 index 00000000000000..deb82ff7af7d54 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-13-17-57-11.gh-issue-112026.WJLJcI.rst @@ -0,0 +1,3 @@ +Add again ```` and ```` includes in ``Python.h``, but +don't include them in the limited C API version 3.13 and newer. Patch by +Victor Stinner. From 58469244beceb515c74cbdbd655256c0da569612 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 17:38:31 +0100 Subject: [PATCH 27/49] gh-112026: Restore removed private C API (#112115) Restore removed private C API functions, macros and structures which have no simple replacement for now: * _PyDict_GetItem_KnownHash() * _PyDict_NewPresized() * _PyHASH_BITS * _PyHASH_IMAG * _PyHASH_INF * _PyHASH_MODULUS * _PyHASH_MULTIPLIER * _PyLong_Copy() * _PyLong_FromDigits() * _PyLong_New() * _PyLong_Sign() * _PyObject_CallMethodId() * _PyObject_CallMethodNoArgs() * _PyObject_CallMethodOneArg() * _PyObject_CallOneArg() * _PyObject_EXTRA_INIT * _PyObject_FastCallDict() * _PyObject_GetAttrId() * _PyObject_Vectorcall() * _PyObject_VectorcallMethod() * _PyStack_AsDict() * _PyThread_CurrentFrames() * _PyUnicodeWriter structure * _PyUnicodeWriter_Dealloc() * _PyUnicodeWriter_Finish() * _PyUnicodeWriter_Init() * _PyUnicodeWriter_Prepare() * _PyUnicodeWriter_PrepareKind() * _PyUnicodeWriter_WriteASCIIString() * _PyUnicodeWriter_WriteChar() * _PyUnicodeWriter_WriteLatin1String() * _PyUnicodeWriter_WriteStr() * _PyUnicodeWriter_WriteSubstring() * _PyUnicode_AsString() * _PyUnicode_FromId() * _PyVectorcall_Function() * _Py_HashDouble() * _Py_HashPointer() * _Py_IDENTIFIER() * _Py_c_abs() * _Py_c_diff() * _Py_c_neg() * _Py_c_pow() * _Py_c_prod() * _Py_c_quot() * _Py_c_sum() * _Py_static_string() * _Py_static_string_init() --- Include/cpython/abstract.h | 32 +++++ Include/cpython/complexobject.h | 10 ++ Include/cpython/dictobject.h | 5 + Include/cpython/longintrepr.h | 10 ++ Include/cpython/longobject.h | 5 + Include/cpython/object.h | 40 ++++++ Include/cpython/pyhash.h | 22 +++ Include/cpython/pystate.h | 5 + Include/cpython/unicodeobject.h | 131 ++++++++++++++++++ Include/internal/pycore_call.h | 19 --- Include/internal/pycore_complexobject.h | 10 -- Include/internal/pycore_dict.h | 5 - Include/internal/pycore_identifier.h | 34 ----- Include/internal/pycore_long.h | 18 --- Include/internal/pycore_pyhash.h | 23 --- Include/internal/pycore_pystate.h | 5 - Include/internal/pycore_unicodeobject.h | 121 ---------------- Include/object.h | 3 + ...-11-15-16-07-57.gh-issue-112026.bnr8dd.rst | 51 +++++++ Objects/dictobject.c | 2 +- 20 files changed, 315 insertions(+), 236 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-11-15-16-07-57.gh-issue-112026.bnr8dd.rst diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 1f495f19df280b..4e7b7a46703a6d 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -2,6 +2,28 @@ # error "this header file must not be included directly" #endif +/* === Object Protocol ================================================== */ + +/* Like PyObject_CallMethod(), but expect a _Py_Identifier* + as the method name. */ +PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( + PyObject *obj, + _Py_Identifier *name, + const char *format, ...); + +/* Convert keyword arguments from the FASTCALL (stack: C array, kwnames: tuple) + format to a Python dictionary ("kwargs" dict). + + The type of kwnames keys is not checked. The final function getting + arguments is responsible to check if all keys are strings, for example using + PyArg_ParseTupleAndKeywords() or PyArg_ValidateKeywordArguments(). + + Duplicate keys are merged using the last value. If duplicate keys must raise + an exception, the caller is responsible to implement an explicit keys on + kwnames. */ +PyAPI_FUNC(PyObject*) _PyStack_AsDict(PyObject *const *values, PyObject *kwnames); + + /* === Vectorcall protocol (PEP 590) ============================= */ // PyVectorcall_NARGS() is exported as a function for the stable ABI. @@ -16,6 +38,16 @@ _PyVectorcall_NARGS(size_t n) PyAPI_FUNC(vectorcallfunc) PyVectorcall_Function(PyObject *callable); +// Backwards compatibility aliases (PEP 590) for API that was provisional +// in Python 3.8 +#define _PyObject_Vectorcall PyObject_Vectorcall +#define _PyObject_VectorcallMethod PyObject_VectorcallMethod +#define _PyObject_FastCallDict PyObject_VectorcallDict +#define _PyVectorcall_Function PyVectorcall_Function +#define _PyObject_CallOneArg PyObject_CallOneArg +#define _PyObject_CallMethodNoArgs PyObject_CallMethodNoArgs +#define _PyObject_CallMethodOneArg PyObject_CallMethodOneArg + /* Same as PyObject_Vectorcall except that keyword arguments are passed as dict, which may be NULL if there are no keyword arguments. */ PyAPI_FUNC(PyObject *) PyObject_VectorcallDict( diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index b524ec42c24371..fbdc6a91fe895c 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,6 +7,16 @@ typedef struct { double imag; } Py_complex; +// Operations on complex numbers. +PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); +PyAPI_FUNC(double) _Py_c_abs(Py_complex); + + /* Complex object interface */ /* diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 64753849d32061..91ddc081e2d013 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -32,6 +32,9 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; +PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, + Py_hash_t hash); + PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); @@ -46,6 +49,8 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); +PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); + PyAPI_FUNC(int) PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result); PyAPI_FUNC(int) PyDict_PopString(PyObject *dict, const char *key, PyObject **result); PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value); diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index fb82f83dc50e42..f5ccbb704e8bb8 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -89,6 +89,16 @@ struct _longobject { _PyLongValue long_value; }; +PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); + +// Return a copy of src. +PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src); + +PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( + int negative, + Py_ssize_t digit_count, + digit *digits); + /* Inline some internals for speed. These should be in pycore_long.h * if user code didn't need them inlined. */ diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index ab885cf5ad03fd..fd1be29ed397d1 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -7,6 +7,11 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); +// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. +// v must not be NULL, and must be a normalized long. +// There are no error cases. +PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); + /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. If n is 0, the integer is 0. Else: diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ede394d9673d7e..762e8a3b86ee1e 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -14,6 +14,44 @@ PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GetRefTotal(PyInterpreterState *); #endif +/********************* String Literals ****************************************/ +/* This structure helps managing static strings. The basic usage goes like this: + Instead of doing + + r = PyObject_CallMethod(o, "foo", "args", ...); + + do + + _Py_IDENTIFIER(foo); + ... + r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...); + + PyId_foo is a static variable, either on block level or file level. On first + usage, the string "foo" is interned, and the structures are linked. On interpreter + shutdown, all strings are released. + + Alternatively, _Py_static_string allows choosing the variable name. + _PyUnicode_FromId returns a borrowed reference to the interned string. + _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. +*/ +typedef struct _Py_Identifier { + const char* string; + // Index in PyInterpreterState.unicode.ids.array. It is process-wide + // unique and must be initialized to -1. + Py_ssize_t index; +} _Py_Identifier; + +#ifndef Py_BUILD_CORE +// For now we are keeping _Py_IDENTIFIER for continued use +// in non-builtin extensions (and naughty PyPI modules). + +#define _Py_static_string_init(value) { .string = (value), .index = -1 } +#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) +#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) + +#endif /* !Py_BUILD_CORE */ + + typedef struct { /* Number implementations must check *both* arguments for proper type and implement the necessary conversions @@ -238,6 +276,8 @@ PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); + PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index 62ae6084bbcf53..6f7113daa5fe4d 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -2,6 +2,28 @@ # error "this header file must not be included directly" #endif +/* Prime multiplier used in string and various other hashes. */ +#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ + +/* Parameters used for the numeric hash implementation. See notes for + _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on + reduction modulo the prime 2**_PyHASH_BITS - 1. */ + +#if SIZEOF_VOID_P >= 8 +# define _PyHASH_BITS 61 +#else +# define _PyHASH_BITS 31 +#endif + +#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define _PyHASH_INF 314159 +#define _PyHASH_IMAG _PyHASH_MULTIPLIER + +/* Helpers for hash functions */ +PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); +PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); + + /* hash function definition */ typedef struct { Py_hash_t (*const hash)(const void *, Py_ssize_t); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 4fad4c60abbafe..7f2378ae76fe5c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -247,6 +247,11 @@ PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); The function returns 1 if _PyGILState_check_enabled is non-zero. */ PyAPI_FUNC(int) PyGILState_Check(void); +/* The implementation of sys._current_frames() Returns a dict mapping + thread id to that thread's current frame. +*/ +PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index cd56a6a74acf51..d9b54bce83202d 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -9,6 +9,7 @@ Py_DEPRECATED(3.13) typedef wchar_t PY_UNICODE_TYPE; Py_DEPRECATED(3.13) typedef wchar_t Py_UNICODE; + /* --- Internal Unicode Operations ---------------------------------------- */ // Static inline functions to work with surrogates @@ -43,6 +44,7 @@ static inline Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) { return (0xDC00 + (ch & 0x3FF)); } + /* --- Unicode Type ------------------------------------------------------- */ /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject @@ -375,6 +377,7 @@ static inline Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *op) #define PyUnicode_MAX_CHAR_VALUE(op) \ PyUnicode_MAX_CHAR_VALUE(_PyObject_CAST(op)) + /* === Public API ========================================================= */ /* With PEP 393, this is the recommended way to allocate a new unicode object. @@ -440,6 +443,123 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( const void *buffer, Py_ssize_t size); + +/* --- _PyUnicodeWriter API ----------------------------------------------- */ + +typedef struct { + PyObject *buffer; + void *data; + int kind; + Py_UCS4 maxchar; + Py_ssize_t size; + Py_ssize_t pos; + + /* minimum number of allocated characters (default: 0) */ + Py_ssize_t min_length; + + /* minimum character (default: 127, ASCII) */ + Py_UCS4 min_char; + + /* If non-zero, overallocate the buffer (default: 0). */ + unsigned char overallocate; + + /* If readonly is 1, buffer is a shared string (cannot be modified) + and size is set to 0. */ + unsigned char readonly; +} _PyUnicodeWriter ; + +// Initialize a Unicode writer. +// +// By default, the minimum buffer size is 0 character and overallocation is +// disabled. Set min_length, min_char and overallocate attributes to control +// the allocation of the buffer. +PyAPI_FUNC(void) +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer); + +/* Prepare the buffer to write 'length' characters + with the specified maximum character. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ + (((MAXCHAR) <= (WRITER)->maxchar \ + && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ + ? 0 \ + : (((LENGTH) == 0) \ + ? 0 \ + : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) + +/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro + instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar); + +/* Prepare the buffer to have at least the kind KIND. + For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will + support characters in range U+000-U+FFFF. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ + ((KIND) <= (WRITER)->kind \ + ? 0 \ + : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) + +/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() + macro instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + int kind); + +/* Append a Unicode character. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, + Py_UCS4 ch + ); + +/* Append a Unicode string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, + PyObject *str /* Unicode string */ + ); + +/* Append a substring of a Unicode string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, + PyObject *str, /* Unicode string */ + Py_ssize_t start, + Py_ssize_t end + ); + +/* Append an ASCII-encoded byte string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, + const char *str, /* ASCII-encoded byte string */ + Py_ssize_t len /* number of bytes, or -1 if unknown */ + ); + +/* Append a latin1-encoded byte string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, + const char *str, /* latin1-encoded byte string */ + Py_ssize_t len /* length in bytes */ + ); + +/* Get the value of the writer as a Unicode string. Clear the + buffer of the writer. Raise an exception and return NULL + on error. */ +PyAPI_FUNC(PyObject *) +_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); + +/* Deallocate memory of a writer (clear its internal buffer). */ +PyAPI_FUNC(void) +_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); + + /* --- Manage the default encoding ---------------------------------------- */ /* Returns a pointer to the default encoding (UTF-8) of the @@ -457,6 +577,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); +// Alias kept for backward compatibility +#define _PyUnicode_AsString PyUnicode_AsUTF8 + + /* === Characters Type APIs =============================================== */ /* These should not be used directly. Use the Py_UNICODE_IS* and @@ -570,3 +694,10 @@ static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { || Py_UNICODE_ISDIGIT(ch) || Py_UNICODE_ISNUMERIC(ch)); } + + +/* === Misc functions ===================================================== */ + +// Return an interned Unicode object for an Identifier; may fail if there is no +// memory. +PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 8846155b38defb..c92028a01299e2 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -31,18 +31,6 @@ PyAPI_FUNC(PyObject*) _Py_CheckFunctionResult( PyObject *result, const char *where); -/* Convert keyword arguments from the FASTCALL (stack: C array, kwnames: tuple) - format to a Python dictionary ("kwargs" dict). - - The type of kwnames keys is not checked. The final function getting - arguments is responsible to check if all keys are strings, for example using - PyArg_ParseTupleAndKeywords() or PyArg_ValidateKeywordArguments(). - - Duplicate keys are merged using the last value. If duplicate keys must raise - an exception, the caller is responsible to implement an explicit keys on - kwnames. */ -extern PyObject* _PyStack_AsDict(PyObject *const *values, PyObject *kwnames); - extern PyObject* _PyObject_Call_Prepend( PyThreadState *tstate, PyObject *callable, @@ -75,13 +63,6 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -/* Like PyObject_CallMethod(), but expect a _Py_Identifier* - as the method name. */ -extern PyObject* _PyObject_CallMethodId( - PyObject *obj, - _Py_Identifier *name, - const char *format, ...); - extern PyObject* _PyObject_CallMethodIdObjArgs( PyObject *obj, _Py_Identifier *name, diff --git a/Include/internal/pycore_complexobject.h b/Include/internal/pycore_complexobject.h index a6fee9d23f3a9f..54713536eedc46 100644 --- a/Include/internal/pycore_complexobject.h +++ b/Include/internal/pycore_complexobject.h @@ -10,16 +10,6 @@ extern "C" { #include "pycore_unicodeobject.h" // _PyUnicodeWriter -// Operations on complex numbers. -// Export functions for 'cmath' shared extension. -PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); -PyAPI_FUNC(double) _Py_c_abs(Py_complex); - /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ extern int _PyComplex_FormatAdvancedWriter( diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 89f30a452c0c64..d96870e9197bbf 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -19,9 +19,6 @@ extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, int (*predicate)(PyObject *value)); // "KnownHash" variants -// Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, PyObject *item, Py_hash_t hash); @@ -44,8 +41,6 @@ extern int _PyDict_HasOnlyStringKeys(PyObject *mp); extern void _PyDict_MaybeUntrack(PyObject *mp); -extern PyObject* _PyDict_NewPresized(Py_ssize_t minused); - // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); diff --git a/Include/internal/pycore_identifier.h b/Include/internal/pycore_identifier.h index 0e015a40c831f4..cda28810a48196 100644 --- a/Include/internal/pycore_identifier.h +++ b/Include/internal/pycore_identifier.h @@ -10,42 +10,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -/* This structure helps managing static strings. The basic usage goes like this: - Instead of doing - - r = PyObject_CallMethod(o, "foo", "args", ...); - - do - - _Py_IDENTIFIER(foo); - ... - r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...); - - PyId_foo is a static variable, either on block level or file level. On first - usage, the string "foo" is interned, and the structures are linked. On interpreter - shutdown, all strings are released. - - Alternatively, _Py_static_string allows choosing the variable name. - _PyUnicode_FromId returns a borrowed reference to the interned string. - _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. -*/ -typedef struct _Py_Identifier { - const char* string; - // Index in PyInterpreterState.unicode.ids.array. It is process-wide - // unique and must be initialized to -1. - Py_ssize_t index; -} _Py_Identifier; - -// For now we are keeping _Py_IDENTIFIER for continued use -// in non-builtin extensions (and naughty PyPI modules). - -#define _Py_static_string_init(value) { .string = (value), .index = -1 } -#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) -#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) - extern PyObject* _PyType_LookupId(PyTypeObject *, _Py_Identifier *); extern PyObject* _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); -extern PyObject* _PyObject_GetAttrId(PyObject *, _Py_Identifier *); extern int _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); #ifdef __cplusplus diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index c4ef6eff37b199..ec27df9e416c58 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -47,24 +47,6 @@ extern "C" { # error "_PY_LONG_DEFAULT_MAX_STR_DIGITS smaller than threshold." #endif -extern PyLongObject* _PyLong_New(Py_ssize_t); - -// Return a copy of src. -extern PyObject* _PyLong_Copy(PyLongObject *src); - -// Export for '_decimal' shared extension -PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( - int negative, - Py_ssize_t digit_count, - digit *digits); - -// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. -// v must not be NULL, and must be a normalized long. -// There are no error cases. -// -// Export for '_pickle' shared extension. -PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); - // _PyLong_NumBits. Return the number of bits needed to represent the // absolute value of a long. For example, this returns 1 for 1 and -1, 2 // for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 78bf0c7d07eb10..c3b72d90de3a69 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -5,35 +5,12 @@ # error "this header requires Py_BUILD_CORE define" #endif -/* Helpers for hash functions */ -extern Py_hash_t _Py_HashDouble(PyObject *, double); - -// Export for '_decimal' shared extension -PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); - // Similar to _Py_HashPointer(), but don't replace -1 with -2 extern Py_hash_t _Py_HashPointerRaw(const void*); // Export for '_datetime' shared extension PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); -/* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ - -/* Parameters used for the numeric hash implementation. See notes for - _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ - -#if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 -#else -# define _PyHASH_BITS 31 -#endif - -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER - /* Hash secret * * memory layout on 64 bit systems diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 7135b1e966feb5..7fa952e371d7b4 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -191,11 +191,6 @@ extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); -/* The implementation of sys._current_frames() Returns a dict mapping - thread id to that thread's current frame. -*/ -extern PyObject* _PyThread_CurrentFrames(void); - /* The implementation of sys._current_exceptions() Returns a dict mapping thread id to that thread's current exception. */ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 23e2670d3a36af..a0d00af92e0f5d 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -73,122 +73,6 @@ extern Py_UCS4 _PyUnicode_FindMaxChar ( /* --- _PyUnicodeWriter API ----------------------------------------------- */ -typedef struct { - PyObject *buffer; - void *data; - int kind; - Py_UCS4 maxchar; - Py_ssize_t size; - Py_ssize_t pos; - - /* minimum number of allocated characters (default: 0) */ - Py_ssize_t min_length; - - /* minimum character (default: 127, ASCII) */ - Py_UCS4 min_char; - - /* If non-zero, overallocate the buffer (default: 0). */ - unsigned char overallocate; - - /* If readonly is 1, buffer is a shared string (cannot be modified) - and size is set to 0. */ - unsigned char readonly; -} _PyUnicodeWriter ; - -// Initialize a Unicode writer. -// -// By default, the minimum buffer size is 0 character and overallocation is -// disabled. Set min_length, min_char and overallocate attributes to control -// the allocation of the buffer. -// -// Export the _PyUnicodeWriter API for '_multibytecodec' shared extension. -PyAPI_FUNC(void) -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer); - -/* Prepare the buffer to write 'length' characters - with the specified maximum character. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ - (((MAXCHAR) <= (WRITER)->maxchar \ - && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ - ? 0 \ - : (((LENGTH) == 0) \ - ? 0 \ - : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) - -/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro - instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar); - -/* Prepare the buffer to have at least the kind KIND. - For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will - support characters in range U+000-U+FFFF. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ - ((KIND) <= (WRITER)->kind \ - ? 0 \ - : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) - -/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() - macro instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, - int kind); - -/* Append a Unicode character. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, - Py_UCS4 ch - ); - -/* Append a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, - PyObject *str /* Unicode string */ - ); - -/* Append a substring of a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, - PyObject *str, /* Unicode string */ - Py_ssize_t start, - Py_ssize_t end - ); - -/* Append an ASCII-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, - const char *str, /* ASCII-encoded byte string */ - Py_ssize_t len /* number of bytes, or -1 if unknown */ - ); - -/* Append a latin1-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, - const char *str, /* latin1-encoded byte string */ - Py_ssize_t len /* length in bytes */ - ); - -/* Get the value of the writer as a Unicode string. Clear the - buffer of the writer. Raise an exception and return NULL - on error. */ -PyAPI_FUNC(PyObject *) -_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) -_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); - - /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ extern int _PyUnicode_FormatAdvancedWriter( @@ -366,11 +250,6 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int); -// Return an interned Unicode object for an Identifier; may fail if there is no -// memory. -// Export for '_testembed' program. -PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); - /* Fast equality check when the inputs are known to be exact unicode types and where the hash values are equal (i.e. a very probable match) */ extern int _PyUnicode_EQ(PyObject *, PyObject *); diff --git a/Include/object.h b/Include/object.h index f6693562d211c5..061b5093fb1a35 100644 --- a/Include/object.h +++ b/Include/object.h @@ -112,6 +112,9 @@ check by comparing the reference count field to the immortality reference count. #define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX #endif +// Kept for backward compatibility. It was needed by Py_TRACE_REFS build. +#define _PyObject_EXTRA_INIT + // Make all internal uses of PyObject_HEAD_INIT immortal while preserving the // C-API expectation that the refcnt will be set to 1. #if defined(Py_NOGIL) diff --git a/Misc/NEWS.d/next/C API/2023-11-15-16-07-57.gh-issue-112026.bnr8dd.rst b/Misc/NEWS.d/next/C API/2023-11-15-16-07-57.gh-issue-112026.bnr8dd.rst new file mode 100644 index 00000000000000..5dbbde88fccd2c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-15-16-07-57.gh-issue-112026.bnr8dd.rst @@ -0,0 +1,51 @@ +Restore removed private C API functions, macros and structures which have no +simple replacement for now: + +* _PyDict_GetItem_KnownHash() +* _PyDict_NewPresized() +* _PyHASH_BITS +* _PyHASH_IMAG +* _PyHASH_INF +* _PyHASH_MODULUS +* _PyHASH_MULTIPLIER +* _PyLong_Copy() +* _PyLong_FromDigits() +* _PyLong_New() +* _PyLong_Sign() +* _PyObject_CallMethodId() +* _PyObject_CallMethodNoArgs() +* _PyObject_CallMethodOneArg() +* _PyObject_CallOneArg() +* _PyObject_EXTRA_INIT +* _PyObject_FastCallDict() +* _PyObject_GetAttrId() +* _PyObject_Vectorcall() +* _PyObject_VectorcallMethod() +* _PyStack_AsDict() +* _PyThread_CurrentFrames() +* _PyUnicodeWriter structure +* _PyUnicodeWriter_Dealloc() +* _PyUnicodeWriter_Finish() +* _PyUnicodeWriter_Init() +* _PyUnicodeWriter_Prepare() +* _PyUnicodeWriter_PrepareKind() +* _PyUnicodeWriter_WriteASCIIString() +* _PyUnicodeWriter_WriteChar() +* _PyUnicodeWriter_WriteLatin1String() +* _PyUnicodeWriter_WriteStr() +* _PyUnicodeWriter_WriteSubstring() +* _PyUnicode_AsString() +* _PyUnicode_FromId() +* _PyVectorcall_Function() +* _Py_IDENTIFIER() +* _Py_c_abs() +* _Py_c_diff() +* _Py_c_neg() +* _Py_c_pow() +* _Py_c_prod() +* _Py_c_quot() +* _Py_c_sum() +* _Py_static_string() +* _Py_static_string_init() + +Patch by Victor Stinner. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d3d16c59f0ea7c..556c74db6f6a23 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -117,7 +117,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_code.h" // stats -#include "pycore_dict.h" // PyDictKeysObject +#include "pycore_dict.h" // export _PyDict_SizeOf() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() From ec055073673d5b949aa81311405d92159bab342e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 15 Nov 2023 17:42:17 +0100 Subject: [PATCH 28/49] gh-96954: Fix `make regen-unicodedata` in out-of-tree builds (#112118) This avoids: python3.13 Tools/unicode/makeunicodedata.py python3.13: can't open file '.../build/debug/Tools/unicode/makeunicodedata.py': [Errno 2] No such file or directory make: *** [Makefile:1498: regen-unicodedata] Error 2 Re-run `make regen-unicodedata` to update the script path in generated files. --- Makefile.pre.in | 2 +- .../next/Build/2023-11-15-16-56-20.gh-issue-96954.6FYvKn.rst | 1 + Modules/unicodedata_db.h | 2 +- Modules/unicodename_db.h | 2 +- Objects/unicodetype_db.h | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-11-15-16-56-20.gh-issue-96954.6FYvKn.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 4fa0337fcc7777..3405975e89040a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1348,7 +1348,7 @@ regen-limited-abi: all .PHONY: regen-unicodedata regen-unicodedata: - $(PYTHON_FOR_REGEN) Tools/unicode/makeunicodedata.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/unicode/makeunicodedata.py ############################################################################ diff --git a/Misc/NEWS.d/next/Build/2023-11-15-16-56-20.gh-issue-96954.6FYvKn.rst b/Misc/NEWS.d/next/Build/2023-11-15-16-56-20.gh-issue-96954.6FYvKn.rst new file mode 100644 index 00000000000000..e9a30e5f4bbaa7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-11-15-16-56-20.gh-issue-96954.6FYvKn.rst @@ -0,0 +1 @@ +Make ``make regen-unicodedata`` work for out-of-tree builds of CPython. diff --git a/Modules/unicodedata_db.h b/Modules/unicodedata_db.h index ed4b0eea9a6c59..3e210863448b78 100644 --- a/Modules/unicodedata_db.h +++ b/Modules/unicodedata_db.h @@ -1,4 +1,4 @@ -/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ #define UNIDATA_VERSION "15.1.0" /* a list of unique database records */ diff --git a/Modules/unicodename_db.h b/Modules/unicodename_db.h index a39dcccd408511..4fe60ad5ed4452 100644 --- a/Modules/unicodename_db.h +++ b/Modules/unicodename_db.h @@ -1,4 +1,4 @@ -/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ #define NAME_MAXLEN 256 diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h index 39a567dc46e89a..e6dbeffbe2aa3e 100644 --- a/Objects/unicodetype_db.h +++ b/Objects/unicodetype_db.h @@ -1,4 +1,4 @@ -/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ /* a list of unique character type descriptors */ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { From bd2f1485b089bd38d123b4d202a37e567f119482 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 18:10:06 +0100 Subject: [PATCH 29/49] gh-112026: Restore removed _PyDict_GetItemStringWithError() (#112119) Restore the removed _PyDict_GetItemStringWithError() function. It is used by numpy. --- Include/cpython/dictobject.h | 2 +- .../2023-11-15-17-10-09.gh-issue-112026.ts9yyn.rst | 2 ++ Objects/dictobject.c | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C API/2023-11-15-17-10-09.gh-issue-112026.ts9yyn.rst diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 91ddc081e2d013..944965fb9e5351 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -34,7 +34,7 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); - +PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); diff --git a/Misc/NEWS.d/next/C API/2023-11-15-17-10-09.gh-issue-112026.ts9yyn.rst b/Misc/NEWS.d/next/C API/2023-11-15-17-10-09.gh-issue-112026.ts9yyn.rst new file mode 100644 index 00000000000000..7d54f6b3626828 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-15-17-10-09.gh-issue-112026.ts9yyn.rst @@ -0,0 +1,2 @@ +Restore the removed ``_PyDict_GetItemStringWithError()`` function. It is +used by numpy. Patch by Victor Stinner. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 556c74db6f6a23..70f424e07ece9a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1842,6 +1842,20 @@ _PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key) return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference } +PyObject * +_PyDict_GetItemStringWithError(PyObject *v, const char *key) +{ + PyObject *kv, *rv; + kv = PyUnicode_FromString(key); + if (kv == NULL) { + return NULL; + } + rv = PyDict_GetItemWithError(v, kv); + Py_DECREF(kv); + return rv; +} + + /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * From e5dfcc2b6e1c8450e47acbc6e7d3fd3380ef0f03 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 18:14:08 +0100 Subject: [PATCH 30/49] gh-96954: Don't run regen-unicodedata in regen-all (#112120) The "make regen-unicodedata" should now be run manually. By the default, it requires an Internet connection, which is not always the case. Some Linux distributions build Linux packages in isolated environment (without network). --- Makefile.pre.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 3405975e89040a..9280d2843693d6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1359,10 +1359,10 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-unicodedata + regen-test-levenshtein regen-global-objects @echo - @echo "Note: make regen-stdlib-module-names, make regen-limited-abi" - @echo "and make regen-configure should be run manually" + @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " + @echo "make regen-configure and make regen-unicodedata should be run manually" ############################################################################ # Special rules for object files From 7e2308aaa29419eadeef3a349c7c4d78e2cbc989 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 19:15:03 +0100 Subject: [PATCH 31/49] gh-112026: Add again _PyThreadState_UncheckedGet() function (#112121) Add again the private _PyThreadState_UncheckedGet() function as an alias to the new public PyThreadState_GetUnchecked() function. --- Include/cpython/pystate.h | 3 +++ .../next/C API/2023-11-15-18-36-21.gh-issue-112026._Yybr5.rst | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2023-11-15-18-36-21.gh-issue-112026._Yybr5.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7f2378ae76fe5c..decafde474b029 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -231,6 +231,9 @@ struct _ts { * if it is NULL. */ PyAPI_FUNC(PyThreadState *) PyThreadState_GetUnchecked(void); +// Alias kept for backward compatibility +#define _PyThreadState_UncheckedGet PyThreadState_GetUnchecked + // Disable tracing and profiling. PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); diff --git a/Misc/NEWS.d/next/C API/2023-11-15-18-36-21.gh-issue-112026._Yybr5.rst b/Misc/NEWS.d/next/C API/2023-11-15-18-36-21.gh-issue-112026._Yybr5.rst new file mode 100644 index 00000000000000..0c39c78a78d1e0 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-15-18-36-21.gh-issue-112026._Yybr5.rst @@ -0,0 +1,3 @@ +Add again the private ``_PyThreadState_UncheckedGet()`` function as an alias +to the new public :c:func:`PyThreadState_GetUnchecked` function. Patch by +Victor Stinner. From d9fd33a869d2be769ff596530f63ee099465b037 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Nov 2023 21:47:14 +0100 Subject: [PATCH 32/49] gh-112088: Run autoreconf in GHA check_generated_files (#112090) The "Check if generated files are up to date" job of GitHub Actions now runs the "autoreconf -ivf -Werror" command instead of the "make regen-configure" command to avoid depending on the external quay.io server. Add Tools/build/regen-configure.sh script to regenerate the configure with an Ubuntu container image. The "quay.io/tiran/cpython_autoconf:271" container image (https://github.com/tiran/cpython_autoconf) is no longer used. --- .github/workflows/build.yml | 11 +- .github/workflows/posix-deps-apt.sh | 2 + Doc/using/configure.rst | 17 ++- Makefile.pre.in | 8 +- ...-11-15-13-40-29.gh-issue-112088.UJQxxh.rst | 5 + Tools/build/regen-configure.sh | 31 +++++ aclocal.m4 | 82 +----------- configure | 120 +++++++++--------- configure.ac | 15 +-- 9 files changed, 131 insertions(+), 160 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-11-15-13-40-29.gh-issue-112088.UJQxxh.rst create mode 100755 Tools/build/regen-configure.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 290b383f44e52c..4210194b978749 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -120,7 +120,9 @@ jobs: check_generated_files: name: 'Check if generated files are up to date' - runs-on: ubuntu-latest + # Don't use ubuntu-latest but a specific version to make the job + # reproducible: to get the same tools versions (autoconf, aclocal, ...) + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -143,15 +145,16 @@ jobs: - name: Check Autoconf and aclocal versions run: | grep "Generated by GNU Autoconf 2.71" configure - grep "aclocal 1.16.4" aclocal.m4 + grep "aclocal 1.16.5" aclocal.m4 grep -q "runstatedir" configure grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 - name: Configure CPython run: | # Build Python with the libpython dynamic library ./configure --config-cache --with-pydebug --enable-shared - - name: Regenerate autoconf files with container image - run: make regen-configure + - name: Regenerate autoconf files + # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) + run: autoreconf -ivf -Werror - name: Build CPython run: | make -j4 regen-all diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index a220896f2cd7be..bbae378f7b994e 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,9 +1,11 @@ #!/bin/sh apt-get update +# autoconf-archive is needed by autoreconf (check_generated_files job) apt-get -yq install \ build-essential \ pkg-config \ + autoconf-archive \ ccache \ gdb \ lcov \ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 69f691dbe1202c..b51546e072a353 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -74,14 +74,21 @@ files. Commands to regenerate all generated files:: The ``Makefile.pre.in`` file documents generated files, their inputs, and tools used to regenerate them. Search for ``regen-*`` make targets. -The ``make regen-configure`` command runs `tiran/cpython_autoconf -`_ container for reproducible build; -see container ``entry.sh`` script. The container is optional, the following -command can be run locally, the generated files depend on autoconf and aclocal -versions:: +configure script +---------------- + +The ``make regen-configure`` command regenerates the ``aclocal.m4`` file and +the ``configure`` script using the ``Tools/build/regen-configure.sh`` shell +script which uses an Ubuntu container to get the same tools versions and have a +reproducible output. + +The container is optional, the following command can be run locally:: autoreconf -ivf -Werror +The generated files can change depending on the exact ``autoconf-archive``, +``aclocal`` and ``pkg-config`` versions. + .. _configure-options: diff --git a/Makefile.pre.in b/Makefile.pre.in index 9280d2843693d6..2174ec3ac56158 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2642,15 +2642,9 @@ recheck: autoconf: (cd $(srcdir); autoreconf -ivf -Werror) -# See https://github.com/tiran/cpython_autoconf container .PHONY: regen-configure regen-configure: - @if command -v podman >/dev/null; then RUNTIME="podman"; else RUNTIME="docker"; fi; \ - if ! command -v $$RUNTIME; then echo "$@ needs either Podman or Docker container runtime." >&2; exit 1; fi; \ - if command -v selinuxenabled >/dev/null && selinuxenabled; then OPT=":Z"; fi; \ - CMD="$$RUNTIME run --rm --pull=always -v $(abs_srcdir):/src$$OPT quay.io/tiran/cpython_autoconf:271"; \ - echo $$CMD; \ - $$CMD || exit $? + $(srcdir)/Tools/build/regen-configure.sh # Create a tags file for vi tags:: diff --git a/Misc/NEWS.d/next/Build/2023-11-15-13-40-29.gh-issue-112088.UJQxxh.rst b/Misc/NEWS.d/next/Build/2023-11-15-13-40-29.gh-issue-112088.UJQxxh.rst new file mode 100644 index 00000000000000..b176d06ec8d749 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-11-15-13-40-29.gh-issue-112088.UJQxxh.rst @@ -0,0 +1,5 @@ +Add ``Tools/build/regen-configure.sh`` script to regenerate the ``configure`` +with an Ubuntu container image. The ``quay.io/tiran/cpython_autoconf:271`` +container image (`tiran/cpython_autoconf +`_) is no longer used. Patch by +Victor Stinner. diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh new file mode 100755 index 00000000000000..e34a36c1a573e5 --- /dev/null +++ b/Tools/build/regen-configure.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e -x + +# The check_generated_files job of .github/workflows/build.yml must kept in +# sync with this script. Use the same container image than the job so the job +# doesn't need to run autoreconf in a container. +IMAGE="ubuntu:22.04" +DEPENDENCIES="autotools-dev autoconf autoconf-archive pkg-config" +AUTORECONF="autoreconf -ivf -Werror" + +WORK_DIR="/src" +SHELL_CMD="apt-get update && apt-get -yq install $DEPENDENCIES && cd $WORK_DIR && $AUTORECONF" + +abs_srcdir=$(cd $(dirname $0)/../..; pwd) + +if podman --version &>/dev/null; then + RUNTIME="podman" +elif docker --version &>/dev/null; then + RUNTIME="docker" +else + echo "$@ needs either Podman or Docker container runtime." >&2 + exit 1 +fi + +PATH_OPT="" +if command -v selinuxenabled >/dev/null && selinuxenabled; then + PATH_OPT=":Z" +fi + +"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" /usr/bin/bash -c "$SHELL_CMD" diff --git a/aclocal.m4 b/aclocal.m4 index da8ee95b9c7f6b..09ae5d1aa8a608 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -1,4 +1,4 @@ -# generated automatically by aclocal 1.16.4 -*- Autoconf -*- +# generated automatically by aclocal 1.16.5 -*- Autoconf -*- # Copyright (C) 1996-2021 Free Software Foundation, Inc. @@ -276,7 +276,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [ ]) # pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -# serial 11 (pkg-config-0.29.1) +# serial 12 (pkg-config-0.29.2) dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson @@ -318,7 +318,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29.1]) +[m4_define([PKG_MACROS_VERSION], [0.29.2]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ @@ -419,7 +419,7 @@ AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl pkg_failed=no -AC_MSG_CHECKING([for $1]) +AC_MSG_CHECKING([for $2]) _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) _PKG_CONFIG([$1][_LIBS], [libs], [$2]) @@ -429,11 +429,11 @@ and $1[]_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details.]) if test $pkg_failed = yes; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` - else + else $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi # Put the nasty error message in config.log where it belongs @@ -450,7 +450,7 @@ installed software in a non-standard prefix. _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full @@ -551,74 +551,6 @@ AS_VAR_COPY([$1], [pkg_cv_][$1]) AS_VAR_IF([$1], [""], [$5], [$4])dnl ])dnl PKG_CHECK_VAR -dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], -dnl [DESCRIPTION], [DEFAULT]) -dnl ------------------------------------------ -dnl -dnl Prepare a "--with-" configure option using the lowercase -dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and -dnl PKG_CHECK_MODULES in a single macro. -AC_DEFUN([PKG_WITH_MODULES], -[ -m4_pushdef([with_arg], m4_tolower([$1])) - -m4_pushdef([description], - [m4_default([$5], [build with ]with_arg[ support])]) - -m4_pushdef([def_arg], [m4_default([$6], [auto])]) -m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) -m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) - -m4_case(def_arg, - [yes],[m4_pushdef([with_without], [--without-]with_arg)], - [m4_pushdef([with_without],[--with-]with_arg)]) - -AC_ARG_WITH(with_arg, - AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, - [AS_TR_SH([with_]with_arg)=def_arg]) - -AS_CASE([$AS_TR_SH([with_]with_arg)], - [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], - [auto],[PKG_CHECK_MODULES([$1],[$2], - [m4_n([def_action_if_found]) $3], - [m4_n([def_action_if_not_found]) $4])]) - -m4_popdef([with_arg]) -m4_popdef([description]) -m4_popdef([def_arg]) - -])dnl PKG_WITH_MODULES - -dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [DESCRIPTION], [DEFAULT]) -dnl ----------------------------------------------- -dnl -dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES -dnl check._[VARIABLE-PREFIX] is exported as make variable. -AC_DEFUN([PKG_HAVE_WITH_MODULES], -[ -PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) - -AM_CONDITIONAL([HAVE_][$1], - [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) -])dnl PKG_HAVE_WITH_MODULES - -dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [DESCRIPTION], [DEFAULT]) -dnl ------------------------------------------------------ -dnl -dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after -dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make -dnl and preprocessor variable. -AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], -[ -PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) - -AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], - [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) -])dnl PKG_HAVE_DEFINE_WITH_MODULES - # AM_CONDITIONAL -*- Autoconf -*- # Copyright (C) 1997-2021 Free Software Foundation, Inc. diff --git a/configure b/configure index dd7a5a3f1fd194..5e86e8255c03d0 100755 --- a/configure +++ b/configure @@ -13040,8 +13040,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBUUID" >&5 -printf %s "checking for LIBUUID... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid >= 2.20" >&5 +printf %s "checking for uuid >= 2.20... " >&6; } if test -n "$LIBUUID_CFLAGS"; then pkg_cv_LIBUUID_CFLAGS="$LIBUUID_CFLAGS" @@ -13081,7 +13081,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -13221,7 +13221,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -13929,8 +13929,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBFFI" >&5 -printf %s "checking for LIBFFI... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libffi" >&5 +printf %s "checking for libffi... " >&6; } if test -n "$LIBFFI_CFLAGS"; then pkg_cv_LIBFFI_CFLAGS="$LIBFFI_CFLAGS" @@ -13970,7 +13970,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -14057,7 +14057,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -14461,8 +14461,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBSQLITE3" >&5 -printf %s "checking for LIBSQLITE3... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3 >= 3.15.2" >&5 +printf %s "checking for sqlite3 >= 3.15.2... " >&6; } if test -n "$LIBSQLITE3_CFLAGS"; then pkg_cv_LIBSQLITE3_CFLAGS="$LIBSQLITE3_CFLAGS" @@ -14502,7 +14502,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -14524,7 +14524,7 @@ fi elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } LIBSQLITE3_CFLAGS=${LIBSQLITE3_CFLAGS-""} @@ -15225,8 +15225,8 @@ for _QUERY in \ pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for TCLTK" >&5 -printf %s "checking for TCLTK... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $_QUERY" >&5 +printf %s "checking for $_QUERY... " >&6; } if test -n "$TCLTK_CFLAGS"; then pkg_cv_TCLTK_CFLAGS="$TCLTK_CFLAGS" @@ -15266,7 +15266,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -15284,7 +15284,7 @@ fi found_tcltk=no elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } found_tcltk=no else @@ -15322,8 +15322,8 @@ case $ac_sys_system in #( pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for X11" >&5 -printf %s "checking for X11... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for x11" >&5 +printf %s "checking for x11... " >&6; } if test -n "$X11_CFLAGS"; then pkg_cv_X11_CFLAGS="$X11_CFLAGS" @@ -15363,7 +15363,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -15390,7 +15390,7 @@ Alternatively, you may set the environment variables X11_CFLAGS and X11_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details." "$LINENO" 5 elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} @@ -19442,8 +19442,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZLIB" >&5 -printf %s "checking for ZLIB... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for zlib >= 1.2.0" >&5 +printf %s "checking for zlib >= 1.2.0... " >&6; } if test -n "$ZLIB_CFLAGS"; then pkg_cv_ZLIB_CFLAGS="$ZLIB_CFLAGS" @@ -19483,7 +19483,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -19626,7 +19626,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -19790,8 +19790,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for BZIP2" >&5 -printf %s "checking for BZIP2... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for bzip2" >&5 +printf %s "checking for bzip2... " >&6; } if test -n "$BZIP2_CFLAGS"; then pkg_cv_BZIP2_CFLAGS="$BZIP2_CFLAGS" @@ -19831,7 +19831,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -19927,7 +19927,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -20018,8 +20018,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBLZMA" >&5 -printf %s "checking for LIBLZMA... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for liblzma" >&5 +printf %s "checking for liblzma... " >&6; } if test -n "$LIBLZMA_CFLAGS"; then pkg_cv_LIBLZMA_CFLAGS="$LIBLZMA_CFLAGS" @@ -20059,7 +20059,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -20155,7 +20155,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -24015,8 +24015,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBREADLINE" >&5 -printf %s "checking for LIBREADLINE... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for readline" >&5 +printf %s "checking for readline... " >&6; } if test -n "$LIBREADLINE_CFLAGS"; then pkg_cv_LIBREADLINE_CFLAGS="$LIBREADLINE_CFLAGS" @@ -24056,7 +24056,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -24149,7 +24149,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -24246,8 +24246,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBEDIT" >&5 -printf %s "checking for LIBEDIT... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libedit" >&5 +printf %s "checking for libedit... " >&6; } if test -n "$LIBEDIT_CFLAGS"; then pkg_cv_LIBEDIT_CFLAGS="$LIBEDIT_CFLAGS" @@ -24287,7 +24287,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -24382,7 +24382,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -25134,8 +25134,8 @@ then : if test "$ac_sys_system" != "Darwin"; then pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CURSES" >&5 -printf %s "checking for CURSES... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncursesw" >&5 +printf %s "checking for ncursesw... " >&6; } if test -n "$CURSES_CFLAGS"; then pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" @@ -25175,7 +25175,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -25253,7 +25253,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -25334,8 +25334,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CURSES" >&5 -printf %s "checking for CURSES... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 +printf %s "checking for ncurses... " >&6; } if test -n "$CURSES_CFLAGS"; then pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" @@ -25375,7 +25375,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -25451,7 +25451,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -25569,8 +25569,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PANEL" >&5 -printf %s "checking for PANEL... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panelw" >&5 +printf %s "checking for panelw... " >&6; } if test -n "$PANEL_CFLAGS"; then pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" @@ -25610,7 +25610,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -25686,7 +25686,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -25765,8 +25765,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PANEL" >&5 -printf %s "checking for PANEL... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panel" >&5 +printf %s "checking for panel... " >&6; } if test -n "$PANEL_CFLAGS"; then pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" @@ -25806,7 +25806,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -25882,7 +25882,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -27788,8 +27788,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBB2" >&5 -printf %s "checking for LIBB2... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libb2" >&5 +printf %s "checking for libb2... " >&6; } if test -n "$LIBB2_CFLAGS"; then pkg_cv_LIBB2_CFLAGS="$LIBB2_CFLAGS" @@ -27829,7 +27829,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -27847,7 +27847,7 @@ fi have_libb2=no elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } have_libb2=no else diff --git a/configure.ac b/configure.ac index 0a622ae9947a8c..f46e19ba4be3f8 100644 --- a/configure.ac +++ b/configure.ac @@ -1,15 +1,12 @@ -dnl *************************************************** -dnl * Please run autoreconf -if to test your changes! * -dnl *************************************************** +dnl ************************************************************ +dnl * Please run autoreconf -ivf -Werror to test your changes! * +dnl ************************************************************ dnl dnl Python's configure script requires autoconf 2.71, autoconf-archive, -dnl pkgconf's m4 macros. +dnl aclocal 1.16, and pkg-config. dnl -dnl It is recommended to use a cpython_autoconf container to regenerate the -dnl configure script: -dnl -dnl podman run --rm --pull=always -v $(pwd):/src:Z quay.io/tiran/cpython_autoconf:271 -dnl docker run --rm --pull=always -v $(pwd):/src quay.io/tiran/cpython_autoconf:271 +dnl It is recommended to use the Tools/build/regen-configure.sh shell script +dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) From 7218bac8c84115a8e9a18a4a8f3146235068facb Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Wed, 15 Nov 2023 23:28:58 +0100 Subject: [PATCH 33/49] gh-111811: Fix test_recursive_repr for WASI (#112130) --- Lib/test/test_xml_etree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 6d413aa68a338d..b9e7937b0bbc00 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2535,6 +2535,7 @@ def __eq__(self, o): e.extend([ET.Element('bar')]) self.assertRaises(ValueError, e.remove, X('baz')) + @support.infinite_recursion(25) def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') From a6465605c1417792ec04ced88340cdf104a402b6 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Thu, 16 Nov 2023 00:53:38 +0100 Subject: [PATCH 34/49] gh-111916: Make hashlib related modules thread-safe without the GIL (#111981) Always use an individual lock on hash objects when in free-threaded builds. Fixes #111916 --- ...-11-14-22-12-11.gh-issue-111916.ZGCayL.rst | 1 + Modules/_blake2/blake2b_impl.c | 33 +++++----- Modules/_blake2/blake2s_impl.c | 33 +++++----- Modules/_hashopenssl.c | 45 +++++-------- Modules/clinic/md5module.c.h | 65 +++++++++++++++++-- Modules/hashlib.h | 28 +++++--- Modules/md5module.c | 32 ++++----- Modules/sha1module.c | 19 +++--- Modules/sha2module.c | 46 +++++++------ Modules/sha3module.c | 23 +++---- 10 files changed, 188 insertions(+), 137 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-11-14-22-12-11.gh-issue-111916.ZGCayL.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-14-22-12-11.gh-issue-111916.ZGCayL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-14-22-12-11.gh-issue-111916.ZGCayL.rst new file mode 100644 index 00000000000000..b3814828e476b9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-14-22-12-11.gh-issue-111916.ZGCayL.rst @@ -0,0 +1 @@ +Make hashlib related modules thread-safe without the GIL diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index c2cac98c7529eb..0c3ae5a2fac275 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -17,6 +17,7 @@ # define Py_BUILD_CORE_MODULE 1 #endif +#include #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() @@ -42,7 +43,8 @@ typedef struct { PyObject_HEAD blake2b_param param; blake2b_state state; - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; } BLAKE2bObject; #include "clinic/blake2b_impl.c.h" @@ -59,9 +61,11 @@ new_BLAKE2bObject(PyTypeObject *type) { BLAKE2bObject *self; self = (BLAKE2bObject *)type->tp_alloc(type, 0); - if (self != NULL) { - self->lock = NULL; + if (self == NULL) { + return NULL; } + HASHLIB_INIT_MUTEX(self); + return self; } @@ -278,18 +282,19 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) - self->lock = PyThread_allocate_lock(); - - if (self->lock != NULL) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - blake2b_update(&self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; + } + if (self->use_mutex) { + Py_BEGIN_ALLOW_THREADS + PyMutex_Lock(&self->mutex); + blake2b_update(&self->state, buf.buf, buf.len); + PyMutex_Unlock(&self->mutex); + Py_END_ALLOW_THREADS } else { blake2b_update(&self->state, buf.buf, buf.len); } + PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -389,10 +394,6 @@ py_blake2b_dealloc(PyObject *self) /* Try not to leave state in memory. */ secure_zero_memory(&obj->param, sizeof(obj->param)); secure_zero_memory(&obj->state, sizeof(obj->state)); - if (obj->lock) { - PyThread_free_lock(obj->lock); - obj->lock = NULL; - } PyTypeObject *type = Py_TYPE(self); PyObject_Free(self); diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 1c47328ece13e8..3014773ab52331 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -17,6 +17,7 @@ # define Py_BUILD_CORE_MODULE 1 #endif +#include #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() @@ -42,7 +43,8 @@ typedef struct { PyObject_HEAD blake2s_param param; blake2s_state state; - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; } BLAKE2sObject; #include "clinic/blake2s_impl.c.h" @@ -59,9 +61,11 @@ new_BLAKE2sObject(PyTypeObject *type) { BLAKE2sObject *self; self = (BLAKE2sObject *)type->tp_alloc(type, 0); - if (self != NULL) { - self->lock = NULL; + if (self == NULL) { + return NULL; } + HASHLIB_INIT_MUTEX(self); + return self; } @@ -278,18 +282,19 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) - self->lock = PyThread_allocate_lock(); - - if (self->lock != NULL) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - blake2s_update(&self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; + } + if (self->use_mutex) { + Py_BEGIN_ALLOW_THREADS + PyMutex_Lock(&self->mutex); + blake2s_update(&self->state, buf.buf, buf.len); + PyMutex_Unlock(&self->mutex); + Py_END_ALLOW_THREADS } else { blake2s_update(&self->state, buf.buf, buf.len); } + PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -389,10 +394,6 @@ py_blake2s_dealloc(PyObject *self) /* Try not to leave state in memory. */ secure_zero_memory(&obj->param, sizeof(obj->param)); secure_zero_memory(&obj->state, sizeof(obj->state)); - if (obj->lock) { - PyThread_free_lock(obj->lock); - obj->lock = NULL; - } PyTypeObject *type = Py_TYPE(self); PyObject_Free(self); diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index ee6fb8b4b03643..0e230f332ff6cb 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -22,6 +22,7 @@ # define Py_BUILD_CORE_MODULE 1 #endif +#include #include "Python.h" #include "pycore_hashtable.h" #include "pycore_pyhash.h" // _Py_HashBytes() @@ -227,16 +228,16 @@ typedef struct { PyObject_HEAD EVP_MD_CTX *ctx; /* OpenSSL message digest context */ // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; /* OpenSSL context lock */ + bool use_mutex; + PyMutex mutex; /* OpenSSL context lock */ } EVPobject; typedef struct { PyObject_HEAD HMAC_CTX *ctx; /* OpenSSL hmac context */ // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; /* HMAC context lock */ + bool use_mutex; + PyMutex mutex; /* HMAC context lock */ } HMACobject; #include "clinic/_hashopenssl.c.h" @@ -414,8 +415,7 @@ newEVPobject(PyTypeObject *type) if (retval == NULL) { return NULL; } - - retval->lock = NULL; + HASHLIB_INIT_MUTEX(retval); retval->ctx = EVP_MD_CTX_new(); if (retval->ctx == NULL) { @@ -453,8 +453,6 @@ static void EVP_dealloc(EVPobject *self) { PyTypeObject *tp = Py_TYPE(self); - if (self->lock != NULL) - PyThread_free_lock(self->lock); EVP_MD_CTX_free(self->ctx); PyObject_Free(self); Py_DECREF(tp); @@ -582,16 +580,14 @@ EVP_update(EVPobject *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &view); - if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - /* fail? lock = NULL and we fail over to non-threaded code. */ + if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); result = EVP_hash(self, view.buf, view.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { result = EVP_hash(self, view.buf, view.len); @@ -1540,7 +1536,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, } self->ctx = ctx; - self->lock = NULL; + HASHLIB_INIT_MUTEX(self); if ((msg_obj != NULL) && (msg_obj != Py_None)) { if (!_hmac_update(self, msg_obj)) @@ -1582,16 +1578,14 @@ _hmac_update(HMACobject *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0); - if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - /* fail? lock = NULL and we fail over to non-threaded code. */ + if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len); @@ -1633,7 +1627,7 @@ _hashlib_HMAC_copy_impl(HMACobject *self) return NULL; } retval->ctx = ctx; - retval->lock = NULL; + HASHLIB_INIT_MUTEX(retval); return (PyObject *)retval; } @@ -1642,9 +1636,6 @@ static void _hmac_dealloc(HMACobject *self) { PyTypeObject *tp = Py_TYPE(self); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } HMAC_CTX_free(self->ctx); PyObject_Free(self); Py_DECREF(tp); diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 1d98c574929f6f..7d4d3108dab9b6 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -2,6 +2,12 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + PyDoc_STRVAR(MD5Type_copy__doc__, "copy($self, /)\n" "--\n" @@ -9,7 +15,7 @@ PyDoc_STRVAR(MD5Type_copy__doc__, "Return a copy of the hash object."); #define MD5TYPE_COPY_METHODDEF \ - {"copy", (PyCFunction)(void(*)(void))MD5Type_copy, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, MD5Type_copy__doc__}, + {"copy", _PyCFunction_CAST(MD5Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, MD5Type_copy__doc__}, static PyObject * MD5Type_copy_impl(MD5object *self, PyTypeObject *cls); @@ -76,25 +82,70 @@ PyDoc_STRVAR(_md5_md5__doc__, "Return a new MD5 hash object; optionally initialized with a string."); #define _MD5_MD5_METHODDEF \ - {"md5", (PyCFunction)(void(*)(void))_md5_md5, METH_VARARGS|METH_KEYWORDS, _md5_md5__doc__}, + {"md5", _PyCFunction_CAST(_md5_md5), METH_FASTCALL|METH_KEYWORDS, _md5_md5__doc__}, static PyObject * _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity); static PyObject * -_md5_md5(PyObject *module, PyObject *args, PyObject *kwargs) +_md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - static char *_keywords[] = {"string", "usedforsecurity", NULL}; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "md5", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *string = NULL; int usedforsecurity = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O$p:md5", _keywords, - &string, &usedforsecurity)) + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: return_value = _md5_md5_impl(module, string, usedforsecurity); exit: return return_value; } -/*[clinic end generated code: output=81702ec915f36236 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bfadda44914804a8 input=a9049054013a1b77]*/ diff --git a/Modules/hashlib.h b/Modules/hashlib.h index a8bad9dd87a939..25b2aaadfb2868 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -1,5 +1,7 @@ /* Common code for use by all hashlib related modules. */ +#include "pycore_lock.h" // PyMutex + /* * Given a PyObject* obj, fill in the Py_buffer* viewp with the result * of PyObject_GetBuffer. Sets an exception and issues the erraction @@ -48,18 +50,28 @@ #include "pythread.h" #define ENTER_HASHLIB(obj) \ - if ((obj)->lock) { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } \ + if ((obj)->use_mutex) { \ + PyMutex_Lock(&(obj)->mutex); \ } #define LEAVE_HASHLIB(obj) \ - if ((obj)->lock) { \ - PyThread_release_lock((obj)->lock); \ + if ((obj)->use_mutex) { \ + PyMutex_Unlock(&(obj)->mutex); \ } +#ifdef Py_NOGIL +#define HASHLIB_INIT_MUTEX(obj) \ + do { \ + (obj)->mutex = (PyMutex){0}; \ + (obj)->use_mutex = true; \ + } while (0) +#else +#define HASHLIB_INIT_MUTEX(obj) \ + do { \ + (obj)->mutex = (PyMutex){0}; \ + (obj)->use_mutex = false; \ + } while (0) +#endif + /* TODO(gpshead): We should make this a module or class attribute * to allow the user to optimize based on the platform they're using. */ #define HASHLIB_GIL_MINSIZE 2048 diff --git a/Modules/md5module.c b/Modules/md5module.c index 7883a8cbe6e246..7d2b3275f213fd 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -16,13 +16,8 @@ /* MD5 objects */ -#ifndef _MSC_VER -#include "pyconfig.h" // Py_NOGIL -#endif - -#ifndef Py_NOGIL -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" @@ -54,8 +49,8 @@ typedef long long MD5_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; Hacl_Streaming_MD5_state *hash_state; } MD5object; @@ -78,7 +73,11 @@ static MD5object * newMD5object(MD5State * st) { MD5object *md5 = (MD5object *)PyObject_GC_New(MD5object, st->md5_type); - md5->lock = NULL; + if (!md5) { + return NULL; + } + HASHLIB_INIT_MUTEX(md5); + PyObject_GC_Track(md5); return md5; } @@ -95,9 +94,6 @@ static void MD5_dealloc(MD5object *ptr) { Hacl_Streaming_MD5_legacy_free(ptr->hash_state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } PyTypeObject *tp = Py_TYPE((PyObject*)ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -202,14 +198,14 @@ MD5Type_update(MD5object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update(self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update(self->hash_state, buf.buf, buf.len); diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 3fd53123229ac4..eda6b5608d52f7 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -49,7 +49,8 @@ typedef long long SHA1_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. + bool use_mutex; + PyMutex mutex; PyThread_type_lock lock; Hacl_Streaming_SHA1_state *hash_state; } SHA1object; @@ -76,7 +77,8 @@ newSHA1object(SHA1State *st) if (sha == NULL) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -94,9 +96,6 @@ static void SHA1_dealloc(SHA1object *ptr) { Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -192,14 +191,14 @@ SHA1Type_update(SHA1object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update(self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update(self->hash_state, buf.buf, buf.len); diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 6ad1ff2e05bfd8..968493ba51b50d 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -53,8 +53,8 @@ typedef struct { PyObject_HEAD int digestsize; // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; Hacl_Streaming_SHA2_state_sha2_256 *state; } SHA256object; @@ -62,8 +62,8 @@ typedef struct { PyObject_HEAD int digestsize; // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; Hacl_Streaming_SHA2_state_sha2_512 *state; } SHA512object; @@ -106,7 +106,8 @@ newSHA224object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -119,7 +120,8 @@ newSHA256object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -132,7 +134,8 @@ newSHA384object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -145,7 +148,8 @@ newSHA512object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -163,9 +167,6 @@ static void SHA256_dealloc(SHA256object *ptr) { Hacl_Streaming_SHA2_free_256(ptr->state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -176,9 +177,6 @@ static void SHA512_dealloc(SHA512object *ptr) { Hacl_Streaming_SHA2_free_512(ptr->state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -376,14 +374,14 @@ SHA256Type_update(SHA256object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update_256(self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update_256(self->state, buf.buf, buf.len); @@ -410,14 +408,14 @@ SHA512Type_update(SHA512object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update_512(self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update_512(self->state, buf.buf, buf.len); diff --git a/Modules/sha3module.c b/Modules/sha3module.c index 558d2005cff617..d9d2f6c385a68b 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -61,8 +61,8 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" typedef struct { PyObject_HEAD // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; Hacl_Streaming_Keccak_state *hash_state; } SHA3object; @@ -76,7 +76,8 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } - newobj->lock = NULL; + HASHLIB_INIT_MUTEX(newobj); + return newobj; } @@ -169,9 +170,6 @@ static void SHA3_dealloc(SHA3object *self) { Hacl_Streaming_Keccak_free(self->hash_state); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -257,19 +255,22 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/ { Py_buffer buf; + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); sha3_update(self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { sha3_update(self->hash_state, buf.buf, buf.len); } + PyBuffer_Release(&buf); Py_RETURN_NONE; } From 985679f05d1b72965bfbed99d1499c22815375e4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 16 Nov 2023 12:05:44 +0100 Subject: [PATCH 35/49] gh-110812: Isolating Extension Modules HOWTO: List GC-related gotchas (GH-111504) Co-authored-by: Hugo van Kemenade Co-authored-by: Eric Snow --- Doc/howto/isolating-extensions.rst | 103 +++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 6 deletions(-) diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 8f3787f2d2f145..835c0afad7c6c8 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -339,12 +339,44 @@ That is, heap types should: - Define a traverse function using ``Py_tp_traverse``, which visits the type (e.g. using :c:expr:`Py_VISIT(Py_TYPE(self))`). -Please refer to the :ref:`the documentation ` of +Please refer to the the documentation of :c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse` for additional considerations. -If your traverse function delegates to the ``tp_traverse`` of its base class -(or another type), ensure that ``Py_TYPE(self)`` is visited only once. +The API for defining heap types grew organically, leaving it +somewhat awkward to use in its current state. +The following sections will guide you through common issues. + + +``tp_traverse`` in Python 3.8 and lower +....................................... + +The requirement to visit the type from ``tp_traverse`` was added in Python 3.9. +If you support Python 3.8 and lower, the traverse function must *not* +visit the type, so it must be more complicated:: + + static int my_traverse(PyObject *self, visitproc visit, void *arg) + { + if (Py_Version >= 0x03090000) { + Py_VISIT(Py_TYPE(self)); + } + return 0; + } + +Unfortunately, :c:data:`Py_Version` was only added in Python 3.11. +As a replacement, use: + +* :c:macro:`PY_VERSION_HEX`, if not using the stable ABI, or +* :py:data:`sys.version_info` (via :c:func:`PySys_GetObject` and + :c:func:`PyArg_ParseTuple`). + + +Delegating ``tp_traverse`` +.......................... + +If your traverse function delegates to the :c:member:`~PyTypeObject.tp_traverse` +of its base class (or another type), ensure that ``Py_TYPE(self)`` is visited +only once. Note that only heap type are expected to visit the type in ``tp_traverse``. For example, if your traverse function includes:: @@ -356,11 +388,70 @@ For example, if your traverse function includes:: if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) { // a heap type's tp_traverse already visited Py_TYPE(self) } else { - Py_VISIT(Py_TYPE(self)); + if (Py_Version >= 0x03090000) { + Py_VISIT(Py_TYPE(self)); + } } -It is not necessary to handle the type's reference count in ``tp_new`` -and ``tp_clear``. +It is not necessary to handle the type's reference count in +:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_clear`. + + +Defining ``tp_dealloc`` +....................... + +If your type has a custom :c:member:`~PyTypeObject.tp_dealloc` function, +it needs to: + +- call :c:func:`PyObject_GC_UnTrack` before any fields are invalidated, and +- decrement the reference count of the type. + +To keep the type valid while ``tp_free`` is called, the type's refcount needs +to be decremented *after* the instance is deallocated. For example:: + + static void my_dealloc(PyObject *self) + { + PyObject_GC_UnTrack(self); + ... + PyTypeObject *type = Py_TYPE(self); + type->tp_free(self); + Py_DECREF(type); + } + +The default ``tp_dealloc`` function does this, so +if your type does *not* override +``tp_dealloc`` you don't need to add it. + + +Not overriding ``tp_free`` +.......................... + +The :c:member:`~PyTypeObject.tp_free` slot of a heap type must be set to +:c:func:`PyObject_GC_Del`. +This is the default; do not override it. + + +Avoiding ``PyObject_New`` +......................... + +GC-tracked objects need to be allocated using GC-aware functions. + +If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: + +- Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible. + That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with:: + + TYPE *o = typeobj->tp_alloc(typeobj, 0); + + Replace ``o = PyObject_NewVar(TYPE, typeobj, size)`` with the same, + but use size instead of the 0. + +- If the above is not possible (e.g. inside a custom ``tp_alloc``), + call :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`:: + + TYPE *o = PyObject_GC_New(TYPE, typeobj); + + TYPE *o = PyObject_GC_NewVar(TYPE, typeobj, size); Module State Access from Classes From 81ab0e8a4add53035c87b040afda6d554cace528 Mon Sep 17 00:00:00 2001 From: Luis Pedro Coelho Date: Thu, 16 Nov 2023 23:04:46 +1000 Subject: [PATCH 36/49] [doc] Make subprocess.wait documentation more precise (#98700) [doc] Make subprocess.wait doc more precise An active loop is only used when the `timeout` parameter is used on POSIX. When no timeout is used, the code calls `os.waitpid` internally (which puts the process on a sleep status). On Windows, the internal Windows API call accepts a timeout parameter, so that is delegated to the OS. --- Doc/library/subprocess.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 7f22a5d1852a89..d6b892a7ed957d 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -791,9 +791,10 @@ Instances of the :class:`Popen` class have the following methods: .. note:: - The function is implemented using a busy loop (non-blocking call and - short sleeps). Use the :mod:`asyncio` module for an asynchronous wait: - see :class:`asyncio.create_subprocess_exec`. + When the ``timeout`` parameter is not ``None``, then (on POSIX) the + function is implemented using a busy loop (non-blocking call and short + sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see + :class:`asyncio.create_subprocess_exec`. .. versionchanged:: 3.3 *timeout* was added. From bd89bca9e2a57779c251ee6fadf4887acb364824 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 16 Nov 2023 14:52:33 +0100 Subject: [PATCH 37/49] gh-111798: Use lower Py_C_RECURSION_LIMIT in debug mode (#112124) * Run again test_ast_recursion_limit() on WASI platform. * Add _testinternalcapi.get_c_recursion_remaining(). * Fix test_ast and test_sys_settrace: test_ast_recursion_limit() and test_trace_unpack_long_sequence() now adjust the maximum recursion depth depending on the the remaining C recursion. --- Include/cpython/pystate.h | 6 +++++- Lib/test/test_ast.py | 8 +++++++- Lib/test/test_sys_settrace.py | 15 ++++++++++++--- ...2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst | 4 ++++ Modules/_testinternalcapi.c | 9 +++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index decafde474b029..34e23830f27e77 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -214,7 +214,11 @@ struct _ts { }; -#ifdef __wasi__ +#ifdef Py_DEBUG + // A debug build is likely built with low optimization level which implies + // higher stack memory usage than a release build: use a lower limit. +# define Py_C_RECURSION_LIMIT 500 +#elif defined(__wasi__) // WASI has limited call stack. Python's recursion limit depends on code // layout, optimization, and WASI runtime. Wasmtime can handle about 700 // recursions, sometimes less. 500 is a more conservative limit. diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 69c356ed88f57b..64fcb02309de77 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -12,6 +12,10 @@ import weakref from functools import partial from textwrap import dedent +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None from test import support from test.support.import_helper import import_fresh_module @@ -1118,12 +1122,14 @@ def next(self): return self enum._test_simple_enum(_Precedence, ast._Precedence) - @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT crash_depth = 100_000 success_depth = 1200 + if _testinternalcapi is not None: + remaining = _testinternalcapi.get_c_recursion_remaining() + success_depth = min(success_depth, remaining) def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 292096383bca63..fc5ca72236b1fb 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -14,6 +14,10 @@ import textwrap import subprocess import warnings +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None support.requires_working_socket(module=True) @@ -3033,16 +3037,21 @@ def test_trace_unpack_long_sequence(self): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): + count = 1000 + if _testinternalcapi is not None: + remaining = _testinternalcapi.get_c_recursion_remaining() + count = min(count, remaining) + code = """if 1: def f(): return ( {} ) - """.format("\n+\n".join(f"var{i}\n" for i in range(1000))) - ns = {f"var{i}": i for i in range(1000)} + """.format("\n+\n".join(f"var{i}\n" for i in range(count))) + ns = {f"var{i}": i for i in range(count)} exec(code, ns) counts = self.count_traces(ns["f"]) - self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1}) + self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1}) class TestEdgeCases(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst new file mode 100644 index 00000000000000..24bb4ec096c2d1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst @@ -0,0 +1,4 @@ +When Python is built in debug mode, set the C recursion limit to 500 instead +of 1500. A debug build is likely built with low optimization level which +implies higher stack memory usage than a release build. Patch by Victor +Stinner. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 604a59e7e19e43..4fc9e853c0cd91 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -109,6 +109,14 @@ get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject* +get_c_recursion_remaining(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return PyLong_FromLong(tstate->c_recursion_remaining); +} + + static PyObject* test_bswap(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -1611,6 +1619,7 @@ perf_trampoline_set_persist_after_fork(PyObject *self, PyObject *args) static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, + {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS}, {"test_bswap", test_bswap, METH_NOARGS}, {"test_popcount", test_popcount, METH_NOARGS}, {"test_bit_length", test_bit_length, METH_NOARGS}, From 12c7e9d573de57343cf018fb4e67521aba46c90f Mon Sep 17 00:00:00 2001 From: Qua27 <92877777+Qua27@users.noreply.github.com> Date: Thu, 16 Nov 2023 18:16:04 +0300 Subject: [PATCH 38/49] GH-112152: Fix typo in `typing.override` docstring (#112158) --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 14845b36028ca1..3493257677c7ff 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -3345,7 +3345,7 @@ def override[F: _Func](method: F, /) -> F: Usage:: class Base: - def method(self) -> None: ... + def method(self) -> None: pass class Child(Base): From 7680da458398c5a08b9c32785b1eeb7b7c0887e4 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 16 Nov 2023 18:40:09 +0300 Subject: [PATCH 39/49] gh-112155: Run `typing.py` doctests as part of `test_typing` (#112156) --- Lib/test/test_typing.py | 6 ++++++ Lib/typing.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1e812e1d525a81..8681e7efba3244 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -9464,5 +9464,11 @@ def test_is_not_instance_of_iterable(self): self.assertNotIsInstance(type_to_test, collections.abc.Iterable) +def load_tests(loader, tests, pattern): + import doctest + tests.addTests(doctest.DocTestSuite(typing)) + return tests + + if __name__ == '__main__': main() diff --git a/Lib/typing.py b/Lib/typing.py index 3493257677c7ff..5e3e1f298295c7 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -3404,8 +3404,8 @@ def get_protocol_members(tp: type, /) -> frozenset[str]: >>> class P(Protocol): ... def a(self) -> str: ... ... b: int - >>> get_protocol_members(P) - frozenset({'a', 'b'}) + >>> get_protocol_members(P) == frozenset({'a', 'b'}) + True Raise a TypeError for arguments that are not Protocols. """ From f66afa395a6d06097ad1ca222ed076e18a7a8126 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 16 Nov 2023 18:57:22 +0100 Subject: [PATCH 40/49] gh-111881: Import lazily zipfile in support.script_helper (#112172) It allows running the test suite when the zlib extension is missing. --- Lib/test/support/script_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index c2b43f4060eb55..7dffe79a0da04e 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -8,7 +8,6 @@ import os.path import subprocess import py_compile -import zipfile from importlib.util import source_from_cache from test import support @@ -226,6 +225,7 @@ def make_script(script_dir, script_basename, source, omit_suffix=False): def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): + import zipfile zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) with zipfile.ZipFile(zip_name, 'w') as zip_file: @@ -252,6 +252,7 @@ def make_pkg(pkg_dir, init_source=''): def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, source, depth=1, compiled=False): + import zipfile unlink = [] init_name = make_script(zip_dir, '__init__', '') unlink.append(init_name) From 446f18a911916eabd2c0ceed0c2a109fc8480727 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 16 Nov 2023 14:19:54 -0500 Subject: [PATCH 41/49] gh-111956: Add thread-safe one-time initialization. (gh-111960) --- Include/internal/pycore_ast_state.h | 6 +- Include/internal/pycore_lock.h | 30 + Include/internal/pycore_modsupport.h | 14 +- Include/internal/pycore_runtime.h | 1 - ...-11-10-10-24-28.gh-issue-111956.ImE6Cx.rst | 2 + Modules/_testinternalcapi/test_lock.c | 32 + Parser/asdl_c.py | 89 +- Python/Python-ast.c | 1735 ++++++++--------- Python/getargs.c | 46 +- Python/lock.c | 58 + Python/pystate.c | 3 +- 11 files changed, 1061 insertions(+), 955 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-11-10-10-24-28.gh-issue-111956.ImE6Cx.rst diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 0c0d53f3e5d7e9..6ffd30aca7b11b 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -2,6 +2,9 @@ #ifndef Py_INTERNAL_AST_STATE_H #define Py_INTERNAL_AST_STATE_H + +#include "pycore_lock.h" // _PyOnceFlag + #ifdef __cplusplus extern "C" { #endif @@ -11,7 +14,8 @@ extern "C" { #endif struct ast_state { - int initialized; + _PyOnceFlag once; + int finalized; int recursion_depth; int recursion_limit; PyObject *AST_type; diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index fe5e21fad221e4..25c3cf5377b778 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -46,6 +46,7 @@ typedef struct _PyMutex PyMutex; #define _Py_UNLOCKED 0 #define _Py_LOCKED 1 #define _Py_HAS_PARKED 2 +#define _Py_ONCE_INITIALIZED 4 // (private) slow path for locking the mutex PyAPI_FUNC(void) _PyMutex_LockSlow(PyMutex *m); @@ -166,6 +167,35 @@ _PyRawMutex_Unlock(_PyRawMutex *m) _PyRawMutex_UnlockSlow(m); } +// A data structure that can be used to run initialization code once in a +// thread-safe manner. The C++11 equivalent is std::call_once. +typedef struct { + uint8_t v; +} _PyOnceFlag; + +// Type signature for one-time initialization functions. The function should +// return 0 on success and -1 on failure. +typedef int _Py_once_fn_t(void *arg); + +// (private) slow path for one time initialization +PyAPI_FUNC(int) +_PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg); + +// Calls `fn` once using `flag`. The `arg` is passed to the call to `fn`. +// +// Returns 0 on success and -1 on failure. +// +// If `fn` returns 0 (success), then subsequent calls immediately return 0. +// If `fn` returns -1 (failure), then subsequent calls will retry the call. +static inline int +_PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) +{ + if (_Py_atomic_load_uint8(&flag->v) == _Py_ONCE_INITIALIZED) { + return 0; + } + return _PyOnceFlag_CallOnceSlow(flag, fn, arg); +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h index e12f3b72c8415b..3d3cd6722528e9 100644 --- a/Include/internal/pycore_modsupport.h +++ b/Include/internal/pycore_modsupport.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_MODSUPPORT_H #define Py_INTERNAL_MODSUPPORT_H + +#include "pycore_lock.h" // _PyOnceFlag + #ifdef __cplusplus extern "C" { #endif @@ -65,15 +68,16 @@ PyAPI_FUNC(void) _PyArg_BadArgument( // --- _PyArg_Parser API --------------------------------------------------- typedef struct _PyArg_Parser { - int initialized; const char *format; const char * const *keywords; const char *fname; const char *custom_msg; - int pos; /* number of positional-only arguments */ - int min; /* minimal number of arguments */ - int max; /* maximal number of positional arguments */ - PyObject *kwtuple; /* tuple of keyword parameter names */ + _PyOnceFlag once; /* atomic one-time initialization flag */ + int is_kwtuple_owned; /* does this parser own the kwtuple object? */ + int pos; /* number of positional-only arguments */ + int min; /* minimal number of arguments */ + int max; /* maximal number of positional arguments */ + PyObject *kwtuple; /* tuple of keyword parameter names */ struct _PyArg_Parser *next; } _PyArg_Parser; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 8fb73dd6b7dc0b..e6efe8b646e86f 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -27,7 +27,6 @@ extern "C" { #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state struct _getargs_runtime_state { - PyThread_type_lock mutex; struct _PyArg_Parser *static_parsers; }; diff --git a/Misc/NEWS.d/next/C API/2023-11-10-10-24-28.gh-issue-111956.ImE6Cx.rst b/Misc/NEWS.d/next/C API/2023-11-10-10-24-28.gh-issue-111956.ImE6Cx.rst new file mode 100644 index 00000000000000..30ee07aa2f1f9b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-10-10-24-28.gh-issue-111956.ImE6Cx.rst @@ -0,0 +1,2 @@ +Add internal-only one-time initialization API: ``_PyOnceFlag`` and +``_PyOnceFlag_CallOnce``. diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 82a0c827deeddf..418f71c1441995 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -341,6 +341,37 @@ test_lock_benchmark(PyObject *module, PyObject *obj) Py_RETURN_NONE; } +static int +init_maybe_fail(void *arg) +{ + int *counter = (int *)arg; + (*counter)++; + if (*counter < 5) { + // failure + return -1; + } + assert(*counter == 5); + return 0; +} + +static PyObject * +test_lock_once(PyObject *self, PyObject *obj) +{ + _PyOnceFlag once = {0}; + int counter = 0; + for (int i = 0; i < 10; i++) { + int res = _PyOnceFlag_CallOnce(&once, init_maybe_fail, &counter); + if (i < 4) { + assert(res == -1); + } + else { + assert(res == 0); + assert(counter == 5); + } + } + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"test_lock_basic", test_lock_basic, METH_NOARGS}, {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, @@ -348,6 +379,7 @@ static PyMethodDef test_methods[] = { {"test_lock_counter_slow", test_lock_counter_slow, METH_NOARGS}, _TESTINTERNALCAPI_BENCHMARK_LOCKS_METHODDEF {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, + {"test_lock_once", test_lock_once, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index ae642e83fb15e1..c9bf08ed2e0f6d 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -518,7 +518,7 @@ def sumTrailer(self, name, add_label=False): if add_label: self.emit("failed:", 1) self.emit("Py_XDECREF(tmp);", 1) - self.emit("return 1;", 1) + self.emit("return -1;", 1) self.emit("}", 0) self.emit("", 0) @@ -529,7 +529,7 @@ def simpleSum(self, sum, name): "state->%s_type);") self.emit(line % (t.name,), 1) self.emit("if (isinstance == -1) {", 1) - self.emit("return 1;", 2) + self.emit("return -1;", 2) self.emit("}", 1) self.emit("if (isinstance) {", 1) self.emit("*out = %s;" % t.name, 2) @@ -558,7 +558,7 @@ def complexSum(self, sum, name): self.emit("tp = state->%s_type;" % (t.name,), 1) self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1) self.emit("if (isinstance == -1) {", 1) - self.emit("return 1;", 2) + self.emit("return -1;", 2) self.emit("}", 1) self.emit("if (isinstance) {", 1) for f in t.fields: @@ -605,7 +605,7 @@ def visitProduct(self, prod, name): self.emit("return 0;", 1) self.emit("failed:", 0) self.emit("Py_XDECREF(tmp);", 1) - self.emit("return 1;", 1) + self.emit("return -1;", 1) self.emit("}", 0) self.emit("", 0) @@ -631,13 +631,13 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): ctype = get_c_type(field.type) line = "if (PyObject_GetOptionalAttr(obj, state->%s, &tmp) < 0) {" self.emit(line % field.name, depth) - self.emit("return 1;", depth+1) + self.emit("return -1;", depth+1) self.emit("}", depth) if field.seq: self.emit("if (tmp == NULL) {", depth) self.emit("tmp = PyList_New(0);", depth+1) self.emit("if (tmp == NULL) {", depth+1) - self.emit("return 1;", depth+2) + self.emit("return -1;", depth+2) self.emit("}", depth+1) self.emit("}", depth) self.emit("{", depth) @@ -647,7 +647,7 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): message = "required field \\\"%s\\\" missing from %s" % (field.name, name) format = "PyErr_SetString(PyExc_TypeError, \"%s\");" self.emit(format % message, depth+1, reflow=False) - self.emit("return 1;", depth+1) + self.emit("return -1;", depth+1) else: self.emit("if (tmp == NULL || tmp == Py_None) {", depth) self.emit("Py_CLEAR(tmp);", depth+1) @@ -968,16 +968,16 @@ def visitModule(self, mod): int i, result; PyObject *s, *l = PyTuple_New(num_fields); if (!l) - return 0; + return -1; for (i = 0; i < num_fields; i++) { s = PyUnicode_InternFromString(attrs[i]); if (!s) { Py_DECREF(l); - return 0; + return -1; } PyTuple_SET_ITEM(l, i, s); } - result = PyObject_SetAttr(type, state->_attributes, l) >= 0; + result = PyObject_SetAttr(type, state->_attributes, l); Py_DECREF(l); return result; } @@ -1052,7 +1052,7 @@ def visitModule(self, mod): { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1061,7 +1061,7 @@ def visitModule(self, mod): { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1071,12 +1071,12 @@ def visitModule(self, mod): int i; if (!PyLong_Check(obj)) { PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); - return 1; + return -1; } i = PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) - return 1; + return -1; *out = i; return 0; } @@ -1102,22 +1102,15 @@ def visitModule(self, mod): static int init_types(struct ast_state *state) { - // init_types() must not be called after _PyAST_Fini() - // has been called - assert(state->initialized >= 0); - - if (state->initialized) { - return 1; - } if (init_identifiers(state) < 0) { - return 0; + return -1; } state->AST_type = PyType_FromSpec(&AST_type_spec); if (!state->AST_type) { - return 0; + return -1; } if (add_ast_fields(state) < 0) { - return 0; + return -1; } ''')) for dfn in mod.dfns: @@ -1125,8 +1118,7 @@ def visitModule(self, mod): self.file.write(textwrap.dedent(''' state->recursion_depth = 0; state->recursion_limit = 0; - state->initialized = 1; - return 1; + return 0; } ''')) @@ -1138,12 +1130,12 @@ def visitProduct(self, prod, name): self.emit('state->%s_type = make_type(state, "%s", state->AST_type, %s, %d,' % (name, name, fields, len(prod.fields)), 1) self.emit('%s);' % reflow_c_string(asdl_of(name, prod), 2), 2, reflow=False) - self.emit("if (!state->%s_type) return 0;" % name, 1) + self.emit("if (!state->%s_type) return -1;" % name, 1) if prod.attributes: - self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" % + self.emit("if (add_attributes(state, state->%s_type, %s_attributes, %d) < 0) return -1;" % (name, name, len(prod.attributes)), 1) else: - self.emit("if (!add_attributes(state, state->%s_type, NULL, 0)) return 0;" % name, 1) + self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1) self.emit_defaults(name, prod.fields, 1) self.emit_defaults(name, prod.attributes, 1) @@ -1151,12 +1143,12 @@ def visitSum(self, sum, name): self.emit('state->%s_type = make_type(state, "%s", state->AST_type, NULL, 0,' % (name, name), 1) self.emit('%s);' % reflow_c_string(asdl_of(name, sum), 2), 2, reflow=False) - self.emit("if (!state->%s_type) return 0;" % name, 1) + self.emit("if (!state->%s_type) return -1;" % name, 1) if sum.attributes: - self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" % + self.emit("if (add_attributes(state, state->%s_type, %s_attributes, %d) < 0) return -1;" % (name, name, len(sum.attributes)), 1) else: - self.emit("if (!add_attributes(state, state->%s_type, NULL, 0)) return 0;" % name, 1) + self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1) self.emit_defaults(name, sum.attributes, 1) simple = is_simple(sum) for t in sum.types: @@ -1170,20 +1162,20 @@ def visitConstructor(self, cons, name, simple): self.emit('state->%s_type = make_type(state, "%s", state->%s_type, %s, %d,' % (cons.name, cons.name, name, fields, len(cons.fields)), 1) self.emit('%s);' % reflow_c_string(asdl_of(cons.name, cons), 2), 2, reflow=False) - self.emit("if (!state->%s_type) return 0;" % cons.name, 1) + self.emit("if (!state->%s_type) return -1;" % cons.name, 1) self.emit_defaults(cons.name, cons.fields, 1) if simple: self.emit("state->%s_singleton = PyType_GenericNew((PyTypeObject *)" "state->%s_type, NULL, NULL);" % (cons.name, cons.name), 1) - self.emit("if (!state->%s_singleton) return 0;" % cons.name, 1) + self.emit("if (!state->%s_singleton) return -1;" % cons.name, 1) def emit_defaults(self, name, fields, depth): for field in fields: if field.opt: self.emit('if (PyObject_SetAttr(state->%s_type, state->%s, Py_None) == -1)' % (name, field.name), depth) - self.emit("return 0;", depth+1) + self.emit("return -1;", depth+1) class ASTModuleVisitor(PickleVisitor): @@ -1279,7 +1271,7 @@ def func_begin(self, name): self.emit("if (++state->recursion_depth > state->recursion_limit) {", 1) self.emit("PyErr_SetString(PyExc_RecursionError,", 2) self.emit('"maximum recursion depth exceeded during ast construction");', 3) - self.emit("return 0;", 2) + self.emit("return NULL;", 2) self.emit("}", 1) def func_end(self): @@ -1400,7 +1392,7 @@ class PartingShots(StaticVisitor): int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { - return 0; + return NULL; } state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; @@ -1414,7 +1406,7 @@ class PartingShots(StaticVisitor): PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", starting_recursion_depth, state->recursion_depth); - return 0; + return NULL; } return result; } @@ -1481,7 +1473,8 @@ def visit(self, object): def generate_ast_state(module_state, f): f.write('struct ast_state {\n') - f.write(' int initialized;\n') + f.write(' _PyOnceFlag once;\n') + f.write(' int finalized;\n') f.write(' int recursion_depth;\n') f.write(' int recursion_limit;\n') for s in module_state: @@ -1501,11 +1494,8 @@ def generate_ast_fini(module_state, f): f.write(textwrap.dedent(""" Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); - #if !defined(NDEBUG) - state->initialized = -1; - #else - state->initialized = 0; - #endif + state->finalized = 1; + state->once = (_PyOnceFlag){0}; } """)) @@ -1544,6 +1534,7 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_ast.h" #include "pycore_ast_state.h" // struct ast_state #include "pycore_ceval.h" // _Py_EnterRecursiveCall + #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() #include @@ -1556,7 +1547,8 @@ def generate_module_def(mod, metadata, f, internal_h): { PyInterpreterState *interp = _PyInterpreterState_GET(); struct ast_state *state = &interp->ast; - if (!init_types(state)) { + assert(!state->finalized); + if (_PyOnceFlag_CallOnce(&state->once, (_Py_once_fn_t *)&init_types, state) < 0) { return NULL; } return state; @@ -1570,8 +1562,8 @@ def generate_module_def(mod, metadata, f, internal_h): for identifier in state_strings: f.write(' if ((state->' + identifier) f.write(' = PyUnicode_InternFromString("') - f.write(identifier + '")) == NULL) return 0;\n') - f.write(' return 1;\n') + f.write(identifier + '")) == NULL) return -1;\n') + f.write(' return 0;\n') f.write('};\n\n') def write_header(mod, metadata, f): @@ -1629,6 +1621,9 @@ def write_internal_h_header(mod, f): print(textwrap.dedent(""" #ifndef Py_INTERNAL_AST_STATE_H #define Py_INTERNAL_AST_STATE_H + + #include "pycore_lock.h" // _PyOnceFlag + #ifdef __cplusplus extern "C" { #endif diff --git a/Python/Python-ast.c b/Python/Python-ast.c index ec132078c1536e..75dc3e156aa945 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -4,6 +4,7 @@ #include "pycore_ast.h" #include "pycore_ast_state.h" // struct ast_state #include "pycore_ceval.h" // _Py_EnterRecursiveCall +#include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() #include @@ -16,7 +17,8 @@ get_ast_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); struct ast_state *state = &interp->ast; - if (!init_types(state)) { + assert(!state->finalized); + if (_PyOnceFlag_CallOnce(&state->once, (_Py_once_fn_t *)&init_types, state) < 0) { return NULL; } return state; @@ -271,102 +273,99 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); -#if !defined(NDEBUG) - state->initialized = -1; -#else - state->initialized = 0; -#endif + state->finalized = 1; + state->once = (_PyOnceFlag){0}; } static int init_identifiers(struct ast_state *state) { - if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0; - if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0; - if ((state->__match_args__ = PyUnicode_InternFromString("__match_args__")) == NULL) return 0; - if ((state->__module__ = PyUnicode_InternFromString("__module__")) == NULL) return 0; - if ((state->_attributes = PyUnicode_InternFromString("_attributes")) == NULL) return 0; - if ((state->_fields = PyUnicode_InternFromString("_fields")) == NULL) return 0; - if ((state->annotation = PyUnicode_InternFromString("annotation")) == NULL) return 0; - if ((state->arg = PyUnicode_InternFromString("arg")) == NULL) return 0; - if ((state->args = PyUnicode_InternFromString("args")) == NULL) return 0; - if ((state->argtypes = PyUnicode_InternFromString("argtypes")) == NULL) return 0; - if ((state->asname = PyUnicode_InternFromString("asname")) == NULL) return 0; - if ((state->ast = PyUnicode_InternFromString("ast")) == NULL) return 0; - if ((state->attr = PyUnicode_InternFromString("attr")) == NULL) return 0; - if ((state->bases = PyUnicode_InternFromString("bases")) == NULL) return 0; - if ((state->body = PyUnicode_InternFromString("body")) == NULL) return 0; - if ((state->bound = PyUnicode_InternFromString("bound")) == NULL) return 0; - if ((state->cases = PyUnicode_InternFromString("cases")) == NULL) return 0; - if ((state->cause = PyUnicode_InternFromString("cause")) == NULL) return 0; - if ((state->cls = PyUnicode_InternFromString("cls")) == NULL) return 0; - if ((state->col_offset = PyUnicode_InternFromString("col_offset")) == NULL) return 0; - if ((state->comparators = PyUnicode_InternFromString("comparators")) == NULL) return 0; - if ((state->context_expr = PyUnicode_InternFromString("context_expr")) == NULL) return 0; - if ((state->conversion = PyUnicode_InternFromString("conversion")) == NULL) return 0; - if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return 0; - if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return 0; - if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return 0; - if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return 0; - if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return 0; - if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return 0; - if ((state->end_lineno = PyUnicode_InternFromString("end_lineno")) == NULL) return 0; - if ((state->exc = PyUnicode_InternFromString("exc")) == NULL) return 0; - if ((state->finalbody = PyUnicode_InternFromString("finalbody")) == NULL) return 0; - if ((state->format_spec = PyUnicode_InternFromString("format_spec")) == NULL) return 0; - if ((state->func = PyUnicode_InternFromString("func")) == NULL) return 0; - if ((state->generators = PyUnicode_InternFromString("generators")) == NULL) return 0; - if ((state->guard = PyUnicode_InternFromString("guard")) == NULL) return 0; - if ((state->handlers = PyUnicode_InternFromString("handlers")) == NULL) return 0; - if ((state->id = PyUnicode_InternFromString("id")) == NULL) return 0; - if ((state->ifs = PyUnicode_InternFromString("ifs")) == NULL) return 0; - if ((state->is_async = PyUnicode_InternFromString("is_async")) == NULL) return 0; - if ((state->items = PyUnicode_InternFromString("items")) == NULL) return 0; - if ((state->iter = PyUnicode_InternFromString("iter")) == NULL) return 0; - if ((state->key = PyUnicode_InternFromString("key")) == NULL) return 0; - if ((state->keys = PyUnicode_InternFromString("keys")) == NULL) return 0; - if ((state->keywords = PyUnicode_InternFromString("keywords")) == NULL) return 0; - if ((state->kind = PyUnicode_InternFromString("kind")) == NULL) return 0; - if ((state->kw_defaults = PyUnicode_InternFromString("kw_defaults")) == NULL) return 0; - if ((state->kwarg = PyUnicode_InternFromString("kwarg")) == NULL) return 0; - if ((state->kwd_attrs = PyUnicode_InternFromString("kwd_attrs")) == NULL) return 0; - if ((state->kwd_patterns = PyUnicode_InternFromString("kwd_patterns")) == NULL) return 0; - if ((state->kwonlyargs = PyUnicode_InternFromString("kwonlyargs")) == NULL) return 0; - if ((state->left = PyUnicode_InternFromString("left")) == NULL) return 0; - if ((state->level = PyUnicode_InternFromString("level")) == NULL) return 0; - if ((state->lineno = PyUnicode_InternFromString("lineno")) == NULL) return 0; - if ((state->lower = PyUnicode_InternFromString("lower")) == NULL) return 0; - if ((state->module = PyUnicode_InternFromString("module")) == NULL) return 0; - if ((state->msg = PyUnicode_InternFromString("msg")) == NULL) return 0; - if ((state->name = PyUnicode_InternFromString("name")) == NULL) return 0; - if ((state->names = PyUnicode_InternFromString("names")) == NULL) return 0; - if ((state->op = PyUnicode_InternFromString("op")) == NULL) return 0; - if ((state->operand = PyUnicode_InternFromString("operand")) == NULL) return 0; - if ((state->ops = PyUnicode_InternFromString("ops")) == NULL) return 0; - if ((state->optional_vars = PyUnicode_InternFromString("optional_vars")) == NULL) return 0; - if ((state->orelse = PyUnicode_InternFromString("orelse")) == NULL) return 0; - if ((state->pattern = PyUnicode_InternFromString("pattern")) == NULL) return 0; - if ((state->patterns = PyUnicode_InternFromString("patterns")) == NULL) return 0; - if ((state->posonlyargs = PyUnicode_InternFromString("posonlyargs")) == NULL) return 0; - if ((state->rest = PyUnicode_InternFromString("rest")) == NULL) return 0; - if ((state->returns = PyUnicode_InternFromString("returns")) == NULL) return 0; - if ((state->right = PyUnicode_InternFromString("right")) == NULL) return 0; - if ((state->simple = PyUnicode_InternFromString("simple")) == NULL) return 0; - if ((state->slice = PyUnicode_InternFromString("slice")) == NULL) return 0; - if ((state->step = PyUnicode_InternFromString("step")) == NULL) return 0; - if ((state->subject = PyUnicode_InternFromString("subject")) == NULL) return 0; - if ((state->tag = PyUnicode_InternFromString("tag")) == NULL) return 0; - if ((state->target = PyUnicode_InternFromString("target")) == NULL) return 0; - if ((state->targets = PyUnicode_InternFromString("targets")) == NULL) return 0; - if ((state->test = PyUnicode_InternFromString("test")) == NULL) return 0; - if ((state->type = PyUnicode_InternFromString("type")) == NULL) return 0; - if ((state->type_comment = PyUnicode_InternFromString("type_comment")) == NULL) return 0; - if ((state->type_ignores = PyUnicode_InternFromString("type_ignores")) == NULL) return 0; - if ((state->type_params = PyUnicode_InternFromString("type_params")) == NULL) return 0; - if ((state->upper = PyUnicode_InternFromString("upper")) == NULL) return 0; - if ((state->value = PyUnicode_InternFromString("value")) == NULL) return 0; - if ((state->values = PyUnicode_InternFromString("values")) == NULL) return 0; - if ((state->vararg = PyUnicode_InternFromString("vararg")) == NULL) return 0; - return 1; + if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return -1; + if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return -1; + if ((state->__match_args__ = PyUnicode_InternFromString("__match_args__")) == NULL) return -1; + if ((state->__module__ = PyUnicode_InternFromString("__module__")) == NULL) return -1; + if ((state->_attributes = PyUnicode_InternFromString("_attributes")) == NULL) return -1; + if ((state->_fields = PyUnicode_InternFromString("_fields")) == NULL) return -1; + if ((state->annotation = PyUnicode_InternFromString("annotation")) == NULL) return -1; + if ((state->arg = PyUnicode_InternFromString("arg")) == NULL) return -1; + if ((state->args = PyUnicode_InternFromString("args")) == NULL) return -1; + if ((state->argtypes = PyUnicode_InternFromString("argtypes")) == NULL) return -1; + if ((state->asname = PyUnicode_InternFromString("asname")) == NULL) return -1; + if ((state->ast = PyUnicode_InternFromString("ast")) == NULL) return -1; + if ((state->attr = PyUnicode_InternFromString("attr")) == NULL) return -1; + if ((state->bases = PyUnicode_InternFromString("bases")) == NULL) return -1; + if ((state->body = PyUnicode_InternFromString("body")) == NULL) return -1; + if ((state->bound = PyUnicode_InternFromString("bound")) == NULL) return -1; + if ((state->cases = PyUnicode_InternFromString("cases")) == NULL) return -1; + if ((state->cause = PyUnicode_InternFromString("cause")) == NULL) return -1; + if ((state->cls = PyUnicode_InternFromString("cls")) == NULL) return -1; + if ((state->col_offset = PyUnicode_InternFromString("col_offset")) == NULL) return -1; + if ((state->comparators = PyUnicode_InternFromString("comparators")) == NULL) return -1; + if ((state->context_expr = PyUnicode_InternFromString("context_expr")) == NULL) return -1; + if ((state->conversion = PyUnicode_InternFromString("conversion")) == NULL) return -1; + if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return -1; + if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return -1; + if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return -1; + if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return -1; + if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return -1; + if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return -1; + if ((state->end_lineno = PyUnicode_InternFromString("end_lineno")) == NULL) return -1; + if ((state->exc = PyUnicode_InternFromString("exc")) == NULL) return -1; + if ((state->finalbody = PyUnicode_InternFromString("finalbody")) == NULL) return -1; + if ((state->format_spec = PyUnicode_InternFromString("format_spec")) == NULL) return -1; + if ((state->func = PyUnicode_InternFromString("func")) == NULL) return -1; + if ((state->generators = PyUnicode_InternFromString("generators")) == NULL) return -1; + if ((state->guard = PyUnicode_InternFromString("guard")) == NULL) return -1; + if ((state->handlers = PyUnicode_InternFromString("handlers")) == NULL) return -1; + if ((state->id = PyUnicode_InternFromString("id")) == NULL) return -1; + if ((state->ifs = PyUnicode_InternFromString("ifs")) == NULL) return -1; + if ((state->is_async = PyUnicode_InternFromString("is_async")) == NULL) return -1; + if ((state->items = PyUnicode_InternFromString("items")) == NULL) return -1; + if ((state->iter = PyUnicode_InternFromString("iter")) == NULL) return -1; + if ((state->key = PyUnicode_InternFromString("key")) == NULL) return -1; + if ((state->keys = PyUnicode_InternFromString("keys")) == NULL) return -1; + if ((state->keywords = PyUnicode_InternFromString("keywords")) == NULL) return -1; + if ((state->kind = PyUnicode_InternFromString("kind")) == NULL) return -1; + if ((state->kw_defaults = PyUnicode_InternFromString("kw_defaults")) == NULL) return -1; + if ((state->kwarg = PyUnicode_InternFromString("kwarg")) == NULL) return -1; + if ((state->kwd_attrs = PyUnicode_InternFromString("kwd_attrs")) == NULL) return -1; + if ((state->kwd_patterns = PyUnicode_InternFromString("kwd_patterns")) == NULL) return -1; + if ((state->kwonlyargs = PyUnicode_InternFromString("kwonlyargs")) == NULL) return -1; + if ((state->left = PyUnicode_InternFromString("left")) == NULL) return -1; + if ((state->level = PyUnicode_InternFromString("level")) == NULL) return -1; + if ((state->lineno = PyUnicode_InternFromString("lineno")) == NULL) return -1; + if ((state->lower = PyUnicode_InternFromString("lower")) == NULL) return -1; + if ((state->module = PyUnicode_InternFromString("module")) == NULL) return -1; + if ((state->msg = PyUnicode_InternFromString("msg")) == NULL) return -1; + if ((state->name = PyUnicode_InternFromString("name")) == NULL) return -1; + if ((state->names = PyUnicode_InternFromString("names")) == NULL) return -1; + if ((state->op = PyUnicode_InternFromString("op")) == NULL) return -1; + if ((state->operand = PyUnicode_InternFromString("operand")) == NULL) return -1; + if ((state->ops = PyUnicode_InternFromString("ops")) == NULL) return -1; + if ((state->optional_vars = PyUnicode_InternFromString("optional_vars")) == NULL) return -1; + if ((state->orelse = PyUnicode_InternFromString("orelse")) == NULL) return -1; + if ((state->pattern = PyUnicode_InternFromString("pattern")) == NULL) return -1; + if ((state->patterns = PyUnicode_InternFromString("patterns")) == NULL) return -1; + if ((state->posonlyargs = PyUnicode_InternFromString("posonlyargs")) == NULL) return -1; + if ((state->rest = PyUnicode_InternFromString("rest")) == NULL) return -1; + if ((state->returns = PyUnicode_InternFromString("returns")) == NULL) return -1; + if ((state->right = PyUnicode_InternFromString("right")) == NULL) return -1; + if ((state->simple = PyUnicode_InternFromString("simple")) == NULL) return -1; + if ((state->slice = PyUnicode_InternFromString("slice")) == NULL) return -1; + if ((state->step = PyUnicode_InternFromString("step")) == NULL) return -1; + if ((state->subject = PyUnicode_InternFromString("subject")) == NULL) return -1; + if ((state->tag = PyUnicode_InternFromString("tag")) == NULL) return -1; + if ((state->target = PyUnicode_InternFromString("target")) == NULL) return -1; + if ((state->targets = PyUnicode_InternFromString("targets")) == NULL) return -1; + if ((state->test = PyUnicode_InternFromString("test")) == NULL) return -1; + if ((state->type = PyUnicode_InternFromString("type")) == NULL) return -1; + if ((state->type_comment = PyUnicode_InternFromString("type_comment")) == NULL) return -1; + if ((state->type_ignores = PyUnicode_InternFromString("type_ignores")) == NULL) return -1; + if ((state->type_params = PyUnicode_InternFromString("type_params")) == NULL) return -1; + if ((state->upper = PyUnicode_InternFromString("upper")) == NULL) return -1; + if ((state->value = PyUnicode_InternFromString("value")) == NULL) return -1; + if ((state->values = PyUnicode_InternFromString("values")) == NULL) return -1; + if ((state->vararg = PyUnicode_InternFromString("vararg")) == NULL) return -1; + return 0; }; GENERATE_ASDL_SEQ_CONSTRUCTOR(mod, mod_ty) @@ -993,16 +992,16 @@ add_attributes(struct ast_state *state, PyObject *type, const char * const *attr int i, result; PyObject *s, *l = PyTuple_New(num_fields); if (!l) - return 0; + return -1; for (i = 0; i < num_fields; i++) { s = PyUnicode_InternFromString(attrs[i]); if (!s) { Py_DECREF(l); - return 0; + return -1; } PyTuple_SET_ITEM(l, i, s); } - result = PyObject_SetAttr(type, state->_attributes, l) >= 0; + result = PyObject_SetAttr(type, state->_attributes, l); Py_DECREF(l); return result; } @@ -1077,7 +1076,7 @@ static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1086,7 +1085,7 @@ static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1096,12 +1095,12 @@ static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* o int i; if (!PyLong_Check(obj)) { PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); - return 1; + return -1; } i = PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) - return 1; + return -1; *out = i; return 0; } @@ -1126,47 +1125,40 @@ static int add_ast_fields(struct ast_state *state) static int init_types(struct ast_state *state) { - // init_types() must not be called after _PyAST_Fini() - // has been called - assert(state->initialized >= 0); - - if (state->initialized) { - return 1; - } if (init_identifiers(state) < 0) { - return 0; + return -1; } state->AST_type = PyType_FromSpec(&AST_type_spec); if (!state->AST_type) { - return 0; + return -1; } if (add_ast_fields(state) < 0) { - return 0; + return -1; } state->mod_type = make_type(state, "mod", state->AST_type, NULL, 0, "mod = Module(stmt* body, type_ignore* type_ignores)\n" " | Interactive(stmt* body)\n" " | Expression(expr body)\n" " | FunctionType(expr* argtypes, expr returns)"); - if (!state->mod_type) return 0; - if (!add_attributes(state, state->mod_type, NULL, 0)) return 0; + if (!state->mod_type) return -1; + if (add_attributes(state, state->mod_type, NULL, 0) < 0) return -1; state->Module_type = make_type(state, "Module", state->mod_type, Module_fields, 2, "Module(stmt* body, type_ignore* type_ignores)"); - if (!state->Module_type) return 0; + if (!state->Module_type) return -1; state->Interactive_type = make_type(state, "Interactive", state->mod_type, Interactive_fields, 1, "Interactive(stmt* body)"); - if (!state->Interactive_type) return 0; + if (!state->Interactive_type) return -1; state->Expression_type = make_type(state, "Expression", state->mod_type, Expression_fields, 1, "Expression(expr body)"); - if (!state->Expression_type) return 0; + if (!state->Expression_type) return -1; state->FunctionType_type = make_type(state, "FunctionType", state->mod_type, FunctionType_fields, 2, "FunctionType(expr* argtypes, expr returns)"); - if (!state->FunctionType_type) return 0; + if (!state->FunctionType_type) return -1; state->stmt_type = make_type(state, "stmt", state->AST_type, NULL, 0, "stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n" " | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n" @@ -1196,160 +1188,161 @@ init_types(struct ast_state *state) " | Pass\n" " | Break\n" " | Continue"); - if (!state->stmt_type) return 0; - if (!add_attributes(state, state->stmt_type, stmt_attributes, 4)) return 0; + if (!state->stmt_type) return -1; + if (add_attributes(state, state->stmt_type, stmt_attributes, 4) < 0) return + -1; if (PyObject_SetAttr(state->stmt_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->stmt_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->FunctionDef_type = make_type(state, "FunctionDef", state->stmt_type, FunctionDef_fields, 7, "FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)"); - if (!state->FunctionDef_type) return 0; + if (!state->FunctionDef_type) return -1; if (PyObject_SetAttr(state->FunctionDef_type, state->returns, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->FunctionDef_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->AsyncFunctionDef_type = make_type(state, "AsyncFunctionDef", state->stmt_type, AsyncFunctionDef_fields, 7, "AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)"); - if (!state->AsyncFunctionDef_type) return 0; + if (!state->AsyncFunctionDef_type) return -1; if (PyObject_SetAttr(state->AsyncFunctionDef_type, state->returns, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->AsyncFunctionDef_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->ClassDef_type = make_type(state, "ClassDef", state->stmt_type, ClassDef_fields, 6, "ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list, type_param* type_params)"); - if (!state->ClassDef_type) return 0; + if (!state->ClassDef_type) return -1; state->Return_type = make_type(state, "Return", state->stmt_type, Return_fields, 1, "Return(expr? value)"); - if (!state->Return_type) return 0; + if (!state->Return_type) return -1; if (PyObject_SetAttr(state->Return_type, state->value, Py_None) == -1) - return 0; + return -1; state->Delete_type = make_type(state, "Delete", state->stmt_type, Delete_fields, 1, "Delete(expr* targets)"); - if (!state->Delete_type) return 0; + if (!state->Delete_type) return -1; state->Assign_type = make_type(state, "Assign", state->stmt_type, Assign_fields, 3, "Assign(expr* targets, expr value, string? type_comment)"); - if (!state->Assign_type) return 0; + if (!state->Assign_type) return -1; if (PyObject_SetAttr(state->Assign_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->TypeAlias_type = make_type(state, "TypeAlias", state->stmt_type, TypeAlias_fields, 3, "TypeAlias(expr name, type_param* type_params, expr value)"); - if (!state->TypeAlias_type) return 0; + if (!state->TypeAlias_type) return -1; state->AugAssign_type = make_type(state, "AugAssign", state->stmt_type, AugAssign_fields, 3, "AugAssign(expr target, operator op, expr value)"); - if (!state->AugAssign_type) return 0; + if (!state->AugAssign_type) return -1; state->AnnAssign_type = make_type(state, "AnnAssign", state->stmt_type, AnnAssign_fields, 4, "AnnAssign(expr target, expr annotation, expr? value, int simple)"); - if (!state->AnnAssign_type) return 0; + if (!state->AnnAssign_type) return -1; if (PyObject_SetAttr(state->AnnAssign_type, state->value, Py_None) == -1) - return 0; + return -1; state->For_type = make_type(state, "For", state->stmt_type, For_fields, 5, "For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)"); - if (!state->For_type) return 0; + if (!state->For_type) return -1; if (PyObject_SetAttr(state->For_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->AsyncFor_type = make_type(state, "AsyncFor", state->stmt_type, AsyncFor_fields, 5, "AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)"); - if (!state->AsyncFor_type) return 0; + if (!state->AsyncFor_type) return -1; if (PyObject_SetAttr(state->AsyncFor_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->While_type = make_type(state, "While", state->stmt_type, While_fields, 3, "While(expr test, stmt* body, stmt* orelse)"); - if (!state->While_type) return 0; + if (!state->While_type) return -1; state->If_type = make_type(state, "If", state->stmt_type, If_fields, 3, "If(expr test, stmt* body, stmt* orelse)"); - if (!state->If_type) return 0; + if (!state->If_type) return -1; state->With_type = make_type(state, "With", state->stmt_type, With_fields, 3, "With(withitem* items, stmt* body, string? type_comment)"); - if (!state->With_type) return 0; + if (!state->With_type) return -1; if (PyObject_SetAttr(state->With_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->AsyncWith_type = make_type(state, "AsyncWith", state->stmt_type, AsyncWith_fields, 3, "AsyncWith(withitem* items, stmt* body, string? type_comment)"); - if (!state->AsyncWith_type) return 0; + if (!state->AsyncWith_type) return -1; if (PyObject_SetAttr(state->AsyncWith_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->Match_type = make_type(state, "Match", state->stmt_type, Match_fields, 2, "Match(expr subject, match_case* cases)"); - if (!state->Match_type) return 0; + if (!state->Match_type) return -1; state->Raise_type = make_type(state, "Raise", state->stmt_type, Raise_fields, 2, "Raise(expr? exc, expr? cause)"); - if (!state->Raise_type) return 0; + if (!state->Raise_type) return -1; if (PyObject_SetAttr(state->Raise_type, state->exc, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->Raise_type, state->cause, Py_None) == -1) - return 0; + return -1; state->Try_type = make_type(state, "Try", state->stmt_type, Try_fields, 4, "Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)"); - if (!state->Try_type) return 0; + if (!state->Try_type) return -1; state->TryStar_type = make_type(state, "TryStar", state->stmt_type, TryStar_fields, 4, "TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)"); - if (!state->TryStar_type) return 0; + if (!state->TryStar_type) return -1; state->Assert_type = make_type(state, "Assert", state->stmt_type, Assert_fields, 2, "Assert(expr test, expr? msg)"); - if (!state->Assert_type) return 0; + if (!state->Assert_type) return -1; if (PyObject_SetAttr(state->Assert_type, state->msg, Py_None) == -1) - return 0; + return -1; state->Import_type = make_type(state, "Import", state->stmt_type, Import_fields, 1, "Import(alias* names)"); - if (!state->Import_type) return 0; + if (!state->Import_type) return -1; state->ImportFrom_type = make_type(state, "ImportFrom", state->stmt_type, ImportFrom_fields, 3, "ImportFrom(identifier? module, alias* names, int? level)"); - if (!state->ImportFrom_type) return 0; + if (!state->ImportFrom_type) return -1; if (PyObject_SetAttr(state->ImportFrom_type, state->module, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->ImportFrom_type, state->level, Py_None) == -1) - return 0; + return -1; state->Global_type = make_type(state, "Global", state->stmt_type, Global_fields, 1, "Global(identifier* names)"); - if (!state->Global_type) return 0; + if (!state->Global_type) return -1; state->Nonlocal_type = make_type(state, "Nonlocal", state->stmt_type, Nonlocal_fields, 1, "Nonlocal(identifier* names)"); - if (!state->Nonlocal_type) return 0; + if (!state->Nonlocal_type) return -1; state->Expr_type = make_type(state, "Expr", state->stmt_type, Expr_fields, 1, "Expr(expr value)"); - if (!state->Expr_type) return 0; + if (!state->Expr_type) return -1; state->Pass_type = make_type(state, "Pass", state->stmt_type, NULL, 0, "Pass"); - if (!state->Pass_type) return 0; + if (!state->Pass_type) return -1; state->Break_type = make_type(state, "Break", state->stmt_type, NULL, 0, "Break"); - if (!state->Break_type) return 0; + if (!state->Break_type) return -1; state->Continue_type = make_type(state, "Continue", state->stmt_type, NULL, 0, "Continue"); - if (!state->Continue_type) return 0; + if (!state->Continue_type) return -1; state->expr_type = make_type(state, "expr", state->AST_type, NULL, 0, "expr = BoolOp(boolop op, expr* values)\n" " | NamedExpr(expr target, expr value)\n" @@ -1378,454 +1371,457 @@ init_types(struct ast_state *state) " | List(expr* elts, expr_context ctx)\n" " | Tuple(expr* elts, expr_context ctx)\n" " | Slice(expr? lower, expr? upper, expr? step)"); - if (!state->expr_type) return 0; - if (!add_attributes(state, state->expr_type, expr_attributes, 4)) return 0; + if (!state->expr_type) return -1; + if (add_attributes(state, state->expr_type, expr_attributes, 4) < 0) return + -1; if (PyObject_SetAttr(state->expr_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->expr_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->BoolOp_type = make_type(state, "BoolOp", state->expr_type, BoolOp_fields, 2, "BoolOp(boolop op, expr* values)"); - if (!state->BoolOp_type) return 0; + if (!state->BoolOp_type) return -1; state->NamedExpr_type = make_type(state, "NamedExpr", state->expr_type, NamedExpr_fields, 2, "NamedExpr(expr target, expr value)"); - if (!state->NamedExpr_type) return 0; + if (!state->NamedExpr_type) return -1; state->BinOp_type = make_type(state, "BinOp", state->expr_type, BinOp_fields, 3, "BinOp(expr left, operator op, expr right)"); - if (!state->BinOp_type) return 0; + if (!state->BinOp_type) return -1; state->UnaryOp_type = make_type(state, "UnaryOp", state->expr_type, UnaryOp_fields, 2, "UnaryOp(unaryop op, expr operand)"); - if (!state->UnaryOp_type) return 0; + if (!state->UnaryOp_type) return -1; state->Lambda_type = make_type(state, "Lambda", state->expr_type, Lambda_fields, 2, "Lambda(arguments args, expr body)"); - if (!state->Lambda_type) return 0; + if (!state->Lambda_type) return -1; state->IfExp_type = make_type(state, "IfExp", state->expr_type, IfExp_fields, 3, "IfExp(expr test, expr body, expr orelse)"); - if (!state->IfExp_type) return 0; + if (!state->IfExp_type) return -1; state->Dict_type = make_type(state, "Dict", state->expr_type, Dict_fields, 2, "Dict(expr* keys, expr* values)"); - if (!state->Dict_type) return 0; + if (!state->Dict_type) return -1; state->Set_type = make_type(state, "Set", state->expr_type, Set_fields, 1, "Set(expr* elts)"); - if (!state->Set_type) return 0; + if (!state->Set_type) return -1; state->ListComp_type = make_type(state, "ListComp", state->expr_type, ListComp_fields, 2, "ListComp(expr elt, comprehension* generators)"); - if (!state->ListComp_type) return 0; + if (!state->ListComp_type) return -1; state->SetComp_type = make_type(state, "SetComp", state->expr_type, SetComp_fields, 2, "SetComp(expr elt, comprehension* generators)"); - if (!state->SetComp_type) return 0; + if (!state->SetComp_type) return -1; state->DictComp_type = make_type(state, "DictComp", state->expr_type, DictComp_fields, 3, "DictComp(expr key, expr value, comprehension* generators)"); - if (!state->DictComp_type) return 0; + if (!state->DictComp_type) return -1; state->GeneratorExp_type = make_type(state, "GeneratorExp", state->expr_type, GeneratorExp_fields, 2, "GeneratorExp(expr elt, comprehension* generators)"); - if (!state->GeneratorExp_type) return 0; + if (!state->GeneratorExp_type) return -1; state->Await_type = make_type(state, "Await", state->expr_type, Await_fields, 1, "Await(expr value)"); - if (!state->Await_type) return 0; + if (!state->Await_type) return -1; state->Yield_type = make_type(state, "Yield", state->expr_type, Yield_fields, 1, "Yield(expr? value)"); - if (!state->Yield_type) return 0; + if (!state->Yield_type) return -1; if (PyObject_SetAttr(state->Yield_type, state->value, Py_None) == -1) - return 0; + return -1; state->YieldFrom_type = make_type(state, "YieldFrom", state->expr_type, YieldFrom_fields, 1, "YieldFrom(expr value)"); - if (!state->YieldFrom_type) return 0; + if (!state->YieldFrom_type) return -1; state->Compare_type = make_type(state, "Compare", state->expr_type, Compare_fields, 3, "Compare(expr left, cmpop* ops, expr* comparators)"); - if (!state->Compare_type) return 0; + if (!state->Compare_type) return -1; state->Call_type = make_type(state, "Call", state->expr_type, Call_fields, 3, "Call(expr func, expr* args, keyword* keywords)"); - if (!state->Call_type) return 0; + if (!state->Call_type) return -1; state->FormattedValue_type = make_type(state, "FormattedValue", state->expr_type, FormattedValue_fields, 3, "FormattedValue(expr value, int conversion, expr? format_spec)"); - if (!state->FormattedValue_type) return 0; + if (!state->FormattedValue_type) return -1; if (PyObject_SetAttr(state->FormattedValue_type, state->format_spec, Py_None) == -1) - return 0; + return -1; state->JoinedStr_type = make_type(state, "JoinedStr", state->expr_type, JoinedStr_fields, 1, "JoinedStr(expr* values)"); - if (!state->JoinedStr_type) return 0; + if (!state->JoinedStr_type) return -1; state->Constant_type = make_type(state, "Constant", state->expr_type, Constant_fields, 2, "Constant(constant value, string? kind)"); - if (!state->Constant_type) return 0; + if (!state->Constant_type) return -1; if (PyObject_SetAttr(state->Constant_type, state->kind, Py_None) == -1) - return 0; + return -1; state->Attribute_type = make_type(state, "Attribute", state->expr_type, Attribute_fields, 3, "Attribute(expr value, identifier attr, expr_context ctx)"); - if (!state->Attribute_type) return 0; + if (!state->Attribute_type) return -1; state->Subscript_type = make_type(state, "Subscript", state->expr_type, Subscript_fields, 3, "Subscript(expr value, expr slice, expr_context ctx)"); - if (!state->Subscript_type) return 0; + if (!state->Subscript_type) return -1; state->Starred_type = make_type(state, "Starred", state->expr_type, Starred_fields, 2, "Starred(expr value, expr_context ctx)"); - if (!state->Starred_type) return 0; + if (!state->Starred_type) return -1; state->Name_type = make_type(state, "Name", state->expr_type, Name_fields, 2, "Name(identifier id, expr_context ctx)"); - if (!state->Name_type) return 0; + if (!state->Name_type) return -1; state->List_type = make_type(state, "List", state->expr_type, List_fields, 2, "List(expr* elts, expr_context ctx)"); - if (!state->List_type) return 0; + if (!state->List_type) return -1; state->Tuple_type = make_type(state, "Tuple", state->expr_type, Tuple_fields, 2, "Tuple(expr* elts, expr_context ctx)"); - if (!state->Tuple_type) return 0; + if (!state->Tuple_type) return -1; state->Slice_type = make_type(state, "Slice", state->expr_type, Slice_fields, 3, "Slice(expr? lower, expr? upper, expr? step)"); - if (!state->Slice_type) return 0; + if (!state->Slice_type) return -1; if (PyObject_SetAttr(state->Slice_type, state->lower, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->Slice_type, state->upper, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->Slice_type, state->step, Py_None) == -1) - return 0; + return -1; state->expr_context_type = make_type(state, "expr_context", state->AST_type, NULL, 0, "expr_context = Load | Store | Del"); - if (!state->expr_context_type) return 0; - if (!add_attributes(state, state->expr_context_type, NULL, 0)) return 0; + if (!state->expr_context_type) return -1; + if (add_attributes(state, state->expr_context_type, NULL, 0) < 0) return -1; state->Load_type = make_type(state, "Load", state->expr_context_type, NULL, 0, "Load"); - if (!state->Load_type) return 0; + if (!state->Load_type) return -1; state->Load_singleton = PyType_GenericNew((PyTypeObject *)state->Load_type, NULL, NULL); - if (!state->Load_singleton) return 0; + if (!state->Load_singleton) return -1; state->Store_type = make_type(state, "Store", state->expr_context_type, NULL, 0, "Store"); - if (!state->Store_type) return 0; + if (!state->Store_type) return -1; state->Store_singleton = PyType_GenericNew((PyTypeObject *)state->Store_type, NULL, NULL); - if (!state->Store_singleton) return 0; + if (!state->Store_singleton) return -1; state->Del_type = make_type(state, "Del", state->expr_context_type, NULL, 0, "Del"); - if (!state->Del_type) return 0; + if (!state->Del_type) return -1; state->Del_singleton = PyType_GenericNew((PyTypeObject *)state->Del_type, NULL, NULL); - if (!state->Del_singleton) return 0; + if (!state->Del_singleton) return -1; state->boolop_type = make_type(state, "boolop", state->AST_type, NULL, 0, "boolop = And | Or"); - if (!state->boolop_type) return 0; - if (!add_attributes(state, state->boolop_type, NULL, 0)) return 0; + if (!state->boolop_type) return -1; + if (add_attributes(state, state->boolop_type, NULL, 0) < 0) return -1; state->And_type = make_type(state, "And", state->boolop_type, NULL, 0, "And"); - if (!state->And_type) return 0; + if (!state->And_type) return -1; state->And_singleton = PyType_GenericNew((PyTypeObject *)state->And_type, NULL, NULL); - if (!state->And_singleton) return 0; + if (!state->And_singleton) return -1; state->Or_type = make_type(state, "Or", state->boolop_type, NULL, 0, "Or"); - if (!state->Or_type) return 0; + if (!state->Or_type) return -1; state->Or_singleton = PyType_GenericNew((PyTypeObject *)state->Or_type, NULL, NULL); - if (!state->Or_singleton) return 0; + if (!state->Or_singleton) return -1; state->operator_type = make_type(state, "operator", state->AST_type, NULL, 0, "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv"); - if (!state->operator_type) return 0; - if (!add_attributes(state, state->operator_type, NULL, 0)) return 0; + if (!state->operator_type) return -1; + if (add_attributes(state, state->operator_type, NULL, 0) < 0) return -1; state->Add_type = make_type(state, "Add", state->operator_type, NULL, 0, "Add"); - if (!state->Add_type) return 0; + if (!state->Add_type) return -1; state->Add_singleton = PyType_GenericNew((PyTypeObject *)state->Add_type, NULL, NULL); - if (!state->Add_singleton) return 0; + if (!state->Add_singleton) return -1; state->Sub_type = make_type(state, "Sub", state->operator_type, NULL, 0, "Sub"); - if (!state->Sub_type) return 0; + if (!state->Sub_type) return -1; state->Sub_singleton = PyType_GenericNew((PyTypeObject *)state->Sub_type, NULL, NULL); - if (!state->Sub_singleton) return 0; + if (!state->Sub_singleton) return -1; state->Mult_type = make_type(state, "Mult", state->operator_type, NULL, 0, "Mult"); - if (!state->Mult_type) return 0; + if (!state->Mult_type) return -1; state->Mult_singleton = PyType_GenericNew((PyTypeObject *)state->Mult_type, NULL, NULL); - if (!state->Mult_singleton) return 0; + if (!state->Mult_singleton) return -1; state->MatMult_type = make_type(state, "MatMult", state->operator_type, NULL, 0, "MatMult"); - if (!state->MatMult_type) return 0; + if (!state->MatMult_type) return -1; state->MatMult_singleton = PyType_GenericNew((PyTypeObject *)state->MatMult_type, NULL, NULL); - if (!state->MatMult_singleton) return 0; + if (!state->MatMult_singleton) return -1; state->Div_type = make_type(state, "Div", state->operator_type, NULL, 0, "Div"); - if (!state->Div_type) return 0; + if (!state->Div_type) return -1; state->Div_singleton = PyType_GenericNew((PyTypeObject *)state->Div_type, NULL, NULL); - if (!state->Div_singleton) return 0; + if (!state->Div_singleton) return -1; state->Mod_type = make_type(state, "Mod", state->operator_type, NULL, 0, "Mod"); - if (!state->Mod_type) return 0; + if (!state->Mod_type) return -1; state->Mod_singleton = PyType_GenericNew((PyTypeObject *)state->Mod_type, NULL, NULL); - if (!state->Mod_singleton) return 0; + if (!state->Mod_singleton) return -1; state->Pow_type = make_type(state, "Pow", state->operator_type, NULL, 0, "Pow"); - if (!state->Pow_type) return 0; + if (!state->Pow_type) return -1; state->Pow_singleton = PyType_GenericNew((PyTypeObject *)state->Pow_type, NULL, NULL); - if (!state->Pow_singleton) return 0; + if (!state->Pow_singleton) return -1; state->LShift_type = make_type(state, "LShift", state->operator_type, NULL, 0, "LShift"); - if (!state->LShift_type) return 0; + if (!state->LShift_type) return -1; state->LShift_singleton = PyType_GenericNew((PyTypeObject *)state->LShift_type, NULL, NULL); - if (!state->LShift_singleton) return 0; + if (!state->LShift_singleton) return -1; state->RShift_type = make_type(state, "RShift", state->operator_type, NULL, 0, "RShift"); - if (!state->RShift_type) return 0; + if (!state->RShift_type) return -1; state->RShift_singleton = PyType_GenericNew((PyTypeObject *)state->RShift_type, NULL, NULL); - if (!state->RShift_singleton) return 0; + if (!state->RShift_singleton) return -1; state->BitOr_type = make_type(state, "BitOr", state->operator_type, NULL, 0, "BitOr"); - if (!state->BitOr_type) return 0; + if (!state->BitOr_type) return -1; state->BitOr_singleton = PyType_GenericNew((PyTypeObject *)state->BitOr_type, NULL, NULL); - if (!state->BitOr_singleton) return 0; + if (!state->BitOr_singleton) return -1; state->BitXor_type = make_type(state, "BitXor", state->operator_type, NULL, 0, "BitXor"); - if (!state->BitXor_type) return 0; + if (!state->BitXor_type) return -1; state->BitXor_singleton = PyType_GenericNew((PyTypeObject *)state->BitXor_type, NULL, NULL); - if (!state->BitXor_singleton) return 0; + if (!state->BitXor_singleton) return -1; state->BitAnd_type = make_type(state, "BitAnd", state->operator_type, NULL, 0, "BitAnd"); - if (!state->BitAnd_type) return 0; + if (!state->BitAnd_type) return -1; state->BitAnd_singleton = PyType_GenericNew((PyTypeObject *)state->BitAnd_type, NULL, NULL); - if (!state->BitAnd_singleton) return 0; + if (!state->BitAnd_singleton) return -1; state->FloorDiv_type = make_type(state, "FloorDiv", state->operator_type, NULL, 0, "FloorDiv"); - if (!state->FloorDiv_type) return 0; + if (!state->FloorDiv_type) return -1; state->FloorDiv_singleton = PyType_GenericNew((PyTypeObject *)state->FloorDiv_type, NULL, NULL); - if (!state->FloorDiv_singleton) return 0; + if (!state->FloorDiv_singleton) return -1; state->unaryop_type = make_type(state, "unaryop", state->AST_type, NULL, 0, "unaryop = Invert | Not | UAdd | USub"); - if (!state->unaryop_type) return 0; - if (!add_attributes(state, state->unaryop_type, NULL, 0)) return 0; + if (!state->unaryop_type) return -1; + if (add_attributes(state, state->unaryop_type, NULL, 0) < 0) return -1; state->Invert_type = make_type(state, "Invert", state->unaryop_type, NULL, 0, "Invert"); - if (!state->Invert_type) return 0; + if (!state->Invert_type) return -1; state->Invert_singleton = PyType_GenericNew((PyTypeObject *)state->Invert_type, NULL, NULL); - if (!state->Invert_singleton) return 0; + if (!state->Invert_singleton) return -1; state->Not_type = make_type(state, "Not", state->unaryop_type, NULL, 0, "Not"); - if (!state->Not_type) return 0; + if (!state->Not_type) return -1; state->Not_singleton = PyType_GenericNew((PyTypeObject *)state->Not_type, NULL, NULL); - if (!state->Not_singleton) return 0; + if (!state->Not_singleton) return -1; state->UAdd_type = make_type(state, "UAdd", state->unaryop_type, NULL, 0, "UAdd"); - if (!state->UAdd_type) return 0; + if (!state->UAdd_type) return -1; state->UAdd_singleton = PyType_GenericNew((PyTypeObject *)state->UAdd_type, NULL, NULL); - if (!state->UAdd_singleton) return 0; + if (!state->UAdd_singleton) return -1; state->USub_type = make_type(state, "USub", state->unaryop_type, NULL, 0, "USub"); - if (!state->USub_type) return 0; + if (!state->USub_type) return -1; state->USub_singleton = PyType_GenericNew((PyTypeObject *)state->USub_type, NULL, NULL); - if (!state->USub_singleton) return 0; + if (!state->USub_singleton) return -1; state->cmpop_type = make_type(state, "cmpop", state->AST_type, NULL, 0, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn"); - if (!state->cmpop_type) return 0; - if (!add_attributes(state, state->cmpop_type, NULL, 0)) return 0; + if (!state->cmpop_type) return -1; + if (add_attributes(state, state->cmpop_type, NULL, 0) < 0) return -1; state->Eq_type = make_type(state, "Eq", state->cmpop_type, NULL, 0, "Eq"); - if (!state->Eq_type) return 0; + if (!state->Eq_type) return -1; state->Eq_singleton = PyType_GenericNew((PyTypeObject *)state->Eq_type, NULL, NULL); - if (!state->Eq_singleton) return 0; + if (!state->Eq_singleton) return -1; state->NotEq_type = make_type(state, "NotEq", state->cmpop_type, NULL, 0, "NotEq"); - if (!state->NotEq_type) return 0; + if (!state->NotEq_type) return -1; state->NotEq_singleton = PyType_GenericNew((PyTypeObject *)state->NotEq_type, NULL, NULL); - if (!state->NotEq_singleton) return 0; + if (!state->NotEq_singleton) return -1; state->Lt_type = make_type(state, "Lt", state->cmpop_type, NULL, 0, "Lt"); - if (!state->Lt_type) return 0; + if (!state->Lt_type) return -1; state->Lt_singleton = PyType_GenericNew((PyTypeObject *)state->Lt_type, NULL, NULL); - if (!state->Lt_singleton) return 0; + if (!state->Lt_singleton) return -1; state->LtE_type = make_type(state, "LtE", state->cmpop_type, NULL, 0, "LtE"); - if (!state->LtE_type) return 0; + if (!state->LtE_type) return -1; state->LtE_singleton = PyType_GenericNew((PyTypeObject *)state->LtE_type, NULL, NULL); - if (!state->LtE_singleton) return 0; + if (!state->LtE_singleton) return -1; state->Gt_type = make_type(state, "Gt", state->cmpop_type, NULL, 0, "Gt"); - if (!state->Gt_type) return 0; + if (!state->Gt_type) return -1; state->Gt_singleton = PyType_GenericNew((PyTypeObject *)state->Gt_type, NULL, NULL); - if (!state->Gt_singleton) return 0; + if (!state->Gt_singleton) return -1; state->GtE_type = make_type(state, "GtE", state->cmpop_type, NULL, 0, "GtE"); - if (!state->GtE_type) return 0; + if (!state->GtE_type) return -1; state->GtE_singleton = PyType_GenericNew((PyTypeObject *)state->GtE_type, NULL, NULL); - if (!state->GtE_singleton) return 0; + if (!state->GtE_singleton) return -1; state->Is_type = make_type(state, "Is", state->cmpop_type, NULL, 0, "Is"); - if (!state->Is_type) return 0; + if (!state->Is_type) return -1; state->Is_singleton = PyType_GenericNew((PyTypeObject *)state->Is_type, NULL, NULL); - if (!state->Is_singleton) return 0; + if (!state->Is_singleton) return -1; state->IsNot_type = make_type(state, "IsNot", state->cmpop_type, NULL, 0, "IsNot"); - if (!state->IsNot_type) return 0; + if (!state->IsNot_type) return -1; state->IsNot_singleton = PyType_GenericNew((PyTypeObject *)state->IsNot_type, NULL, NULL); - if (!state->IsNot_singleton) return 0; + if (!state->IsNot_singleton) return -1; state->In_type = make_type(state, "In", state->cmpop_type, NULL, 0, "In"); - if (!state->In_type) return 0; + if (!state->In_type) return -1; state->In_singleton = PyType_GenericNew((PyTypeObject *)state->In_type, NULL, NULL); - if (!state->In_singleton) return 0; + if (!state->In_singleton) return -1; state->NotIn_type = make_type(state, "NotIn", state->cmpop_type, NULL, 0, "NotIn"); - if (!state->NotIn_type) return 0; + if (!state->NotIn_type) return -1; state->NotIn_singleton = PyType_GenericNew((PyTypeObject *)state->NotIn_type, NULL, NULL); - if (!state->NotIn_singleton) return 0; + if (!state->NotIn_singleton) return -1; state->comprehension_type = make_type(state, "comprehension", state->AST_type, comprehension_fields, 4, "comprehension(expr target, expr iter, expr* ifs, int is_async)"); - if (!state->comprehension_type) return 0; - if (!add_attributes(state, state->comprehension_type, NULL, 0)) return 0; + if (!state->comprehension_type) return -1; + if (add_attributes(state, state->comprehension_type, NULL, 0) < 0) return + -1; state->excepthandler_type = make_type(state, "excepthandler", state->AST_type, NULL, 0, "excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)"); - if (!state->excepthandler_type) return 0; - if (!add_attributes(state, state->excepthandler_type, - excepthandler_attributes, 4)) return 0; + if (!state->excepthandler_type) return -1; + if (add_attributes(state, state->excepthandler_type, + excepthandler_attributes, 4) < 0) return -1; if (PyObject_SetAttr(state->excepthandler_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->excepthandler_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->ExceptHandler_type = make_type(state, "ExceptHandler", state->excepthandler_type, ExceptHandler_fields, 3, "ExceptHandler(expr? type, identifier? name, stmt* body)"); - if (!state->ExceptHandler_type) return 0; + if (!state->ExceptHandler_type) return -1; if (PyObject_SetAttr(state->ExceptHandler_type, state->type, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->ExceptHandler_type, state->name, Py_None) == -1) - return 0; + return -1; state->arguments_type = make_type(state, "arguments", state->AST_type, arguments_fields, 7, "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults)"); - if (!state->arguments_type) return 0; - if (!add_attributes(state, state->arguments_type, NULL, 0)) return 0; + if (!state->arguments_type) return -1; + if (add_attributes(state, state->arguments_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->arguments_type, state->vararg, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arguments_type, state->kwarg, Py_None) == -1) - return 0; + return -1; state->arg_type = make_type(state, "arg", state->AST_type, arg_fields, 3, "arg(identifier arg, expr? annotation, string? type_comment)"); - if (!state->arg_type) return 0; - if (!add_attributes(state, state->arg_type, arg_attributes, 4)) return 0; + if (!state->arg_type) return -1; + if (add_attributes(state, state->arg_type, arg_attributes, 4) < 0) return + -1; if (PyObject_SetAttr(state->arg_type, state->annotation, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arg_type, state->type_comment, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arg_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arg_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->keyword_type = make_type(state, "keyword", state->AST_type, keyword_fields, 2, "keyword(identifier? arg, expr value)"); - if (!state->keyword_type) return 0; - if (!add_attributes(state, state->keyword_type, keyword_attributes, 4)) - return 0; + if (!state->keyword_type) return -1; + if (add_attributes(state, state->keyword_type, keyword_attributes, 4) < 0) + return -1; if (PyObject_SetAttr(state->keyword_type, state->arg, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->keyword_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->keyword_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->alias_type = make_type(state, "alias", state->AST_type, alias_fields, 2, "alias(identifier name, identifier? asname)"); - if (!state->alias_type) return 0; - if (!add_attributes(state, state->alias_type, alias_attributes, 4)) return - 0; + if (!state->alias_type) return -1; + if (add_attributes(state, state->alias_type, alias_attributes, 4) < 0) + return -1; if (PyObject_SetAttr(state->alias_type, state->asname, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->alias_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->alias_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->withitem_type = make_type(state, "withitem", state->AST_type, withitem_fields, 2, "withitem(expr context_expr, expr? optional_vars)"); - if (!state->withitem_type) return 0; - if (!add_attributes(state, state->withitem_type, NULL, 0)) return 0; + if (!state->withitem_type) return -1; + if (add_attributes(state, state->withitem_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->withitem_type, state->optional_vars, Py_None) == -1) - return 0; + return -1; state->match_case_type = make_type(state, "match_case", state->AST_type, match_case_fields, 3, "match_case(pattern pattern, expr? guard, stmt* body)"); - if (!state->match_case_type) return 0; - if (!add_attributes(state, state->match_case_type, NULL, 0)) return 0; + if (!state->match_case_type) return -1; + if (add_attributes(state, state->match_case_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->match_case_type, state->guard, Py_None) == -1) - return 0; + return -1; state->pattern_type = make_type(state, "pattern", state->AST_type, NULL, 0, "pattern = MatchValue(expr value)\n" " | MatchSingleton(constant value)\n" @@ -1835,93 +1831,92 @@ init_types(struct ast_state *state) " | MatchStar(identifier? name)\n" " | MatchAs(pattern? pattern, identifier? name)\n" " | MatchOr(pattern* patterns)"); - if (!state->pattern_type) return 0; - if (!add_attributes(state, state->pattern_type, pattern_attributes, 4)) - return 0; + if (!state->pattern_type) return -1; + if (add_attributes(state, state->pattern_type, pattern_attributes, 4) < 0) + return -1; state->MatchValue_type = make_type(state, "MatchValue", state->pattern_type, MatchValue_fields, 1, "MatchValue(expr value)"); - if (!state->MatchValue_type) return 0; + if (!state->MatchValue_type) return -1; state->MatchSingleton_type = make_type(state, "MatchSingleton", state->pattern_type, MatchSingleton_fields, 1, "MatchSingleton(constant value)"); - if (!state->MatchSingleton_type) return 0; + if (!state->MatchSingleton_type) return -1; state->MatchSequence_type = make_type(state, "MatchSequence", state->pattern_type, MatchSequence_fields, 1, "MatchSequence(pattern* patterns)"); - if (!state->MatchSequence_type) return 0; + if (!state->MatchSequence_type) return -1; state->MatchMapping_type = make_type(state, "MatchMapping", state->pattern_type, MatchMapping_fields, 3, "MatchMapping(expr* keys, pattern* patterns, identifier? rest)"); - if (!state->MatchMapping_type) return 0; + if (!state->MatchMapping_type) return -1; if (PyObject_SetAttr(state->MatchMapping_type, state->rest, Py_None) == -1) - return 0; + return -1; state->MatchClass_type = make_type(state, "MatchClass", state->pattern_type, MatchClass_fields, 4, "MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)"); - if (!state->MatchClass_type) return 0; + if (!state->MatchClass_type) return -1; state->MatchStar_type = make_type(state, "MatchStar", state->pattern_type, MatchStar_fields, 1, "MatchStar(identifier? name)"); - if (!state->MatchStar_type) return 0; + if (!state->MatchStar_type) return -1; if (PyObject_SetAttr(state->MatchStar_type, state->name, Py_None) == -1) - return 0; + return -1; state->MatchAs_type = make_type(state, "MatchAs", state->pattern_type, MatchAs_fields, 2, "MatchAs(pattern? pattern, identifier? name)"); - if (!state->MatchAs_type) return 0; + if (!state->MatchAs_type) return -1; if (PyObject_SetAttr(state->MatchAs_type, state->pattern, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->MatchAs_type, state->name, Py_None) == -1) - return 0; + return -1; state->MatchOr_type = make_type(state, "MatchOr", state->pattern_type, MatchOr_fields, 1, "MatchOr(pattern* patterns)"); - if (!state->MatchOr_type) return 0; + if (!state->MatchOr_type) return -1; state->type_ignore_type = make_type(state, "type_ignore", state->AST_type, NULL, 0, "type_ignore = TypeIgnore(int lineno, string tag)"); - if (!state->type_ignore_type) return 0; - if (!add_attributes(state, state->type_ignore_type, NULL, 0)) return 0; + if (!state->type_ignore_type) return -1; + if (add_attributes(state, state->type_ignore_type, NULL, 0) < 0) return -1; state->TypeIgnore_type = make_type(state, "TypeIgnore", state->type_ignore_type, TypeIgnore_fields, 2, "TypeIgnore(int lineno, string tag)"); - if (!state->TypeIgnore_type) return 0; + if (!state->TypeIgnore_type) return -1; state->type_param_type = make_type(state, "type_param", state->AST_type, NULL, 0, "type_param = TypeVar(identifier name, expr? bound)\n" " | ParamSpec(identifier name)\n" " | TypeVarTuple(identifier name)"); - if (!state->type_param_type) return 0; - if (!add_attributes(state, state->type_param_type, type_param_attributes, - 4)) return 0; + if (!state->type_param_type) return -1; + if (add_attributes(state, state->type_param_type, type_param_attributes, 4) + < 0) return -1; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, TypeVar_fields, 2, "TypeVar(identifier name, expr? bound)"); - if (!state->TypeVar_type) return 0; + if (!state->TypeVar_type) return -1; if (PyObject_SetAttr(state->TypeVar_type, state->bound, Py_None) == -1) - return 0; + return -1; state->ParamSpec_type = make_type(state, "ParamSpec", state->type_param_type, ParamSpec_fields, 1, "ParamSpec(identifier name)"); - if (!state->ParamSpec_type) return 0; + if (!state->ParamSpec_type) return -1; state->TypeVarTuple_type = make_type(state, "TypeVarTuple", state->type_param_type, TypeVarTuple_fields, 1, "TypeVarTuple(identifier name)"); - if (!state->TypeVarTuple_type) return 0; + if (!state->TypeVarTuple_type) return -1; state->recursion_depth = 0; state->recursion_limit = 0; - state->initialized = 1; - return 1; + return 0; } static int obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, @@ -3786,7 +3781,7 @@ ast2obj_mod(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case Module_kind: @@ -3864,7 +3859,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case FunctionDef_kind: @@ -4470,7 +4465,7 @@ ast2obj_expr(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case BoolOp_kind: @@ -5048,7 +5043,7 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->comprehension_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -5094,7 +5089,7 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case ExceptHandler_kind: @@ -5160,7 +5155,7 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->arguments_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -5221,7 +5216,7 @@ ast2obj_arg(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->arg_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -5282,7 +5277,7 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->keyword_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -5338,7 +5333,7 @@ ast2obj_alias(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->alias_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -5394,7 +5389,7 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->withitem_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -5430,7 +5425,7 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->match_case_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -5471,7 +5466,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case MatchValue_kind: @@ -5633,7 +5628,7 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case TypeIgnore_kind: @@ -5673,7 +5668,7 @@ ast2obj_type_param(struct ast_state *state, void* _o) if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case TypeVar_kind: @@ -5757,19 +5752,19 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->Module_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; asdl_type_ignore_seq* type_ignores; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5802,12 +5797,12 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_ignores, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5846,18 +5841,18 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->Interactive_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5896,17 +5891,17 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->Expression_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty body; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Expression"); - return 1; + return -1; } else { int res; @@ -5925,19 +5920,19 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->FunctionType_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* argtypes; expr_ty returns; if (PyObject_GetOptionalAttr(obj, state->argtypes, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5970,11 +5965,11 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->returns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"returns\" missing from FunctionType"); - return 1; + return -1; } else { int res; @@ -5994,7 +5989,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) PyErr_Format(PyExc_TypeError, "expected some sort of mod, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -6015,11 +6010,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from stmt"); - return 1; + return -1; } else { int res; @@ -6032,11 +6027,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from stmt"); - return 1; + return -1; } else { int res; @@ -6049,7 +6044,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6066,7 +6061,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6085,7 +6080,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->FunctionDef_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; @@ -6097,11 +6092,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_type_param_seq* type_params; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from FunctionDef"); - return 1; + return -1; } else { int res; @@ -6114,11 +6109,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from FunctionDef"); - return 1; + return -1; } else { int res; @@ -6131,12 +6126,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6169,12 +6164,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->decorator_list, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6207,7 +6202,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->returns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6224,7 +6219,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6241,12 +6236,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6288,7 +6283,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AsyncFunctionDef_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; @@ -6300,11 +6295,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_type_param_seq* type_params; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from AsyncFunctionDef"); - return 1; + return -1; } else { int res; @@ -6317,11 +6312,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from AsyncFunctionDef"); - return 1; + return -1; } else { int res; @@ -6334,12 +6329,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6372,12 +6367,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->decorator_list, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6410,7 +6405,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->returns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6427,7 +6422,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6444,12 +6439,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6491,7 +6486,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->ClassDef_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; @@ -6502,11 +6497,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_type_param_seq* type_params; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from ClassDef"); - return 1; + return -1; } else { int res; @@ -6519,12 +6514,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->bases, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6557,12 +6552,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->keywords, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6595,12 +6590,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6633,12 +6628,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->decorator_list, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6671,12 +6666,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6717,13 +6712,13 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Return_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6747,18 +6742,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Delete_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* targets; if (PyObject_GetOptionalAttr(obj, state->targets, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6798,7 +6793,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Assign_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* targets; @@ -6806,12 +6801,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->targets, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6844,11 +6839,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Assign"); - return 1; + return -1; } else { int res; @@ -6861,7 +6856,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6885,7 +6880,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->TypeAlias_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty name; @@ -6893,11 +6888,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* expr_ty value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from TypeAlias"); - return 1; + return -1; } else { int res; @@ -6910,12 +6905,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6948,11 +6943,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from TypeAlias"); - return 1; + return -1; } else { int res; @@ -6972,7 +6967,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AugAssign_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -6980,11 +6975,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* expr_ty value; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AugAssign"); - return 1; + return -1; } else { int res; @@ -6997,11 +6992,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from AugAssign"); - return 1; + return -1; } else { int res; @@ -7014,11 +7009,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from AugAssign"); - return 1; + return -1; } else { int res; @@ -7038,7 +7033,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AnnAssign_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -7047,11 +7042,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* int simple; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AnnAssign"); - return 1; + return -1; } else { int res; @@ -7064,11 +7059,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->annotation, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"annotation\" missing from AnnAssign"); - return 1; + return -1; } else { int res; @@ -7081,7 +7076,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7098,11 +7093,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->simple, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"simple\" missing from AnnAssign"); - return 1; + return -1; } else { int res; @@ -7122,7 +7117,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->For_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -7132,11 +7127,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from For"); - return 1; + return -1; } else { int res; @@ -7149,11 +7144,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->iter, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from For"); - return 1; + return -1; } else { int res; @@ -7166,12 +7161,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7204,12 +7199,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7242,7 +7237,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7266,7 +7261,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AsyncFor_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -7276,11 +7271,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AsyncFor"); - return 1; + return -1; } else { int res; @@ -7293,11 +7288,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->iter, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from AsyncFor"); - return 1; + return -1; } else { int res; @@ -7310,12 +7305,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7348,12 +7343,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7386,7 +7381,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7411,7 +7406,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->While_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; @@ -7419,11 +7414,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* orelse; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from While"); - return 1; + return -1; } else { int res; @@ -7436,12 +7431,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7474,12 +7469,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7519,7 +7514,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->If_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; @@ -7527,11 +7522,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* orelse; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from If"); - return 1; + return -1; } else { int res; @@ -7544,12 +7539,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7582,12 +7577,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7627,7 +7622,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->With_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_withitem_seq* items; @@ -7635,12 +7630,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->items, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7673,12 +7668,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7711,7 +7706,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7735,7 +7730,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AsyncWith_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_withitem_seq* items; @@ -7743,12 +7738,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->items, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7781,12 +7776,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7819,7 +7814,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7843,18 +7838,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Match_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty subject; asdl_match_case_seq* cases; if (PyObject_GetOptionalAttr(obj, state->subject, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"subject\" missing from Match"); - return 1; + return -1; } else { int res; @@ -7867,12 +7862,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->cases, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7912,14 +7907,14 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Raise_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty exc; expr_ty cause; if (PyObject_GetOptionalAttr(obj, state->exc, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7936,7 +7931,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->cause, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7960,7 +7955,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Try_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; @@ -7969,12 +7964,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* finalbody; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8007,12 +8002,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->handlers, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8045,12 +8040,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8083,12 +8078,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->finalbody, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8128,7 +8123,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->TryStar_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; @@ -8137,12 +8132,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* finalbody; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8175,12 +8170,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->handlers, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8213,12 +8208,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8251,12 +8246,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->finalbody, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8296,18 +8291,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Assert_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; expr_ty msg; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from Assert"); - return 1; + return -1; } else { int res; @@ -8320,7 +8315,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->msg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8344,18 +8339,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Import_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_alias_seq* names; if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8395,7 +8390,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->ImportFrom_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier module; @@ -8403,7 +8398,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* int level; if (PyObject_GetOptionalAttr(obj, state->module, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8420,12 +8415,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8458,7 +8453,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->level, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8482,18 +8477,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Global_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_identifier_seq* names; if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8533,18 +8528,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Nonlocal_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_identifier_seq* names; if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8584,17 +8579,17 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Expr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Expr"); - return 1; + return -1; } else { int res; @@ -8614,7 +8609,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Pass_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { @@ -8626,7 +8621,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Break_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { @@ -8638,7 +8633,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Continue_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { @@ -8651,7 +8646,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* PyErr_Format(PyExc_TypeError, "expected some sort of stmt, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -8672,11 +8667,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from expr"); - return 1; + return -1; } else { int res; @@ -8689,11 +8684,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from expr"); - return 1; + return -1; } else { int res; @@ -8706,7 +8701,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8723,7 +8718,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8742,18 +8737,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->BoolOp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { boolop_ty op; asdl_expr_seq* values; if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BoolOp"); - return 1; + return -1; } else { int res; @@ -8766,12 +8761,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->values, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8811,18 +8806,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->NamedExpr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; expr_ty value; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from NamedExpr"); - return 1; + return -1; } else { int res; @@ -8835,11 +8830,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NamedExpr"); - return 1; + return -1; } else { int res; @@ -8859,7 +8854,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->BinOp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty left; @@ -8867,11 +8862,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty right; if (PyObject_GetOptionalAttr(obj, state->left, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from BinOp"); - return 1; + return -1; } else { int res; @@ -8884,11 +8879,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BinOp"); - return 1; + return -1; } else { int res; @@ -8901,11 +8896,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->right, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"right\" missing from BinOp"); - return 1; + return -1; } else { int res; @@ -8925,18 +8920,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->UnaryOp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { unaryop_ty op; expr_ty operand; if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from UnaryOp"); - return 1; + return -1; } else { int res; @@ -8949,11 +8944,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->operand, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"operand\" missing from UnaryOp"); - return 1; + return -1; } else { int res; @@ -8973,18 +8968,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Lambda_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { arguments_ty args; expr_ty body; if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Lambda"); - return 1; + return -1; } else { int res; @@ -8997,11 +8992,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Lambda"); - return 1; + return -1; } else { int res; @@ -9021,7 +9016,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->IfExp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; @@ -9029,11 +9024,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty orelse; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from IfExp"); - return 1; + return -1; } else { int res; @@ -9046,11 +9041,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from IfExp"); - return 1; + return -1; } else { int res; @@ -9063,11 +9058,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from IfExp"); - return 1; + return -1; } else { int res; @@ -9087,19 +9082,19 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Dict_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* keys; asdl_expr_seq* values; if (PyObject_GetOptionalAttr(obj, state->keys, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9132,12 +9127,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->values, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9177,18 +9172,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Set_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* elts; if (PyObject_GetOptionalAttr(obj, state->elts, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9228,18 +9223,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->ListComp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty elt; asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->elt, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from ListComp"); - return 1; + return -1; } else { int res; @@ -9252,12 +9247,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9297,18 +9292,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->SetComp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty elt; asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->elt, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from SetComp"); - return 1; + return -1; } else { int res; @@ -9321,12 +9316,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9366,7 +9361,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->DictComp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty key; @@ -9374,11 +9369,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->key, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"key\" missing from DictComp"); - return 1; + return -1; } else { int res; @@ -9391,11 +9386,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from DictComp"); - return 1; + return -1; } else { int res; @@ -9408,12 +9403,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9453,18 +9448,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->GeneratorExp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty elt; asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->elt, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from GeneratorExp"); - return 1; + return -1; } else { int res; @@ -9477,12 +9472,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9522,17 +9517,17 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Await_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Await"); - return 1; + return -1; } else { int res; @@ -9552,13 +9547,13 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Yield_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -9582,17 +9577,17 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->YieldFrom_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from YieldFrom"); - return 1; + return -1; } else { int res; @@ -9612,7 +9607,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Compare_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty left; @@ -9620,11 +9615,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* asdl_expr_seq* comparators; if (PyObject_GetOptionalAttr(obj, state->left, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from Compare"); - return 1; + return -1; } else { int res; @@ -9637,12 +9632,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ops, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9675,12 +9670,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->comparators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9720,7 +9715,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Call_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty func; @@ -9728,11 +9723,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* asdl_keyword_seq* keywords; if (PyObject_GetOptionalAttr(obj, state->func, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"func\" missing from Call"); - return 1; + return -1; } else { int res; @@ -9745,12 +9740,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9783,12 +9778,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->keywords, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9828,7 +9823,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->FormattedValue_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; @@ -9836,11 +9831,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty format_spec; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue"); - return 1; + return -1; } else { int res; @@ -9853,11 +9848,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->conversion, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"conversion\" missing from FormattedValue"); - return 1; + return -1; } else { int res; @@ -9870,7 +9865,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->format_spec, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -9895,18 +9890,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->JoinedStr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* values; if (PyObject_GetOptionalAttr(obj, state->values, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9946,18 +9941,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Constant_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { constant value; string kind; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant"); - return 1; + return -1; } else { int res; @@ -9970,7 +9965,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kind, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -9994,7 +9989,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Attribute_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; @@ -10002,11 +9997,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Attribute"); - return 1; + return -1; } else { int res; @@ -10019,11 +10014,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->attr, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"attr\" missing from Attribute"); - return 1; + return -1; } else { int res; @@ -10036,11 +10031,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Attribute"); - return 1; + return -1; } else { int res; @@ -10060,7 +10055,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Subscript_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; @@ -10068,11 +10063,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Subscript"); - return 1; + return -1; } else { int res; @@ -10085,11 +10080,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->slice, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"slice\" missing from Subscript"); - return 1; + return -1; } else { int res; @@ -10102,11 +10097,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Subscript"); - return 1; + return -1; } else { int res; @@ -10126,18 +10121,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Starred_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Starred"); - return 1; + return -1; } else { int res; @@ -10150,11 +10145,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Starred"); - return 1; + return -1; } else { int res; @@ -10174,18 +10169,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Name_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier id; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->id, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"id\" missing from Name"); - return 1; + return -1; } else { int res; @@ -10198,11 +10193,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Name"); - return 1; + return -1; } else { int res; @@ -10222,19 +10217,19 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->List_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* elts; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->elts, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -10267,11 +10262,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from List"); - return 1; + return -1; } else { int res; @@ -10291,19 +10286,19 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Tuple_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* elts; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->elts, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -10336,11 +10331,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Tuple"); - return 1; + return -1; } else { int res; @@ -10360,7 +10355,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Slice_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty lower; @@ -10368,7 +10363,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty step; if (PyObject_GetOptionalAttr(obj, state->lower, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10385,7 +10380,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->upper, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10402,7 +10397,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->step, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10427,7 +10422,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -10438,7 +10433,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* isinstance = PyObject_IsInstance(obj, state->Load_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Load; @@ -10446,7 +10441,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* } isinstance = PyObject_IsInstance(obj, state->Store_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Store; @@ -10454,7 +10449,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* } isinstance = PyObject_IsInstance(obj, state->Del_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Del; @@ -10462,7 +10457,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* } PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %R", obj); - return 1; + return -1; } int @@ -10473,7 +10468,7 @@ obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* isinstance = PyObject_IsInstance(obj, state->And_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = And; @@ -10481,7 +10476,7 @@ obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Or_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Or; @@ -10489,7 +10484,7 @@ obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* } PyErr_Format(PyExc_TypeError, "expected some sort of boolop, but got %R", obj); - return 1; + return -1; } int @@ -10500,7 +10495,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, isinstance = PyObject_IsInstance(obj, state->Add_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Add; @@ -10508,7 +10503,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Sub_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Sub; @@ -10516,7 +10511,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Mult_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Mult; @@ -10524,7 +10519,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->MatMult_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = MatMult; @@ -10532,7 +10527,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Div_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Div; @@ -10540,7 +10535,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Mod_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Mod; @@ -10548,7 +10543,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Pow_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Pow; @@ -10556,7 +10551,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->LShift_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = LShift; @@ -10564,7 +10559,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->RShift_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = RShift; @@ -10572,7 +10567,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->BitOr_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = BitOr; @@ -10580,7 +10575,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->BitXor_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = BitXor; @@ -10588,7 +10583,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->BitAnd_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = BitAnd; @@ -10596,7 +10591,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->FloorDiv_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = FloorDiv; @@ -10604,7 +10599,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } PyErr_Format(PyExc_TypeError, "expected some sort of operator, but got %R", obj); - return 1; + return -1; } int @@ -10615,7 +10610,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, isinstance = PyObject_IsInstance(obj, state->Invert_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Invert; @@ -10623,7 +10618,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } isinstance = PyObject_IsInstance(obj, state->Not_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Not; @@ -10631,7 +10626,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } isinstance = PyObject_IsInstance(obj, state->UAdd_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = UAdd; @@ -10639,7 +10634,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } isinstance = PyObject_IsInstance(obj, state->USub_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = USub; @@ -10647,7 +10642,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } PyErr_Format(PyExc_TypeError, "expected some sort of unaryop, but got %R", obj); - return 1; + return -1; } int @@ -10658,7 +10653,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* isinstance = PyObject_IsInstance(obj, state->Eq_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Eq; @@ -10666,7 +10661,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->NotEq_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = NotEq; @@ -10674,7 +10669,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Lt_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Lt; @@ -10682,7 +10677,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->LtE_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = LtE; @@ -10690,7 +10685,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Gt_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Gt; @@ -10698,7 +10693,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->GtE_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = GtE; @@ -10706,7 +10701,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Is_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Is; @@ -10714,7 +10709,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->IsNot_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = IsNot; @@ -10722,7 +10717,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->In_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = In; @@ -10730,7 +10725,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->NotIn_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = NotIn; @@ -10738,7 +10733,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } PyErr_Format(PyExc_TypeError, "expected some sort of cmpop, but got %R", obj); - return 1; + return -1; } int @@ -10752,11 +10747,11 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* int is_async; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from comprehension"); - return 1; + return -1; } else { int res; @@ -10769,11 +10764,11 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->iter, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from comprehension"); - return 1; + return -1; } else { int res; @@ -10786,12 +10781,12 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ifs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -10824,11 +10819,11 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->is_async, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"is_async\" missing from comprehension"); - return 1; + return -1; } else { int res; @@ -10845,7 +10840,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -10866,11 +10861,11 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from excepthandler"); - return 1; + return -1; } else { int res; @@ -10883,11 +10878,11 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from excepthandler"); - return 1; + return -1; } else { int res; @@ -10900,7 +10895,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10917,7 +10912,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10936,7 +10931,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* tp = state->ExceptHandler_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty type; @@ -10944,7 +10939,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* asdl_stmt_seq* body; if (PyObject_GetOptionalAttr(obj, state->type, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10961,7 +10956,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10978,12 +10973,12 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11024,7 +11019,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* PyErr_Format(PyExc_TypeError, "expected some sort of excepthandler, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11041,12 +11036,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, asdl_expr_seq* defaults; if (PyObject_GetOptionalAttr(obj, state->posonlyargs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11079,12 +11074,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11117,7 +11112,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->vararg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11134,12 +11129,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwonlyargs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11172,12 +11167,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kw_defaults, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11210,7 +11205,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwarg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11227,12 +11222,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->defaults, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11270,7 +11265,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11286,11 +11281,11 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) int end_col_offset; if (PyObject_GetOptionalAttr(obj, state->arg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from arg"); - return 1; + return -1; } else { int res; @@ -11303,7 +11298,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->annotation, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11320,7 +11315,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11337,11 +11332,11 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg"); - return 1; + return -1; } else { int res; @@ -11354,11 +11349,11 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg"); - return 1; + return -1; } else { int res; @@ -11371,7 +11366,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11388,7 +11383,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11410,7 +11405,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11426,7 +11421,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, int end_col_offset; if (PyObject_GetOptionalAttr(obj, state->arg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11443,11 +11438,11 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from keyword"); - return 1; + return -1; } else { int res; @@ -11460,11 +11455,11 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from keyword"); - return 1; + return -1; } else { int res; @@ -11477,11 +11472,11 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from keyword"); - return 1; + return -1; } else { int res; @@ -11494,7 +11489,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11511,7 +11506,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11533,7 +11528,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11549,11 +11544,11 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* int end_col_offset; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from alias"); - return 1; + return -1; } else { int res; @@ -11566,7 +11561,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->asname, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11583,11 +11578,11 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from alias"); - return 1; + return -1; } else { int res; @@ -11600,11 +11595,11 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from alias"); - return 1; + return -1; } else { int res; @@ -11617,7 +11612,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11634,7 +11629,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11656,7 +11651,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11668,11 +11663,11 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, expr_ty optional_vars; if (PyObject_GetOptionalAttr(obj, state->context_expr, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem"); - return 1; + return -1; } else { int res; @@ -11685,7 +11680,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->optional_vars, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11706,7 +11701,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11719,11 +11714,11 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, asdl_stmt_seq* body; if (PyObject_GetOptionalAttr(obj, state->pattern, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"pattern\" missing from match_case"); - return 1; + return -1; } else { int res; @@ -11736,7 +11731,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->guard, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11753,12 +11748,12 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11795,7 +11790,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11816,11 +11811,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11833,11 +11828,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11850,11 +11845,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_lineno\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11867,11 +11862,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_col_offset\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11886,17 +11881,17 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchValue_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchValue"); - return 1; + return -1; } else { int res; @@ -11916,17 +11911,17 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchSingleton_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { constant value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchSingleton"); - return 1; + return -1; } else { int res; @@ -11946,18 +11941,18 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchSequence_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_pattern_seq* patterns; if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11997,7 +11992,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchMapping_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* keys; @@ -12005,12 +12000,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, identifier rest; if (PyObject_GetOptionalAttr(obj, state->keys, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12043,12 +12038,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12081,7 +12076,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->rest, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12105,7 +12100,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchClass_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty cls; @@ -12114,11 +12109,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, asdl_pattern_seq* kwd_patterns; if (PyObject_GetOptionalAttr(obj, state->cls, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"cls\" missing from MatchClass"); - return 1; + return -1; } else { int res; @@ -12131,12 +12126,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12169,12 +12164,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwd_attrs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12207,12 +12202,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwd_patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12253,13 +12248,13 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchStar_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12283,14 +12278,14 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchAs_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { pattern_ty pattern; identifier name; if (PyObject_GetOptionalAttr(obj, state->pattern, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12307,7 +12302,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12331,18 +12326,18 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchOr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_pattern_seq* patterns; if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12383,7 +12378,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, PyErr_Format(PyExc_TypeError, "expected some sort of pattern, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -12402,18 +12397,18 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* tp = state->TypeIgnore_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { int lineno; string tag; if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from TypeIgnore"); - return 1; + return -1; } else { int res; @@ -12426,11 +12421,11 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->tag, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore"); - return 1; + return -1; } else { int res; @@ -12450,7 +12445,7 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* PyErr_Format(PyExc_TypeError, "expected some sort of type_ignore, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -12471,11 +12466,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12488,11 +12483,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12505,11 +12500,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_lineno\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12522,11 +12517,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_col_offset\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12541,18 +12536,18 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, tp = state->TypeVar_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; expr_ty bound; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from TypeVar"); - return 1; + return -1; } else { int res; @@ -12565,7 +12560,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->bound, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12589,17 +12584,17 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, tp = state->ParamSpec_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from ParamSpec"); - return 1; + return -1; } else { int res; @@ -12619,17 +12614,17 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, tp = state->TypeVarTuple_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from TypeVarTuple"); - return 1; + return -1; } else { int res; @@ -12650,7 +12645,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, PyErr_Format(PyExc_TypeError, "expected some sort of type_param, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } @@ -13093,7 +13088,7 @@ PyObject* PyAST_mod2obj(mod_ty t) int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { - return 0; + return NULL; } state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; @@ -13107,7 +13102,7 @@ PyObject* PyAST_mod2obj(mod_ty t) PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", starting_recursion_depth, state->recursion_depth); - return 0; + return NULL; } return result; } diff --git a/Python/getargs.c b/Python/getargs.c index 5a12ca8def74fa..4f873e5182e6a6 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1877,8 +1877,9 @@ new_kwtuple(const char * const *keywords, int total, int pos) } static int -_parser_init(struct _PyArg_Parser *parser) +_parser_init(void *arg) { + struct _PyArg_Parser *parser = (struct _PyArg_Parser *)arg; const char * const *keywords = parser->keywords; assert(keywords != NULL); assert(parser->pos == 0 && @@ -1889,7 +1890,7 @@ _parser_init(struct _PyArg_Parser *parser) int len, pos; if (scan_keywords(keywords, &len, &pos) < 0) { - return 0; + return -1; } const char *fname, *custommsg = NULL; @@ -1898,7 +1899,7 @@ _parser_init(struct _PyArg_Parser *parser) assert(parser->fname == NULL); if (parse_format(parser->format, len, pos, &fname, &custommsg, &min, &max) < 0) { - return 0; + return -1; } } else { @@ -1911,7 +1912,7 @@ _parser_init(struct _PyArg_Parser *parser) if (kwtuple == NULL) { kwtuple = new_kwtuple(keywords, len, pos); if (kwtuple == NULL) { - return 0; + return -1; } owned = 1; } @@ -1925,40 +1926,27 @@ _parser_init(struct _PyArg_Parser *parser) parser->min = min; parser->max = max; parser->kwtuple = kwtuple; - parser->initialized = owned ? 1 : -1; + parser->is_kwtuple_owned = owned; assert(parser->next == NULL); - parser->next = _PyRuntime.getargs.static_parsers; - _PyRuntime.getargs.static_parsers = parser; - return 1; + parser->next = _Py_atomic_load_ptr(&_PyRuntime.getargs.static_parsers); + do { + // compare-exchange updates parser->next on failure + } while (_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers, + &parser->next, parser)); + return 0; } static int parser_init(struct _PyArg_Parser *parser) { - // volatile as it can be modified by other threads - // and should not be optimized or reordered by compiler - if (*((volatile int *)&parser->initialized)) { - assert(parser->kwtuple != NULL); - return 1; - } - PyThread_acquire_lock(_PyRuntime.getargs.mutex, WAIT_LOCK); - // Check again if another thread initialized the parser - // while we were waiting for the lock. - if (*((volatile int *)&parser->initialized)) { - assert(parser->kwtuple != NULL); - PyThread_release_lock(_PyRuntime.getargs.mutex); - return 1; - } - int ret = _parser_init(parser); - PyThread_release_lock(_PyRuntime.getargs.mutex); - return ret; + return _PyOnceFlag_CallOnce(&parser->once, &_parser_init, parser); } static void parser_clear(struct _PyArg_Parser *parser) { - if (parser->initialized == 1) { + if (parser->is_kwtuple_owned) { Py_CLEAR(parser->kwtuple); } } @@ -2025,7 +2013,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, return 0; } - if (!parser_init(parser)) { + if (parser_init(parser) < 0) { return 0; } @@ -2258,7 +2246,7 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, args = buf; } - if (!parser_init(parser)) { + if (parser_init(parser) < 0) { return NULL; } @@ -2435,7 +2423,7 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, args = buf; } - if (!parser_init(parser)) { + if (parser_init(parser) < 0) { return NULL; } diff --git a/Python/lock.c b/Python/lock.c index 3dad2aa93b5cc9..bc43b1ab192fea 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -295,3 +295,61 @@ PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns) return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED; } } + +static int +unlock_once(_PyOnceFlag *o, int res) +{ + // On success (res=0), we set the state to _Py_ONCE_INITIALIZED. + // On failure (res=-1), we reset the state to _Py_UNLOCKED. + uint8_t new_value; + switch (res) { + case -1: new_value = _Py_UNLOCKED; break; + case 0: new_value = _Py_ONCE_INITIALIZED; break; + default: { + Py_FatalError("invalid result from _PyOnceFlag_CallOnce"); + Py_UNREACHABLE(); + break; + } + } + + uint8_t old_value = _Py_atomic_exchange_uint8(&o->v, new_value); + if ((old_value & _Py_HAS_PARKED) != 0) { + // wake up anyone waiting on the once flag + _PyParkingLot_UnparkAll(&o->v); + } + return res; +} + +int +_PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) +{ + uint8_t v = _Py_atomic_load_uint8(&flag->v); + for (;;) { + if (v == _Py_UNLOCKED) { + if (!_Py_atomic_compare_exchange_uint8(&flag->v, &v, _Py_LOCKED)) { + continue; + } + int res = fn(arg); + return unlock_once(flag, res); + } + + if (v == _Py_ONCE_INITIALIZED) { + return 0; + } + + // The once flag is initializing (locked). + assert((v & _Py_LOCKED)); + if (!(v & _Py_HAS_PARKED)) { + // We are the first waiter. Set the _Py_HAS_PARKED flag. + uint8_t new_value = v | _Py_HAS_PARKED; + if (!_Py_atomic_compare_exchange_uint8(&flag->v, &v, new_value)) { + continue; + } + v = new_value; + } + + // Wait for initialization to finish. + _PyParkingLot_Park(&flag->v, &v, sizeof(v), -1, NULL, 1); + v = _Py_atomic_load_uint8(&flag->v); + } +} diff --git a/Python/pystate.c b/Python/pystate.c index 991d8d204a1c25..89e9bddc6de060 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -379,12 +379,11 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); _Py_COMP_DIAG_POP -#define NUMLOCKS 9 +#define NUMLOCKS 8 #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ &(runtime)->xi.registry.mutex, \ - &(runtime)->getargs.mutex, \ &(runtime)->unicode_state.ids.lock, \ &(runtime)->imports.extensions.mutex, \ &(runtime)->ceval.pending_mainthread.lock, \ From 762eb58220992d1ab809b9a281d47c0cd48a5aec Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 16 Nov 2023 22:21:23 +0300 Subject: [PATCH 42/49] Remove `imp_dummy_def` from `Tools/c-analyzer/cpython/ignored.tsv` (gh-112122) It was removed in 3.12, no need to keep the ignore. --- Tools/c-analyzer/cpython/ignored.tsv | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 68d32b78eed18e..d59e0ddcdfde4e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -565,7 +565,6 @@ Modules/_testmultiphase.c - def_nonascii_latin - Modules/_testmultiphase.c - def_nonmodule - Modules/_testmultiphase.c - def_nonmodule_with_exec_slots - Modules/_testmultiphase.c - def_nonmodule_with_methods - -Modules/_testmultiphase.c - imp_dummy_def - Modules/_testmultiphase.c - main_def - Modules/_testmultiphase.c - main_slots - Modules/_testmultiphase.c - meth_state_access_slots - From 974847be443e9798615e197ec6642e546a71a6b0 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 16 Nov 2023 22:47:38 +0300 Subject: [PATCH 43/49] gh-111800: Fix `test_recursive_repr` from `test_io` under WASI to not recurse so deeply (GH-112150) --- Lib/test/test_io.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index ab3389215e93d0..09cced9baef99b 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1234,11 +1234,9 @@ def test_recursive_repr(self): # Issue #25455 raw = self.MockRawIO() b = self.tp(raw) - with support.swap_attr(raw, 'name', b): - try: + with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(b) # Should not crash - except RuntimeError: - pass def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush @@ -2801,11 +2799,9 @@ def test_recursive_repr(self): # Issue #25455 raw = self.BytesIO() t = self.TextIOWrapper(raw, encoding="utf-8") - with support.swap_attr(raw, 'name', t): - try: + with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(t) # Should not crash - except RuntimeError: - pass def test_line_buffering(self): r = self.BytesIO() From ceefa0b0795b5cc7adef89bd036ce843b5c78d3e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 17 Nov 2023 00:00:16 +0100 Subject: [PATCH 44/49] gh-111482: Fix time_clockid_converter() on AIX (#112170) clockid_t is defined as long long on AIX. --- Modules/timemodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index e82f6eb98ebaf3..bc3901e0d7a621 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -236,8 +236,8 @@ _PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) static int time_clockid_converter(PyObject *obj, clockid_t *p) { -#if defined(_AIX) && (SIZEOF_LONG == 8) - long clk_id = PyLong_AsLong(obj); +#ifdef _AIX + long long clk_id = PyLong_AsLongLong(obj); #else int clk_id = PyLong_AsInt(obj); #endif From f92ea63f6f2c37917fc095a1bc036a8b0c45a084 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 17 Nov 2023 02:12:27 +0300 Subject: [PATCH 45/49] gh-111799: Fix `testRecursiveRepr` from `test_fileio` on WASI (GH-112181) --- Lib/test/test_fileio.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index ebfcffd1829174..f490485cdaf3eb 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,8 @@ from functools import wraps from test.support import ( - cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi + cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi, + infinite_recursion, ) from test.support.os_helper import ( TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, @@ -183,6 +184,7 @@ def testReprNoCloseFD(self): finally: os.close(fd) + @infinite_recursion(25) def testRecursiveRepr(self): # Issue #25455 with swap_attr(self.f, 'name', self.f): From 8cd70eefc7f3363cfa0d43f34522c3072fa9e160 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 16 Nov 2023 20:05:16 -0500 Subject: [PATCH 46/49] gh-112165: Fix typo in `__main__.py` (#112183) Change '[2]' to '[1]' to get second argument. --- Doc/library/__main__.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index 24a32b30bba673..c999253f781b10 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -227,7 +227,7 @@ students:: import sys from .student import search_students - student_name = sys.argv[2] if len(sys.argv) >= 2 else '' + student_name = sys.argv[1] if len(sys.argv) >= 2 else '' print(f'Found student: {search_students(student_name)}') Note that ``from .student import search_students`` is an example of a relative From 0ee2d77331f2362fcaab20cc678530b18e467e3c Mon Sep 17 00:00:00 2001 From: Wanderxjtu Date: Fri, 17 Nov 2023 13:03:02 +0800 Subject: [PATCH 47/49] gh-112070: make `functools.lru_cache` threadsafe in --disable-gil build (gh-112111) * gh-112070: make `functools.lrucacle` threadsafe in --disable-gil build * gh-112070: update generate `functoolsmodule` files * gh-112070: add NEWS file * Delete Misc/NEWS.d/next/Library/2023-11-15-20-19-45.gh-issue-112070.q6OhcU.rst * gh-112070: reformat functoolsmodule.c --------- Co-authored-by: Sam Gross --- Modules/_functoolsmodule.c | 13 ++++++++++--- Modules/clinic/_functoolsmodule.c.h | 18 +++++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index ca440e4c70c0d9..2592c4d7f75631 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION #include "pycore_dict.h" // _PyDict_Pop_KnownHash() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() @@ -1274,7 +1275,11 @@ lru_cache_dealloc(lru_cache_object *obj) static PyObject * lru_cache_call(lru_cache_object *self, PyObject *args, PyObject *kwds) { - return self->wrapper(self, args, kwds); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(self); + result = self->wrapper(self, args, kwds); + Py_END_CRITICAL_SECTION(); + return result; } static PyObject * @@ -1287,6 +1292,7 @@ lru_cache_descr_get(PyObject *self, PyObject *obj, PyObject *type) } /*[clinic input] +@critical_section _functools._lru_cache_wrapper.cache_info Report cache statistics @@ -1294,7 +1300,7 @@ Report cache statistics static PyObject * _functools__lru_cache_wrapper_cache_info_impl(PyObject *self) -/*[clinic end generated code: output=cc796a0b06dbd717 input=f05e5b6ebfe38645]*/ +/*[clinic end generated code: output=cc796a0b06dbd717 input=00e1acb31aa21ecc]*/ { lru_cache_object *_self = (lru_cache_object *) self; if (_self->maxsize == -1) { @@ -1308,6 +1314,7 @@ _functools__lru_cache_wrapper_cache_info_impl(PyObject *self) } /*[clinic input] +@critical_section _functools._lru_cache_wrapper.cache_clear Clear the cache and cache statistics @@ -1315,7 +1322,7 @@ Clear the cache and cache statistics static PyObject * _functools__lru_cache_wrapper_cache_clear_impl(PyObject *self) -/*[clinic end generated code: output=58423b35efc3e381 input=6ca59dba09b12584]*/ +/*[clinic end generated code: output=58423b35efc3e381 input=dfa33acbecf8b4b2]*/ { lru_cache_object *_self = (lru_cache_object *) self; lru_list_elem *list = lru_cache_unlink_list(_self); diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 94f7a23e8f217a..11fe0439c08a5e 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -81,7 +81,13 @@ _functools__lru_cache_wrapper_cache_info_impl(PyObject *self); static PyObject * _functools__lru_cache_wrapper_cache_info(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _functools__lru_cache_wrapper_cache_info_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _functools__lru_cache_wrapper_cache_info_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_functools__lru_cache_wrapper_cache_clear__doc__, @@ -99,6 +105,12 @@ _functools__lru_cache_wrapper_cache_clear_impl(PyObject *self); static PyObject * _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _functools__lru_cache_wrapper_cache_clear_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _functools__lru_cache_wrapper_cache_clear_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } -/*[clinic end generated code: output=231403340a20e31b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e3207fa0d28cdb1 input=a9049054013a1b77]*/ From fb4cddb0cc6c9b94929f846da8e95aeec3849212 Mon Sep 17 00:00:00 2001 From: Ori Avtalion Date: Fri, 17 Nov 2023 14:30:47 +0200 Subject: [PATCH 48/49] gh-94309: "What's new in Python 3.12": improve deprecation notice for typing.Hashable and typing.Sized (#112196) --- Doc/whatsnew/3.12.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 72277a023b3b2e..8b7a043d068e5c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1289,8 +1289,9 @@ Deprecated * :mod:`typing`: - * :class:`typing.Hashable` and :class:`typing.Sized` aliases for :class:`collections.abc.Hashable` - and :class:`collections.abc.Sized`. (:gh:`94309`.) + * :class:`typing.Hashable` and :class:`typing.Sized`, aliases for + :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are + deprecated. (:gh:`94309`.) * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. From 7c50800241997f93647a57354c052de2cec25d04 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 17 Nov 2023 15:09:19 +0100 Subject: [PATCH 49/49] gh-110481, doc: Add "immortal" term to the glossary (#112180) --- Doc/c-api/bool.rst | 12 ++++++------ Doc/c-api/init.rst | 2 +- Doc/c-api/init_config.rst | 2 +- Doc/c-api/none.rst | 6 +++--- Doc/c-api/refcounting.rst | 10 ++++++---- Doc/c-api/slice.rst | 3 +-- Doc/glossary.rst | 12 +++++++++++- Doc/library/sys.rst | 6 +++--- 8 files changed, 32 insertions(+), 21 deletions(-) diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index b14fa6a0a982e2..b4dc4849d044e1 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -26,19 +26,19 @@ are available, however. .. c:var:: PyObject* Py_False The Python ``False`` object. This object has no methods and is - `immortal `_. + :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_False` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_False` is :term:`immortal`. .. c:var:: PyObject* Py_True The Python ``True`` object. This object has no methods and is - `immortal `_. + :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_True` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_True` is :term:`immortal`. .. c:macro:: Py_RETURN_FALSE diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index d164d1a752e295..e89641f74c7491 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1485,7 +1485,7 @@ otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared because of the refcount. One simple but less-efficient approach around this is to use a global lock around all use of some state (or object). Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them "immortal". +can be made safe in spite of their refcounts by making them :term:`immortal`. In fact, this has been done for the builtin singletons, small integers, and a number of other builtin objects. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0c2ed8af33d430..47a8fbb2cd9c97 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1170,7 +1170,7 @@ PyConfig .. c:member:: int show_ref_count - Show total reference count at exit (excluding immortal objects)? + Show total reference count at exit (excluding :term:`immortal` objects)? Set to ``1`` by :option:`-X showrefcount <-X>` command line option. diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index dd8bfb56104251..f1941fc4bc4e85 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -16,10 +16,10 @@ same reason. .. c:var:: PyObject* Py_None The Python ``None`` object, denoting lack of value. This object has no methods - and is `immortal `_. + and is :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_None` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_None` is :term:`immortal`. .. c:macro:: Py_RETURN_NONE diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 118af7a1a8cf90..68119a27b18ec2 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -17,7 +17,7 @@ of Python objects. Note that the returned value may not actually reflect how many references to the object are actually held. For example, some - objects are "immortal" and have a very high refcount that does not + objects are :term:`immortal` and have a very high refcount that does not reflect the actual number of references. Consequently, do not rely on the returned value to be accurate, other than a value of 0 or 1. @@ -34,9 +34,7 @@ of Python objects. Set the object *o* reference counter to *refcnt*. - Note that this function has no effect on - `immortal `_ - objects. + This function has no effect on :term:`immortal` objects. .. versionadded:: 3.9 @@ -49,6 +47,8 @@ of Python objects. Indicate taking a new :term:`strong reference` to object *o*, indicating it is in use and should not be destroyed. + This function has no effect on :term:`immortal` objects. + This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. @@ -113,6 +113,8 @@ of Python objects. Release a :term:`strong reference` to object *o*, indicating the reference is no longer used. + This function has no effect on :term:`immortal` objects. + Once the last :term:`strong reference` is released (i.e. the object's reference count reaches 0), the object's type's deallocation diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 9e880c6b7f25ad..27a1757c745d8b 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -119,8 +119,7 @@ Ellipsis Object .. c:var:: PyObject *Py_Ellipsis The Python ``Ellipsis`` object. This object has no methods. Like - :c:data:`Py_None`, it is an `immortal `_. - singleton object. + :c:data:`Py_None`, it is an :term:`immortal` singleton object. .. versionchanged:: 3.12 :c:data:`Py_Ellipsis` is immortal. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index dad745348f9b4b..6b517b95f97013 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -579,6 +579,16 @@ Glossary :ref:`idle` is a basic editor and interpreter environment which ships with the standard distribution of Python. + immortal + If an object is immortal, its reference count is never modified, and + therefore it is never deallocated. + + Built-in strings and singletons are immortal objects. For example, + :const:`True` and :const:`None` singletons are immmortal. + + See `PEP 683 – Immortal Objects, Using a Fixed Refcount + `_ for more information. + immutable An object with a fixed value. Immutable objects include numbers, strings and tuples. Such an object cannot be altered. A new object has to @@ -1056,7 +1066,7 @@ Glossary reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are - "immortal" and have reference counts that are never modified, and + :term:`immortal` and have reference counts that are never modified, and therefore the objects are never deallocated. Reference counting is generally not visible to Python code, but it is a key element of the :term:`CPython` implementation. Programmers can call the diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 4d24606a1653e5..bf9aaca2a696de 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -831,7 +831,7 @@ always available. Note that the returned value may not actually reflect how many references to the object are actually held. For example, some - objects are "immortal" and have a very high refcount that does not + objects are :term:`immortal` and have a very high refcount that does not reflect the actual number of references. Consequently, do not rely on the returned value to be accurate, other than a value of 0 or 1. @@ -1182,8 +1182,8 @@ always available. names used in Python programs are automatically interned, and the dictionaries used to hold module, class or instance attributes have interned keys. - Interned strings are not immortal; you must keep a reference to the return - value of :func:`intern` around to benefit from it. + Interned strings are not :term:`immortal`; you must keep a reference to the + return value of :func:`intern` around to benefit from it. .. function:: is_finalizing()