diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 1a702f7bf28d61..b1dd600756b6c7 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -460,7 +460,7 @@ def test_predictions(self): INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); _PyStackRef res; - DEOPT_IF(xxx, OP1); + DEOPT_IF(xxx, OP1, INLINE_CACHE_ENTRIES_OP1); res = Py_None; stack_pointer[-1] = res; DISPATCH(); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ce139c0d5c2b00..72d820d3dfab57 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,7 +45,7 @@ #include "ceval_macros.h" /* Flow control macros */ -#define GO_TO_INSTRUCTION(instname) ((void)0) +#define GO_TO_INSTRUCTION(instname, size) ((void)0) #define inst(name, ...) case name: #define op(name, ...) /* NAME is ignored */ @@ -2014,7 +2014,7 @@ dummy_func( // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); } family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { @@ -4313,7 +4313,7 @@ dummy_func( frame, this_instr, function, arg); ERROR_IF(err, error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) { @@ -4539,7 +4539,7 @@ dummy_func( _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 0); } op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index efadf30157d3b2..259cdd677bb078 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -326,18 +326,18 @@ GETITEM(PyObject *v, Py_ssize_t i) { PyStackRef_XCLOSE(tmp); } while (0) #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE -#define GO_TO_INSTRUCTION(op) do { \ +#define GO_TO_INSTRUCTION(op, size) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg, lltrace); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - size, opcode, oparg, lltrace); \ } while (0) #else -#define GO_TO_INSTRUCTION(op) do { \ +#define GO_TO_INSTRUCTION(op, size) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - size, opcode, oparg); \ } while (0) #endif #else -#define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) +#define GO_TO_INSTRUCTION(op, size) goto PREDICT_ID(op) #endif #ifdef Py_STATS @@ -354,12 +354,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define UPDATE_MISS_STATS(INSTNAME) ((void)0) #endif -#define DEOPT_IF(COND, INSTNAME) \ +#define DEOPT_IF(COND, INSTNAME, SIZE) \ if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - GO_TO_INSTRUCTION(INSTNAME); \ + GO_TO_INSTRUCTION(INSTNAME, SIZE); \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c7a5223024aad9..4467f06202f3fa 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -73,8 +73,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -111,8 +111,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -148,8 +148,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -193,7 +193,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP); + DEOPT_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -230,8 +230,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -248,7 +248,7 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -294,8 +294,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -332,8 +332,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -369,8 +369,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -407,8 +407,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -540,7 +540,7 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -574,22 +574,22 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -634,19 +634,19 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -674,14 +674,14 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -706,12 +706,12 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1101,7 +1101,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1112,17 +1112,17 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1199,14 +1199,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1225,9 +1225,9 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1236,15 +1236,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1302,7 +1302,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1310,11 +1310,11 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -1403,7 +1403,7 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -1411,7 +1411,7 @@ arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1488,8 +1488,8 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1572,8 +1572,8 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1656,11 +1656,11 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1933,9 +1933,9 @@ arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2132,7 +2132,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2140,11 +2140,11 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -2238,8 +2238,8 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -2329,16 +2329,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -2422,9 +2422,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2465,10 +2465,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2511,11 +2511,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2596,12 +2596,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2681,16 +2681,16 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2753,16 +2753,16 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2820,8 +2820,8 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -2905,16 +2905,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -2923,15 +2923,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -2986,16 +2986,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -3071,8 +3071,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3122,8 +3122,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3171,8 +3171,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3376,8 +3376,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3414,16 +3414,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3456,8 +3456,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3540,7 +3540,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3568,7 +3568,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4075,14 +4075,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4123,7 +4123,7 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -4175,7 +4175,7 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -4217,7 +4217,7 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -4637,7 +4637,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 0); } TARGET(INSTRUMENTED_CALL_KW) { @@ -4661,7 +4661,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } TARGET(INSTRUMENTED_END_FOR) { @@ -4851,7 +4851,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); } TARGET(INSTRUMENTED_NOT_TAKEN) { @@ -5394,9 +5394,9 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5429,16 +5429,16 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5469,17 +5469,17 @@ PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -5507,14 +5507,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -5522,10 +5522,10 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -5557,7 +5557,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5565,7 +5565,7 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -5600,7 +5600,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5636,14 +5636,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -5651,7 +5651,7 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5686,11 +5686,11 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -5701,11 +5701,11 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -5737,7 +5737,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5768,14 +5768,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -5783,7 +5783,7 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5808,7 +5808,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5816,7 +5816,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5826,10 +5826,10 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -5878,7 +5878,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5886,10 +5886,10 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR); + DEOPT_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -5921,14 +5921,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR); + DEOPT_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -5937,26 +5937,26 @@ uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6319,18 +6319,18 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -6339,10 +6339,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6370,9 +6370,9 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -6382,10 +6382,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6604,8 +6604,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6640,8 +6640,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7167,16 +7167,16 @@ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif DISPATCH(); } @@ -7346,15 +7346,15 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -7544,11 +7544,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -7559,7 +7559,7 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -7602,14 +7602,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -7639,7 +7639,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7648,12 +7648,12 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -7661,17 +7661,17 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -7907,7 +7907,7 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -7935,16 +7935,16 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -8034,7 +8034,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -8055,7 +8055,7 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -8071,7 +8071,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -8096,7 +8096,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -8115,7 +8115,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -8133,7 +8133,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -8269,11 +8269,11 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -8298,8 +8298,8 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -8323,8 +8323,8 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index e8c0fb1fe55be0..6149f9da769989 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -252,8 +252,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -308,8 +308,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -363,8 +363,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -426,7 +426,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP); + DEOPT_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -481,8 +481,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -499,7 +499,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -563,8 +563,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -619,8 +619,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -674,8 +674,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -730,8 +730,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -917,7 +917,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -969,22 +969,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -1047,19 +1047,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -1105,14 +1105,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -1155,12 +1155,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1712,7 +1712,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1723,17 +1723,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1828,14 +1828,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1854,9 +1854,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1865,15 +1865,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1949,7 +1949,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1957,11 +1957,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -2068,7 +2068,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -2076,7 +2076,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -2171,8 +2171,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -2273,8 +2273,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2375,11 +2375,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -2724,9 +2724,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2959,7 +2959,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2967,11 +2967,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -3083,8 +3083,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -3192,16 +3192,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -3303,9 +3303,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -3364,10 +3364,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -3428,11 +3428,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3531,12 +3531,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3634,16 +3634,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3724,16 +3724,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3809,8 +3809,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -3912,16 +3912,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3930,15 +3930,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -4011,16 +4011,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -4114,8 +4114,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -4183,8 +4183,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -4250,8 +4250,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -4545,8 +4545,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -4601,16 +4601,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -4661,8 +4661,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -4781,7 +4781,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -4827,7 +4827,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5712,14 +5712,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -5778,7 +5778,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -5848,7 +5848,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -5908,7 +5908,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -6508,7 +6508,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 0); } pop_4_error: TAIL_CALL(pop_4_error); @@ -6550,7 +6550,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } pop_4_error: TAIL_CALL(pop_4_error); @@ -6884,7 +6884,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); } pop_4_error: TAIL_CALL(pop_4_error); @@ -7751,9 +7751,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -7804,16 +7804,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -7862,17 +7862,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -7918,14 +7918,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -7933,10 +7933,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -7986,7 +7986,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -7994,7 +7994,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -8047,7 +8047,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -8101,14 +8101,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -8116,7 +8116,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -8169,11 +8169,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -8184,11 +8184,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -8238,7 +8238,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -8287,14 +8287,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -8302,7 +8302,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -8345,7 +8345,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -8353,7 +8353,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -8363,10 +8363,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -8433,7 +8433,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -8441,10 +8441,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR); + DEOPT_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -8494,14 +8494,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR); + DEOPT_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -8510,26 +8510,26 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -9144,18 +9144,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -9164,10 +9164,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -9213,9 +9213,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -9225,10 +9225,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -9555,8 +9555,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9609,8 +9609,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -10550,16 +10550,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif } DISPATCH(); @@ -10801,15 +10801,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -11107,11 +11107,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -11122,7 +11122,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -11183,14 +11183,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -11238,7 +11238,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -11247,12 +11247,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -11260,17 +11260,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -11668,7 +11668,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -11714,16 +11714,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -11867,7 +11867,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -11906,7 +11906,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); } DISPATCH(); @@ -11940,7 +11940,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -11983,7 +11983,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -12020,7 +12020,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -12056,7 +12056,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -12300,11 +12300,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -12347,8 +12347,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -12390,8 +12390,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index f54afbb880d2fa..1b9f09800d09c7 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -160,6 +160,8 @@ def deopt_if( assert inst is not None assert inst.family is not None self.out.emit(inst.family.name) + self.out.emit(", ") + self.out.emit(inst.family.size) self.out.emit(");\n") return not always_true(first_tkn)