diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 429847b19d2703..7c06ec7a6f37d0 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -390,6 +390,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -1338,28 +1339,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef start; - _PyStackRef stop; - _PyStackRef step = PyStackRef_NULL; + _PyStackRef *args; _PyStackRef slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (slice_o == NULL) { - stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -1679,6 +1676,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1795,6 +1793,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1916,6 +1915,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2427,18 +2427,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM (void)this_instr; _PyStackRef func; _PyStackRef callargs; - _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef kwargs_in; _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef kwargs_out; _PyStackRef func_st; _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef kwargs_st; _PyStackRef result; // _MAKE_CALLARGS_A_TUPLE { - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -2480,8 +2480,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -2524,7 +2524,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2 - (oparg & 1); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -2545,8 +2545,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2556,11 +2556,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(callargs_st); PyStackRef_CLOSE(func_st); - if (result_o == NULL) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + if (result_o == NULL) goto pop_4_error; result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -2568,19 +2564,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; - stack_pointer += 2 + (oparg & 1); + stack_pointer += 3; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -2925,6 +2921,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -3161,6 +3158,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -3882,6 +3880,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3981,6 +3980,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -4456,7 +4456,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ } OPCODE_DEFERRED_INC(COMPARE_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _COMPARE_OP { @@ -7395,11 +7395,74 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + // Need to re-dispatch so the warmup counter isn't off by one: + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); @@ -7411,16 +7474,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ if (err != 0) goto error; } } - // _JUMP_BACKWARD + // _JUMP_BACKWARD_NO_INTERRUPT { - uint16_t the_counter = read_u16(&this_instr[1].cache); - (void)the_counter; + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); + } + // _JIT + { #ifdef _Py_TIER2 - #if ENABLE_SPECIALIZATION _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ while (oparg > 255) { @@ -7447,8 +7515,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } - #endif /* ENABLE_SPECIALIZATION */ - #endif /* _Py_TIER2 */ + #endif } } DISPATCH(); @@ -7480,6 +7547,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ + assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); } DISPATCH(); @@ -7501,6 +7569,54 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ TAIL_CALL(resume_with_error); } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; @@ -7626,7 +7742,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ (void)this_instr; _PyStackRef owner; _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; + _PyStackRef *self_or_null; // _SPECIALIZE_LOAD_ATTR { owner = stack_pointer[-1]; @@ -7648,6 +7764,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ /* Skip 8 cache entries */ // _LOAD_ATTR { + self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); PyObject *attr_o; if (oparg & 1) { @@ -7662,7 +7779,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ meth | self | arg1 | ... | argN */ assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + self_or_null[0] = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -7673,7 +7790,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ */ PyStackRef_CLOSE(owner); if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; + self_or_null[0] = PyStackRef_NULL; } } else { @@ -7683,14 +7800,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) goto pop_1_error; - /* We need to define self_or_null on all paths */ - self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); + stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -7715,6 +7829,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7738,9 +7853,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -7768,6 +7886,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7797,9 +7916,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -7827,6 +7949,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7881,6 +8004,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7918,10 +8042,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -7949,12 +8076,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8010,12 +8138,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8064,12 +8193,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8132,6 +8262,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8171,10 +8302,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -8202,6 +8336,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8251,6 +8386,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8313,6 +8449,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8396,6 +8533,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8425,10 +8563,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -8456,6 +8597,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8510,10 +8652,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -8633,11 +8778,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ * marshalling can intern strings and make them immortal. */ PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT + #ifdef Py_GIL_DISABLED + uint8_t expected = LOAD_CONST; + if (!_Py_atomic_compare_exchange_uint8( + &this_instr->op.code, &expected, + _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { + // We might lose a race with instrumentation, which we don't care about. + assert(expected >= MIN_INSTRUMENTED_OPCODE); + } + #else if (this_instr->op.code == LOAD_CONST) { this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; } #endif + #endif stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -9081,6 +9236,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } if (oparg & 1) stack_pointer[1] = null; @@ -9109,6 +9267,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -9149,6 +9308,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } stack_pointer[0] = res; @@ -9178,6 +9340,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -9210,6 +9373,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } stack_pointer[0] = res; @@ -9487,6 +9653,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } stack_pointer[0] = attr; @@ -11069,6 +11238,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -11145,6 +11315,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -11200,6 +11371,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -11828,6 +12000,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -12657,7 +12830,9 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, [IS_OP] = _TAIL_CALL_IS_OP, [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, @@ -12791,8 +12966,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [146] = _TAIL_CALL_UNKNOWN_OPCODE, [147] = _TAIL_CALL_UNKNOWN_OPCODE, [148] = _TAIL_CALL_UNKNOWN_OPCODE, - [230] = _TAIL_CALL_UNKNOWN_OPCODE, - [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, [234] = _TAIL_CALL_UNKNOWN_OPCODE,