Integrate Tier 2 into _PyEval_EvalFrameDefault

Python/bytecodes.c

@@ -2318,13 +2318,16 @@ dummy_func(
             JUMPBY(1-original_oparg);
             frame->instr_ptr = next_instr;
             Py_INCREF(executor);
+            if (executor->execute == _PyUopExecute) {
+                self = (_PyUOpExecutorObject *)executor;
+                goto enter_tier_two;
+            }
             frame = executor->execute(executor, frame, stack_pointer);
             if (frame == NULL) {
                 frame = tstate->current_frame;
                 goto resume_with_error;
             }
-            next_instr = frame->instr_ptr;
-            goto resume_frame;
+            goto enter_tier_one;
         }
 
         inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) {
@@ -3947,18 +3950,18 @@ dummy_func(
 
         op(_POP_JUMP_IF_FALSE, (flag -- )) {
             if (Py_IsFalse(flag)) {
-                pc = oparg;
+                next_uop = self->trace + oparg;
             }
         }
 
         op(_POP_JUMP_IF_TRUE, (flag -- )) {
             if (Py_IsTrue(flag)) {
-                pc = oparg;
+                next_uop = self->trace + oparg;
             }
         }
 
         op(_JUMP_TO_TOP, (--)) {
-            pc = 0;
+            next_uop = self->trace;
             CHECK_EVAL_BREAKER();
         }
 
@@ -3981,7 +3984,7 @@ dummy_func(
             _PyFrame_SetStackPointer(frame, stack_pointer);
             Py_DECREF(self);
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
-            return frame;
+            goto enter_tier_one;
         }
 
         op(_INSERT, (unused[oparg], top -- top, unused[oparg])) {

Python/ceval.c

@@ -679,9 +679,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #ifdef Py_STATS
     int lastopcode = 0;
 #endif
-    // opcode is an 8-bit value to improve the code generated by MSVC
-    // for the big switch below (in combination with the EXTRA_CASES macro).
-    uint8_t opcode;        /* Current opcode */
+    int opcode;        /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
 #ifdef LLTRACE
     int lltrace = 0;
@@ -729,6 +727,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
         goto resume_with_error;
     }
 
+    /* State shared between Tier 1 and Tier 2 interpreter */
+    _PyUOpExecutorObject *self = NULL;
+
     /* Local "register" variables.
      * These are cached values from the frame and code object.  */
 
@@ -765,7 +766,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     /* Start instructions */
 #if !USE_COMPUTED_GOTOS
     dispatch_opcode:
-        switch (opcode)
+        // Cast to an 8-bit value to improve the code generated by MSVC
+        // (in combination with the EXTRA_CASES macro).
+        switch ((uint8_t)opcode)
 #endif
         {
 
@@ -942,6 +945,123 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     stack_pointer = _PyFrame_GetStackPointer(frame);
     goto error;
 
+
+
+// The Tier 2 interpreter is also here!
+enter_tier_two:
+
+#undef LOAD_IP
+#define LOAD_IP(UNUSED) \
+do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0)
+
+#undef GOTO_ERROR
+#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two
+
+#undef DEOPT_IF
+#define DEOPT_IF(COND, INSTNAME) \
+    if ((COND)) {                \
+        goto deoptimize_tier_two;\
+    }
+
+#ifdef Py_STATS
+// Disable these macros that apply to Tier 1 stats when we are in Tier 2
+#undef STAT_INC
+#define STAT_INC(opname, name) ((void)0)
+#undef STAT_DEC
+#define STAT_DEC(opname, name) ((void)0)
+#undef CALL_STAT_INC
+#define CALL_STAT_INC(name) ((void)0)
+#endif
+
+#undef ENABLE_SPECIALIZATION
+#define ENABLE_SPECIALIZATION 0
+
+#ifdef Py_DEBUG
+    #define DPRINTF(level, ...) \
+        if (lltrace >= (level)) { printf(__VA_ARGS__); }
+#else
+    #define DPRINTF(level, ...)
+#endif
+
+    CHECK_EVAL_BREAKER();
+
+    OPT_STAT_INC(traces_executed);
+    _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
+    _PyUOpInstruction *next_uop = self->trace;
+    uint64_t operand;
+#ifdef Py_STATS
+    uint64_t trace_uop_execution_counter = 0;
+#endif
+
+    for (;;) {
+        opcode = next_uop->opcode;
+        oparg = next_uop->oparg;
+        operand = next_uop->operand;
+        DPRINTF(3,
+                "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
+                (int)(next_uop - self->trace),
+                opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
+                oparg,
+                operand,
+                (int)(stack_pointer - _PyFrame_Stackbase(frame)));
+        next_uop++;
+        OPT_STAT_INC(uops_executed);
+        UOP_EXE_INC(opcode);
+#ifdef Py_STATS
+        trace_uop_execution_counter++;
+#endif
+
+        switch (opcode) {
+
+#undef TIER_ONE
+#define TIER_TWO 2
+#include "executor_cases.c.h"
+
+            default:
+            {
+                fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 "\n",
+                        opcode, oparg, operand);
+                Py_FatalError("Unknown uop");
+            }
+
+        }
+    }
+
+unbound_local_error_tier_two:
+    _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+        UNBOUNDLOCAL_ERROR_MSG,
+        PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+    );
+    goto error_tier_two;
+
+pop_4_error_tier_two:
+    STACK_SHRINK(1);
+pop_3_error_tier_two:
+    STACK_SHRINK(1);
+pop_2_error_tier_two:
+    STACK_SHRINK(1);
+pop_1_error_tier_two:
+    STACK_SHRINK(1);
+error_tier_two:
+    DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+    OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+    frame->return_offset = 0;  // Don't leave this random
+    _PyFrame_SetStackPointer(frame, stack_pointer);
+    Py_DECREF(self);
+    goto resume_with_error;
+
+deoptimize_tier_two:
+    // On DEOPT_IF we just repeat the last instruction.
+    // This presumes nothing was popped from the stack (nor pushed).
+    DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+    OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+    frame->return_offset = 0;  // Dispatch to frame->instr_ptr
+    _PyFrame_SetStackPointer(frame, stack_pointer);
+    Py_DECREF(self);
+enter_tier_one:
+    next_instr = frame->instr_ptr;
+    goto resume_frame;
+
 }
 #if defined(__GNUC__)
 #  pragma GCC diagnostic pop

Python/executor.c

@@ -47,6 +47,7 @@
 _PyInterpreterFrame *
 _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)
 {
+    Py_FatalError("Tier 2 is now inlined into Tier 1");
 #ifdef Py_DEBUG
     char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
     int lltrace = 0;
@@ -74,7 +75,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
 
     OPT_STAT_INC(traces_executed);
     _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
-    int pc = 0;
+    _PyUOpInstruction *next_uop = self->trace;
     int opcode;
     int oparg;
     uint64_t operand;
@@ -83,17 +84,17 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
 #endif
 
     for (;;) {
-        opcode = self->trace[pc].opcode;
-        oparg = self->trace[pc].oparg;
-        operand = self->trace[pc].operand;
+        opcode = next_uop->opcode;
+        oparg = next_uop->oparg;
+        operand = next_uop->operand;
         DPRINTF(3,
                 "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
-                pc,
+                (int)(next_uop - self->trace),
                 opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
                 oparg,
                 operand,
                 (int)(stack_pointer - _PyFrame_Stackbase(frame)));
-        pc++;
+        next_uop++;
         OPT_STAT_INC(uops_executed);
         UOP_EXE_INC(opcode);
 #ifdef Py_STATS
@@ -146,5 +147,6 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
     frame->return_offset = 0;  // Dispatch to frame->instr_ptr
     _PyFrame_SetStackPointer(frame, stack_pointer);
     Py_DECREF(self);
+enter_tier_one:
     return frame;
 }