GH-113710: Add a peephole optimization pass.

Include/internal/pycore_uop_metadata.h

@@ -202,6 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
     [_EXIT_TRACE] = HAS_DEOPT_FLAG,
     [_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
+    [_LOAD_CONST_INLINE] = 0,
     [_LOAD_CONST_INLINE_BORROW] = 0,
     [_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
 };
@@ -329,6 +330,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT",
     [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS",
     [_LOAD_CONST] = "_LOAD_CONST",
+    [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE",
     [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW",
     [_LOAD_DEREF] = "_LOAD_DEREF",
     [_LOAD_FAST] = "_LOAD_FAST",

Python/bytecodes.c

@@ -4070,6 +4070,10 @@ dummy_func(
             DEOPT_IF(!current_executor->vm_data.valid);
         }
 
+        op(_LOAD_CONST_INLINE, (ptr/4 -- value)) {
+            value = Py_NewRef(ptr);
+        }
+
         op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) {
             value = ptr;
         }

Python/optimizer.c

@@ -588,6 +588,9 @@ translate_bytecode_to_trace(
                         ADD_TO_TRACE(uop, oparg, operand, target);
                         if (uop == _POP_FRAME) {
                             TRACE_STACK_POP();
+                            /* Set the operand to the code object returned to,
+                             * to assist optimization passes */
+                            trace[trace_length-1].operand = (uintptr_t)code;
                             DPRINTF(2,
                                 "Returning to %s (%s:%d) at byte offset %d\n",
                                 PyUnicode_AsUTF8(code->co_qualname),
@@ -629,6 +632,9 @@ translate_bytecode_to_trace(
                                 instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1;
                                 TRACE_STACK_PUSH();
                                 _Py_BloomFilter_Add(dependencies, new_code);
+                                /* Set the operand to the callee's code object,
+                                * to assist optimization passes */
+                                trace[trace_length-1].operand = (uintptr_t)new_code;
                                 code = new_code;
                                 instr = _PyCode_CODE(code);
                                 DPRINTF(2,

Python/optimizer_analysis.c

@@ -12,6 +12,39 @@
 #include <stddef.h>
 #include "pycore_optimizer.h"
 
+static void
+peephole_opt(PyCodeObject *co, _PyUOpInstruction *buffer, int buffer_size)
+{
+    for (int pc = 0; pc < buffer_size; pc++) {
+        int opcode = buffer[pc].opcode;
+        switch(opcode) {
+            case _LOAD_CONST: {
+                assert(co != NULL);
+                PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg);
+                buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE;
+                buffer[pc].operand = (uintptr_t)val;
+                break;
+            }
+            case _CHECK_PEP_523:
+            {
+                /* Setting the eval frame function invalidates
+                 * all executors, so no need to check dynamically */
+                if (_PyInterpreterState_GET()->eval_frame == NULL) {
+                    buffer[pc].opcode = _NOP;
+                }
+                break;
+            }
+            case _PUSH_FRAME:
+            case _POP_FRAME:
+                co = (PyCodeObject *)buffer[pc].operand;
+                break;
+            case _JUMP_TO_TOP:
+            case _EXIT_TRACE:
+                return;
+        }
+    }
+}
+
 static void
 remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
 {
@@ -59,6 +92,7 @@ _Py_uop_analyze_and_optimize(
     int curr_stacklen
 )
 {
+    peephole_opt(co, buffer, buffer_size);
     remove_unneeded_uops(buffer, buffer_size);
     return 0;
 }

Python/pystate.c

@@ -2367,11 +2367,15 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp,
                                      _PyFrameEvalFunction eval_frame)
 {
     if (eval_frame == _PyEval_EvalFrameDefault) {
-        interp->eval_frame = NULL;
+        eval_frame = NULL;
     }
-    else {
-        interp->eval_frame = eval_frame;
+    if (eval_frame == interp->eval_frame) {
+        return;
+    }
+    if (eval_frame != NULL) {
+        _Py_Executors_InvalidateAll(interp);
     }
+    interp->eval_frame = eval_frame;
 }