pythonGH-115816: Make tier2 optimizer symbols testable, and add a few…

… tests. (pythonGH-115953)

Include/internal/pycore_optimizer.h

@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_uop_ids.h"
+#include <stdbool.h>
 
 // This is the length of the trace we project initially.
 #define UOP_MAX_TRACE_LENGTH 512
@@ -25,6 +26,90 @@ extern PyTypeObject _PyDefaultOptimizer_Type;
 extern PyTypeObject _PyUOpExecutor_Type;
 extern PyTypeObject _PyUOpOptimizer_Type;
 
+/* Symbols */
+
+struct _Py_UOpsSymType {
+    int flags;
+    PyTypeObject *typ;
+    // constant propagated value (might be NULL)
+    PyObject *const_val;
+};
+
+// Holds locals, stack, locals, stack ... co_consts (in that order)
+#define MAX_ABSTRACT_INTERP_SIZE 4096
+
+#define OVERALLOCATE_FACTOR 5
+
+#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * OVERALLOCATE_FACTOR)
+
+// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH())
+#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2)
+
+typedef struct _Py_UOpsSymType _Py_UOpsSymType;
+
+struct _Py_UOpsAbstractFrame {
+    // Max stacklen
+    int stack_len;
+    int locals_len;
+
+    _Py_UOpsSymType **stack_pointer;
+    _Py_UOpsSymType **stack;
+    _Py_UOpsSymType **locals;
+};
+
+typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame;
+
+typedef struct ty_arena {
+    int ty_curr_number;
+    int ty_max_number;
+    _Py_UOpsSymType arena[TY_ARENA_SIZE];
+} ty_arena;
+
+struct _Py_UOpsAbstractInterpContext {
+    PyObject_HEAD
+    // The current "executing" frame.
+    _Py_UOpsAbstractFrame *frame;
+    _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH];
+    int curr_frame_depth;
+
+    // Arena for the symbolic types.
+    ty_arena t_arena;
+
+    _Py_UOpsSymType **n_consumed;
+    _Py_UOpsSymType **limit;
+    _Py_UOpsSymType *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
+};
+
+typedef struct _Py_UOpsAbstractInterpContext _Py_UOpsAbstractInterpContext;
+
+extern bool _Py_uop_sym_is_null(_Py_UOpsSymType *sym);
+extern bool _Py_uop_sym_is_not_null(_Py_UOpsSymType *sym);
+extern bool _Py_uop_sym_is_const(_Py_UOpsSymType *sym);
+extern PyObject *_Py_uop_sym_get_const(_Py_UOpsSymType *sym);
+extern _Py_UOpsSymType *_Py_uop_sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx);
+extern _Py_UOpsSymType *_Py_uop_sym_new_not_null(_Py_UOpsAbstractInterpContext *ctx);
+extern _Py_UOpsSymType *_Py_uop_sym_new_type(
+    _Py_UOpsAbstractInterpContext *ctx, PyTypeObject *typ);
+extern _Py_UOpsSymType *_Py_uop_sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val);
+extern _Py_UOpsSymType *_Py_uop_sym_new_null(_Py_UOpsAbstractInterpContext *ctx);
+extern bool _Py_uop_sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ);
+extern void _Py_uop_sym_set_null(_Py_UOpsSymType *sym);
+extern void _Py_uop_sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp);
+
+extern int _Py_uop_abstractcontext_init(_Py_UOpsAbstractInterpContext *ctx);
+extern void _Py_uop_abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx);
+
+extern _Py_UOpsAbstractFrame *_Py_uop_ctx_frame_new(
+    _Py_UOpsAbstractInterpContext *ctx,
+    PyCodeObject *co,
+    _Py_UOpsSymType **localsplus_start,
+    int n_locals_already_filled,
+    int curr_stackentries);
+extern int _Py_uop_ctx_frame_pop(_Py_UOpsAbstractInterpContext *ctx);
+
+PyAPI_FUNC(PyObject *)
+_Py_uop_symbols_test(PyObject *self, PyObject *ignored);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_generated_cases.py

@@ -900,7 +900,7 @@ def test_overridden_abstract_args(self):
 
         case OP2: {
             _Py_UOpsSymType *out;
-            out = sym_new_unknown(ctx);
+            out = _Py_uop_sym_new_unknown(ctx);
             if (out == NULL) goto out_of_space;
             stack_pointer[-1] = out;
             break;
@@ -925,7 +925,7 @@ def test_no_overridden_case(self):
         output = """
         case OP: {
             _Py_UOpsSymType *out;
-            out = sym_new_unknown(ctx);
+            out = _Py_uop_sym_new_unknown(ctx);
             if (out == NULL) goto out_of_space;
             stack_pointer[-1] = out;
             break;

Lib/test/test_optimizer.py

@@ -77,5 +77,12 @@ def func(x=0):
                 _testinternalcapi.get_rare_event_counters()["func_modification"]
             )
 
+
+class TestOptimizerSymbols(unittest.TestCase):
+
+    def test_optimizer_symbols(self):
+        _testinternalcapi.uop_symbols_test()
+
+
 if __name__ == "__main__":
     unittest.main()

Makefile.pre.in

@@ -446,6 +446,7 @@ PYTHON_OBJS=	\
 		Python/object_stack.o \
 		Python/optimizer.o \
 		Python/optimizer_analysis.o \
+		Python/optimizer_symbols.o \
 		Python/parking_lot.o \
 		Python/pathconfig.o \
 		Python/preconfig.o \

Modules/_testinternalcapi.c

@@ -24,6 +24,7 @@
 #include "pycore_interp.h"        // _PyInterpreterState_GetConfigCopy()
 #include "pycore_long.h"          // _PyLong_Sign()
 #include "pycore_object.h"        // _PyObject_IsFreed()
+#include "pycore_optimizer.h"     // _Py_UOpsSymType, etc.
 #include "pycore_pathconfig.h"    // _PyPathConfig_ClearGlobal()
 #include "pycore_pyerrors.h"      // _PyErr_ChainExceptions1()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
@@ -1677,7 +1678,6 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored))
 }
 #endif
 
-
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -1747,6 +1747,7 @@ static PyMethodDef module_functions[] = {
 #ifdef Py_GIL_DISABLED
     {"py_thread_id", get_py_thread_id, METH_NOARGS},
 #endif
+    {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };
 

PCbuild/_freeze_module.vcxproj

@@ -235,6 +235,7 @@
     <ClCompile Include="..\Python\object_stack.c" />
     <ClCompile Include="..\Python\optimizer.c" />
     <ClCompile Include="..\Python\optimizer_analysis.c" />
+    <ClCompile Include="..\Python\optimizer_symbols.c" />
     <ClCompile Include="..\Python\parking_lot.c" />
     <ClCompile Include="..\Python\pathconfig.c" />
     <ClCompile Include="..\Python\perf_trampoline.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -310,6 +310,9 @@
     <ClCompile Include="..\Python\optimizer_analysis.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\optimizer_symbols.c">
+      <Filter>Python</Filter>
+    </ClCompile>
     <ClCompile Include="..\Parser\parser.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -601,6 +601,7 @@
     <ClCompile Include="..\Python\object_stack.c" />
     <ClCompile Include="..\Python\optimizer.c" />
     <ClCompile Include="..\Python\optimizer_analysis.c" />
+    <ClCompile Include="..\Python\optimizer_symbols.c" />
     <ClCompile Include="..\Python\parking_lot.c" />
     <ClCompile Include="..\Python\pathconfig.c" />
     <ClCompile Include="..\Python\perf_trampoline.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -1382,6 +1382,9 @@
     <ClCompile Include="..\Python\optimizer_analysis.c">
       <Filter>Python</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\optimizer_symbols.c">
+      <Filter>Python</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\parking_lot.c">
       <Filter>Python</Filter>
     </ClCompile>