From 99b565a140d916e64da845e89e9f01c1b739bb2e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Feb 2025 13:52:12 +0000 Subject: [PATCH 1/5] Perform stack spilling in labels. WIP --- Python/bytecodes.c | 21 +++--- Python/generated_cases.c.h | 21 ++++-- Tools/cases_generator/analyzer.py | 58 ++++++++++------ Tools/cases_generator/generators_common.py | 73 ++++++++++++-------- Tools/cases_generator/lexer.py | 2 + Tools/cases_generator/optimizer_generator.py | 5 +- Tools/cases_generator/parser.py | 1 + Tools/cases_generator/parsing.py | 6 +- Tools/cases_generator/stack.py | 2 +- Tools/cases_generator/tier1_generator.py | 12 +++- Tools/cases_generator/tier2_generator.py | 18 ++--- 11 files changed, 145 insertions(+), 74 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index effc8e0b6f6578..c98489ce75ddf1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5213,6 +5213,7 @@ dummy_func( } label(error) { + SAVE_STACK(); /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -5235,11 +5236,12 @@ dummy_func( goto exception_unwind; } - label(exception_unwind) { + spilled label(exception_unwind) { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); + if (handled == 0) { // No handlers, so exit. assert(_PyErr_Occurred(tstate)); @@ -5276,7 +5278,8 @@ dummy_func( PUSH(PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + int err = monitor_handled(tstate, frame, next_instr, exc); + if (err < 0) { goto exception_unwind; } /* Resume normal execution */ @@ -5285,10 +5288,11 @@ dummy_func( lltrace_resume_frame(frame); } #endif + RELOAD_STACK(); DISPATCH(); } - label(exit_unwind) { + spilled label(exit_unwind) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -5304,16 +5308,16 @@ dummy_func( return NULL; } next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); + RELOAD_STACK(); goto error; } - label(start_frame) { - if (_Py_EnterRecursivePy(tstate)) { + spilled label(start_frame) { + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { goto exit_unwind; } next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE { @@ -5332,6 +5336,7 @@ dummy_func( assert(!_PyErr_Occurred(tstate)); #endif + RELOAD_STACK(); DISPATCH(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 38ea63d71ab044..8ef27a8d203e3e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3291,6 +3291,7 @@ _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); goto exception_unwind; } stack_pointer[-3] = none; @@ -3849,6 +3850,7 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); goto exception_unwind; } stack_pointer += -2; @@ -7164,6 +7166,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); goto exception_unwind; } goto error; @@ -7209,6 +7212,7 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); goto exception_unwind; } @@ -8615,6 +8619,7 @@ error: { + _PyFrame_SetStackPointer(frame, stack_pointer); /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -8639,10 +8644,12 @@ exception_unwind: { + /* STACK SPILLED */ /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); + if (handled == 0) { // No handlers, so exit. assert(_PyErr_Occurred(tstate)); /* Pop remaining stack entries. */ @@ -8675,7 +8682,8 @@ PyObject *exc = _PyErr_GetRaisedException(tstate); PUSH(PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + int err = monitor_handled(tstate, frame, next_instr, exc); + if (err < 0) { goto exception_unwind; } /* Resume normal execution */ @@ -8684,11 +8692,13 @@ lltrace_resume_frame(frame); } #endif + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } exit_unwind: { + /* STACK SPILLED */ assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -8710,11 +8720,12 @@ start_frame: { - if (_Py_EnterRecursivePy(tstate)) { + /* STACK SPILLED */ + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { goto exit_unwind; } next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE { int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); @@ -8731,7 +8742,7 @@ caller loses its exception */ assert(!_PyErr_Occurred(tstate)); #endif - + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index acf9458019fb4b..c9dc76dc802f90 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -120,6 +120,8 @@ def size(self) -> int: return 0 + + @dataclass class StackItem: name: str @@ -218,7 +220,24 @@ def is_super(self) -> bool: return False +class Label: + + def __init__(self, name: str, spilled: bool, body: list[lexer.Token],properties: Properties): + self.name = name + self.spilled = spilled + self.body = body + self.properties = properties + + size:int = 0 + output_stores: list[lexer.Token] = [] + instruction_size = None + + def __str__(self) -> str: + return f"label({self.name})" + + Part = Uop | Skip | Flush +CodeSection = Uop | Label @dataclass @@ -258,12 +277,6 @@ def is_super(self) -> bool: return False -@dataclass -class Label: - name: str - body: list[lexer.Token] - - @dataclass class PseudoInstruction: name: str @@ -481,22 +494,24 @@ def in_frame_push(idx: int) -> bool: return refs -def variable_used(node: parser.InstDef, name: str) -> bool: +def variable_used(node: parser.CodeDef, name: str) -> bool: """Determine whether a variable with a given name is used in a node.""" return any( token.kind == "IDENTIFIER" and token.text == name for token in node.block.tokens ) -def oparg_used(node: parser.InstDef) -> bool: +def oparg_used(node: parser.CodeDef) -> bool: """Determine whether `oparg` is used in a node.""" return any( token.kind == "IDENTIFIER" and token.text == "oparg" for token in node.tokens ) -def tier_variable(node: parser.InstDef) -> int | None: +def tier_variable(node: parser.CodeDef) -> int | None: """Determine whether a tier variable is used in a node.""" + if isinstance(node, parser.LabelDef): + return None for token in node.tokens: if token.kind == "ANNOTATION": if token.text == "specializing": @@ -506,7 +521,7 @@ def tier_variable(node: parser.InstDef) -> int | None: return None -def has_error_with_pop(op: parser.InstDef) -> bool: +def has_error_with_pop(op: parser.CodeDef) -> bool: return ( variable_used(op, "ERROR_IF") or variable_used(op, "pop_1_error") @@ -514,7 +529,7 @@ def has_error_with_pop(op: parser.InstDef) -> bool: ) -def has_error_without_pop(op: parser.InstDef) -> bool: +def has_error_without_pop(op: parser.CodeDef) -> bool: return ( variable_used(op, "ERROR_NO_POP") or variable_used(op, "pop_1_error") @@ -644,7 +659,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "restart_backoff_counter", ) -def find_stmt_start(node: parser.InstDef, idx: int) -> lexer.Token: +def find_stmt_start(node: parser.CodeDef, idx: int) -> lexer.Token: assert idx < len(node.block.tokens) while True: tkn = node.block.tokens[idx-1] @@ -657,7 +672,7 @@ def find_stmt_start(node: parser.InstDef, idx: int) -> lexer.Token: return node.block.tokens[idx] -def find_stmt_end(node: parser.InstDef, idx: int) -> lexer.Token: +def find_stmt_end(node: parser.CodeDef, idx: int) -> lexer.Token: assert idx < len(node.block.tokens) while True: idx += 1 @@ -665,7 +680,7 @@ def find_stmt_end(node: parser.InstDef, idx: int) -> lexer.Token: if tkn.kind == "SEMI": return node.block.tokens[idx+1] -def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, tuple[lexer.Token, lexer.Token]]) -> None: +def check_escaping_calls(instr: parser.CodeDef, escapes: dict[lexer.Token, tuple[lexer.Token, lexer.Token]]) -> None: calls = {escapes[t][0] for t in escapes} in_if = 0 tkn_iter = iter(instr.block.tokens) @@ -684,7 +699,7 @@ def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, tuple elif tkn in calls and in_if: raise analysis_error(f"Escaping call '{tkn.text} in condition", tkn) -def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, tuple[lexer.Token, lexer.Token]]: +def find_escaping_api_calls(instr: parser.CodeDef) -> dict[lexer.Token, tuple[lexer.Token, lexer.Token]]: result: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] = {} tokens = instr.block.tokens for idx, tkn in enumerate(tokens): @@ -736,7 +751,7 @@ def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, tuple[le } -def always_exits(op: parser.InstDef) -> bool: +def always_exits(op: parser.CodeDef) -> bool: depth = 0 tkn_iter = iter(op.tokens) for tkn in tkn_iter: @@ -795,7 +810,7 @@ def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: return False -def compute_properties(op: parser.InstDef) -> Properties: +def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) has_free = ( variable_used(op, "PyCell_New") @@ -826,6 +841,8 @@ def compute_properties(op: parser.InstDef) -> Properties: variable_used(op, "PyStackRef_CLEAR") or variable_used(op, "SETLOCAL") ) + pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations + no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations return Properties( escaping_calls=escaping_calls, escapes=escapes, @@ -844,8 +861,8 @@ def compute_properties(op: parser.InstDef) -> Properties: uses_locals=(variable_used(op, "GETLOCAL") or variable_used(op, "SETLOCAL")) and not has_free, has_free=has_free, - pure="pure" in op.annotations, - no_save_ip="no_save_ip" in op.annotations, + pure=pure, + no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), ) @@ -1024,7 +1041,8 @@ def add_label( label: parser.LabelDef, labels: dict[str, Label], ) -> None: - labels[label.name] = Label(label.name, label.block.tokens) + properties = compute_properties(label) + labels[label.name] = Label(label.name, label.spilled, label.block.tokens, properties) def assign_opcodes( diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index f1f166ae104ba5..0e2e146428326e 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -7,6 +7,8 @@ Properties, StackItem, analysis_error, + Label, + CodeSection, ) from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable @@ -90,7 +92,7 @@ def emit_to(out: CWriter, tkn_iter: TokenIterator, end: str) -> Token: ReplacementFunctionType = Callable[ - [Token, TokenIterator, Uop, Storage, Instruction | None], bool + [Token, TokenIterator, CodeSection, Storage, Instruction | None], bool ] def always_true(tkn: Token | None) -> bool: @@ -106,9 +108,10 @@ def always_true(tkn: Token | None) -> bool: class Emitter: out: CWriter + labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] - def __init__(self, out: CWriter): + def __init__(self, out: CWriter, labels: dict[str, Label]): self._replacers = { "EXIT_IF": self.exit_if, "DEOPT_IF": self.deopt_if, @@ -130,12 +133,13 @@ def __init__(self, out: CWriter): "GO_TO_INSTRUCTION": self.go_to_instruction, } self.out = out + self.labels = labels def dispatch( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -146,7 +150,7 @@ def deopt_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -170,7 +174,7 @@ def error_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -219,7 +223,7 @@ def error_no_pop( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -233,7 +237,7 @@ def decref_inputs( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -269,7 +273,7 @@ def kill_inputs( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -284,7 +288,7 @@ def kill( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -324,7 +328,7 @@ def stackref_close( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -344,7 +348,7 @@ def stackref_close_specialized( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -374,7 +378,7 @@ def stackref_steal( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -394,7 +398,7 @@ def sync_sp( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -410,7 +414,7 @@ def go_to_instruction( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -423,6 +427,19 @@ def go_to_instruction( self.emit(f"goto PREDICTED_{name.text};\n") return True + def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: + if label.text not in self.labels: + print(self.labels.keys()) + raise analysis_error(f"Label '{label.text}' does not exist", label) + label_node = self.labels[label.text] + if label_node.spilled: + if not storage.spilled: + self.emit_save(storage) + elif storage.spilled: + raise analysis_error("Cannot goto spilled label without saving the stack", goto) + self.out.emit(goto) + self.out.emit(label) + def emit_save(self, storage: Storage) -> None: storage.save(self.out) self._print_storage(storage) @@ -431,7 +448,7 @@ def save_stack( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -445,7 +462,7 @@ def pop_input( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -472,7 +489,7 @@ def reload_stack( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -485,7 +502,7 @@ def reload_stack( def instruction_size(self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -504,7 +521,7 @@ def _print_storage(self, storage: Storage) -> None: def _emit_if( self, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> tuple[bool, Token, Storage]: @@ -564,7 +581,7 @@ def _emit_if( def _emit_block( self, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, emit_first_brace: bool @@ -605,8 +622,9 @@ def _emit_block( return reachable, tkn, storage self.out.emit(tkn) elif tkn.kind == "GOTO": + label_tkn = next(tkn_iter) + self.goto_label(tkn, label_tkn, storage) reachable = False; - self.out.emit(tkn) elif tkn.kind == "IDENTIFIER": if tkn.text in self._replacers: if not self._replacers[tkn.text](tkn, tkn_iter, uop, storage, inst): @@ -637,17 +655,18 @@ def _emit_block( def emit_tokens( self, - uop: Uop, + code: CodeSection, storage: Storage, inst: Instruction | None, ) -> Storage: - tkn_iter = TokenIterator(uop.body) + tkn_iter = TokenIterator(code.body) self.out.start_line() - _, rbrace, storage = self._emit_block(tkn_iter, uop, storage, inst, False) + reachable, rbrace, storage = self._emit_block(tkn_iter, code, storage, inst, False) try: - self._print_storage(storage) - storage.push_outputs() - self._print_storage(storage) + if reachable: + self._print_storage(storage) + storage.push_outputs() + self._print_storage(storage) except StackError as ex: raise analysis_error(ex.args[0], rbrace) from None return storage diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index cf3c39762f29cb..6afca750be9b19 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -216,6 +216,8 @@ def choice(*opts: str) -> str: # A label in the DSL LABEL = "LABEL" kwds.append(LABEL) +SPILLED = "SPILLED" +kwds.append(SPILLED) keywords = {name.lower(): name for name in kwds} ANNOTATION = "ANNOTATION" diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 5cfec4bfecbf07..6c33debd58e1fe 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -112,6 +112,9 @@ def emit_save(self, storage: Storage) -> None: def emit_reload(self, storage: Storage) -> None: pass + def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: + self.out.emit(goto) + self.out.emit(label) def write_uop( override: Uop | None, @@ -145,7 +148,7 @@ def write_uop( cast = f"uint{cache.size*16}_t" out.emit(f"{type}{cache.name} = ({cast})this_instr->operand0;\n") if override: - emitter = OptimizerEmitter(out) + emitter = OptimizerEmitter(out, {}) # No reference management of inputs needed. for var in storage.inputs: # type: ignore[possibly-undefined] var.defined = False diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 68bbb88719e682..696c5c16432990 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -13,6 +13,7 @@ AstNode, ) +CodeDef = InstDef | LabelDef def prettify_filename(filename: str) -> str: # Make filename more user-friendly and less platform-specific, diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index eb8c8a7ecd32e8..011f34de288871 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -153,6 +153,7 @@ class Pseudo(Node): @dataclass class LabelDef(Node): name: str + spilled: bool block: Block @@ -176,12 +177,15 @@ def definition(self) -> AstNode | None: @contextual def label_def(self) -> LabelDef | None: + spilled = False + if self.expect(lx.SPILLED): + spilled = True if self.expect(lx.LABEL): if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.RPAREN): if block := self.block(): - return LabelDef(tkn.text, block) + return LabelDef(tkn.text, spilled, block) return None @contextual diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 5121837ed8334b..729973f1e32758 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -570,7 +570,7 @@ def copy(self) -> "Storage": assert [v.name for v in inputs] == [v.name for v in self.inputs], (inputs, self.inputs) return Storage( new_stack, inputs, - self.copy_list(self.outputs), self.copy_list(self.peeks) + self.copy_list(self.outputs), self.copy_list(self.peeks), self.spilled ) def sanity_check(self) -> None: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 13430524b26dcd..ab3b5b87f5127b 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -184,19 +184,25 @@ def generate_tier1_labels( analysis: Analysis, outfile: TextIO, lines: bool ) -> None: out = CWriter(outfile, 2, lines) + emitter = Emitter(out, analysis.labels) out.emit("\n") for name, label in analysis.labels.items(): out.emit(f"{name}:\n") - for tkn in label.body: - out.emit(tkn) + out.emit("{\n") + storage = Storage(Stack(), [], [], []) + if label.spilled: + storage.spilled = 1 + out.emit("/* STACK SPILLED */\n") + emitter.emit_tokens(label, storage, None) out.emit("\n") + out.emit("}\n") out.emit("\n") def generate_tier1_cases( analysis: Analysis, outfile: TextIO, lines: bool ) -> None: out = CWriter(outfile, 2, lines) - emitter = Emitter(out) + emitter = Emitter(out, analysis.labels) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): needs_this = uses_this(inst) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index dd16a1a7eb28b5..0618b38712b347 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -9,6 +9,8 @@ Analysis, Instruction, Uop, + Label, + CodeSection, analyze_files, StackItem, analysis_error, @@ -65,15 +67,15 @@ def declare_variables(uop: Uop, out: CWriter) -> None: class Tier2Emitter(Emitter): - def __init__(self, out: CWriter): - super().__init__(out) + def __init__(self, out: CWriter, labels: dict[str, Label]): + super().__init__(out, labels) self._replacers["oparg"] = self.oparg def error_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -95,7 +97,7 @@ def error_no_pop( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -109,7 +111,7 @@ def deopt_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -130,7 +132,7 @@ def exit_if( # type: ignore[override] self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -150,7 +152,7 @@ def oparg( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: CodeSection, storage: Storage, inst: Instruction | None, ) -> bool: @@ -210,7 +212,7 @@ def generate_tier2( """ ) out = CWriter(outfile, 2, lines) - emitter = Tier2Emitter(out) + emitter = Tier2Emitter(out, analysis.labels) out.emit("\n") for name, uop in analysis.uops.items(): if uop.properties.tier == 1: From 878cfb2683ca171db25db9aeed466db2850dc99d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Feb 2025 15:10:43 +0000 Subject: [PATCH 2/5] Fix up details including rejecting use of stack_pointer when spilled --- Include/internal/pycore_optimizer.h | 2 +- Python/bytecodes.c | 26 +++++++++---------- Python/executor_cases.c.h | 4 +-- Python/generated_cases.c.h | 29 ++++++++++++++-------- Python/optimizer.c | 3 ++- Tools/cases_generator/generators_common.py | 18 +++++++++++++- 6 files changed, 53 insertions(+), 29 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e806e306d2d57f..00fc4338b0a412 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -282,7 +282,7 @@ extern int _Py_uop_frame_pop(JitOptContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); static inline int is_terminator(const _PyUOpInstruction *uop) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 32eba865a65ce3..1cf0906f53c2c3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2808,7 +2808,7 @@ dummy_func( start--; } _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); ERROR_IF(optimized < 0, error); @@ -5021,7 +5021,7 @@ dummy_func( } else { int chain_depth = current_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, chain_depth); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { @@ -5122,7 +5122,7 @@ dummy_func( exit->temperature = advance_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); } - int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, 0); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, 0); if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); if (optimized < 0) { @@ -5208,7 +5208,6 @@ dummy_func( } label(error) { - SAVE_STACK(); /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -5239,22 +5238,21 @@ dummy_func( if (handled == 0) { // No handlers, so exit. assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + while (frame->stackpointer > stackbase) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); monitor_unwind(tstate, frame, next_instr-1); goto exit_unwind; } - assert(STACK_LEVEL() >= level); _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + assert(frame->stackpointer >= new_top); + while (frame->stackpointer > new_top) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); @@ -5262,7 +5260,7 @@ dummy_func( if (lasti == NULL) { goto exception_unwind; } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); } /* Make the raw exception data @@ -5270,7 +5268,7 @@ dummy_func( so a program can emulate the Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyFrame_GetBytecode(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c1c2c8fda20a7a..14ca341b739bb2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6335,7 +6335,7 @@ else { int chain_depth = current_executor->vm_data.chain_depth + 1; _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, chain_depth); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); @@ -6502,7 +6502,7 @@ GOTO_TIER_ONE(target); } _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, 0); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f9925acf1d54a3..874a5db6a267ad 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5461,7 +5461,7 @@ } _PyExecutorObject *executor; _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); @@ -8930,12 +8930,13 @@ error: { - _PyFrame_SetStackPointer(frame, stack_pointer); /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "error return without exception set"); + stack_pointer = _PyFrame_GetStackPointer(frame); } #else assert(_PyErr_Occurred(tstate)); @@ -8944,12 +8945,19 @@ /* Log traceback info. */ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyFrameObject *f = _PyFrame_GetFrameObject(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); if (f != NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyTraceBack_Here(f); + stack_pointer = _PyFrame_GetStackPointer(frame); } } + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, next_instr-1); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); goto exception_unwind; } @@ -8965,18 +8973,19 @@ assert(_PyErr_Occurred(tstate)); /* Pop remaining stack entries. */ _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + while (frame->stackpointer > stackbase) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); monitor_unwind(tstate, frame, next_instr-1); goto exit_unwind; } assert(STACK_LEVEL() >= level); _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + assert(frame->stackpointer >= new_top); + while (frame->stackpointer > new_top) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); @@ -8984,14 +8993,14 @@ if (lasti == NULL) { goto exception_unwind; } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); } /* Make the raw exception data available to the handler, so a program can emulate the Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyFrame_GetBytecode(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { diff --git a/Python/optimizer.c b/Python/optimizer.c index d71abd3224240b..97831f58098c95 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -105,8 +105,9 @@ uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, int _PyOptimizer_Optimize( _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - _PyStackRef *stack_pointer, _PyExecutorObject **executor_ptr, int chain_depth) + _PyExecutorObject **executor_ptr, int chain_depth) { + _PyStackRef *stack_pointer = frame->stackpointer; assert(_PyInterpreterState_GET()->jit); // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 3ed0894d2c8bea..f791cbd839b687 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -129,6 +129,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label]): "INSTRUCTION_SIZE": self.instruction_size, "POP_INPUT": self.pop_input, "GO_TO_INSTRUCTION": self.go_to_instruction, + "stack_pointer": self.stack_pointer, } self.out = out self.labels = labels @@ -141,6 +142,8 @@ def dispatch( storage: Storage, inst: Instruction | None, ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) self.emit(tkn) return False @@ -402,6 +405,19 @@ def go_to_instruction( self.emit(f"goto PREDICTED_{name.text};\n") return True + def stack_pointer( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer is invalid when stack is spilled to memory", tkn) + self.emit(tkn) + return True + def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: if label.text not in self.labels: print(self.labels.keys()) @@ -411,7 +427,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: if not storage.spilled: self.emit_save(storage) elif storage.spilled: - raise analysis_error("Cannot goto spilled label without saving the stack", goto) + raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.emit(goto) self.out.emit(label) From 5a03ec39cfffc3af111e1970476f2654aba85f04 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Feb 2025 15:35:52 +0000 Subject: [PATCH 3/5] Update tests --- Lib/test/test_generated_cases.py | 90 ++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 35600ce5486642..d2b33706ea6b75 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -286,7 +286,7 @@ def run_cases_test(self, input: str, expected: str): instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) - actual = instructions + labels + actual = instructions.strip() + "\n\n " + labels.strip() # if actual.strip() != expected.strip(): # print("Actual:") # print(actual) @@ -652,6 +652,9 @@ def test_cache_effect(self): def test_suppress_dispatch(self): input = """ + label(somewhere) { + } + inst(OP, (--)) { goto somewhere; } @@ -663,6 +666,11 @@ def test_suppress_dispatch(self): INSTRUCTION_STATS(OP); goto somewhere; } + + somewhere: + { + + } """ self.run_cases_test(input, output) @@ -1768,9 +1776,15 @@ def test_kill_in_wrong_order(self): def test_complex_label(self): input = """ + label(other_label) { + } + + label(other_label2) { + } + label(my_label) { // Comment - do_thing() + do_thing(); if (complex) { goto other_label; } @@ -1779,10 +1793,22 @@ def test_complex_label(self): """ output = """ + other_label: + { + + } + + other_label2: + { + + } + my_label: { // Comment - do_thing() + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing(); + stack_pointer = _PyFrame_GetStackPointer(frame); if (complex) { goto other_label; } @@ -1791,6 +1817,60 @@ def test_complex_label(self): """ self.run_cases_test(input, output) + def test_spilled_label(self): + input = """ + spilled label(one) { + RELOAD_STACK(); + goto two; + } + + label(two) { + SAVE_STACK(); + goto one; + } + """ + + output = """ + one: + { + /* STACK SPILLED */ + stack_pointer = _PyFrame_GetStackPointer(frame); + goto two; + } + + two: + { + _PyFrame_SetStackPointer(frame, stack_pointer); + goto one; + } + """ + self.run_cases_test(input, output) + + + def test_incorrect_spills(self): + input1 = """ + spilled label(one) { + goto two; + } + + label(two) { + } + """ + + input2 = """ + spilled label(one) { + } + + label(two) { + goto one; + } + """ + with self.assertRaisesRegex(SyntaxError, ".*reload.*"): + self.run_cases_test(input1, "") + with self.assertRaisesRegex(SyntaxError, ".*spill.*"): + self.run_cases_test(input2, "") + + def test_multiple_labels(self): input = """ label(my_label_1) { @@ -1802,7 +1882,7 @@ def test_multiple_labels(self): label(my_label_2) { // Comment do_thing2(); - goto my_label_3; + goto my_label_1; } """ @@ -1818,7 +1898,7 @@ def test_multiple_labels(self): { // Comment do_thing2(); - goto my_label_3; + goto my_label_1; } """ From 632f984b02266b01f0fc221a63e867186a27fbe3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Feb 2025 16:45:34 +0000 Subject: [PATCH 4/5] Fix whitespace --- Tools/cases_generator/analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index cd773f20fdd8e5..776e071a2d1028 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -232,7 +232,7 @@ def is_super(self) -> bool: class Label: - def __init__(self, name: str, spilled: bool, body: list[lexer.Token],properties: Properties): + def __init__(self, name: str, spilled: bool, body: list[lexer.Token], properties: Properties): self.name = name self.spilled = spilled self.body = body From 0dea9b63d2fcbfe3c55e993bf524e570c2498fdd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Feb 2025 09:11:26 +0000 Subject: [PATCH 5/5] Fix merge artifact --- Tools/cases_generator/analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 776e071a2d1028..724fba5f953a4e 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -720,7 +720,7 @@ def check_escaping_calls(instr: parser.CodeDef, escapes: dict[lexer.Token, Escap elif tkn in calls and in_if: raise analysis_error(f"Escaping call '{tkn.text} in condition", tkn) -def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, EscapingCall]: +def find_escaping_api_calls(instr: parser.CodeDef) -> dict[lexer.Token, EscapingCall]: result: dict[lexer.Token, EscapingCall] = {} tokens = instr.block.tokens for idx, tkn in enumerate(tokens):