diff --git a/Makefile b/Makefile index 53090f1e..ad2a25cf 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ PYTHON3 ?= python3 RM ?= rm LINT = flake8 +IS_PYPY = $(shell $(PYTHON) -c 'import platform; print("pypy" if platform.python_implementation() == "PyPy" else "")') +PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2 | head -1)$(IS_PYPY) + #EXTRA_DIST=ipython/ipy_trepan.py trepan PHONY=all test check clean distcheck pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc @@ -20,8 +23,7 @@ all: check #: Run all tests test check: - @PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \ - $(MAKE) check-$$PYTHON_VERSION + $(MAKE) check-$(PYTHON_VERSION) #: Run all quick tests check-short: pytest @@ -35,6 +37,11 @@ check-3.9 check-3.10: pytest check-3.9 check-3.10: pytest +#: Run working tests from Python 3.8 +check-3.8pypy: + $(MAKE) -C test $@ + + # FIXME #: pypy3.8-7.3.7 7.3: diff --git a/decompyle3/parsers/main.py b/decompyle3/parsers/main.py index c6a8792e..13096d62 100644 --- a/decompyle3/parsers/main.py +++ b/decompyle3/parsers/main.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023 Rocky Bernstein +# Copyright (c) 2019-2024 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -42,6 +42,13 @@ Python38ParserLambda, Python38ParserSingle, ) +from decompyle3.parsers.p38pypy.heads import ( + Python38PyPyParserEval, + Python38PyPyParserExec, + Python38PyPyParserExpr, + Python38PyPyParserLambda, + Python38PyPyParserSingle, +) from decompyle3.parsers.treenode import SyntaxTree from decompyle3.show import maybe_show_asm @@ -99,17 +106,36 @@ def get_python_parser( p = Python37ParserSingle(debug_parser) elif version == (3, 8): if compile_mode == "exec": - p = Python38ParserExec(debug_parser=debug_parser) + if is_pypy: + p = Python38PyPyParserExec(debug_parser=debug_parser) + else: + p = Python38ParserExec(debug_parser=debug_parser) + elif compile_mode == "single": - p = Python38ParserSingle(debug_parser=debug_parser) + if is_pypy: + p = Python38PyPyParserSingle(debug_parser=debug_parser) + else: + p = Python38ParserSingle(debug_parser=debug_parser) elif compile_mode == "lambda": - p = Python38ParserLambda(debug_parser=debug_parser) + if is_pypy: + p = Python38PyPyParserLambda(debug_parser=debug_parser) + else: + p = Python38ParserLambda(debug_parser=debug_parser) elif compile_mode == "eval": - p = Python38ParserEval(debug_parser=debug_parser) + if is_pypy: + p = Python38PyPyParserEval(debug_parser=debug_parser) + else: + p = Python38ParserEval(debug_parser=debug_parser) elif compile_mode == "expr": - p = Python38ParserExpr(debug_parser=debug_parser) + if is_pypy: + p = Python38PyPyParserExpr(debug_parser=debug_parser) + else: + p = Python38ParserExpr(debug_parser=debug_parser) + elif is_pypy: + p = Python38PyPyParserSingle(debug_parser) else: p = Python38ParserSingle(debug_parser) + elif version > (3, 8): raise RuntimeError( f"""Version {version_tuple_to_str(version)} is not supported.""" diff --git a/decompyle3/parsers/p38pypy/__init__.py b/decompyle3/parsers/p38pypy/__init__.py new file mode 100644 index 00000000..20e86334 --- /dev/null +++ b/decompyle3/parsers/p38pypy/__init__.py @@ -0,0 +1,4 @@ +""" +Here we have Python 3.8 PyPy grammars and associated customization +for the both full language and the subset used in lambda expressions. +""" diff --git a/decompyle3/parsers/p38pypy/base.py b/decompyle3/parsers/p38pypy/base.py new file mode 100644 index 00000000..9d2257b2 --- /dev/null +++ b/decompyle3/parsers/p38pypy/base.py @@ -0,0 +1,50 @@ +# Copyright (c) 2020-2022, 2024 Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG + +from decompyle3.parsers.parse_heads import PythonBaseParser +from decompyle3.parsers.reduce_check import ( + break_invalid, + for38_invalid, + forelse38_invalid, + pop_return_check, + whilestmt38_check, + whileTruestmt38_check, +) + + +class Python38PyPyBaseParser(PythonBaseParser): + def __init__(self, start_symbol, debug_parser: dict = PARSER_DEFAULT_DEBUG): + super(Python38PyPyBaseParser, self).__init__( + start_symbol=start_symbol, debug_parser=debug_parser + ) + + def customize_grammar_rules38(self, tokens, customize): + self.customize_grammar_rules37(tokens, customize) + self.check_reduce["break"] = "tokens" + self.check_reduce["for38"] = "tokens" + self.check_reduce["forelsestmt38"] = "AST" + self.check_reduce["pop_return"] = "tokens" + self.check_reduce["whileTruestmt38"] = "AST" + self.check_reduce["whilestmt38"] = "tokens" + self.check_reduce["try_elsestmtl38"] = "AST" + + self.reduce_check_table["break"] = break_invalid + self.reduce_check_table["for38"] = for38_invalid + self.reduce_check_table["forelsestmt38"] = forelse38_invalid + self.reduce_check_table["pop_return"] = pop_return_check + self.reduce_check_table["whilestmt38"] = whilestmt38_check + self.reduce_check_table["whileTruestmt38"] = whileTruestmt38_check diff --git a/decompyle3/parsers/p38pypy/full.py b/decompyle3/parsers/p38pypy/full.py new file mode 100644 index 00000000..bcb68e4d --- /dev/null +++ b/decompyle3/parsers/p38pypy/full.py @@ -0,0 +1,682 @@ +# Copyright (c) 2017-2024 Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +spark grammar for Python 3.8 PyPy +""" + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from spark_parser.spark import rule2str + +from decompyle3.parsers.p37.full import Python37Parser +from decompyle3.parsers.p38pypy.full_custom import Python38PyPyFullCustom +from decompyle3.parsers.p38pypy.lambda_expr import Python38PyPyLambdaParser +from decompyle3.parsers.parse_heads import ParserError +from decompyle3.scanners.tok import Token + + +class Python38PyPyParser( + Python38PyPyLambdaParser, Python38PyPyFullCustom, Python37Parser +): + def __init__(self, start_symbol: str = "stmts", debug_parser=PARSER_DEFAULT_DEBUG): + Python38PyPyLambdaParser.__init__(self, start_symbol, debug_parser) + self.customized = {} + + def customize_grammar_rules(self, tokens, customize): + self.customize_grammar_rules_full38(tokens, customize) + + ############################################### + # Python 3.8 grammar rules with statements + ############################################### + + def p_38_full_if_ifelse(self, args): + """ + # cf_pt introduced to keep indices the same in ifelsestmtc + cf_pt ::= COME_FROM POP_TOP + ifelsestmtc ::= testexpr c_stmts cf_pt else_suite + + # 3.8 can push a looping JUMP_LOOP into a JUMP_ from a statement that jumps to + # it + lastc_stmt ::= ifpoplaststmtc + ifpoplaststmtc ::= testexpr POP_TOP c_stmts_opt + ifelsestmtc ::= testexpr c_stmts_opt jb_cfs else_suitec JUMP_LOOP + come_froms + + testtrue ::= or_in_ifexp POP_JUMP_IF_TRUE + + + # The below ifelsetmtc is a really weird one for the inner if/else in: + # if a: + # while i: + # if c: + # j = j + 1 + # # A JUMP_LOOP is here... + # else: + # break + # # but also a JUMP_LOOP is inserted here! + # else: + # j = 10 + + ifelsestmtc ::= testexpr c_stmts_opt JUMP_LOOP else_suitec JUMP_LOOP + """ + + def p_38_full_stmt(self, args): + """ + stmt ::= async_with_stmt38 + stmt ::= for38 + stmt ::= forelselaststmt38 + stmt ::= forelselaststmtc38 + stmt ::= forelsestmt38 + stmt ::= try_elsestmtl38 + stmt ::= try_except38 + stmt ::= try_except38r + stmt ::= try_except38r2 + stmt ::= try_except38r3 + stmt ::= try_except38r4 + stmt ::= try_except38r5 + stmt ::= try_except38r6 + stmt ::= try_except38r7 + stmt ::= try_except_as + stmt ::= try_except_ret38 + stmt ::= try_except_ret38a + stmt ::= tryfinallystmt_break + stmt ::= tryfinally38astmt + stmt ::= tryfinally38rstmt + stmt ::= tryfinally38rstmt2 + stmt ::= tryfinally38rstmt3 + stmt ::= tryfinally38rstmt4 + stmt ::= tryfinally38rstmt5 + stmt ::= tryfinally38stmt + stmt ::= tryfinally38_return + stmt ::= tryfinally38a_return + stmt ::= tryfinally38rstmt2 + stmt ::= whileTruestmt38 + stmt ::= whilestmt38 + + # FIXME: "break"" should be isolated to loops + stmt ::= break + + break ::= POP_BLOCK BREAK_LOOP + break ::= POP_BLOCK POP_TOP BREAK_LOOP + break ::= POP_TOP BREAK_LOOP + break ::= POP_EXCEPT BREAK_LOOP + break ::= POP_TOP CONTINUE JUMP_LOOP + + # An except with nothing other than a single break + break_except ::= POP_EXCEPT POP_TOP BREAK_LOOP + + # FIXME: this should be restricted to being inside a try block + stmt ::= except_ret38 + stmt ::= except_ret38a + + async_with_stmt38 ::= expr + BEFORE_ASYNC_WITH + GET_AWAITABLE + LOAD_CONST + YIELD_FROM + SETUP_ASYNC_WITH + POP_TOP + c_stmts_opt + POP_BLOCK + BEGIN_FINALLY + COME_FROM_ASYNC_WITH + WITH_CLEANUP_START + GET_AWAITABLE + LOAD_CONST + YIELD_FROM + WITH_CLEANUP_FINISH + END_FINALLY + + async_with_stmt38 ::= expr + BEFORE_ASYNC_WITH + GET_AWAITABLE + LOAD_CONST + YIELD_FROM + SETUP_ASYNC_WITH + POP_TOP + c_stmts_opt + POP_BLOCK + BEGIN_FINALLY + WITH_CLEANUP_START + GET_AWAITABLE + LOAD_CONST + YIELD_FROM + WITH_CLEANUP_FINISH + POP_FINALLY + + async_with_stmt38 ::= expr + BEFORE_ASYNC_WITH + GET_AWAITABLE + LOAD_CONST + YIELD_FROM + SETUP_ASYNC_WITH + POP_TOP + c_stmts_opt + POP_BLOCK + BEGIN_FINALLY + WITH_CLEANUP_START + GET_AWAITABLE + LOAD_CONST + YIELD_FROM + WITH_CLEANUP_FINISH + POP_FINALLY + JUMP_LOOP + + # Seems to be used to discard values before a return in a "for" loop + discard_top ::= ROT_TWO POP_TOP + discard_tops ::= discard_top+ + pop_tops ::= POP_TOP+ + + return ::= return_expr + discard_tops RETURN_VALUE + + return ::= pop_return + return ::= popb_return + return ::= pop_ex_return + except_stmt ::= except_with_break + except_stmt ::= except_with_break2 + except_stmt ::= pop_ex_return + except_stmt ::= pop3_except_return38 + except_stmt ::= pop3_rot4_except_return38 + except_stmt ::= except_cond_pop3_rot4_except_return38 + + except_stmts ::= except_stmt+ + except_stmts_opt ::= except_stmt* + + pop_return ::= POP_TOP return_expr RETURN_VALUE + popb_return ::= return_expr POP_BLOCK RETURN_VALUE + + # Return from exception where value is on stack + pop_ex_return ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE + + # Return from exception where value is no on stack but is computed + pop_ex_return2 ::= POP_EXCEPT expr RETURN_VALUE + + + # The below are 3.8 "except:" (no except condition) + + pop3_except_return38 ::= POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK + CALL_FINALLY return + + except_return38 ::= POP_BLOCK + CALL_FINALLY POP_TOP return + + pop3_rot4_except_return38 ::= POP_TOP POP_TOP POP_TOP + except_stmts_opt return_expr ROT_FOUR + POP_EXCEPT POP_BLOCK CALL_FINALLY RETURN_VALUE + + + pop3_rot4_except_return38 ::= POP_TOP POP_TOP POP_TOP + except_stmts_opt return_expr ROT_FOUR + POP_EXCEPT POP_BLOCK ROT_TWO POP_TOP + CALL_FINALLY RETURN_VALUE + END_FINALLY COME_FROM POP_BLOCK + BEGIN_FINALLY COME_FROM + + # The above but with an except condition name e.g. "except Exception:" + except_cond_pop3_rot4_except_return38 ::= except_cond1 + except_stmts_opt return_expr ROT_FOUR + POP_EXCEPT POP_BLOCK CALL_FINALLY RETURN_VALUE + COME_FROM + + except_stmt ::= except_cond1 except_suite come_from_opt + except_stmt ::= except_cond2 except_ret38b + + get_iter ::= expr GET_ITER + for38 ::= expr get_iter store for_block JUMP_LOOP _come_froms + for38 ::= expr get_for_iter store for_block JUMP_LOOP _come_froms + for38 ::= expr get_for_iter store for_block JUMP_LOOP _come_froms + POP_BLOCK + for38 ::= expr get_for_iter store for_block _come_froms + + forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suite + _come_froms + forelsestmt38 ::= expr get_for_iter store for_block JUMP_LOOP _come_froms + else_suite _come_froms + + c_stmt ::= c_forelsestmt38 + c_stmt ::= pop_tops return + c_forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec + c_forelsestmt38 ::= expr get_for_iter store for_block JUMP_LOOP _come_froms + else_suitec + + # continue is a weird one. In 3.8, CONTINUE_LOOP was removed. + # Inside an loop we can have this, which can only appear in side a try/except + # And it can also appear at the end of the try except. + continue ::= POP_EXCEPT JUMP_LOOP + + forelselaststmt38 ::= expr get_for_iter store for_block else_suitec + _come_froms + forelselaststmtc38 ::= expr get_for_iter store for_block else_suitec + _come_froms + # forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec + # forelselaststmtc38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec + + returns_in_except ::= _stmts except_return_value + returns_in_except2 ::= _stmts except_return_value2 + + except_return_value ::= POP_BLOCK return + except_return_value ::= expr POP_BLOCK RETURN_VALUE + except_return_value2 ::= POP_BLOCK return + + whilestmt38 ::= _come_froms testexpr c_stmts_opt COME_FROM JUMP_LOOP + POP_BLOCK + whilestmt38 ::= _come_froms testexpr c_stmts_opt JUMP_LOOP POP_BLOCK + whilestmt38 ::= _come_froms testexpr c_stmts_opt JUMP_LOOP come_froms + whilestmt38 ::= _come_froms testexprc c_stmts_opt come_froms JUMP_LOOP + _come_froms + whilestmt38 ::= _come_froms testexpr returns POP_BLOCK + whilestmt38 ::= _come_froms testexpr c_stmts JUMP_LOOP _come_froms + whilestmt38 ::= _come_froms testexpr c_stmts come_froms + whilestmt38 ::= _come_froms bool_op c_stmts JUMP_LOOP _come_froms + + # while1elsestmt ::= c_stmts JUMP_LOOP + whileTruestmt ::= _come_froms c_stmts JUMP_LOOP _come_froms + POP_BLOCK + while1stmt ::= _come_froms c_stmts COME_FROM_LOOP + while1stmt ::= _come_froms c_stmts COME_FROM JUMP_LOOP COME_FROM_LOOP + whileTruestmt38 ::= _come_froms c_stmts JUMP_LOOP _come_froms + whileTruestmt38 ::= _come_froms c_stmts JUMP_LOOP COME_FROM_EXCEPT_CLAUSE + whileTruestmt38 ::= _come_froms pass JUMP_LOOP + + for_block ::= _come_froms c_stmts_opt come_from_loops JUMP_LOOP + + # Note there is a 3.7 except_cond1 that doesn't have the final POP_EXCEPT + except_cond1 ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE + POP_TOP POP_TOP POP_TOP + POP_EXCEPT + + except_suite ::= c_stmts_opt + POP_EXCEPT POP_TOP JUMP_FORWARD POP_EXCEPT + jump_except + + try_elsestmtl38 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + except_handler38 COME_FROM + else_suitec opt_come_from_except + try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + except_handler38 + try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + except_handler38 + jump_excepts + come_from_except_clauses + + c_try_except ::= SETUP_FINALLY c_suite_stmts POP_BLOCK + except_handler38 + + c_stmt ::= c_tryfinallystmt38 + c_stmt ::= c_tryfinallybstmt38 + + c_tryfinallystmt38 ::= SETUP_FINALLY c_suite_stmts_opt + POP_BLOCK + CALL_FINALLY + POP_BLOCK + POP_EXCEPT + CALL_FINALLY + JUMP_FORWARD + POP_BLOCK BEGIN_FINALLY + COME_FROM + COME_FROM_FINALLY + c_suite_stmts_opt END_FINALLY + + # try: + # .. + # break + # finally: + c_tryfinallybstmt38 ::= SETUP_FINALLY c_suite_stmts_opt + POP_BLOCK + CALL_FINALLY + POP_BLOCK + POP_EXCEPT + CALL_FINALLY + BREAK_LOOP + POP_BLOCK BEGIN_FINALLY + COME_FROM + COME_FROM_FINALLY + c_suite_stmts_opt END_FINALLY + + c_tryfinallystmt38 ::= SETUP_FINALLY c_suite_stmts_opt + POP_BLOCK BEGIN_FINALLY COME_FROM COME_FROM_FINALLY + c_suite_stmts_opt END_FINALLY + + try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt + except_handler38a + + # suite_stmts has a return + try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts + except_handler38b + try_except38r ::= SETUP_FINALLY return_except + except_handler38b + return_except ::= stmts POP_BLOCK return + + + # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there + # is a final return in the "except" block. + # So we treat the "return" separate from the other statements + cond_except_stmt ::= except_cond1 except_stmts + cond_except_stmts_opt ::= cond_except_stmt* + + try_except38r2 ::= SETUP_FINALLY + suite_stmts_opt + POP_BLOCK JUMP_FORWARD + COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + cond_except_stmts_opt + POP_EXCEPT return + END_FINALLY + COME_FROM + + try_except38r3 ::= SETUP_FINALLY + suite_stmts_opt + POP_BLOCK JUMP_FORWARD + COME_FROM_FINALLY + cond_except_stmts_opt + POP_EXCEPT return + COME_FROM + END_FINALLY + COME_FROM + + + # I think this can be combined with the r5 + try_except38r4 ::= SETUP_FINALLY + returns_in_except + COME_FROM_FINALLY + except_cond1 + return + COME_FROM + END_FINALLY + + try_except38r5 ::= SETUP_FINALLY + returns_in_except + COME_FROM_FINALLY + except_cond1 + except_ret38d + COME_FROM + END_FINALLY + + try_except38r5 ::= SETUP_FINALLY + returns_in_except + COME_FROM_FINALLY + except_cond1 + except_suite + COME_FROM + END_FINALLY + COME_FROM + + try_except38r5 ::= SETUP_FINALLY + returns_in_except + COME_FROM_FINALLY + except_cond2 + except_ret38b + END_FINALLY COME_FROM + + try_except38r6 ::= SETUP_FINALLY + returns_in_except2 + COME_FROM_FINALLY + POP_TOP POP_TOP POP_TOP + except_ret38d + END_FINALLY + + + try_except38r7 ::= SETUP_FINALLY + suite_stmts_opt + POP_BLOCK JUMP_FORWARD + COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + return_expr + ROT_FOUR POP_EXCEPT POP_BLOCK ROT_TWO POP_TOP + CALL_FINALLY RETURN_VALUE + END_FINALLY + COME_FROM POP_BLOCK + BEGIN_FINALLY + COME_FROM + COME_FROM_FINALLY + + + try_except_as ::= SETUP_FINALLY POP_BLOCK suite_stmts + except_handler_as END_FINALLY COME_FROM + try_except_as ::= SETUP_FINALLY suite_stmts + except_handler_as END_FINALLY COME_FROM + + + try_except_ret38 ::= SETUP_FINALLY returns except_ret38a + try_except_ret38a ::= SETUP_FINALLY returns except_handler38c + END_FINALLY come_from_opt + + # Note: there is a suite_stmts_opt which seems + # to be bookkeeping which is not expressed in source code + except_ret38 ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT + CALL_FINALLY RETURN_VALUE COME_FROM + COME_FROM_FINALLY + suite_stmts_opt END_FINALLY + except_ret38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + expr ROT_FOUR + POP_EXCEPT RETURN_VALUE END_FINALLY + + except_ret38b ::= SETUP_FINALLY suite_stmts expr + ROT_FOUR POP_BLOCK POP_EXCEPT + CALL_FINALLY RETURN_VALUE COME_FROM + COME_FROM_FINALLY + suite_stmts_opt END_FINALLY + POP_EXCEPT JUMP_FORWARD COME_FROM + + except_ret38c ::= SETUP_FINALLY suite_stmts expr + ROT_FOUR POP_BLOCK POP_EXCEPT + CALL_FINALLY POP_BLOCK CALL_FINALLY RETURN_VALUE + COME_FROM + COME_FROM_FINALLY + expr STORE_FAST DELETE_FAST END_FINALLY + POP_EXCEPT JUMP_FORWARD COME_FROM + END_FINALLY come_any_froms + + except_ret38d ::= suite_stmts_opt + expr ROT_FOUR + POP_EXCEPT RETURN_VALUE + + except_handler38 ::= jump COME_FROM_FINALLY + except_stmts + END_FINALLY + opt_come_from_except + + except_handler38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + POP_EXCEPT POP_TOP stmts END_FINALLY + except_handler38b ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + POP_EXCEPT returns END_FINALLY + except_handler38c ::= COME_FROM_FINALLY except_cond1 except_stmts + COME_FROM + except_handler38c ::= COME_FROM_FINALLY except_cond1 except_stmts + POP_EXCEPT JUMP_FORWARD COME_FROM + + except_handler_as ::= COME_FROM_FINALLY except_cond2 tryfinallystmt + POP_EXCEPT JUMP_FORWARD COME_FROM + + except ::= POP_TOP POP_TOP POP_TOP c_stmts_opt break + POP_EXCEPT + + # Except of a try inside a loop + except ::= POP_TOP POP_TOP POP_TOP c_stmts_opt break + POP_EXCEPT JUMP_LOOP + + except ::= POP_TOP POP_TOP POP_TOP c_stmts_opt + POP_EXCEPT JUMP_LOOP + + except_with_break ::= POP_TOP POP_TOP POP_TOP c_stmts break_except + POP_EXCEPT JUMP_LOOP + + # Just except: break, no statements + except_with_break2 ::= POP_TOP POP_TOP POP_TOP break_except + POP_EXCEPT JUMP_LOOP + + except_with_return38 ::= POP_TOP POP_TOP POP_TOP stmts pop_ex_return2 + except_with_return38 ::= POP_TOP POP_TOP POP_TOP pop_ex_return2 + + except_stmt ::= except_with_return38 + + + # In 3.8 any POP_EXCEPT comes before the "break" loop. + # We should add a rule to check that JUMP_FORWARD is indeed a "break". + break ::= POP_EXCEPT JUMP_FORWARD + break ::= POP_BLOCK POP_TOP JUMP_FORWARD + + tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt + END_FINALLY + + tryfinallystmt_break ::= + SETUP_FINALLY suite_stmts_opt POP_BLOCK POP_EXCEPT + CALL_FINALLY + JUMP_FORWARD POP_BLOCK + BEGIN_FINALLY COME_FROM COME_FROM_FINALLY suite_stmts_opt + END_FINALLY + + + lc_setup_finally ::= LOAD_CONST SETUP_FINALLY + call_finally_pt ::= CALL_FINALLY POP_TOP + cf_cf_finally ::= come_from_opt COME_FROM_FINALLY + pop_finally_pt ::= POP_FINALLY POP_TOP + ss_end_finally ::= suite_stmts END_FINALLY + sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns + sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK POP_EXCEPT CALL_FINALLY returns + + suite_stmts_return ::= suite_stmts expr + suite_stmts_return ::= expr + + + # FIXME: DRY rules below + tryfinally38rstmt ::= sf_pb_call_returns + cf_cf_finally + ss_end_finally + tryfinally38rstmt ::= sf_pb_call_returns + cf_cf_finally END_FINALLY + suite_stmts + tryfinally38rstmt ::= sf_pb_call_returns + cf_cf_finally POP_FINALLY + ss_end_finally + tryfinally38rstmt ::= sf_pb_call_returns + COME_FROM_FINALLY POP_FINALLY + ss_end_finally + + tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt + returns + cf_cf_finally pop_finally_pt + ss_end_finally POP_TOP + + tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE + COME_FROM COME_FROM_FINALLY + ss_end_finally + + tryfinally38rstmt4 ::= lc_setup_finally suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_FINALLY + suite_stmts_return + POP_FINALLY ROT_TWO POP_TOP + RETURN_VALUE + END_FINALLY POP_TOP + + + tryfinally38rstmt5 ::= lc_setup_finally try_except38r7 expr + POP_FINALLY ROT_TWO POP_TOP + RETURN_VALUE + END_FINALLY POP_TOP + + tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_FINALLY + POP_FINALLY suite_stmts_opt END_FINALLY + + tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM + COME_FROM_FINALLY + suite_stmts_opt END_FINALLY + + # try: .. finally: ending with return ... + tryfinally38_return ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + JUMP_FORWARD + COME_FROM_FINALLY except_cond2 except_ret38c + + + tryfinally38a_return ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt except_return38 + COME_FROM COME_FROM_FINALLY + suite_stmts_opt pop_finally_pt return + END_FINALLY POP_TOP + + + tryfinally38astmt ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_FINALLY + POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP + """ + + def p_38_full_walrus(self, args): + """ + # named_expr is also known as the "walrus op" := + expr ::= named_expr + named_expr ::= expr DUP_TOP store + """ + + # FIXME: try this + def reduce_is_invalid(self, rule, ast, tokens, first, last): + lhs = rule[0] + if lhs == "call_kw": + # Make sure we don't derive call_kw + nt = ast[0] + while not isinstance(nt, Token): + if nt[0] == "call_kw": + return True + nt = nt[0] + pass + pass + n = len(tokens) + last = min(last, n - 1) + fn = self.reduce_check_table.get(lhs, None) + try: + if fn: + return fn(self, lhs, n, rule, ast, tokens, first, last) + except Exception: + import sys + import traceback + + print( + f"Exception in {fn.__name__} {sys.exc_info()[1]}\n" + + f"rule: {rule2str(rule)}\n" + + f"offsets {tokens[first].offset} .. {tokens[last].offset}" + ) + print(traceback.print_tb(sys.exc_info()[2], -1)) + raise ParserError(tokens[last], tokens[last].off2int(), self.debug["rules"]) + + if lhs in ("aug_assign1", "aug_assign2") and ast[0][0] == "and": + return True + elif lhs == "annotate_tuple": + return not isinstance(tokens[first].attr, tuple) + elif lhs == "import_from37": + importlist37 = ast[3] + alias37 = importlist37[0] + if importlist37 == "importlist37" and alias37 == "alias37": + store = alias37[1] + assert store == "store" + return alias37[0].attr != store[0].attr + return False + + return False + + return False + + +if __name__ == "__main__": + # Check grammar + from decompyle3.parsers.dump import dump_and_check + + p = Python38PyPyParser() + modified_tokens = set( + """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST + """.split() + ) + + dump_and_check(p, (3, 8), modified_tokens) diff --git a/decompyle3/parsers/p38pypy/full_custom.py b/decompyle3/parsers/p38pypy/full_custom.py new file mode 100644 index 00000000..a0f54c01 --- /dev/null +++ b/decompyle3/parsers/p38pypy/full_custom.py @@ -0,0 +1,1309 @@ +# Copyright (c) 2021-2024 Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# from decompyle3.parsers.reduce_check.import_from37 import import_from37_ok +from decompyle3.parsers.p37.base import Python37BaseParser +from decompyle3.parsers.p38.lambda_custom import Python38LambdaCustom +from decompyle3.parsers.parse_heads import PythonBaseParser, nop_func +from decompyle3.parsers.reduce_check import ( # joined_str_check, + break_invalid, + for38_invalid, + forelse38_invalid, + if_not_stmtc_invalid, + pop_return_check, + whilestmt38_check, + whileTruestmt38_check, +) + +# from decompyle3.parsers.reduce_check.ifelsestmt_check import ifelsestmt_ok +from decompyle3.parsers.reduce_check.ifstmt import ifstmt +from decompyle3.parsers.reduce_check.or_cond_check import or_cond_check_invalid + + +class Python38PyPyFullCustom(Python38LambdaCustom, PythonBaseParser): + def add_make_function_rule(self, rule, opname, attr, customize): + """Python 3.3 added an additional LOAD_STR before MAKE_FUNCTION and + this has an effect on many rules. + """ + new_rule = rule % "LOAD_STR " + self.add_unique_rule(new_rule, opname, attr, customize) + + @staticmethod + def call_fn_name(token): + """Customize CALL_FUNCTION to add the number of positional arguments""" + if token.attr is not None: + return f"{token.kind}_{token.attr}" + else: + return f"{token.kind}_0" + + def remove_rules_38(self): + self.remove_rules( + """ + stmt ::= async_for_stmt37 + stmt ::= for + stmt ::= forelsestmt + stmt ::= try_except36 + stmt ::= async_forelse_stmt + + # There is no SETUP_LOOP + setup_loop ::= SETUP_LOOP _come_froms + forelselaststmt ::= SETUP_LOOP expr get_for_iter store + for_block POP_BLOCK else_suitec _come_froms + + forelsestmt ::= SETUP_LOOP expr get_for_iter store + whileTruestmt ::= SETUP_LOOP c_stmts_opt JUMP_LOOP COME_FROM_LOOP + for_block POP_BLOCK else_suite _come_froms + + # async_for_stmt ::= setup_loop expr + # GET_AITER + # SETUP_EXCEPT GET_ANEXT LOAD_CONST + # YIELD_FROM + # store + # POP_BLOCK JUMP_FORWARD bb_end_start DUP_TOP + # LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + # END_FINALLY bb_end_start + # for_block + # COME_FROM + # POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK + # COME_FROM_LOOP + + # async_for_stmt37 ::= setup_loop expr + # GET_AITER + # SETUP_EXCEPT GET_ANEXT + # LOAD_CONST YIELD_FROM + # store + # POP_BLOCK JUMP_LOOP COME_FROM_EXCEPT DUP_TOP + # LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + # END_FINALLY for_block COME_FROM + # POP_TOP POP_TOP POP_TOP POP_EXCEPT + # POP_TOP POP_BLOCK + # COME_FROM_LOOP + + # async_forelse_stmt ::= setup_loop expr + # GET_AITER + # SETUP_EXCEPT GET_ANEXT LOAD_CONST + # YIELD_FROM + # store + # POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP + # LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + # END_FINALLY COME_FROM + # for_block + # COME_FROM + # POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK + # else_suite COME_FROM_LOOP + + for ::= setup_loop expr get_for_iter store for_block POP_BLOCK + for ::= setup_loop expr get_for_iter store for_block POP_BLOCK NOP + + for_block ::= c_stmts_opt COME_FROM_LOOP JUMP_LOOP + forelsestmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suite + forelselaststmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec + forelselaststmtc ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec + + try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler opt_come_from_except + + tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + LOAD_CONST COME_FROM_FINALLY suite_stmts_opt + END_FINALLY + tryfinally36 ::= SETUP_FINALLY returns + COME_FROM_FINALLY suite_stmts_opt END_FINALLY + tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + LOAD_CONST COME_FROM_FINALLY + """ + ) + + # def custom_classfunc_rule(self, opname, token, customize, next_token): + # """ + # call ::= expr {expr}^n CALL_FUNCTION_n + # call ::= expr {expr}^n CALL_FUNCTION_VAR_n + # call ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n + # call ::= expr {expr}^n CALL_FUNCTION_KW_n + + # classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n + # """ + # args_pos, args_kw = self.get_pos_kw(token) + + # # Additional exprs for * and ** args: + # # 0 if neither + # # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW + # # 2 for * and ** args (CALL_FUNCTION_VAR_KW). + # # Yes, this computation based on instruction name is a little bit hoaky. + # nak = (len(opname) - len("CALL_FUNCTION")) // 3 + # uniq_param = args_kw + args_pos + # if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops): + # rule = ( + # "async_call ::= expr " + # + ("expr " * args_pos) + # + ("kwarg " * args_kw) + # + "expr " * nak + # + token.kind + # + " GET_AWAITABLE LOAD_CONST YIELD_FROM" + # ) + # self.add_unique_rule(rule, token.kind, uniq_param, customize) + # self.add_unique_rule( + # "expr ::= async_call", token.kind, uniq_param, customize + # ) + + # if opname.startswith("CALL_FUNCTION_VAR"): + # token.kind = self.call_fn_name(token) + # if opname.endswith("KW"): + # kw = "expr " + # else: + # kw = "" + # rule = ( + # "call ::= expr expr " + # + ("expr " * args_pos) + # + ("kwarg " * args_kw) + # + kw + # + token.kind + # ) + + # # Note: semantic actions make use of the fact of whether "args_pos" + # # zero or not in creating a template rule. + # self.add_unique_rule(rule, token.kind, args_pos, customize) + # else: + # token.kind = self.call_fn_name(token) + # uniq_param = args_kw + args_pos + + # # Note: 3.5+ have subclassed this method; so we don't handle + # # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here. + # rule = ( + # "call ::= expr " + # + ("expr " * args_pos) + # + ("kwarg " * args_kw) + # + "expr " * nak + # + token.kind + # ) + + # self.add_unique_rule(rule, token.kind, uniq_param, customize) + + # if "LOAD_BUILD_CLASS" in self.seen_ops: + # if ( + # next_token == "CALL_FUNCTION" + # and next_token.attr == 1 + # and args_pos > 1 + # ): + # rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % ( + # ("expr " * (args_pos - 1)), + # opname, + # args_pos, + # ) + # self.add_unique_rule(rule, token.kind, uniq_param, customize) + + def customize_grammar_rules_full38(self, tokens, customize): + + self.customize_grammar_rules_lambda38(tokens, customize) + self.customize_reduce_checks_full38(tokens, customize) + self.remove_rules_38() + + # include instructions that don't need customization, + # but we'll do a finer check after the rough breakout. + customize_instruction_basenames = frozenset( + ( + "BEFORE", + "BUILD", + "CALL", + "CONTINUE", + "DELETE", + "FORMAT", + "GET", + "JUMP", + "LOAD", + "LOOKUP", + "MAKE", + "RETURN", + "RAISE", + "SETUP", + "UNPACK", + "WITH", + ) + ) + + # Opcode names in the custom_ops_processed set have rules that get added + # unconditionally and the rules are constant. So they need to be done + # only once and if we see the opcode a second we don't have to consider + # adding more rules. + # + custom_ops_processed = set() + + # A set of instruction operation names that exist in the token stream. + # We use this to customize the grammar that we create. + # 2.6-compatible set comprehensions + + # The initial initialization is done in lambea_expr.py + self.seen_ops = frozenset([t.kind for t in tokens]) + self.seen_op_basenames = frozenset( + [opname[: opname.rfind("_")] for opname in self.seen_ops] + ) + + # Loop over instructions adding custom grammar rules based on + # a specific instruction seen. + + if "PyPy" in customize: + self.addRule( + """ + stmt ::= assign3_pypy + stmt ::= assign2_pypy + assign3_pypy ::= expr expr expr store store store + assign2_pypy ::= expr expr store store + """, + nop_func, + ) + + for i, token in enumerate(tokens): + opname = token.kind + + # Do a quick breakout before testing potentially + # each of the dozen or so instruction in if elif. + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_ops_processed + ): + continue + + opname_base = opname[: opname.rfind("_")] + + # The order of opname listed is roughly sorted below + + if opname == "LOAD_ASSERT" and "PyPy" in customize: + rules_str = """ + stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM + """ + self.add_unique_doc_rules(rules_str, customize) + + elif opname == "BEFORE_ASYNC_WITH": + rules_str = """ + stmt ::= async_with_stmt + stmt ::= async_with_as_stmt + c_stmt ::= c_async_with_stmt + """ + + if self.version < (3, 8): + rules_str += """ + stmt ::= async_with_stmt SETUP_ASYNC_WITH + c_stmt ::= c_async_with_stmt SETUP_ASYNC_WITH + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + c_async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + async_with_post + c_async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts_opt + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store + c_suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + async_with_post + """ + else: + rules_str += """ + async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH + async_with_post ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts + POP_TOP POP_BLOCK + async_with_post + c_async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts + POP_TOP POP_BLOCK + async_with_post + c_async_with_stmt ::= async_with_stmt + async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts + POP_BLOCK + BEGIN_FINALLY + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD + POP_BLOCK + BEGIN_FINALLY + COME_FROM_ASYNC_WITH + WITH_AWAITABLE + LOAD_CONST + YEILD_FROM + WITH_CLEANUP_FINISH + END_FINALLY + + async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_TOP POP_BLOCK + async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_TOP POP_BLOCK + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_BLOCK async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_BLOCK async_with_post + """ + self.addRule(rules_str, nop_func) + + elif opname == "BUILD_STRING_2": + self.addRule( + """ + expr ::= formatted_value_debug + formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2 + formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2 + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "BUILD_STRING_3": + self.addRule( + """ + expr ::= formatted_value_debug + formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3 + formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3 + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname in frozenset( + ( + "CALL_FUNCTION", + "CALL_FUNCTION_EX_KW", + "CALL_FUNCTION_VAR_KW", + "CALL_FUNCTION_VAR", + "CALL_FUNCTION_VAR_KW", + ) + ) or opname.startswith("CALL_FUNCTION_KW"): + + if opname == "CALL_FUNCTION" and token.attr == 1: + rule = """ + classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1 + classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1 + """ + self.addRule(rule, nop_func) + + # self.custom_classfunc_rule(opname, token, customize, tokens[i + 1]) + # Note: don't add to custom_ops_processed. + + elif opname_base == "CALL_METHOD": + # PyPy and Python 3.7+ only - DRY with parse2 + + if opname == "CALL_METHOD_KW": + args_kw = token.attr + rules_str = """ + expr ::= call_kw_pypy37 + pypy_kw_keys ::= LOAD_CONST + """ + self.add_unique_doc_rules(rules_str, customize) + rule = ( + "call_kw_pypy37 ::= expr " + + ("expr " * args_kw) + + " pypy_kw_keys " + + opname + ) + else: + args_pos, args_kw = self.get_pos_kw(token) + # number of apply equiv arguments: + nak = (len(opname_base) - len("CALL_METHOD")) // 3 + rule = ( + "call ::= expr " + + ("expr " * args_pos) + + ("kwarg " * args_kw) + + "expr " * nak + + opname + ) + + self.add_unique_rule(rule, opname, token.attr, customize) + + elif opname == "CONTINUE": + self.addRule("continue ::= CONTINUE", nop_func) + custom_ops_processed.add(opname) + elif opname == "CONTINUE_LOOP": + self.addRule("continue ::= CONTINUE_LOOP", nop_func) + custom_ops_processed.add(opname) + elif opname == "DELETE_ATTR": + self.addRule("delete ::= expr DELETE_ATTR", nop_func) + custom_ops_processed.add(opname) + elif opname == "DELETE_DEREF": + self.addRule( + """ + stmt ::= del_deref_stmt + del_deref_stmt ::= DELETE_DEREF + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "DELETE_SUBSCR": + self.addRule( + """ + delete ::= delete_subscript + delete_subscript ::= expr expr DELETE_SUBSCR + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "FORMAT_VALUE_ATTR": + self.addRule( + """ + expr ::= formatted_value_debug + formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2 + expr FORMAT_VALUE_ATTR + formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2 + formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2 + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "GET_AITER": + self.addRule( + """ + async_for ::= GET_AITER _come_froms + SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK + + async_for_stmt38 ::= expr async_for + store for_block + COME_FROM_FINALLY + END_ASYNC_FOR + + # FIXME: come froms after the else_suite or + # END_ASYNC_FOR distinguish which of for / forelse + # is used. Add come froms and check of add up + # control-flow detection phase. + # async_forelse_stmt38 ::= expr async_for store + # for_block COME_FROM_FINALLY END_ASYNC_FOR + # else_suite + + async_forelse_stmt38 ::= expr async_for + store for_block + COME_FROM_FINALLY + END_ASYNC_FOR + else_suite + POP_TOP COME_FROM + + stmt ::= async_for_stmt38 + stmt ::= async_forelse_stmt38 + stmt ::= generator_exp_async + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "GET_ANEXT": + self.addRule( + """ + stmt ::= genexpr_func_async + stmt ::= BUILD_SET_0 genexpr_func_async + RETURN_VALUE + _come_froms + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "JUMP_IF_NOT_DEBUG": + self.addRule( + """ + stmt ::= assert_pypy + stmt ::= assert2_pypy", nop_func) + assert_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE + LOAD_ASSERT RAISE_VARARGS_1 COME_FROM + assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr POP_JUMP_IF_TRUE + LOAD_ASSERT expr CALL_FUNCTION_1 + RAISE_VARARGS_1 COME_FROM + assert2_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE + LOAD_ASSERT expr CALL_FUNCTION_1 + RAISE_VARARGS_1 COME_FROM, + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "LOAD_CLASSDEREF": + # Python 3.4+ + self.addRule("expr ::= LOAD_CLASSDEREF", nop_func) + custom_ops_processed.add(opname) + + elif opname == "LOAD_CLASSNAME": + self.addRule("expr ::= LOAD_CLASSNAME", nop_func) + custom_ops_processed.add(opname) + + elif opname == "RAISE_VARARGS_0": + self.addRule( + """ + stmt ::= raise_stmt0 + last_stmt ::= raise_stmt0 + raise_stmt0 ::= RAISE_VARARGS_0 + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "RAISE_VARARGS_1": + self.addRule( + """ + stmt ::= raise_stmt1 + last_stmt ::= raise_stmt1 + raise_stmt1 ::= expr RAISE_VARARGS_1 + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "RAISE_VARARGS_2": + self.addRule( + """ + stmt ::= raise_stmt2 + last_stmt ::= raise_stmt2 + raise_stmt2 ::= expr expr RAISE_VARARGS_2 + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "RETURN_VALUE_LAMBDA": + self.addRule( + """ + return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "SETUP_EXCEPT": + self.addRule( + """ + try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler opt_come_from_except + c_try_except ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK + c_except_handler opt_come_from_except + stmt ::= tryelsestmt3 + tryelsestmt3 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler COME_FROM else_suite + opt_come_from_except + + tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler else_suite come_from_except_clauses + + tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler else_suite come_froms + + c_stmt ::= c_tryelsestmt + c_tryelsestmt ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK + c_except_handler + come_any_froms else_suitec + come_from_except_clauses + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "WITH_CLEANUP_START": + rules_str = """ + stmt ::= with_null + with_null ::= with_suffix + with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + """ + self.addRule(rules_str, nop_func) + + # FIXME: reconcile with same code in lambda_custom.py + elif opname == "SETUP_WITH": + rules_str = """ + stmt ::= with + stmt ::= withasstmt + c_stmt ::= c_with + + c_with ::= expr SETUP_WITH POP_TOP + c_suite_stmts_opt + COME_FROM_WITH + with_suffix + c_with ::= expr SETUP_WITH POP_TOP + c_suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + + with ::= expr SETUP_WITH POP_TOP + suite_stmts_opt + COME_FROM_WITH + with_suffix + + withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH + with_suffix + + with ::= expr + SETUP_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + withasstmt ::= expr + SETUP_WITH store suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + + with ::= expr + SETUP_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + withasstmt ::= expr + SETUP_WITH store suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + """ + if self.version < (3, 8): + rules_str += """ + with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + LOAD_CONST + with_suffix + """ + else: + rules_str += """ + # A return at the end of a withas stmt can be this. + # FIXME: should this be a different kind of return? + return ::= return_expr POP_BLOCK + ROT_TWO + BEGIN_FINALLY + WITH_CLEANUP_START + WITH_CLEANUP_FINISH + POP_FINALLY + RETURN_VALUE + + with ::= expr + SETUP_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + + + withasstmt ::= expr + SETUP_WITH store suite_stmts + POP_BLOCK LOAD_CONST COME_FROM_WITH + + withasstmt ::= expr + SETUP_WITH store suite_stmts + POP_BLOCK BEGIN_FINALLY COME_FROM_WITH + with_suffix + + # withasstmt ::= expr SETUP_WITH store suite_stmts + # COME_FROM expr COME_FROM POP_BLOCK ROT_TWO + # BEGIN_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH + # POP_FINALLY RETURN_VALUE COME_FROM_WITH + # WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + + with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_WITH + with_suffix + """ + self.addRule(rules_str, nop_func) + pass + + return + + def customize_reduce_checks_full38(self, tokens, customize): + """ + Extra tests when a reduction is made in the full grammar. + + Reductions here are extended from those used in the lambda grammar + """ + self.remove_rules_38() + + self.check_reduce["and"] = "AST" + self.check_reduce["and_cond"] = "AST" + self.check_reduce["and_not"] = "AST" + self.check_reduce["annotate_tuple"] = "tokens" + self.check_reduce["aug_assign1"] = "AST" + self.check_reduce["aug_assign2"] = "AST" + self.check_reduce["c_forelsestmt38"] = "AST" + self.check_reduce["c_try_except"] = "AST" + self.check_reduce["c_tryelsestmt"] = "AST" + self.check_reduce["if_and_stmt"] = "AST" + self.check_reduce["if_and_elsestmtc"] = "AST" + self.check_reduce["if_not_stmtc"] = "AST" + self.check_reduce["ifelsestmt"] = "AST" + self.check_reduce["ifelsestmtc"] = "AST" + self.check_reduce["iflaststmt"] = "AST" + self.check_reduce["iflaststmtc"] = "AST" + self.check_reduce["ifstmt"] = "AST" + self.check_reduce["ifstmtc"] = "AST" + self.check_reduce["ifstmts_jump"] = "AST" + self.check_reduce["ifstmts_jumpc"] = "AST" + self.check_reduce["import_as37"] = "tokens" + self.check_reduce["import_from37"] = "AST" + self.check_reduce["import_from_as37"] = "tokens" + self.check_reduce["lastc_stmt"] = "tokens" + self.check_reduce["list_if_not"] = "AST" + self.check_reduce["while1elsestmt"] = "tokens" + self.check_reduce["while1stmt"] = "tokens" + self.check_reduce["whilestmt"] = "tokens" + self.check_reduce["not_or"] = "AST" + self.check_reduce["or"] = "AST" + self.check_reduce["or_cond"] = "tokens" + self.check_reduce["testtrue"] = "tokens" + self.check_reduce["testfalsec"] = "tokens" + + self.check_reduce["break"] = "tokens" + self.check_reduce["forelselaststmt38"] = "AST" + self.check_reduce["forelselaststmtc38"] = "AST" + self.check_reduce["for38"] = "tokens" + self.check_reduce["ifstmt"] = "AST" + self.check_reduce["joined_str"] = "AST" + self.check_reduce["pop_return"] = "tokens" + self.check_reduce["whileTruestmt38"] = "AST" + self.check_reduce["whilestmt38"] = "tokens" + self.check_reduce["try_elsestmtl38"] = "AST" + + self.reduce_check_table["break"] = break_invalid + self.reduce_check_table["if_not_stmtc"] = if_not_stmtc_invalid + self.reduce_check_table["for38"] = for38_invalid + self.reduce_check_table["c_forelsestmt38"] = forelse38_invalid + self.reduce_check_table["forelselaststmt38"] = forelse38_invalid + self.reduce_check_table["forelselaststmtc38"] = forelse38_invalid + # self.reduce_check_table["joined_str"] = joined_str_check.joined_str_invalid + self.reduce_check_table["or"] = or_cond_check_invalid + self.reduce_check_table["pop_return"] = pop_return_check + self.reduce_check_table["whilestmt38"] = whilestmt38_check + self.reduce_check_table["whileTruestmt38"] = whileTruestmt38_check + + # Use update we don't destroy entries from lambda. + self.reduce_check_table.update( + { + # "ifelsestmt": ifelsestmt_ok, + "ifstmt": ifstmt, + # "import_from37": import_from37_ok, + } + ) + + self.check_reduce["ifelsestmt"] = "AST" + self.check_reduce["ifelsestmtc"] = "AST" + self.check_reduce["ifstmt"] = "AST" + # self.check_reduce["import_from37"] = "AST" + + def customize_grammar_rules38(self, tokens, customize): + Python37BaseParser.customize_grammar_rules37(self, tokens, customize) + self.customize_reduce_checks_lambda38() + self.customize_reduce_checks_full38(tokens, customize) + + # include instructions that don't need customization, + # but we'll do a finer check after the rough breakout. + customize_instruction_basenames = frozenset( + ( + "BEFORE", + "BUILD", + "CALL", + "CONTINUE", + "DELETE", + "FORMAT", + "GET", + "JUMP", + "LOAD", + "LOOKUP", + "MAKE", + "RETURN", + "RAISE", + "SETUP", + "UNPACK", + "WITH", + ) + ) + + # Opcode names in the custom_ops_processed set have rules that get added + # unconditionally and the rules are constant. So they need to be done + # only once and if we see the opcode a second we don't have to consider + # adding more rules. + # + custom_ops_processed = set() + + # A set of instruction operation names that exist in the token stream. + # We use this to customize the grammar that we create. + # 2.6-compatible set comprehensions + + # The initial initialization is done in lambda_expr.py + self.seen_ops = frozenset([t.kind for t in tokens]) + self.seen_op_basenames = frozenset( + [opname[: opname.rfind("_")] for opname in self.seen_ops] + ) + + # Loop over instructions adding custom grammar rules based on + # a specific instruction seen. + + if "PyPy" in customize: + self.addRule( + """ + stmt ::= assign3_pypy + stmt ::= assign2_pypy + assign3_pypy ::= expr expr expr store store store + assign2_pypy ::= expr expr store store + """, + nop_func, + ) + + for i, token in enumerate(tokens): + opname = token.kind + + # Do a quick breakout before testing potentially + # each of the dozen or so instruction in if elif. + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_ops_processed + ): + continue + + opname_base = opname[: opname.rfind("_")] + + # The order of opname listed is roughly sorted below + + if opname == "LOAD_ASSERT" and "PyPy" in customize: + rules_str = """ + stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM + """ + self.add_unique_doc_rules(rules_str, customize) + + elif opname == "BEFORE_ASYNC_WITH": + rules_str = """ + stmt ::= async_with_stmt + stmt ::= async_with_as_stmt + c_stmt ::= c_async_with_stmt + """ + if self.version < (3, 8): + rules_str += """ + async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM + SETUP_ASYNC_WITH + stmt ::= async_with_stmt SETUP_ASYNC_WITH + c_stmt ::= c_async_with_stmt SETUP_ASYNC_WITH + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + c_async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + async_with_post + c_async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts_opt + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store + c_suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + async_with_post + """ + else: + rules_str += """ + async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH + async_with_post ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts + POP_TOP POP_BLOCK + async_with_post + c_async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts + POP_TOP POP_BLOCK + async_with_post + c_async_with_stmt ::= async_with_stmt + async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts + POP_BLOCK + BEGIN_FINALLY + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD + POP_BLOCK + BEGIN_FINALLY + COME_FROM_ASYNC_WITH + WITH_AWAITABLE + LOAD_CONST + YEILD_FROM + WITH_CLEANUP_FINISH + END_FINALLY + + async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_TOP POP_BLOCK + async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_TOP POP_BLOCK + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_BLOCK async_with_post + c_async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_BLOCK async_with_post + """ + self.addRule(rules_str, nop_func) + + elif opname in frozenset( + ( + "CALL_FUNCTION", + "CALL_FUNCTION_EX_KW", + "CALL_FUNCTION_VAR_KW", + "CALL_FUNCTION_VAR", + "CALL_FUNCTION_VAR_KW", + ) + ) or opname.startswith("CALL_FUNCTION_KW"): + + if opname == "CALL_FUNCTION" and token.attr == 1: + rule = """ + classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1 + classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1 + """ + self.addRule(rule, nop_func) + + # self.custom_classfunc_rule(opname, token, customize, tokens[i + 1]) + # Note: don't add to custom_ops_processed. + + elif opname_base == "CALL_METHOD": + # PyPy and Python 3.7+ only - DRY with parse2 + + if opname == "CALL_METHOD_KW": + args_kw = token.attr + rules_str = """ + expr ::= call_kw_pypy37 + pypy_kw_keys ::= LOAD_CONST + """ + self.add_unique_doc_rules(rules_str, customize) + rule = ( + "call_kw_pypy37 ::= expr " + + ("expr " * args_kw) + + " pypy_kw_keys " + + opname + ) + else: + args_pos, args_kw = self.get_pos_kw(token) + # number of apply equiv arguments: + nak = (len(opname_base) - len("CALL_METHOD")) // 3 + rule = ( + "call ::= expr " + + ("expr " * args_pos) + + ("kwarg " * args_kw) + + "expr " * nak + + opname + ) + + self.add_unique_rule(rule, opname, token.attr, customize) + + elif opname == "CONTINUE": + self.addRule("continue ::= CONTINUE", nop_func) + custom_ops_processed.add(opname) + elif opname == "CONTINUE_LOOP": + self.addRule("continue ::= CONTINUE_LOOP", nop_func) + custom_ops_processed.add(opname) + elif opname == "DELETE_ATTR": + self.addRule("delete ::= expr DELETE_ATTR", nop_func) + custom_ops_processed.add(opname) + elif opname == "DELETE_DEREF": + self.addRule( + """ + stmt ::= del_deref_stmt + del_deref_stmt ::= DELETE_DEREF + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "DELETE_SUBSCR": + self.addRule( + """ + delete ::= delete_subscript + delete_subscript ::= expr expr DELETE_SUBSCR + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "GET_AITER": + self.addRule( + """ + stmt ::= generator_exp_async + stmt ::= genexpr_func_async + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "GET_ANEXT": + self.addRule( + """ + stmt ::= BUILD_SET_0 genexpr_func_async + RETURN_VALUE + bb_doms_end_opt + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "JUMP_IF_NOT_DEBUG": + self.addRule( + """ + stmt ::= assert_pypy + stmt ::= assert2_pypy", nop_func) + assert_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE + LOAD_ASSERT RAISE_VARARGS_1 COME_FROM + assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr POP_JUMP_IF_TRUE + LOAD_ASSERT expr CALL_FUNCTION_1 + RAISE_VARARGS_1 COME_FROM + assert2_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE + LOAD_ASSERT expr CALL_FUNCTION_1 + RAISE_VARARGS_1 COME_FROM, + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "LOAD_CLASSDEREF": + # Python 3.4+ + self.addRule("expr ::= LOAD_CLASSDEREF", nop_func) + custom_ops_processed.add(opname) + + elif opname == "LOAD_CLASSNAME": + self.addRule("expr ::= LOAD_CLASSNAME", nop_func) + custom_ops_processed.add(opname) + + elif opname == "RAISE_VARARGS_0": + self.addRule( + """ + stmt ::= raise_stmt0 + last_stmt ::= raise_stmt0 + raise_stmt0 ::= RAISE_VARARGS_0 + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "RAISE_VARARGS_1": + self.addRule( + """ + stmt ::= raise_stmt1 + last_stmt ::= raise_stmt1 + raise_stmt1 ::= expr RAISE_VARARGS_1 + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "RAISE_VARARGS_2": + self.addRule( + """ + stmt ::= raise_stmt2 + last_stmt ::= raise_stmt2 + raise_stmt2 ::= expr expr RAISE_VARARGS_2 + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "RETURN_VALUE_LAMBDA": + self.addRule( + """ + return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "SETUP_EXCEPT": + self.addRule( + """ + try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler opt_come_from_except + c_try_except ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK + c_except_handler opt_come_from_except + stmt ::= tryelsestmt3 + tryelsestmt3 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler COME_FROM else_suite + opt_come_from_except + + tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler else_suite come_from_except_clauses + + tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler else_suite come_froms + + c_stmt ::= c_tryelsestmt + c_tryelsestmt ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK + c_except_handler + come_any_froms else_suitec + come_from_except_clauses + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "WITH_CLEANUP_START": + rules_str = """ + stmt ::= with_null + with_null ::= with_suffix + with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + """ + self.addRule(rules_str, nop_func) + + # FIXME: reconcile with same code in lambda_custom.py + elif opname == "SETUP_WITH": + rules_str = """ + stmt ::= with + stmt ::= withasstmt + c_stmt ::= c_with + + c_with ::= expr SETUP_WITH POP_TOP + c_suite_stmts_opt + COME_FROM_WITH + with_suffix + c_with ::= expr SETUP_WITH POP_TOP + c_suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + + with ::= expr SETUP_WITH POP_TOP + suite_stmts_opt + COME_FROM_WITH + with_suffix + + withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH + with_suffix + + with ::= expr + SETUP_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + withasstmt ::= expr + SETUP_WITH store suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + + with ::= expr + SETUP_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + withasstmt ::= expr + SETUP_WITH store suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + """ + if self.version < (3, 8): + rules_str += """ + with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + LOAD_CONST + with_suffix + """ + else: + rules_str += """ + # A return at the end of a withas stmt can be this. + # FIXME: should this be a different kind of return? + return ::= return_expr POP_BLOCK + ROT_TWO + BEGIN_FINALLY + WITH_CLEANUP_START + WITH_CLEANUP_FINISH + POP_FINALLY + RETURN_VALUE + + with ::= expr + SETUP_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_WITH + with_suffix + + + withasstmt ::= expr + SETUP_WITH store suite_stmts + POP_BLOCK LOAD_CONST COME_FROM_WITH + + withasstmt ::= expr + SETUP_WITH store suite_stmts + POP_BLOCK BEGIN_FINALLY COME_FROM_WITH + with_suffix + + # withasstmt ::= expr SETUP_WITH store suite_stmts + # COME_FROM expr COME_FROM POP_BLOCK ROT_TWO + # BEGIN_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH + # POP_FINALLY RETURN_VALUE COME_FROM_WITH + # WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + + with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_WITH + with_suffix + """ + self.addRule(rules_str, nop_func) + pass + + return diff --git a/decompyle3/parsers/p38pypy/heads.py b/decompyle3/parsers/p38pypy/heads.py new file mode 100644 index 00000000..a5e91997 --- /dev/null +++ b/decompyle3/parsers/p38pypy/heads.py @@ -0,0 +1,53 @@ +""" +All of the specific kinds of canned parsers for Python 3.8 + +These are derived from "compile-modes" but we have others that +can be used to parse common part of a larger grammar. + +For example: +* a basic-block expression (no branching) +* an unadorned expression (no POP_TOP needed afterwards) +* A non-compound statement +""" +from decompyle3.parsers.p38pypy.full import Python38PyPyParser +from decompyle3.parsers.p38pypy.lambda_expr import Python38PyPyLambdaParser +from decompyle3.parsers.parse_heads import ( # FIXME: add; PythonParserSimpleStmt; PythonParserStmt + PythonParserEval, + PythonParserExec, + PythonParserExpr, + PythonParserLambda, + PythonParserSingle, +) + +# Make sure to list Python38... classes first so we prefer to inherit methods from that first. +# In particular methods like reduce_is_invalid() need to come from there rather than +# a more generic place. + + +class Python38PyPyParserEval(Python38PyPyLambdaParser, PythonParserEval): + def __init__(self, debug_parser): + PythonParserEval.__init__(self, debug_parser) + + +class Python38PyPyParserExec(Python38PyPyParser, PythonParserExec): + def __init__(self, debug_parser): + PythonParserExec.__init__(self, debug_parser) + + +class Python38PyPyParserExpr(Python38PyPyParser, PythonParserExpr): + def __init__(self, debug_parser): + PythonParserExpr.__init__(self, debug_parser) + + +# Understand: Python38LambdaParser has to come before PythonParserLambda or we get a +# MRO failure +class Python38PyPyParserLambda(Python38PyPyLambdaParser, PythonParserLambda): + def __init__(self, debug_parser): + PythonParserLambda.__init__(self, debug_parser) + + +# These classes are here just to get parser doc-strings for the +# various classes inherited properly and start_symbols set properly. +class Python38PyPyParserSingle(Python38PyPyParser, PythonParserSingle): + def __init__(self, debug_parser): + PythonParserSingle.__init__(self, debug_parser) diff --git a/decompyle3/parsers/p38pypy/lambda_custom.py b/decompyle3/parsers/p38pypy/lambda_custom.py new file mode 100644 index 00000000..2bc6397d --- /dev/null +++ b/decompyle3/parsers/p38pypy/lambda_custom.py @@ -0,0 +1,767 @@ +# Copyright (c) 2020-2024 Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Grammar Customization rules for Python 3.8's Lambda expression grammar. +""" + +from decompyle3.parsers.p37.base import Python37BaseParser +from decompyle3.parsers.p38pypy.base import Python38PyPyBaseParser +from decompyle3.parsers.parse_heads import nop_func + + +class Python38PyPyLambdaCustom(Python38PyPyBaseParser): + def __init__(self): + self.new_rules = set() + self.customized = {} + + def remove_rules_pypy38(self): + self.remove_rules( + """ + """ + ) + + def customize_grammar_rules_lambda38(self, tokens, customize): + Python38PyPyBaseParser.customize_grammar_rules38(self, tokens, customize) + + # self.remove_rules_pypy38() + self.check_reduce["call_kw"] = "AST" + + # For a rough break out on the first word. This may + # include instructions that don't need customization, + # but we'll do a finer check after the rough breakout. + customize_instruction_basenames = frozenset( + ( + "BEFORE", + "BUILD", + "CALL", + "DICT", + "GET", + "FORMAT", + "LIST", + "LOAD", + "MAKE", + "SETUP", + "UNPACK", + ) + ) + + # Opcode names in the custom_ops_processed set have rules that get added + # unconditionally and the rules are constant. So they need to be done + # only once and if we see the opcode a second we don't have to consider + # adding more rules. + # + custom_ops_processed = frozenset() + + # A set of instruction operation names that exist in the token stream. + # We use this customize the grammar that we create. + # 2.6-compatible set comprehensions + self.seen_ops = frozenset([t.kind for t in tokens]) + self.seen_op_basenames = frozenset( + [opname[: opname.rfind("_")] for opname in self.seen_ops] + ) + + custom_ops_processed = set(["DICT_MERGE"]) + + # Loop over instructions adding custom grammar rules based on + # a specific instruction seen. + + if "PyPy" in customize: + self.addRule( + """ + stmt ::= assign3_pypy + stmt ::= assign2_pypy + assign3_pypy ::= expr expr expr store store store + assign2_pypy ::= expr expr store store + """, + nop_func, + ) + + n = len(tokens) + + # Determine if we have an iteration CALL_FUNCTION_1. + has_get_iter_call_function1 = False + for i, token in enumerate(tokens): + if ( + token == "GET_ITER" + and i < n - 2 + and self.call_fn_name(tokens[i + 1]) == "CALL_FUNCTION_1" + ): + has_get_iter_call_function1 = True + + for i, token in enumerate(tokens): + opname = token.kind + + # Do a quick breakout before testing potentially + # each of the dozen or so instruction in if elif. + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_ops_processed + ): + continue + + opname_base = opname[: opname.rfind("_")] + + # Do a quick breakout before testing potentially + # each of the dozen or so instruction in if elif. + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_ops_processed + ): + continue + + if opname == "BEFORE_ASYNC_WITH": + rules_str = """ + async_with_post ::= COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + + stmt ::= async_with_as_stmt + async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + + async_with_stmt ::= expr + async_with_pre + POP_TOP + c_suite_stmts + POP_BLOCK + BEGIN_FINALLY + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD + POP_BLOCK + BEGIN_FINALLY + COME_FROM_ASYNC_WITH + WITH_CLEANUP_START + GET_AWAITABLE + LOAD_CONST + YIELD_FROM + WITH_CLEANUP_FINISH + END_FINALLY + + + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + async_with_post + """ + self.addRule(rules_str, nop_func) + + elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"): + if opname == "BUILD_CONST_DICT": + rule = f""" + add_consts ::= ADD_VALUE* + const_list ::= COLLECTION_START add_consts {opname} + dict ::= const_list + expr ::= dict + """ + else: + rule = f""" + add_consts ::= ADD_VALUE* + const_list ::= COLLECTION_START add_consts {opname} + expr ::= const_list + """ + self.addRule(rule, nop_func) + elif opname_base in ( + "BUILD_LIST", + "BUILD_SET", + "BUILD_SET_UNPACK", + "BUILD_TUPLE", + "BUILD_TUPLE_UNPACK", + ): + v = token.attr + + is_LOAD_CLOSURE = False + if opname_base == "BUILD_TUPLE": + # If is part of a "load_closure", then it is not part of a + # "list". + is_LOAD_CLOSURE = True + for j in range(v): + if tokens[i - j - 1].kind != "LOAD_CLOSURE": + is_LOAD_CLOSURE = False + break + if is_LOAD_CLOSURE: + rule = "load_closure ::= %s%s" % (("LOAD_CLOSURE " * v), opname) + self.add_unique_rule(rule, opname, token.attr, customize) + + elif opname_base == "BUILD_LIST": + v = token.attr + if v == 0: + rule_str = """ + list ::= BUILD_LIST_0 + list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND + list ::= list_unpack + """ + self.add_unique_doc_rules(rule_str, customize) + + elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL": + # FIXME: should this be parameterized by EX value? + self.addRule( + """expr ::= call_ex_kw3 + call_ex_kw3 ::= expr + build_tuple_unpack_with_call + expr + CALL_FUNCTION_EX_KW + """, + nop_func, + ) + + if not is_LOAD_CLOSURE or v == 0: + # We do this complicated test to speed up parsing of + # pathelogically long literals, especially those over 1024. + build_count = token.attr + thousands = build_count // 1024 + thirty32s = (build_count // 32) % 32 + if thirty32s > 0: + rule = "expr32 ::=%s" % (" expr" * 32) + self.add_unique_rule(rule, opname_base, build_count, customize) + pass + if thousands > 0: + self.add_unique_rule( + "expr1024 ::=%s" % (" expr32" * 32), + opname_base, + build_count, + customize, + ) + pass + collection = opname_base[opname_base.find("_") + 1 :].lower() + rule = ( + ("%s ::= " % collection) + + "expr1024 " * thousands + + "expr32 " * thirty32s + + "expr " * (build_count % 32) + + opname + ) + self.add_unique_rules(["expr ::= %s" % collection, rule], customize) + continue + continue + + elif opname.startswith("BUILD_STRING"): + v = token.attr + rules_str = """ + expr ::= joined_str + joined_str ::= %sBUILD_STRING_%d + """ % ( + "expr " * v, + v, + ) + self.add_unique_doc_rules(rules_str, customize) + if "FORMAT_VALUE_ATTR" in self.seen_ops: + rules_str = """ + formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING + expr ::= formatted_value_attr + """ + self.add_unique_doc_rules(rules_str, customize) + elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"): + v = token.attr + rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname) + self.addRule(rule, nop_func) + elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"): + v = token.attr + rule = ( + "build_tuple_unpack_with_call ::= " + + "expr1024 " * int(v // 1024) + + "expr32 " * int((v // 32) % 32) + + "expr " * (v % 32) + + opname + ) + self.addRule(rule, nop_func) + rule = "starred ::= %s %s" % ("expr " * v, opname) + self.addRule(rule, nop_func) + + elif opname == "FORMAT_VALUE": + rules_str = """ + expr ::= formatted_value1 + formatted_value1 ::= expr FORMAT_VALUE + """ + self.add_unique_doc_rules(rules_str, customize) + elif opname == "FORMAT_VALUE_ATTR": + rules_str = """ + expr ::= formatted_value2 + formatted_value2 ::= expr expr FORMAT_VALUE_ATTR + """ + self.add_unique_doc_rules(rules_str, customize) + + elif opname == "GET_AITER": + self.add_unique_doc_rules("get_aiter ::= expr GET_AITER", customize) + + if not {"MAKE_FUNCTION_0", "MAKE_FUNCTION_CLOSURE"} in self.seen_ops: + self.addRule( + """ + expr ::= dict_comp_async + expr ::= generator_exp_async + expr ::= list_comp_async + + dict_comp_async ::= LOAD_DICTCOMP + LOAD_STR + MAKE_FUNCTION_0 + get_aiter + CALL_FUNCTION_1 + + dict_comp_async ::= BUILD_MAP_0 LOAD_ARG + dict_comp_async + + generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0 + get_aiter CALL_FUNCTION_1 + + list_comp_async ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0 + get_aiter CALL_FUNCTION_1 + await + + list_comp_async ::= LOAD_CLOSURE + BUILD_TUPLE_1 + LOAD_LISTCOMP + LOAD_STR MAKE_FUNCTION_CLOSURE + get_aiter CALL_FUNCTION_1 + await + + set_comp_async ::= LOAD_SETCOMP + LOAD_STR + MAKE_FUNCTION_0 + get_aiter + CALL_FUNCTION_1 + + set_comp_async ::= LOAD_CLOSURE + BUILD_TUPLE_1 + LOAD_SETCOMP + LOAD_STR MAKE_FUNCTION_CLOSURE + get_aiter CALL_FUNCTION_1 + await + """, + nop_func, + ) + custom_ops_processed.add(opname) + + self.addRule( + """ + dict_comp_async ::= BUILD_MAP_0 LOAD_ARG + dict_comp_async + + expr ::= dict_comp_async + expr ::= generator_exp_async + expr ::= list_comp_async + expr ::= set_comp_async + + func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT + DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + END_FINALLY _come_froms + + # async_iter ::= block_break SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM + + get_aiter ::= expr GET_AITER + + list_afor ::= get_aiter list_afor2 + + list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2 + list_iter ::= list_afor + + + set_afor ::= get_aiter set_afor2 + set_iter ::= set_afor + + set_comp_async ::= BUILD_SET_0 LOAD_ARG + set_comp_async + + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "GET_ANEXT": + self.addRule( + """ + expr ::= genexpr_func_async + expr ::= BUILD_MAP_0 genexpr_func_async + expr ::= list_comp_async + + dict_comp_async ::= BUILD_MAP_0 genexpr_func_async + + async_iter ::= _come_froms + SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK + + func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM + + genexpr_func_async ::= LOAD_ARG async_iter + store + comp_iter + JUMP_LOOP + COME_FROM_FINALLY + END_ASYNC_FOR + + genexpr_func_async ::= LOAD_ARG func_async_prefix + store func_async_middle comp_iter + JUMP_LOOP COME_FROM + POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP + + list_afor2 ::= async_iter + store + list_iter + JUMP_LOOP + COME_FROM_FINALLY + END_ASYNC_FOR + + list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2 + + set_afor2 ::= async_iter + store + set_iter + JUMP_LOOP + COME_FROM_FINALLY + END_ASYNC_FOR + + set_afor2 ::= expr_or_arg + set_iter_async + + set_comp_async ::= BUILD_SET_0 set_afor2 + + set_iter_async ::= async_iter + store + set_iter + JUMP_LOOP + _come_froms + END_ASYNC_FOR + + return_expr_lambda ::= genexpr_func_async + LOAD_CONST RETURN_VALUE + RETURN_VALUE_LAMBDA + + return_expr_lambda ::= BUILD_SET_0 genexpr_func_async + RETURN_VALUE_LAMBDA + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "GET_AWAITABLE": + rule_str = """ + await ::= GET_AWAITABLE LOAD_CONST YIELD_FROM + await_expr ::= expr await + expr ::= await_expr + """ + self.add_unique_doc_rules(rule_str, customize) + + elif opname == "GET_ITER": + self.addRule( + """ + expr ::= get_iter + get_iter ::= expr GET_ITER + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "LOAD_ASSERT": + if "PyPy" in customize: + rules_str = """ + stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM + """ + self.add_unique_doc_rules(rules_str, customize) + + elif opname == "LOAD_ATTR": + self.addRule( + """ + expr ::= attribute + attribute ::= expr LOAD_ATTR + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "LOAD_CLOSURE": + self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func) + + elif opname == "LOAD_DICTCOMP": + if has_get_iter_call_function1: + rule_pat = "dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1" + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + pass + custom_ops_processed.add(opname) + + elif opname == "LOAD_GENEXPR": + self.addRule("load_genexpr ::= LOAD_GENEXPR", nop_func) + custom_ops_processed.add(opname) + + elif opname == "LOAD_LISTCOMP": + self.add_unique_rule( + "expr ::= list_comp", opname, token.attr, customize + ) + custom_ops_processed.add(opname) + + elif opname == "LOAD_NAME": + if ( + token.attr == "__annotations__" + and "SETUP_ANNOTATIONS" in self.seen_ops + ): + token.kind = "LOAD_ANNOTATION" + self.addRule( + """ + stmt ::= SETUP_ANNOTATIONS + stmt ::= ann_assign + ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR + """, + nop_func, + ) + pass + elif opname == "LOAD_SETCOMP": + # Should this be generalized and put under MAKE_FUNCTION? + if has_get_iter_call_function1: + self.addRule("expr ::= set_comp", nop_func) + rule_pat = "set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1" + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + pass + custom_ops_processed.add(opname) + elif opname == "LOOKUP_METHOD": + # A PyPy speciality - DRY with parse3 + self.addRule( + """ + expr ::= attribute + attribute ::= expr LOOKUP_METHOD + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "MAKE_FUNCTION_CLOSURE": + if "LOAD_DICTCOMP" in self.seen_ops: + # Is there something general going on here? + rule = """ + dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR + MAKE_FUNCTION_CLOSURE expr + GET_ITER CALL_FUNCTION_1 + """ + self.addRule(rule, nop_func) + elif "LOAD_SETCOMP" in self.seen_ops: + rule = """ + set_comp ::= load_closure LOAD_SETCOMP LOAD_STR + MAKE_FUNCTION_CLOSURE expr + GET_ITER CALL_FUNCTION_1 + """ + self.addRule(rule, nop_func) + + elif opname == "MAKE_FUNCTION_CLOSURE_POS": + + args_pos, args_kw, annotate_args, closure = token.attr + stack_count = args_pos + args_kw + annotate_args + + if closure: + + if args_pos: + # This was seen ion line 447 of Python 3.8 + # This is needed for Python 3.8 line 447 of site-packages/nltk/tgrep.py + # line 447: + # lambda i: lambda n, m=None, l=None: ... + # which has + # L. 447 0 LOAD_CONST (None, None) + # 2 LOAD_CLOSURE 'i' + # 4 LOAD_CLOSURE 'predicate' + # 6 BUILD_TUPLE_2 2 + # 8 LOAD_LAMBDA '>' + # 10 LOAD_STR '_tgrep_relation_action....' + # 12 MAKE_FUNCTION_CLOSURE_POS 'default, closure' + # FIXME: Possibly we need to generalize for more nested lambda's of lambda's? + rule = """ + expr ::= lambda_body + lambda_body ::= %s%s%s%s + """ % ( + "expr " * stack_count, + "load_closure " * closure, + "BUILD_TUPLE_2 LOAD_LAMBDA LOAD_STR ", + opname, + ) + self.add_unique_rule(rule, opname, token.attr, customize) + rule = """ + expr ::= lambda_body + lambda_body ::= %s%s%s%s + """ % ( + "expr " * stack_count, + "load_closure " * closure, + "LOAD_LAMBDA LOAD_STR ", + opname, + ) + + else: + rule = """ + expr ::= lambda_body + lambda_body ::= %s%s%s""" % ( + "load_closure " * closure, + "LOAD_LAMBDA LOAD_STR ", + opname, + ) + self.add_unique_rule(rule, opname, token.attr, customize) + + elif opname == "SETUP_WITH": + rules_str = """ + with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH + WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + + # Removes POP_BLOCK LOAD_CONST from 3.6- + withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH + WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + """ + if self.version < (3, 8): + rules_str += """ + with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + LOAD_CONST + WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + """ + else: + rules_str += """ + with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_WITH + WITH_CLEANUP_START WITH_CLEANUP_FINISH + END_FINALLY + """ + self.addRule(rules_str, nop_func) + pass + pass + + def custom_classfunc_rule(self, opname, token, customize, next_token): + + args_pos, args_kw = self.get_pos_kw(token) + + # Additional exprs for * and ** args: + # 0 if neither + # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW + # 2 for * and ** args (CALL_FUNCTION_VAR_KW). + # Yes, this computation based on instruction name is a little bit hoaky. + nak = (len(opname) - len("CALL_FUNCTION")) // 3 + uniq_param = args_kw + args_pos + + if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops): + rule_str = """ + await ::= GET_AWAITABLE LOAD_CONST YIELD_FROM + await_expr ::= expr await + expr ::= await_expr + """ + self.add_unique_doc_rules(rule_str, customize) + rule = ( + "async_call ::= expr " + + ("expr " * args_pos) + + ("kwarg " * args_kw) + + "expr " * nak + + token.kind + + " GET_AWAITABLE LOAD_CONST YIELD_FROM" + ) + self.add_unique_rule(rule, token.kind, uniq_param, customize) + self.add_unique_rule( + "expr ::= async_call", token.kind, uniq_param, customize + ) + + if opname.startswith("CALL_FUNCTION_KW"): + self.addRule("expr ::= call_kw36", nop_func) + values = "expr " * token.attr + rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals()) + self.add_unique_rule(rule, token.kind, token.attr, customize) + elif opname == "CALL_FUNCTION_EX_KW": + # Note that we don't add to customize token.kind here. Instead, the non-terminal + # names call_ex_kw... are is in semantic actions. + self.addRule( + """expr ::= call_ex_kw4 + call_ex_kw4 ::= expr + expr + expr + CALL_FUNCTION_EX_KW + """, + nop_func, + ) + if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames: + self.addRule( + """expr ::= call_ex_kw + call_ex_kw ::= expr expr build_map_unpack_with_call + CALL_FUNCTION_EX_KW + """, + nop_func, + ) + if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames: + # FIXME: should this be parameterized by EX value? + self.addRule( + """expr ::= call_ex_kw3 + call_ex_kw3 ::= expr + build_tuple_unpack_with_call + expr + CALL_FUNCTION_EX_KW + """, + nop_func, + ) + if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames: + # FIXME: should this be parameterized by EX value? + self.addRule( + """expr ::= call_ex_kw2 + call_ex_kw2 ::= expr + build_tuple_unpack_with_call + build_map_unpack_with_call + CALL_FUNCTION_EX_KW + """, + nop_func, + ) + + elif opname == "CALL_FUNCTION_EX": + self.addRule( + """ + expr ::= call_ex + starred ::= expr + call_ex ::= expr starred CALL_FUNCTION_EX + """, + nop_func, + ) + if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops: + self.addRule( + """ + expr ::= call_ex_kw + call_ex_kw ::= expr expr + build_map_unpack_with_call CALL_FUNCTION_EX + """, + nop_func, + ) + if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops: + self.addRule( + """ + expr ::= call_ex_kw3 + call_ex_kw3 ::= expr + build_tuple_unpack_with_call + %s + CALL_FUNCTION_EX + """ + % "expr " + * token.attr, + nop_func, + ) + pass + + # FIXME: Is this right? + self.addRule( + """ + expr ::= call_ex_kw4 + call_ex_kw4 ::= expr + expr + expr + CALL_FUNCTION_EX + """, + nop_func, + ) + pass + else: + Python37BaseParser.custom_classfunc_rule( + self, opname, token, customize, next_token + ) diff --git a/decompyle3/parsers/p38pypy/lambda_expr.py b/decompyle3/parsers/p38pypy/lambda_expr.py new file mode 100644 index 00000000..76c1872d --- /dev/null +++ b/decompyle3/parsers/p38pypy/lambda_expr.py @@ -0,0 +1,96 @@ +# Copyright (c) 2020-2024 Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Differences over Python 3.7 for Python 3.8 in the Earley-algorithm lambda grammar +""" + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG + +from decompyle3.parsers.p37.lambda_expr import Python37LambdaParser +from decompyle3.parsers.p38pypy.lambda_custom import Python38PyPyLambdaCustom +from decompyle3.parsers.parse_heads import PythonBaseParser, PythonParserLambda + + +class Python38PyPyLambdaParser( + Python38PyPyLambdaCustom, Python37LambdaParser, PythonParserLambda +): + def p_38walrus(self, args): + """ + # named_expr is also known as the "walrus op" := + expr ::= named_expr + named_expr ::= expr DUP_TOP store + """ + + def p_lambda_start(self, args): + """ + return_expr_lambda ::= genexpr_func LOAD_CONST RETURN_VALUE_LAMBDA + + """ + + def p_pypy38_comprehension(self, args): + """ + list_comp ::= LOAD_ARG + BUILD_LIST_FROM_ARG + COME_FROM FOR_ITER + store lc_body + JUMP_LOOP _come_froms + + lc_body ::= expr LIST_APPEND + """ + + def p_expr38(self, args): + """ + expr ::= if_exp_compare38 + + if_exp_compare38 ::= or_in_ifexp jump_if_false_cf expr jf_cfs expr come_froms + + list_iter ::= list_if_not38 + list_if_not38 ::= expr pjump_ift expr pjump_ift _come_froms list_iter + come_from_opt + + or_in_ifexp ::= expr_pjit expr + or_in_ifexp ::= or_in_ifexp POP_JUMP_IF_TRUE expr + """ + + def __init__( + self, + start_symbol: str = "lambda_start", + debug_parser: dict = PARSER_DEFAULT_DEBUG, + ): + PythonParserLambda.__init__( + self, debug_parser=debug_parser, start_symbol=start_symbol + ) + PythonBaseParser.__init__( + self, start_symbol=start_symbol, debug_parser=debug_parser + ) + Python38PyPyLambdaCustom.__init__(self) + + def customize_grammar_rules(self, tokens, customize): + self.customize_grammar_rules_lambda38(tokens, customize) + + +if __name__ == "__main__": + # Check grammar + from decompyle3.parsers.dump import dump_and_check + + p = Python38PyPyLambdaParser() + modified_tokens = set( + """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST + """.split() + ) + + dump_and_check(p, (3, 8), modified_tokens) diff --git a/decompyle3/scanners/scanner37base.py b/decompyle3/scanners/scanner37base.py index 549edaa8..b0a33ef7 100644 --- a/decompyle3/scanners/scanner37base.py +++ b/decompyle3/scanners/scanner37base.py @@ -227,20 +227,19 @@ def bound_collection( new_tokens.append( Token( opname="COLLECTION_START", - optype="pseudo", attr=collection_enum, pattr=collection_type, offset=f"{start_offset}_0", has_arg=True, opc=self.opc, has_extended_arg=False, + optype=None, ) ) for j in range(collection_start, i): new_tokens.append( Token( opname="ADD_VALUE", - optype="pseudo", attr=tokens[j].attr, pattr=tokens[j].pattr, offset=tokens[j].offset, @@ -248,6 +247,7 @@ def bound_collection( linestart=tokens[j].linestart, opc=self.opc, has_extended_arg=False, + optype=tokens[j].optype, ) ) new_tokens.append( @@ -371,20 +371,21 @@ def tokens_append(j, token): # but the operator and operand properties come from the other # instruction self.insts[i] = Instruction( - jump_inst.opname, - jump_inst.opcode, - jump_inst.optype, - jump_inst.inst_size, - jump_inst.arg, - jump_inst.argval, - jump_inst.argrepr, - jump_inst.has_arg, - inst.offset, - inst.starts_line, - inst.is_jump_target, - inst.has_extended_arg, - None, - None, + opcode=jump_inst.opcode, + opname=jump_inst.opname, + arg=jump_inst.arg, + argval=jump_inst.argval, + argrepr=jump_inst.argrepr, + offset=inst.offset, + starts_line=inst.starts_line, + is_jump_target=inst.is_jump_target, + positions=None, + optype=jump_inst.optype, + has_arg=jump_inst.has_arg, + inst_size=jump_inst.inst_size, + has_extended_arg=inst.has_extended_arg, + tos_str=None, + start_offset=None, ) # Get jump targets @@ -431,13 +432,13 @@ def tokens_append(j, token): j, Token( opname=come_from_name, - optype="pseudo", attr=jump_offset, pattr=repr(jump_offset), offset="%s_%s" % (inst.offset, jump_idx), has_arg=True, opc=self.opc, has_extended_arg=False, + optype=inst.optype, ), ) jump_idx += 1 @@ -515,6 +516,7 @@ def tokens_append(j, token): has_arg=inst.has_arg, opc=self.opc, has_extended_arg=inst.has_extended_arg, + optype=inst.optype, ), ) continue @@ -1091,5 +1093,8 @@ def next_except_jump(self, start): for t in tokens: print(t) else: - print(f"Need to be Python 3.7 to demo; I am version {version_tuple_to_str()}.") + print( + "Need to be Python 3.7..3.8 to demo; " + f"I am version {version_tuple_to_str()}." + ) pass diff --git a/decompyle3/scanners/tok.py b/decompyle3/scanners/tok.py index 18a6fa99..5e480751 100644 --- a/decompyle3/scanners/tok.py +++ b/decompyle3/scanners/tok.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2020, 2023 by Rocky Bernstein +# Copyright (c) 2016-2020, 2023-2024 by Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock # @@ -17,7 +17,7 @@ import re import sys -from typing import Optional +from typing import Optional, Union def off2int(offset: int, prefer_last=True) -> int: @@ -60,10 +60,9 @@ class Token: def __init__( self, opname: str, - optype: Optional[str] = None, attr=None, pattr=None, - offset=-1, + offset: Union[int, str] = -1, linestart=None, op=None, has_arg=None, @@ -76,6 +75,7 @@ def __init__( has_extended_arg=False, tos_str=None, start_offset=None, + optype: Optional[str] = None, ): self.attr = attr self.has_arg = has_arg diff --git a/decompyle3/semantics/consts.py b/decompyle3/semantics/consts.py index 301fb664..c948b707 100644 --- a/decompyle3/semantics/consts.py +++ b/decompyle3/semantics/consts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2020, 2022-2023 by Rocky Bernstein +# Copyright (c) 2017-2020, 2022-2024 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -141,7 +141,7 @@ Token(doc_load, pattr=doc_string, attr=doc_string), ] ), - SyntaxTree("store", [Token("STORE_NAME", pattr="__doc__")]), + SyntaxTree("store", [Token("STORE_NAME", pattr="__doc__", optype="name")]), ], ) @@ -149,10 +149,10 @@ "assign", [ SyntaxTree( - "expr", [Token("LOAD_NAME", pattr="__name__", offset=0, has_arg=True)] + "expr", [Token("LOAD_NAME", pattr="__name__", offset=0, has_arg=True, optype="name")] ), SyntaxTree( - "store", [Token("STORE_NAME", pattr="__module__", offset=3, has_arg=True)] + "store", [Token("STORE_NAME", pattr="__module__", offset=3, has_arg=True, optype="name")] ), ], ) diff --git a/decompyle3/semantics/gencomp.py b/decompyle3/semantics/gencomp.py index d74b8dba..6a5d2233 100644 --- a/decompyle3/semantics/gencomp.py +++ b/decompyle3/semantics/gencomp.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023 by Rocky Bernstein +# Copyright (c) 2022-2024 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -608,7 +608,7 @@ def comprehension_walk_newer( # Here is where we handle nested list iterations which # includes their corresponding "if" conditions. - if tree in ("list_comp", "set_comp"): + if tree in ("list_comp", "set_comp") and not self.is_pypy: list_iter = tree[1] assert list_iter in ("list_iter", "set_iter") list_for = list_iter[0] diff --git a/decompyle3/semantics/n_actions.py b/decompyle3/semantics/n_actions.py index 918481db..8ea6aea6 100644 --- a/decompyle3/semantics/n_actions.py +++ b/decompyle3/semantics/n_actions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023 by Rocky Bernstein +# Copyright (c) 2022-2024 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -228,9 +228,16 @@ def n_const_list(self, node: SyntaxTree): keys = flat_elems[-1].attr assert isinstance(keys, tuple) assert len(keys) == len(flat_elems) - 1 + for i, elem in enumerate(flat_elems[:-1]): assert elem.kind == "ADD_VALUE" - value = elem.pattr + if elem.optype in ("local", "name"): + value = elem.attr + elif elem.optype == "const" and not isinstance(elem.attr, str): + value = elem.attr + else: + value = elem.pattr + if elem.linestart is not None: if elem.linestart != self.line_number: sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1] @@ -243,7 +250,13 @@ def n_const_list(self, node: SyntaxTree): else: for elem in flat_elems: assert elem.kind == "ADD_VALUE" - value = elem.pattr + if elem.optype in ("local", "name"): + value = elem.attr + elif elem.optype == "const" and not isinstance(elem.attr, str): + value = elem.attr + else: + value = elem.pattr + if elem.linestart is not None: if elem.linestart != self.line_number: sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1] @@ -945,7 +958,7 @@ def n_list_comp(self, node): # there isn't an iter_index at the top level list_iter_index = None else: - list_iter_index = 1 + list_iter_index = 5 if self.is_pypy else 1 self.comprehension_walk_newer(node, list_iter_index, 0) self.write("]") self.prune() diff --git a/decompyle3/semantics/transform.py b/decompyle3/semantics/transform.py index 7a68ea59..d1032dae 100644 --- a/decompyle3/semantics/transform.py +++ b/decompyle3/semantics/transform.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2020, 2022-2023 by Rocky Bernstein +# Copyright (c) 2019-2020, 2022-2024 by Rocky Bernstein # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -551,18 +551,11 @@ def transform( try: for i in range(n): if is_docstring(self.ast[i], code.co_consts): - load_const = self.ast[i].first_child() + load_const = copy(self.ast[i].first_child()) + store_name = copy(self.ast[i].last_child()) docstring_ast = SyntaxTree( "docstring", - [ - Token( - "LOAD_STR", - has_arg=True, - offset=0, - attr=load_const.attr, - pattr=load_const.pattr, - ) - ], + [load_const, store_name], transformed_by="transform", ) del self.ast[i] diff --git a/test/Makefile b/test/Makefile index a4135c16..35a173f8 100644 --- a/test/Makefile +++ b/test/Makefile @@ -10,8 +10,9 @@ GIT2CL ?= git2cl PYTHON ?= python PYTHONLIB_OPTS ?= -PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2 | head -1) -NATIVE_CHECK = check-$(PYTHON_VERSION) +IS_PYPY = $(shell $(PYTHON) -c 'import platform; print("pypy" if platform.python_implementation() == "PyPy" else "")') +PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2 | head -1)$(IS_PYPY) +NATIVE_CHECK = check-$(PYTHON_VERSION)$(IS_PYPY) # Set COMPILE='--compile' to force compilation before check COMPILE ?= @@ -24,7 +25,7 @@ check-short: #: Run tests for a given version check: check-code-fragments - $(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2 | head -1`; \ + $(PYTHON) -V && \ $(MAKE) check-$(PYTHON_VERSION) #: Run working tests from Python 3.7 @@ -40,6 +41,10 @@ check-3.8: $(PYTHON) test_pythonlib.py --bytecode-3.8 $(PYTHONLIB_OPTS) $(COMPILE) --run --verify $(PYTHON) test_pythonlib.py --bytecode-3.7 $(COMPILE) +#: Run working tests from Python 3.8 +check-3.8pypy: + $(PYTHON) test_pythonlib.py --bytecode-3.8pypy $(PYTHONLIB_OPTS) $(COMPILE) --run --verify + check-3.9: check-bytecode @echo "Note that we do not support decompiling Python 3.9 bytecode - no 3.9 tests run" $(PYTHON) test_pythonlib.py --bytecode-3.7 $(COMPILE) @@ -63,9 +68,14 @@ check-bytecode-38: check-code-fragments check-bytecode-38-run $(PYTHON) test_pythonlib.py \ --bytecode-3.7 --bytecode-3.8 +#: Check deparsing bytecode PYPY 3.8 only +check-bytecode-38pypy: check-bytecode-38pypy-run + $(PYTHON) test_pythonlib.py \ + --bytecode-3.8pypy + check-bytecode-3-list-comprehension: $(PYTHON) test_pythonlib.py \ - --bytecode-3.8 --list-comprehension + --bytecode-3.8$(IS_PYPY) --list-comprehension check-bytecode-3-run: check-bytecode-37-run check-bytecode-38-run @@ -77,6 +87,10 @@ check-bytecode-38-run: $(PYTHON) test_pythonlib.py \ --bytecode-3.8 --run --verify-run +check-bytecode-38pypy-run: + $(PYTHON) test_pythonlib.py \ + --bytecode-3.8pypy --run --verify-run + check-code-fragments: \ check-bytecode-3-list-comprehension diff --git a/test/bytecode_3.8pypy/code-fragment/list-comprehension/01_list_comprehensionpy38.pyc b/test/bytecode_3.8pypy/code-fragment/list-comprehension/01_list_comprehensionpy38.pyc new file mode 100644 index 00000000..c94f7097 Binary files /dev/null and b/test/bytecode_3.8pypy/code-fragment/list-comprehension/01_list_comprehensionpy38.pyc differ diff --git a/test/bytecode_3.8pypy/exec/00_while_true_passpy38.pyc b/test/bytecode_3.8pypy/exec/00_while_true_passpy38.pyc new file mode 100644 index 00000000..69c61e7c Binary files /dev/null and b/test/bytecode_3.8pypy/exec/00_while_true_passpy38.pyc differ diff --git a/test/bytecode_3.8pypy/run/00_importpy38.pyc b/test/bytecode_3.8pypy/run/00_importpy38.pyc new file mode 100644 index 00000000..92f22e7d Binary files /dev/null and b/test/bytecode_3.8pypy/run/00_importpy38.pyc differ diff --git a/test/stdlib/3.8-exclude.sh b/test/stdlib/3.8-exclude.sh index 742d3917..e61e7b09 100644 --- a/test/stdlib/3.8-exclude.sh +++ b/test/stdlib/3.8-exclude.sh @@ -98,7 +98,7 @@ SKIP_TESTS=( [test_cgi.py]=1 # parse error [test_clinic.py]=1 # it fails on its own - [test_cmath.py]=1 # parse error + [test_cmath.py]=pytest [test_cmd_line.py]=1 # Interactive? [test_cmd_line_script.py]=1 # test check failures [test_codecs.py]=1 # test takes too long to run - probabl wrong python decompiled diff --git a/test/stdlib/runtests3.sh b/test/stdlib/runtests3.sh index 674799b2..0af0e3e1 100755 --- a/test/stdlib/runtests3.sh +++ b/test/stdlib/runtests3.sh @@ -18,10 +18,10 @@ function displaytime { local H=$((T/60/60%24)) local M=$((T/60%60)) local S=$((T%60)) - (( $D > 0 )) && printf '%d days ' $D - (( $H > 0 )) && printf '%d hours ' $H - (( $M > 0 )) && printf '%d minutes ' $M - (( $D > 0 || $H > 0 || $M > 0 )) && printf 'and ' + (( D > 0 )) && printf '%d days ' $D + (( H > 0 )) && printf '%d hours ' $H + (( M > 0 )) && printf '%d minutes ' $M + (( D > 0 || H > 0 || M > 0 )) && printf 'and ' printf '%d seconds\n' $S } @@ -125,7 +125,11 @@ if [[ -n $1 ]] ; then files=$@ typeset -a files_ary=( $(echo $@) ) if (( ${#files_ary[@]} == 1 || DONT_SKIP_TESTS == 1 )) ; then - SKIP_TESTS=() + for file in $files; do + if (( SKIP_TESTS[$file] != "pytest" )); then + SKIP_TESTS[$file]=1; + fi + done fi else files=$(echo test_*.py) @@ -139,9 +143,14 @@ NOT_INVERTED_TESTS=${NOT_INVERTED_TESTS:-1} for file in $files; do # AIX bash doesn't grok [[ -v SKIP... ]] [[ -z ${SKIP_TESTS[$file]} ]] && SKIP_TESTS[$file]=0 - if [[ ${SKIP_TESTS[$file]} == ${NOT_INVERTED_TESTS} ]] ; then - ((skipped++)) - continue + + if [[ ${SKIP_TESTS[$file]} == "pytest" ]]; then + PYTHON=pytest + else + if [[ ${SKIP_TESTS[$file]}s == ${NOT_INVERTED_TESTS} ]] ; then + ((skipped++)) + continue + fi fi # If the fails *before* decompiling, skip it! @@ -156,7 +165,7 @@ for file in $files; do typeset -i ENDTIME=$(date +%s) typeset -i time_diff (( time_diff = ENDTIME - STARTTIME)) - if (( time_diff > $timeout )) ; then + if (( time_diff > timeout )) ; then echo "Skipping test $file -- test takes too long to run: $time_diff seconds" continue fi diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index 83f27bd2..d133793e 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -13,18 +13,23 @@ Step 1) Edit this file and add a new entry to 'test_options', eg. test_options['mylib'] = ('/usr/lib/mylib', PYOC, 'mylib') Step 2: Run the test: - test_pyenvlib --mylib # decompile 'mylib' - test_pyenvlib --mylib --syntax-verify # decompile verify 'mylib' + test_pyenvlib --mylib # decompile 'mylib' + test_pyenvlib --mylib --syntax-verify # decompile verify 'mylib' """ from __future__ import print_function -import os, time, re, shutil, sys +import os +import re +import shutil +import sys +import time from fnmatch import fnmatch -from decompyle3 import main import xdis.magics as magics +from decompyle3 import main + # ----- configure this for your needs python_versions = [v for v in magics.python_versions if re.match("^[0-9.]+$", v)] @@ -136,7 +141,8 @@ def visitor(files, dirname, names): if __name__ == "__main__": - import getopt, sys + import getopt + import sys do_coverage = do_verify = False test_dirs = [] @@ -147,7 +153,14 @@ def visitor(files, dirname, names): opts, args = getopt.getopt( sys.argv[1:], "", - ["start-with=", "verify-run", "syntax-verify", "max=", "coverage", "all",] + [ + "start-with=", + "verify-run", + "syntax-verify", + "max=", + "coverage", + "all", + ] + test_options_keys, ) vers = "" diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index 2fa5ff57..321d3f33 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -24,11 +24,18 @@ test_pythonlib.py --mylib --syntax-verify # decompile verify 'mylib' """ -import getopt, os, py_compile, sys, shutil, tempfile, time +import getopt +import os +import py_compile +import shutil +import sys +import tempfile +import time +from fnmatch import fnmatch from xdis.version_info import version_tuple_to_str + from decompyle3.main import main -from fnmatch import fnmatch def get_srcdir(): @@ -56,7 +63,7 @@ def get_srcdir(): "test": ("test", PYC, "test"), } -for vers in (3.7, 3.8): +for vers in ("3.7", "3.8", "3.8pypy"): bytecode = "bytecode_%s" % vers key = "bytecode-%s" % vers test_options[key] = (bytecode, PYC, bytecode, vers)