From ce588574066b4d240256e80a6ec7b35d29524fe7 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 29 Feb 2024 23:07:25 -0500 Subject: [PATCH 1/4] WOrk on 3.8 to 3.8 PyPy code changes --- decompyle3/bin/decompile_tokens.py | 2 +- decompyle3/parsers/p38/lambda_custom.py | 6 ++-- decompyle3/parsers/p38pypy/lambda_custom.py | 32 ++++++++++++--------- decompyle3/parsers/p38pypy/lambda_expr.py | 6 +++- decompyle3/semantics/gencomp.py | 7 +++-- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/decompyle3/bin/decompile_tokens.py b/decompyle3/bin/decompile_tokens.py index fdb916cf..72e3b160 100755 --- a/decompyle3/bin/decompile_tokens.py +++ b/decompyle3/bin/decompile_tokens.py @@ -29,7 +29,7 @@ from decompyle3.disas import disassemble_file from decompyle3.version import __version__ -program, ext = os.path.splitext(os.path.basename(__file__)) +program = "decompile-tokens" __doc__ = """ Usage: diff --git a/decompyle3/parsers/p38/lambda_custom.py b/decompyle3/parsers/p38/lambda_custom.py index 7d2f6a16..d8321fce 100644 --- a/decompyle3/parsers/p38/lambda_custom.py +++ b/decompyle3/parsers/p38/lambda_custom.py @@ -356,10 +356,12 @@ def customize_grammar_rules_lambda38(self, tokens, customize): expr ::= set_comp_async func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT - DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + DUP_TOP LOAD_GLOBAL COMPARE_OP + POP_JUMP_IF_TRUE END_FINALLY _come_froms - # async_iter ::= block_break SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM + # async_iter ::= block_break SETUP_EXCEPT GET_ANEXT + LOAD_CONST YIELD_FROM get_aiter ::= expr GET_AITER diff --git a/decompyle3/parsers/p38pypy/lambda_custom.py b/decompyle3/parsers/p38pypy/lambda_custom.py index 2bc6397d..624a1174 100644 --- a/decompyle3/parsers/p38pypy/lambda_custom.py +++ b/decompyle3/parsers/p38pypy/lambda_custom.py @@ -364,16 +364,24 @@ def customize_grammar_rules_lambda38(self, tokens, customize): expr ::= set_comp_async func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT - DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + DUP_TOP LOAD_GLOBAL COMPARE_OP + POP_JUMP_IF_TRUE END_FINALLY _come_froms - # async_iter ::= block_break SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM + # async_iter ::= block_break SETUP_EXCEPT GET_ANEXT + LOAD_CONST YIELD_FROM get_aiter ::= expr GET_AITER list_afor ::= get_aiter list_afor2 - list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2 + return_expr_lambda ::= list_comp_async + + + list_afor2 ::= async_iter store list_iter + JUMP_LOOP COME_FROM_EXCEPT + END_ASYNC_FOR + list_iter ::= list_afor @@ -398,9 +406,14 @@ def customize_grammar_rules_lambda38(self, tokens, customize): dict_comp_async ::= BUILD_MAP_0 genexpr_func_async async_iter ::= _come_froms - SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK + SETUP_EXCEPT + GET_ANEXT + LOAD_CONST + YIELD_FROM + POP_BLOCK - func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM + func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT + LOAD_CONST YIELD_FROM genexpr_func_async ::= LOAD_ARG async_iter store @@ -414,14 +427,7 @@ def customize_grammar_rules_lambda38(self, tokens, customize): JUMP_LOOP COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP - list_afor2 ::= async_iter - store - list_iter - JUMP_LOOP - COME_FROM_FINALLY - END_ASYNC_FOR - - list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2 + list_comp_async ::= LOAD_ARG BUILD_LIST_FROM_ARG list_afor2 set_afor2 ::= async_iter store diff --git a/decompyle3/parsers/p38pypy/lambda_expr.py b/decompyle3/parsers/p38pypy/lambda_expr.py index 76c1872d..46b37451 100644 --- a/decompyle3/parsers/p38pypy/lambda_expr.py +++ b/decompyle3/parsers/p38pypy/lambda_expr.py @@ -36,7 +36,6 @@ def p_38walrus(self, args): def p_lambda_start(self, args): """ return_expr_lambda ::= genexpr_func LOAD_CONST RETURN_VALUE_LAMBDA - """ def p_pypy38_comprehension(self, args): @@ -47,6 +46,11 @@ def p_pypy38_comprehension(self, args): store lc_body JUMP_LOOP _come_froms + list_afor2 ::= async_iter store list_iter + JUMP_LOOP COME_FROM_EXCEPT + END_ASYNC_FOR + + lc_body ::= expr LIST_APPEND """ diff --git a/decompyle3/semantics/gencomp.py b/decompyle3/semantics/gencomp.py index 6a5d2233..e468790d 100644 --- a/decompyle3/semantics/gencomp.py +++ b/decompyle3/semantics/gencomp.py @@ -332,7 +332,8 @@ def comprehension_walk_newer( # list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2 if tree[0] == "expr" and tree[0][0] == "list_comp_async": tree = tree[0][0] - if tree[0] == "BUILD_LIST_0": + # PyPy 3.8 has LOAD_ARG + if tree[0] in ("BUILD_LIST_0", "LOAD_ARG"): list_afor2 = tree[2] assert list_afor2 == "list_afor2" store = list_afor2[1] @@ -717,7 +718,9 @@ def get_comprehension_function(self, node, code_index: int): if tree == "lambda_start": tree = tree[0] - while len(tree) == 1 or (tree in ("stmt", "sstmt", "return", "return_expr")): + while len(tree) == 1 or ( + tree in ("stmt", "sstmt", "return", "return_expr", "return_expr_lambda") + ): self.prec = 100 tree = tree[0] return tree From 7ffd28d6d6dd7db9a12f02907fce657d695b1fe1 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 1 Mar 2024 06:06:49 -0500 Subject: [PATCH 2/4] Handle unicode in Python 3... Adjust async generators more --- decompyle3/parsers/p38/lambda_custom.py | 6 ++++-- decompyle3/parsers/p38pypy/lambda_custom.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/decompyle3/parsers/p38/lambda_custom.py b/decompyle3/parsers/p38/lambda_custom.py index d8321fce..54a8c0a1 100644 --- a/decompyle3/parsers/p38/lambda_custom.py +++ b/decompyle3/parsers/p38/lambda_custom.py @@ -392,9 +392,11 @@ def customize_grammar_rules_lambda38(self, tokens, customize): dict_comp_async ::= BUILD_MAP_0 genexpr_func_async async_iter ::= _come_froms - SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK + SETUP_FINALLY GET_ANEXT LOAD_CONST + YIELD_FROM POP_BLOCK - func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM + func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT + LOAD_CONST YIELD_FROM genexpr_func_async ::= LOAD_ARG async_iter store diff --git a/decompyle3/parsers/p38pypy/lambda_custom.py b/decompyle3/parsers/p38pypy/lambda_custom.py index 624a1174..5c978117 100644 --- a/decompyle3/parsers/p38pypy/lambda_custom.py +++ b/decompyle3/parsers/p38pypy/lambda_custom.py @@ -419,7 +419,7 @@ def customize_grammar_rules_lambda38(self, tokens, customize): store comp_iter JUMP_LOOP - COME_FROM_FINALLY + COME_FROM_EXCEPT END_ASYNC_FOR genexpr_func_async ::= LOAD_ARG func_async_prefix From eb88b18e0cb1d3b7aa9b5962589c886d8a136f22 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 1 Mar 2024 12:06:33 -0500 Subject: [PATCH 3/4] Correct "if" detection in async list comprehension --- decompyle3/semantics/gencomp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/decompyle3/semantics/gencomp.py b/decompyle3/semantics/gencomp.py index e468790d..2477e3bb 100644 --- a/decompyle3/semantics/gencomp.py +++ b/decompyle3/semantics/gencomp.py @@ -498,7 +498,7 @@ def comprehension_walk_newer( "comp_if_not", ): if n in ("list_if37", "list_if37_not", "comp_if"): - if n == "comp_if": + if n in ("comp_if", "list_if37"): if_nodes.append(n[0]) n = n[1] else: @@ -632,7 +632,7 @@ def comprehension_walk_newer( comp_store = None pass - if tree == "set_comp_func": + elif tree == "set_comp_func": # Handle nested comp_for iterations. comp_iter = tree[4] assert comp_iter in ("comp_iter", "await_expr") @@ -653,6 +653,7 @@ def comprehension_walk_newer( if if_node != "comp_if_or": self.write(" if ") if if_node in ( + "c_compare_chained37_false", "comp_if_not_and", "comp_if_not_or", "comp_if_or", From 8a162149902148a9c49b77151124f4f210945978 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 1 Mar 2024 18:03:32 -0500 Subject: [PATCH 4/4] Finish PyPy 3.8 support for list comprehension --- decompyle3/parsers/p38pypy/lambda_custom.py | 1 + decompyle3/semantics/gencomp.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/decompyle3/parsers/p38pypy/lambda_custom.py b/decompyle3/parsers/p38pypy/lambda_custom.py index 5c978117..9c58a512 100644 --- a/decompyle3/parsers/p38pypy/lambda_custom.py +++ b/decompyle3/parsers/p38pypy/lambda_custom.py @@ -428,6 +428,7 @@ def customize_grammar_rules_lambda38(self, tokens, customize): POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP list_comp_async ::= LOAD_ARG BUILD_LIST_FROM_ARG list_afor2 + list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2 set_afor2 ::= async_iter store diff --git a/decompyle3/semantics/gencomp.py b/decompyle3/semantics/gencomp.py index 2477e3bb..47966fd8 100644 --- a/decompyle3/semantics/gencomp.py +++ b/decompyle3/semantics/gencomp.py @@ -390,7 +390,13 @@ def comprehension_walk_newer( elif node == "set_comp" and tree[1] == "set_iter": n = tree[1] else: - n = tree[iter_index] + for k in tree: + if k.kind in ("comp_iter", "list_iter", "set_iter", "lc_body"): + n = k + break + pass + else: + n = tree[iter_index] if tree in ( "dict_comp_func", @@ -498,7 +504,7 @@ def comprehension_walk_newer( "comp_if_not", ): if n in ("list_if37", "list_if37_not", "comp_if"): - if n in ("comp_if", "list_if37"): + if n in ("comp_if", "list_if37", "list_if"): if_nodes.append(n[0]) n = n[1] else: