From 7eb4735a5188aa7fd75e849e09fce92b8e0fea48 Mon Sep 17 00:00:00 2001 From: Claire Foster Date: Wed, 15 May 2024 18:24:52 +1000 Subject: [PATCH] Lowering of macro definitions --- src/ast.jl | 90 +++++++++++++++------------- src/desugaring.jl | 60 +++++++++++++++++-- src/eval.jl | 2 +- src/macro_expansion.jl | 6 ++ src/syntax_graph.jl | 8 +++ test/demo.jl | 90 ++++++++++++++-------------- test/runtests.jl | 129 +++++++++++++++++++++++++---------------- 7 files changed, 245 insertions(+), 140 deletions(-) diff --git a/src/ast.jl b/src/ast.jl index 0b8c7bd..30bf50f 100644 --- a/src/ast.jl +++ b/src/ast.jl @@ -331,19 +331,28 @@ function is_function_def(ex) (k == K"=" && numchildren(ex) == 2 && is_eventually_call(ex[1])) end -function identifier_name(ex) - kind(ex) == K"var" ? ex[1] : ex -end - function is_valid_name(ex) n = identifier_name(ex).name_val n !== "ccall" && n !== "cglobal" end +function identifier_name(ex) + kind(ex) == K"var" ? ex[1] : ex +end + function decl_var(ex) kind(ex) == K"::" ? ex[1] : ex end +# Remove empty parameters block, eg, in the arg list of `f(x, y;)` +function remove_empty_parameters(args) + i = length(args) + while i > 0 && kind(args[i]) == K"parameters" && numchildren(args[i]) == 0 + i -= 1 + end + args[1:i] +end + # given a complex assignment LHS, return the symbol that will ultimately be assigned to function assigned_name(ex) k = kind(ex) @@ -355,48 +364,49 @@ function assigned_name(ex) end #------------------------------------------------------------------------------- -# @chk: AST structure checking tool -function _chk_code(ex, cond) - cond_str = string(cond) +# @chk: Basic AST structure checking tool +# +# Check a condition involving an expression, throwing a LoweringError if it +# doesn't evaluate to true. Does some very simple pattern matching to attempt +# to extract the expression variable from the left hand side. +# +# Forms: +# @chk pred(ex) +# @chk pred(ex) msg +# @chk pred(ex) (msg_display_ex, msg) +macro chk(cond, msg=nothing) + if Meta.isexpr(msg, :tuple) + ex = msg.args[1] + msg = msg.args[2] + else + ex = cond + while true + if ex isa Symbol + break + elseif ex.head == :call + ex = ex.args[2] + elseif ex.head == :ref + ex = ex.args[1] + elseif ex.head == :. + ex = ex.args[1] + elseif ex.head in (:(==), :(in), :<, :>) + ex = ex.args[1] + else + error("Can't analyze $cond") + end + end + end quote ex = $(esc(ex)) @assert ex isa SyntaxTree - try - ok = $(esc(cond)) - if !ok - throw(LoweringError(ex, "Expected `$($cond_str)`")) - end + ok = try + $(esc(cond)) catch - throw(LoweringError(ex, "Structure error evaluating `$($cond_str)`")) + false end - end -end - -# Internal error checking macro. -# Check a condition involving an expression, throwing a LoweringError if it -# doesn't evaluate to true. Does some very simple pattern matching to attempt -# to extract the expression variable from the left hand side. -macro chk(cond) - ex = cond - while true - if ex isa Symbol - break - elseif ex.head == :call - ex = ex.args[2] - elseif ex.head == :ref - ex = ex.args[1] - elseif ex.head == :. - ex = ex.args[1] - elseif ex.head in (:(==), :(in), :<, :>) - ex = ex.args[1] - else - error("Can't analyze $cond") + if !ok + throw(LoweringError(ex, $(isnothing(msg) ? "expected `$cond`" : esc(msg)))) end end - _chk_code(ex, cond) -end - -macro chk(ex, cond) - _chk_code(ex, cond) end diff --git a/src/desugaring.jl b/src/desugaring.jl index a5c8dfa..bf2472c 100644 --- a/src/desugaring.jl +++ b/src/desugaring.jl @@ -155,7 +155,7 @@ function analyze_function_arg(full_ex) end break elseif k == K"..." - @chk full_ex !is_slurp + @chk !is_slurp (full_ex,"nested `...` in function argument") @chk numchildren(ex) == 1 is_slurp = true ex = ex[1] @@ -298,6 +298,43 @@ function expand_function_def(ctx, ex) end end +function _make_macro_name(ctx, name) + @chk kind(name) == K"Identifier" (name, "invalid macro name") + ex = mapleaf(ctx, name, K"Identifier") + ex.name_val = "@$(name.name_val)" + ex +end + +# flisp: expand-macro-def +function expand_macro_def(ctx, ex) + @chk numchildren(ex) >= 1 (ex,"invalid macro definition") + if numchildren(ex) == 1 + name = ex[1] + # macro with zero methods + # `macro m end` + return @ast ctx ex [K"function" _make_macro_name(ctx, name)] + end + # TODO: Making this manual pattern matching robust is such a pain!!! + sig = ex[1] + @chk (kind(sig) == K"call" && numchildren(sig) >= 1) (sig, "invalid macro signature") + name = sig[1] + args = remove_empty_parameters(children(sig)) + @chk kind(args[end]) != K"parameters" (args[end], "macros cannot accept keyword arguments") + ret = @ast ctx ex [K"function" + [K"call"(sig) + _make_macro_name(ctx, name) + [K"::" + "__context__"::K"Identifier"(scope_layer=name.scope_layer) + MacroContext::K"Value" + ] + # flisp: We don't mark these @nospecialize because all arguments to + # new macros will be of type SyntaxTree + args[2:end]... + ] + ex[2] + ] +end + function _append_importpath(ctx, path_spec, path) prev_was_dot = true for component in children(path) @@ -460,13 +497,17 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree) elseif k == K"." @chk numchildren(ex) == 2 @chk kind(ex[2]) == K"Identifier" - @ast ctx ex [K"call" - "getproperty"::K"top" - ex[1] - ex[2]=>K"Symbol" - ] + expand_forms_2(ctx, + @ast ctx ex [K"call" + "getproperty"::K"top" + ex[1] + ex[2]=>K"Symbol" + ] + ) elseif k == K"function" expand_forms_2(ctx, expand_function_def(ctx, ex)) + elseif k == K"macro" + expand_forms_2(ctx, expand_macro_def(ctx, ex)) elseif k == K"let" expand_forms_2(ctx, expand_let(ctx, ex)) elseif k == K"local" || k == K"global" @@ -504,6 +545,11 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree) expand_import(ctx, ex) elseif k == K"export" || k == K"public" TODO(ex) + elseif k == K"ref" + if numchildren(ex) > 2 + TODO(ex, "ref expansion") + end + expand_forms_2(ctx, @ast ctx ex [K"call" "getindex"::K"top" ex[1] ex[2]]) elseif k == K"toplevel" # The toplevel form can't be lowered here - it needs to just be quoted # and passed through to a call to eval. @@ -514,6 +560,8 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree) ctx.mod ::K"Value" [K"inert" ex] ] + elseif k == K"inert" + ex elseif !haschildren(ex) ex else diff --git a/src/eval.jl b/src/eval.jl index 2c6e96b..6e4179f 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -199,7 +199,7 @@ end function InterpolationContext() graph = SyntaxGraph() ensure_attributes!(graph, kind=Kind, syntax_flags=UInt16, source=SourceAttrType, - value=Any, name_val=String) + value=Any, name_val=String, scope_layer=LayerId) InterpolationContext(freeze_attrs(graph)) end diff --git a/src/macro_expansion.jl b/src/macro_expansion.jl index 9c4e24a..24b59c8 100644 --- a/src/macro_expansion.jl +++ b/src/macro_expansion.jl @@ -240,6 +240,12 @@ function expand_macro(ctx, ex) end if expanded isa SyntaxTree + if syntax_graph(expanded) !== syntax_graph(ctx) + # If the macro has produced syntax outside the macro context, copy it over. + # TODO: Do we expect this always to happen? What is the API for access + # to the macro expansion context? + expanded = copy_ast(ctx, expanded) + end new_layer = new_scope_layer(ctx.scope_layers, parentmodule(macfunc)) ctx2 = MacroExpansionContext(ctx.graph, ctx.next_var_id, ctx.scope_layers, new_layer) expand_forms_1(ctx2, expanded) diff --git a/src/syntax_graph.jl b/src/syntax_graph.jl index b9f3e52..c6731b0 100644 --- a/src/syntax_graph.jl +++ b/src/syntax_graph.jl @@ -173,6 +173,10 @@ function Base.getproperty(tree::SyntaxTree, name::Symbol) end end +function Base.setproperty!(tree::SyntaxTree, name::Symbol, val) + return setattr!(tree.graph, tree.id; name=>val) +end + function Base.propertynames(tree::SyntaxTree) attrnames(tree) end @@ -449,6 +453,10 @@ Base.IndexStyle(::Type{<:SyntaxList}) = IndexLinear() Base.getindex(v::SyntaxList, i::Int) = SyntaxTree(v.graph, v.ids[i]) +function Base.getindex(v::SyntaxList, r::UnitRange) + SyntaxList(v.graph, view(v.ids, r)) +end + function Base.setindex!(v::SyntaxList, tree::SyntaxTree, i::Int) v.graph === tree.graph || error("Mismatching syntax graphs") v.ids[i] = tree.id diff --git a/test/demo.jl b/test/demo.jl index 6fdf0ae..9e4c321 100644 --- a/test/demo.jl +++ b/test/demo.jl @@ -82,42 +82,38 @@ end # end # """ +JuliaLowering.include_string(Main, """ module M - using JuliaLowering: @ast, @chk + using JuliaLowering: JuliaLowering, @ast, @chk using JuliaSyntax - const someglobal = "global in M" + # Introspection + macro __MODULE__() + __context__.mod + end + + macro __FILE__() + JuliaLowering.filename(__context__.macroname) + end + + macro __LINE__() + JuliaLowering.source_location(__context__.macroname)[1] + end - # TODO: macrocall in macro call - # module A - # function var"@bar"(mctx, ex) - # end - # end + someglobal = "global in module M" # Macro with local variables - function var"@foo"(mctx, ex) - # :(let x = "local in @asdf expansion" - # (x, someglobal, $ex) - # end) - @ast mctx (@HERE) [K"let" - [K"block"(@HERE) - [K"="(@HERE) - "x"::K"Identifier"(@HERE) - "local in @asdf expansion"::K"String"(@HERE) - ] - ] - [K"block"(@HERE) - [K"tuple"(@HERE) - "x"::K"Identifier"(@HERE) - "someglobal"::K"Identifier"(@HERE) - ex - ] - ] - ] + macro foo(ex) + :(let x = "`x` from @foo" + (x, someglobal, \$ex) + end) end +end +""") +Base.eval(M, quote # Recursive macro call - function var"@recursive"(mctx, N) + function var"@recursive"(__context__, N) @chk kind(N) == K"Integer" Nval = N.value::Int if Nval < 1 @@ -127,7 +123,7 @@ module M # x = $N # (@recursive $(Nval-1), x) # end - @ast mctx (@HERE) [K"block" + @ast __context__ (@HERE) [K"block" [K"="(@HERE) "x"::K"Identifier"(@HERE) N @@ -141,6 +137,15 @@ module M ] ] end +end) + +function wrapscope(ex, scope_type) + makenode(ex, ex, K"scope_block", ex; scope_type=scope_type) +end + +function softscope_test(ex) + g = JuliaLowering.ensure_attributes(ex.graph, scope_type=Symbol) + wrapscope(wrapscope(JuliaLowering.reparent(g, ex), :neutral), :soft) end # src = """ @@ -154,27 +159,26 @@ src = """ M.@recursive 3 """ -src = """ -begin - x = 2 -end -""" - -function wrapscope(ex, scope_type) - makenode(ex, ex, K"scope_block", ex; scope_type=scope_type) -end +# src = """ +# macro mmm(a; b=2) +# end +# macro A.b(ex) +# end +# """ -function softscope_test(ex) - g = JuliaLowering.ensure_attributes(ex.graph, scope_type=Symbol) - wrapscope(wrapscope(JuliaLowering.reparent(g, ex), :neutral), :soft) -end +# TODO: +# "hygiene bending" / (being unhygenic, or bending hygiene to the context of a +# macro argument on purpose) +# * bend to macro name to get to parent layer? +# * already needed in `#self#` argument -ex = softscope_test(parsestmt(SyntaxTree, src, filename="foo.jl")) +ex = parsestmt(SyntaxTree, src, filename="foo.jl") +#ex = softscope_test(ex) @info "Input code" ex in_mod = Main ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex) -# @info "Macro expanded" ex_macroexpand +@info "Macro expanded" ex_macroexpand ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) @info "Desugared" ex_desugar diff --git a/test/runtests.jl b/test/runtests.jl index 3b1d0dc..0e378f4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -145,67 +145,58 @@ end @test sourcetext(ex[1][2]) == "x+1" @test sourcetext(ex[1][3]) == "g(z)" +# Test expression flags are preserved during interpolation +@test JuliaSyntax.is_infix_op_call(JuliaLowering.include_string(test_mod, """ +let + x = 1 + :(\$x + \$x) +end +""")) + #------------------------------------------------------------------------------- # Macro expansion -Base.eval(test_mod, :( +JuliaLowering.include_string(test_mod, """ module M - using JuliaLowering: @ast, @chk + using JuliaLowering: JuliaLowering, @ast, @chk using JuliaSyntax - const someglobal = "global in module M" + # Introspection + macro __MODULE__() + __context__.mod + end - # Macro with local variables - function var"@foo"(mctx, ex) - # TODO - # :(let x = "local in @foo expansion" - # (x, someglobal, $ex) - # end) - @ast mctx (@HERE) [K"let" - [K"block"(@HERE) - [K"="(@HERE) - "x"::K"Identifier"(@HERE) - "`x` from @foo"::K"String"(@HERE) - ] - ] - [K"block"(@HERE) - [K"tuple"(@HERE) - "x"::K"Identifier"(@HERE) - "someglobal"::K"Identifier"(@HERE) - ex - ] - ] - ] + macro __FILE__() + JuliaLowering.filename(__context__.macroname) end - # Recursive macro call - function var"@recursive"(mctx, N) - @chk kind(N) == K"Integer" - Nval = N.value::Int - if Nval < 1 - return N - end - # TODO - # quote - # x = $N - # (@recursive $(Nval-1), x) - # end - @ast mctx (@HERE) [K"block" - [K"="(@HERE) - "x"::K"Identifier"(@HERE) - N - ] - [K"tuple"(@HERE) - "x"::K"Identifier"(@HERE) - [K"macrocall"(@HERE) - "@recursive"::K"Identifier" - (Nval-1)::K"Integer" - ] - ] - ] + macro __LINE__() + JuliaLowering.source_location(__context__.macroname)[1] + end + + someglobal = "global in module M" + + # Macro with local variables + macro foo(ex) + :(let x = "`x` from @foo" + (x, someglobal, \$ex) + end) end + + # # Recursive macro call + # # TODO: Need branching! + # macro recursive(N) + # Nval = N.value #::Int + # if Nval < 1 + # return N + # end + # quote + # x = \$N + # (@recursive \$(Nval-1), x) + # end + # end end -)) +""") @test JuliaLowering.include_string(test_mod, """ let @@ -214,9 +205,47 @@ let end """) == ("`x` from @foo", "global in module M", "`x` from outer scope") +@test JuliaLowering.include_string(test_mod, """ +#line1 +(M.@__MODULE__(), M.@__FILE__(), M.@__LINE__()) +""", "foo.jl") == (test_mod, "foo.jl", 2) + +Base.eval(test_mod.M, :( +# Recursive macro call +function var"@recursive"(mctx, N) + @chk kind(N) == K"Integer" + Nval = N.value::Int + if Nval < 1 + return N + end + @ast mctx (@HERE) [K"block" + [K"="(@HERE) + "x"::K"Identifier"(@HERE) + N + ] + [K"tuple"(@HERE) + "x"::K"Identifier"(@HERE) + [K"macrocall"(@HERE) + "@recursive"::K"Identifier" + (Nval-1)::K"Integer" + ] + ] + ] +end +)) @test JuliaLowering.include_string(test_mod, """ M.@recursive 3 """) == (3, (2, (1, 0))) +@test_throws JuliaLowering.LoweringError JuliaLowering.include_string(test_mod, """ +macro mmm(a; b=2) +end +""") + +@test_throws JuliaLowering.LoweringError JuliaLowering.include_string(test_mod, """ +macro A.b(ex) +end +""") + end