Lowering of macro definitions

src/ast.jl

@@ -331,19 +331,28 @@ function is_function_def(ex)
         (k == K"=" && numchildren(ex) == 2 && is_eventually_call(ex[1]))
 end
 
-function identifier_name(ex)
-    kind(ex) == K"var" ? ex[1] : ex
-end
-
 function is_valid_name(ex)
     n = identifier_name(ex).name_val
     n !== "ccall" && n !== "cglobal"
 end
 
+function identifier_name(ex)
+    kind(ex) == K"var" ? ex[1] : ex
+end
+
 function decl_var(ex)
     kind(ex) == K"::" ? ex[1] : ex
 end
 
+# Remove empty parameters block, eg, in the arg list of `f(x, y;)`
+function remove_empty_parameters(args)
+    i = length(args)
+    while i > 0 && kind(args[i]) == K"parameters" && numchildren(args[i]) == 0
+        i -= 1
+    end
+    args[1:i]
+end
+
 # given a complex assignment LHS, return the symbol that will ultimately be assigned to
 function assigned_name(ex)
     k = kind(ex)
@@ -355,48 +364,49 @@ function assigned_name(ex)
 end
 
 #-------------------------------------------------------------------------------
-# @chk: AST structure checking tool
-function _chk_code(ex, cond)
-    cond_str = string(cond)
+# @chk: Basic AST structure checking tool
+#
+# Check a condition involving an expression, throwing a LoweringError if it
+# doesn't evaluate to true. Does some very simple pattern matching to attempt
+# to extract the expression variable from the left hand side.
+#
+# Forms:
+# @chk pred(ex)
+# @chk pred(ex) msg
+# @chk pred(ex) (msg_display_ex, msg)
+macro chk(cond, msg=nothing)
+    if Meta.isexpr(msg, :tuple)
+        ex = msg.args[1]
+        msg = msg.args[2]
+    else
+        ex = cond
+        while true
+            if ex isa Symbol
+                break
+            elseif ex.head == :call
+                ex = ex.args[2]
+            elseif ex.head == :ref
+                ex = ex.args[1]
+            elseif ex.head == :.
+                ex = ex.args[1]
+            elseif ex.head in (:(==), :(in), :<, :>)
+                ex = ex.args[1]
+            else
+                error("Can't analyze $cond")
+            end
+        end
+    end
     quote
         ex = $(esc(ex))
         @assert ex isa SyntaxTree
-        try
-            ok = $(esc(cond))
-            if !ok
-                throw(LoweringError(ex, "Expected `$($cond_str)`"))
-            end
+        ok = try
+            $(esc(cond))
         catch
-            throw(LoweringError(ex, "Structure error evaluating `$($cond_str)`"))
+            false
         end
-    end
-end
-
-# Internal error checking macro.
-# Check a condition involving an expression, throwing a LoweringError if it
-# doesn't evaluate to true. Does some very simple pattern matching to attempt
-# to extract the expression variable from the left hand side.
-macro chk(cond)
-    ex = cond
-    while true
-        if ex isa Symbol
-            break
-        elseif ex.head == :call
-            ex = ex.args[2]
-        elseif ex.head == :ref
-            ex = ex.args[1]
-        elseif ex.head == :.
-            ex = ex.args[1]
-        elseif ex.head in (:(==), :(in), :<, :>)
-            ex = ex.args[1]
-        else
-            error("Can't analyze $cond")
+        if !ok
+            throw(LoweringError(ex, $(isnothing(msg) ? "expected `$cond`" : esc(msg))))
         end
     end
-    _chk_code(ex, cond)
-end
-
-macro chk(ex, cond)
-    _chk_code(ex, cond)
 end
 

src/desugaring.jl

@@ -155,7 +155,7 @@ function analyze_function_arg(full_ex)
             end
             break
         elseif k == K"..."
-            @chk full_ex !is_slurp
+            @chk !is_slurp (full_ex,"nested `...` in function argument")
             @chk numchildren(ex) == 1
             is_slurp = true
             ex = ex[1]
@@ -298,6 +298,43 @@ function expand_function_def(ctx, ex)
     end
 end
 
+function _make_macro_name(ctx, name)
+    @chk kind(name) == K"Identifier" (name, "invalid macro name")
+    ex = mapleaf(ctx, name, K"Identifier")
+    ex.name_val = "@$(name.name_val)"
+    ex
+end
+
+# flisp: expand-macro-def
+function expand_macro_def(ctx, ex)
+    @chk numchildren(ex) >= 1 (ex,"invalid macro definition")
+    if numchildren(ex) == 1
+        name = ex[1]
+        # macro with zero methods
+        # `macro m end`
+        return @ast ctx ex [K"function" _make_macro_name(ctx, name)]
+    end
+    # TODO: Making this manual pattern matching robust is such a pain!!!
+    sig = ex[1]
+    @chk (kind(sig) == K"call" && numchildren(sig) >= 1) (sig, "invalid macro signature")
+    name = sig[1]
+    args = remove_empty_parameters(children(sig))
+    @chk kind(args[end]) != K"parameters" (args[end], "macros cannot accept keyword arguments")
+    ret = @ast ctx ex [K"function"
+        [K"call"(sig)
+            _make_macro_name(ctx, name)
+            [K"::"
+                "__context__"::K"Identifier"(scope_layer=name.scope_layer)
+                MacroContext::K"Value"
+            ]
+            # flisp: We don't mark these @nospecialize because all arguments to
+            # new macros will be of type SyntaxTree
+            args[2:end]...
+        ]
+        ex[2]
+    ]
+end
+
 function _append_importpath(ctx, path_spec, path)
     prev_was_dot = true
     for component in children(path)
@@ -460,13 +497,17 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree)
     elseif k == K"."
         @chk numchildren(ex) == 2
         @chk kind(ex[2]) == K"Identifier"
-        @ast ctx ex [K"call"
-            "getproperty"::K"top" 
-            ex[1]
-            ex[2]=>K"Symbol"
-        ]
+        expand_forms_2(ctx,
+            @ast ctx ex [K"call"
+                "getproperty"::K"top" 
+                ex[1]
+                ex[2]=>K"Symbol"
+            ]
+        )
     elseif k == K"function"
         expand_forms_2(ctx, expand_function_def(ctx, ex))
+    elseif k == K"macro"
+        expand_forms_2(ctx, expand_macro_def(ctx, ex))
     elseif k == K"let"
         expand_forms_2(ctx, expand_let(ctx, ex))
     elseif k == K"local" || k == K"global"
@@ -504,6 +545,11 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree)
         expand_import(ctx, ex)
     elseif k == K"export" || k == K"public"
         TODO(ex)
+    elseif k == K"ref"
+        if numchildren(ex) > 2
+            TODO(ex, "ref expansion")
+        end
+        expand_forms_2(ctx, @ast ctx ex [K"call" "getindex"::K"top" ex[1] ex[2]])
     elseif k == K"toplevel"
         # The toplevel form can't be lowered here - it needs to just be quoted
         # and passed through to a call to eval.
@@ -514,6 +560,8 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree)
             ctx.mod       ::K"Value"
             [K"inert" ex]
         ]
+    elseif k == K"inert"
+        ex
     elseif !haschildren(ex)
         ex
     else

src/eval.jl

@@ -199,7 +199,7 @@ end
 function InterpolationContext()
     graph = SyntaxGraph()
     ensure_attributes!(graph, kind=Kind, syntax_flags=UInt16, source=SourceAttrType,
-                       value=Any, name_val=String)
+                       value=Any, name_val=String, scope_layer=LayerId)
     InterpolationContext(freeze_attrs(graph))
 end
 

src/macro_expansion.jl

@@ -240,6 +240,12 @@ function expand_macro(ctx, ex)
     end
 
     if expanded isa SyntaxTree
+        if syntax_graph(expanded) !== syntax_graph(ctx)
+            # If the macro has produced syntax outside the macro context, copy it over.
+            # TODO: Do we expect this always to happen?  What is the API for access
+            # to the macro expansion context?
+            expanded = copy_ast(ctx, expanded)
+        end
         new_layer = new_scope_layer(ctx.scope_layers, parentmodule(macfunc))
         ctx2 = MacroExpansionContext(ctx.graph, ctx.next_var_id, ctx.scope_layers, new_layer)
         expand_forms_1(ctx2, expanded)

src/syntax_graph.jl

@@ -173,6 +173,10 @@ function Base.getproperty(tree::SyntaxTree, name::Symbol)
     end
 end
 
+function Base.setproperty!(tree::SyntaxTree, name::Symbol, val)
+    return setattr!(tree.graph, tree.id; name=>val)
+end
+
 function Base.propertynames(tree::SyntaxTree)
     attrnames(tree)
 end
@@ -449,6 +453,10 @@ Base.IndexStyle(::Type{<:SyntaxList}) = IndexLinear()
 
 Base.getindex(v::SyntaxList, i::Int) = SyntaxTree(v.graph, v.ids[i])
 
+function Base.getindex(v::SyntaxList, r::UnitRange)
+    SyntaxList(v.graph, view(v.ids, r))
+end
+
 function Base.setindex!(v::SyntaxList, tree::SyntaxTree, i::Int)
     v.graph === tree.graph || error("Mismatching syntax graphs")
     v.ids[i] = tree.id

test/demo.jl

@@ -82,42 +82,38 @@ end
 # end
 # """
 
+JuliaLowering.include_string(Main, """
 module M
-    using JuliaLowering: @ast, @chk
+    using JuliaLowering: JuliaLowering, @ast, @chk
     using JuliaSyntax
 
-    const someglobal = "global in M"
+    # Introspection
+    macro __MODULE__()
+        __context__.mod
+    end
+
+    macro __FILE__()
+        JuliaLowering.filename(__context__.macroname)
+    end
+
+    macro __LINE__()
+        JuliaLowering.source_location(__context__.macroname)[1]
+    end
 
-    # TODO: macrocall in macro call
-    # module A
-    #     function var"@bar"(mctx, ex)
-    #     end
-    # end
+    someglobal = "global in module M"
 
     # Macro with local variables
-    function var"@foo"(mctx, ex)
-        # :(let x = "local in @asdf expansion"
-        #     (x, someglobal, $ex)
-        # end)
-        @ast mctx (@HERE) [K"let"
-            [K"block"(@HERE)
-                [K"="(@HERE)
-                    "x"::K"Identifier"(@HERE)
-                    "local in @asdf expansion"::K"String"(@HERE)
-                ]
-            ]
-            [K"block"(@HERE)
-                [K"tuple"(@HERE)
-                    "x"::K"Identifier"(@HERE)
-                    "someglobal"::K"Identifier"(@HERE)
-                    ex
-                ]
-            ]
-        ]
+    macro foo(ex)
+        :(let x = "`x` from @foo"
+            (x, someglobal, \$ex)
+        end)
     end
+end
+""")
 
+Base.eval(M, quote
     # Recursive macro call
-    function var"@recursive"(mctx, N)
+    function var"@recursive"(__context__, N)
         @chk kind(N) == K"Integer"
         Nval = N.value::Int
         if Nval < 1
@@ -127,7 +123,7 @@ module M
         #     x = $N
         #     (@recursive $(Nval-1), x)
         # end
-        @ast mctx (@HERE) [K"block"
+        @ast __context__ (@HERE) [K"block"
             [K"="(@HERE)
                 "x"::K"Identifier"(@HERE)
                 N
@@ -141,6 +137,15 @@ module M
             ]
         ]
     end
+end)
+
+function wrapscope(ex, scope_type)
+    makenode(ex, ex, K"scope_block", ex; scope_type=scope_type)
+end
+
+function softscope_test(ex)
+    g = JuliaLowering.ensure_attributes(ex.graph, scope_type=Symbol)
+    wrapscope(wrapscope(JuliaLowering.reparent(g, ex), :neutral), :soft)
 end
 
 # src = """
@@ -154,27 +159,26 @@ src = """
 M.@recursive 3
 """
 
-src = """
-begin
-    x = 2
-end
-"""
-
-function wrapscope(ex, scope_type)
-    makenode(ex, ex, K"scope_block", ex; scope_type=scope_type)
-end
+# src = """
+# macro mmm(a; b=2)
+# end
+# macro A.b(ex)
+# end
+# """
 
-function softscope_test(ex)
-    g = JuliaLowering.ensure_attributes(ex.graph, scope_type=Symbol)
-    wrapscope(wrapscope(JuliaLowering.reparent(g, ex), :neutral), :soft)
-end
+# TODO:
+# "hygiene bending" / (being unhygenic, or bending hygiene to the context of a
+# macro argument on purpose)
+# * bend to macro name to get to parent layer?
+# * already needed in `#self#` argument
 
-ex = softscope_test(parsestmt(SyntaxTree, src, filename="foo.jl"))
+ex = parsestmt(SyntaxTree, src, filename="foo.jl")
+#ex = softscope_test(ex)
 @info "Input code" ex
 
 in_mod = Main
 ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex)
-# @info "Macro expanded" ex_macroexpand
+@info "Macro expanded" ex_macroexpand
 
 ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand)
 @info "Desugared" ex_desugar