Incremental lowering API

src/desugaring.jl

@@ -19,7 +19,7 @@ function DesugaringContext(ctx, expr_compat_mode::Bool)
     DesugaringContext(graph,
                       ctx.bindings,
                       ctx.scope_layers,
-                      first(ctx.scope_layers).mod,
+                      current_layer(ctx).mod,
                       expr_compat_mode)
 end
 
@@ -4270,39 +4270,6 @@ function expand_public(ctx, ex)
     ]
 end
 
-#-------------------------------------------------------------------------------
-# Expand module definitions
-
-function expand_module(ctx, ex::SyntaxTree)
-    modname_ex = ex[1]
-    @chk kind(modname_ex) == K"Identifier"
-    modname = modname_ex.name_val
-
-    std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
-
-    body = ex[2]
-    @chk kind(body) == K"block"
-
-    @ast ctx ex [K"block"
-        [K"assert"
-            "global_toplevel_only"::K"Symbol"
-            [K"inert" ex]
-        ]
-        [K"call"
-            eval_module           ::K"Value"
-            ctx.mod               ::K"Value"
-            modname               ::K"String"
-            std_defs              ::K"Bool"
-            ctx.expr_compat_mode  ::K"Bool"
-            [K"inert"(body)
-                [K"toplevel"
-                    children(body)...
-                ]
-            ]
-        ]
-    ]
-end
-
 #-------------------------------------------------------------------------------
 # Expand docstring-annotated expressions
 
@@ -4477,7 +4444,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
     elseif k == K"$"
         throw(LoweringError(ex, "`\$` expression outside string or quote block"))
     elseif k == K"module"
-        expand_module(ctx, ex)
+        throw(LoweringError(ex, "`module` is only allowed at top level"))
     elseif k == K"import" || k == K"using"
         expand_import_or_using(ctx, ex)
     elseif k == K"export" || k == K"public"

src/eval.jl

@@ -1,3 +1,6 @@
+# Non-incremental lowering API for non-toplevel non-module expressions.
+# May be removed?
+
 function lower(mod::Module, ex0; expr_compat_mode=false, world=Base.get_world_counter())
     ctx1, ex1 = expand_forms_1(  mod,  ex0, expr_compat_mode, world)
     ctx2, ex2 = expand_forms_2(  ctx1, ex1)
@@ -12,6 +15,96 @@ function macroexpand(mod::Module, ex; expr_compat_mode=false, world=Base.get_wor
     ex1
 end
 
+# Incremental lowering API which can manage toplevel and module expressions.
+#
+# This iteration API is oddly bespoke and arguably somewhat non-Julian for two
+# reasons:
+#
+# * Lowering knows when new modules are required, and may request them with
+#   `:begin_module`. However `eval()` generates those modules so they need to
+#   be passed back into lowering. So we can't just use `Base.iterate()`. (Put a
+#   different way, we have a situation which is suited to coroutines but we
+#   don't want to use full Julia `Task`s for this.)
+# * We might want to implement this `eval()` in Julia's C runtime code or early
+#   in bootstrap. Hence using SimpleVector and Symbol as the return values of
+#   `lower_step()`
+#
+# We might consider changing at least the second of these choices, depending on
+# how we end up putting this into Base.
+
+struct LoweringIterator{GraphType}
+    ctx::MacroExpansionContext{GraphType}
+    todo::Vector{Tuple{SyntaxTree{GraphType}, Bool, Int}}
+end
+
+function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_mode::Bool=false)
+    graph = ensure_macro_attributes(syntax_graph(ex))
+    ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
+    ex = reparent(ctx, ex)
+    LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)])
+end
+
+function lower_step(iter, push_mod=nothing)
+    if !isnothing(push_mod)
+        push_layer!(iter.ctx, push_mod, false)
+    end
+
+    if isempty(iter.todo)
+        return Core.svec(:done)
+    end
+
+    ex, is_module_body, child_idx = pop!(iter.todo)
+    if child_idx > 0
+        next_child = child_idx + 1
+        if child_idx <= numchildren(ex)
+            push!(iter.todo, (ex, is_module_body, next_child))
+            ex = ex[child_idx]
+        else
+            if is_module_body
+                pop_layer!(iter.ctx)
+                return Core.svec(:end_module)
+            else
+                return lower_step(iter)
+            end
+        end
+    end
+
+    k = kind(ex)
+    if !(k in KSet"toplevel module")
+        ex = expand_forms_1(iter.ctx, ex)
+        k = kind(ex)
+    end
+    if k == K"toplevel"
+        push!(iter.todo, (ex, false, 1))
+        return lower_step(iter)
+    elseif k == K"module"
+        name = ex[1]
+        if kind(name) != K"Identifier"
+            throw(LoweringError(name, "Expected module name"))
+        end
+        newmod_name = Symbol(name.name_val)
+        body = ex[2]
+        if kind(body) != K"block"
+            throw(LoweringError(body, "Expected block in module body"))
+        end
+        std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
+        loc = source_location(LineNumberNode, ex)
+        push!(iter.todo, (body, true, 1))
+        return Core.svec(:begin_module, newmod_name, std_defs, loc)
+    else
+        # Non macro expansion parts of lowering
+        ctx2, ex2 = expand_forms_2(iter.ctx, ex)
+        ctx3, ex3 = resolve_scopes(ctx2, ex2)
+        ctx4, ex4 = convert_closures(ctx3, ex3)
+        ctx5, ex5 = linearize_ir(ctx4, ex4)
+        thunk = to_lowered_expr(ex5)
+        return Core.svec(:thunk, thunk)
+    end
+end
+
+
+#-------------------------------------------------------------------------------
+
 function codeinfo_has_image_globalref(@nospecialize(e))
     if e isa GlobalRef
         return 0x00 !== @ccall jl_object_in_image(e.mod::Any)::UInt8
@@ -274,7 +367,7 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int)
     elseif k == K"code_info"
         ir = to_code_info(ex[1], ex.slots)
         if ex.is_toplevel_thunk
-            Expr(:thunk, ir)
+            Expr(:thunk, ir) # TODO: Maybe nice to just return a CodeInfo here?
         else
             ir
         end
@@ -357,19 +450,74 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int)
 end
 
 #-------------------------------------------------------------------------------
-# Our version of eval takes our own data structures
-@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree; expr_compat_mode::Bool=false)
-    k = kind(ex)
-    if k == K"toplevel"
-        x = nothing
-        for e in children(ex)
-            x = eval(mod, e; expr_compat_mode)
+# Our version of eval - should be upstreamed though?
+@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree;
+                                macro_world::UInt=Base.get_world_counter(),
+                                opts...)
+    iter = lower_init(ex, mod, macro_world; opts...)
+    _eval(mod, iter)
+end
+
+if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604
+
+function _eval(mod, iter)
+    modules = Module[]
+    new_mod = nothing
+    result = nothing
+    while true
+        thunk = lower_step(iter, new_mod)::Core.SimpleVector
+        new_mod = nothing
+        type = thunk[1]::Symbol
+        if type == :done
+            break
+        elseif type == :begin_module
+            push!(modules, mod)
+            mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint,
+                                             thunk[4].file::Cstring, thunk[4].line::Cint)::Module
+            new_mod = mod
+        elseif type == :end_module
+            @ccall jl_end_new_module(mod::Module)::Cvoid
+            result = mod
+            mod = pop!(modules)
+        else
+            @assert type == :thunk
+            result = Core.eval(mod, thunk[2])
+        end
+    end
+    @assert isempty(modules)
+    return result
+end
+
+else
+
+function _eval(mod, iter, new_mod=nothing)
+    in_new_mod = !isnothing(new_mod)
+    result = nothing
+    while true
+        thunk = lower_step(iter, new_mod)::Core.SimpleVector
+        new_mod = nothing
+        type = thunk[1]::Symbol
+        if type == :done
+            @assert !in_new_mod
+            break
+        elseif type == :begin_module
+            name = thunk[2]::Symbol
+            std_defs = thunk[3]
+            result = Core.eval(mod,
+                Expr(:module, std_defs, name,
+                     Expr(:block, thunk[4], Expr(:call, m->_eval(m, iter, m), name)))
+            )
+        elseif type == :end_module
+            @assert in_new_mod
+            return mod
+        else
+            @assert type == :thunk
+            result = Core.eval(mod, thunk[2])
         end
-        return x
     end
-    linear_ir = lower(mod, ex; expr_compat_mode)
-    thunk = to_lowered_expr(linear_ir)
-    Core.eval(mod, thunk)
+    return result
+end
+
 end
 
 """

src/macro_expansion.jl

@@ -29,6 +29,16 @@ function MacroExpansionContext(graph::SyntaxGraph, mod::Module, expr_compat_mode
     MacroExpansionContext(graph, Bindings(), layers, LayerId[length(layers)], expr_compat_mode, world)
 end
 
+function push_layer!(ctx::MacroExpansionContext, mod::Module, is_macro_expansion::Bool)
+    new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod,
+                           current_layer_id(ctx), is_macro_expansion)
+    push!(ctx.scope_layers, new_layer)
+    push!(ctx.scope_layer_stack, new_layer.id)
+end
+function pop_layer!(ctx::MacroExpansionContext)
+    pop!(ctx.scope_layer_stack)
+end
+
 current_layer(ctx::MacroExpansionContext) = ctx.scope_layers[last(ctx.scope_layer_stack)]
 current_layer_id(ctx::MacroExpansionContext) = last(ctx.scope_layer_stack)
 
@@ -342,10 +352,9 @@ function expand_macro(ctx, ex)
         expanded = fix_toplevel_expansion(ctx, expanded, mod_for_ast, macro_loc)
         new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod_for_ast,
                                current_layer_id(ctx), true)
-        push!(ctx.scope_layers, new_layer)
-        push!(ctx.scope_layer_stack, new_layer.id)
+        push_layer!(ctx, mod_for_ast, true)
         expanded = expand_forms_1(ctx, expanded)
-        pop!(ctx.scope_layer_stack)
+        pop_layer!(ctx)
     end
     return expanded
 end
@@ -520,23 +529,28 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
     end
 end
 
+function ensure_macro_attributes(graph)
+    ensure_attributes(graph,
+                      var_id=IdTag,
+                      scope_layer=LayerId,
+                      __macro_ctx__=Nothing,
+                      meta=CompileHints)
+end
+
 @fzone "JL: macroexpand" function expand_forms_1(mod::Module, ex::SyntaxTree, expr_compat_mode::Bool, macro_world::UInt)
     if kind(ex) == K"local"
         # This error assumes we're expanding the body of a top level thunk but
         # we might want to make that more explicit in the pass system.
         throw(LoweringError(ex, "local declarations have no effect outside a scope"))
     end
-    graph = ensure_attributes(syntax_graph(ex),
-                              var_id=IdTag,
-                              scope_layer=LayerId,
-                              __macro_ctx__=Nothing,
-                              meta=CompileHints)
+    graph = ensure_macro_attributes(syntax_graph(ex))
     ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
     ex2 = expand_forms_1(ctx, reparent(ctx, ex))
     graph2 = delete_attributes(graph, :__macro_ctx__)
     # TODO: Returning the context with pass-specific mutable data is a bad way
     # to carry state into the next pass. We might fix this by attaching such
     # data to the graph itself as global attributes?
-    ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[], expr_compat_mode, macro_world)
+    ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, ctx.scope_layer_stack,
+                                 expr_compat_mode, macro_world)
     return ctx2, reparent(ctx2, ex2)
 end

src/runtime.jl

@@ -207,35 +207,6 @@ end
 #--------------------------------------------------
 # Functions which create modules or mutate their bindings
 
-# Construct new bare module including only the "default names"
-#
-#     using Core
-#     const modname = modval
-#     public modname
-#
-# And run statments in the toplevel expression `body`
-function eval_module(parentmod::Module, modname::AbstractString, std_defs::Bool,
-                     expr_compat_mode::Bool, body::SyntaxTree)
-    # Here we just use `eval()` with an Expr.
-    # If we wanted to avoid this we'd need to reproduce a lot of machinery from
-    # jl_eval_module_expr()
-    #
-    # 1. Register / deparent toplevel modules
-    # 2. Set binding in parent module
-    # 3. Deal with replacing modules
-    #    * Warn if replacing
-    #    * Root old module being replaced
-    # 4. Run __init__
-    #    * Also run __init__ for any children after parent is defined
-    # mod = @ccall jl_new_module(Symbol(modname)::Symbol, parentmod::Module)::Any
-    # ...
-    name = Symbol(modname)
-    eval_module_body(mod) = eval(mod, body; expr_compat_mode=expr_compat_mode)
-    Core.eval(parentmod,
-        Expr(:module, std_defs, name,
-             Expr(:block, Expr(:call, eval_module_body, name))))
-end
-
 const _Base_has_eval_import = isdefined(Base, :_eval_import)
 
 function eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...)
@@ -357,11 +328,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
     graph = ensure_attributes(SyntaxGraph(), kind=Kind, syntax_flags=UInt16, source=SourceAttrType,
                               value=Any, name_val=String)
     # Attributes for macro expansion
-    graph = ensure_attributes(graph,
-                              var_id=IdTag,
-                              scope_layer=LayerId,
-                              __macro_ctx__=Nothing,
-                              meta=CompileHints,
+    graph = ensure_attributes(ensure_macro_attributes(graph),
                               # Additional attribute for resolve_scopes, for
                               # adding our custom lambda below
                               is_toplevel_thunk=Bool
@@ -393,7 +360,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
     ex1 = expand_forms_1(ctx1, reparent(ctx1, ex0))
     ctx1 = MacroExpansionContext(delete_attributes(graph, :__macro_ctx__),
                                  ctx1.bindings, ctx1.scope_layers,
-                                 LayerId[], false, macro_world)
+                                 ctx1.scope_layer_stack, false, macro_world)
     ex1 = reparent(ctx1, ex1)
 
     # Desugaring

test/demo.jl

@@ -74,7 +74,7 @@ end
 
 #-------------------------------------------------------------------------------
 # Module containing macros used in the demo.
-define_macros = true
+define_macros = false
 if !define_macros
     eval(:(module M end))
 else
@@ -893,7 +893,7 @@ ex = parsestmt(SyntaxTree, src, filename="foo.jl")
  ctx3, ex_scoped,
  ctx4, ex_converted,
  ctx5, ex_compiled,
- ex_expr, eval_result) = debug_lower(M, ex; verbose=true)
+ ex_expr, eval_result) = debug_lower(M, ex; verbose=true, do_eval=true)
 
 # Automatic test reduction
 # bad = reduce_any_failing_toplevel(JuliaLowering, joinpath(@__DIR__, "../src/desugaring.jl"))