Incremental lowering API

Julia's incrementally evaluated top level semantics make it rather
tricky to design a lowering interface for top level and module level
expressions. Currently these expressions are effectively *interpreted*
by eval rather than ever being processed by lowering.

However, I'd like a cleaner separation between "low level evaluation"
and lowering, such that Core can contain only the low level eval "driver
function". I'd like to propose the split as follows:

* "Low level" evaluation is about executing a sequence of thunks
  represented as `CodeInfo`, and potentially about creating modules for
  those to be executed inside.
* Lowering is about expression processing.

In principle, the runtime's view of `eval()` shouldn't know about `Expr`
or `SyntaxTree` (or whatever AST we use) - that should be left to the
compiler frontend. A useful way to think about the duties of the
frontend is to consider the question "What if we wanted to host another
language on top of the Julia runtime?". If we can eventually achieve
that without ever generating Julia `Expr` then we will have succeeded in
separating the frontend.

To implement all this I've recast lowering as an incremental iterative
API in this change. Thus it's the job of `eval()` to simply evaluate
thunks and create new modules as driven by lowering. (Perhaps we'd move
this definition of `eval()` over to the Julia runtime before 1.13.)

Depends on JuliaLang/julia#59604

Author: c42f

src/eval.jl

@@ -1,3 +1,6 @@
+# Non-incremental lowering API for non-toplevel non-module expressions.
+# May be removed?
+
 function lower(mod::Module, ex0; expr_compat_mode=false, world=Base.get_world_counter())
     ctx1, ex1 = expand_forms_1(  mod,  ex0, expr_compat_mode, world)
     ctx2, ex2 = expand_forms_2(  ctx1, ex1)
@@ -12,6 +15,81 @@ function macroexpand(mod::Module, ex; expr_compat_mode=false, world=Base.get_wor
     ex1
 end
 
+# Incremental lowering API which can manage toplevel and module expressions.
+
+struct LoweringIterator{GraphType}
+    ctx::MacroExpansionContext{GraphType}
+    todo::Vector{Tuple{SyntaxTree{GraphType}, Bool, Int}}
+end
+
+function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_mode::Bool=false)
+    graph = ensure_macro_attributes(syntax_graph(ex))
+    ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
+    ex = reparent(ctx, ex)
+    LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)])
+end
+
+function lower_step(iter, push_mod=nothing)
+    if !isnothing(push_mod)
+        push_layer!(iter.ctx, push_mod, false)
+    end
+
+    if isempty(iter.todo)
+        return Core.svec(:done)
+    end
+
+    ex, is_module_body, child_idx = pop!(iter.todo)
+    if child_idx > 0
+        next_child = child_idx + 1
+        if child_idx <= numchildren(ex)
+            push!(iter.todo, (ex, is_module_body, next_child))
+            ex = ex[child_idx]
+        else
+            if is_module_body
+                pop_layer!(iter.ctx)
+                return Core.svec(:end_module)
+            else
+                return lower_step(iter)
+            end
+        end
+    end
+
+    k = kind(ex)
+    if !(k in KSet"toplevel module")
+        ex = expand_forms_1(iter.ctx, ex)
+        k = kind(ex)
+    end
+    if k == K"toplevel"
+        push!(iter.todo, (ex, false, 1))
+        return lower_step(iter)
+    elseif k == K"module"
+        name = ex[1]
+        if kind(name) != K"Identifier"
+            throw(LoweringError(name, "Expected module name"))
+        end
+        newmod_name = Symbol(name.name_val)
+        body = ex[2]
+        if kind(body) != K"block"
+            throw(LoweringError(body, "Expected block in module body"))
+        end
+        std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
+        loc = source_location(LineNumberNode, ex)
+        push!(iter.todo, (body, true, 1))
+        return Core.svec(:begin_module, newmod_name, std_defs, loc)
+    else
+        # Non macro expansion parts of lowering
+        ctx2, ex2 = expand_forms_2(iter.ctx, ex)
+        ctx3, ex3 = resolve_scopes(ctx2, ex2)
+        ctx4, ex4 = convert_closures(ctx3, ex3)
+        ctx5, ex5 = linearize_ir(ctx4, ex4)
+        thunk = to_lowered_expr(ex5)
+        return Core.svec(:thunk, thunk)
+    end
+end
+
+
+#-------------------------------------------------------------------------------
+
 function codeinfo_has_image_globalref(@nospecialize(e))
     if e isa GlobalRef
         return 0x00 !== @ccall jl_object_in_image(e.mod::Any)::UInt8
@@ -274,7 +352,7 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int)
     elseif k == K"code_info"
         ir = to_code_info(ex[1], ex.slots)
         if ex.is_toplevel_thunk
-            Expr(:thunk, ir)
+            Expr(:thunk, ir) # TODO: Maybe nice to just return a CodeInfo here?
         else
             ir
         end
@@ -357,74 +435,39 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int)
 end
 
 #-------------------------------------------------------------------------------
-# Our version of eval takes our own data structures
+# Our version of eval - should be upstreamed though?
 @fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree;
-        expr_compat_mode::Bool=false, macro_world=Base.get_world_counter())
-    graph = ensure_macro_attributes(syntax_graph(ex))
-    ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
-    _eval(ctx, ex)
-end
-
-function _eval_stmts(ctx, exs)
-    res = nothing
-    for ex in exs
-        res = _eval(ctx, ex)
-    end
-    res
-end
-
-function _eval_module_body(ctx, mod, ex)
-    new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod,
-                           current_layer_id(ctx), false)
-    push!(ctx.scope_layers, new_layer)
-    push!(ctx.scope_layer_stack, new_layer.id)
-    stmts = kind(ex[2]) == K"block" ? ex[2][1:end] : ex[2:2]
-    _eval_stmts(ctx, stmts)
-    pop!(ctx.scope_layer_stack)
-end
-
-function _eval_module(ctx, ex)
-    std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
-    newmod_name = Symbol(ex[1].name_val)
-    parent = current_layer(ctx).mod
-
-    # Core.eval(parent,
-    #           Expr(:module, std_defs, newmod_name,
-    #                Expr(:block, Expr(:call,
-    #                                  newmod->_eval_module_body(ctx, newmod, ex),
-    #                                  newmod_name))))
-
-    # https://github.com/JuliaLang/julia/pull/59604
-    loc = source_location(LineNumberNode, ex)
-    newmod = @ccall jl_begin_new_module(parent::Any, newmod_name::Symbol, std_defs::Cint,
-                                        loc.file::Cstring, loc.line::Cint)::Module
-    _eval_module_body(ctx, newmod, ex)
-    @ccall jl_end_new_module(newmod::Module)::Cvoid
-
-    return newmod
-end
-
-function _eval(ctx, ex::SyntaxTree)
-    k = kind(ex)
-    if k == K"toplevel"
-        _eval_stmts(ctx, children(ex))
-    elseif k == K"module"
-        _eval_module(ctx, ex)
-    else
-        ex1 = expand_forms_1(ctx, ex)
-        if kind(ex1) in KSet"toplevel module"
-            _eval(ctx, ex1)
+                                macro_world::UInt=Base.get_world_counter(),
+                                opts...)
+    modules = Module[]
+    iter = lower_init(ex, mod, macro_world; opts...)
+    new_mod = nothing
+    result = nothing
+    while true
+        thunk = lower_step(iter, new_mod)::Core.SimpleVector
+        new_mod = nothing
+        type = thunk[1]::Symbol
+        if type == :done
+            break
+        elseif type == :begin_module
+            push!(modules, mod)
+            mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint,
+                                             thunk[4].file::Cstring, thunk[4].line::Cint)::Module
+            new_mod = mod
+        elseif type == :end_module
+            @ccall jl_end_new_module(mod::Module)::Cvoid
+            result = mod
+            mod = pop!(modules)
         else
-            ctx2, ex2 = expand_forms_2(ctx, ex1)
-            ctx3, ex3 = resolve_scopes(ctx2, ex2)
-            ctx4, ex4 = convert_closures(ctx3, ex3)
-            ctx5, ex5 = linearize_ir(ctx4, ex4)
-            thunk = to_lowered_expr(ex5)
-            Core.eval(current_layer(ctx).mod, thunk)
+            @assert type == :thunk
+            result = Core.eval(mod, thunk[2])
         end
     end
+    @assert isempty(modules)
+    return result
 end
 
+
 """
     include(mod::Module, path::AbstractString)
 

src/macro_expansion.jl

@@ -29,6 +29,16 @@ function MacroExpansionContext(graph::SyntaxGraph, mod::Module, expr_compat_mode
     MacroExpansionContext(graph, Bindings(), layers, LayerId[length(layers)], expr_compat_mode, world)
 end
 
+function push_layer!(ctx::MacroExpansionContext, mod::Module, is_macro_expansion::Bool)
+    new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod,
+                           current_layer_id(ctx), is_macro_expansion)
+    push!(ctx.scope_layers, new_layer)
+    push!(ctx.scope_layer_stack, new_layer.id)
+end
+function pop_layer!(ctx::MacroExpansionContext)
+    pop!(ctx.scope_layer_stack)
+end
+
 current_layer(ctx::MacroExpansionContext) = ctx.scope_layers[last(ctx.scope_layer_stack)]
 current_layer_id(ctx::MacroExpansionContext) = last(ctx.scope_layer_stack)
 
@@ -342,10 +352,9 @@ function expand_macro(ctx, ex)
         expanded = fix_toplevel_expansion(ctx, expanded, mod_for_ast, macro_loc)
         new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod_for_ast,
                                current_layer_id(ctx), true)
-        push!(ctx.scope_layers, new_layer)
-        push!(ctx.scope_layer_stack, new_layer.id)
+        push_layer!(ctx, mod_for_ast, true)
         expanded = expand_forms_1(ctx, expanded)
-        pop!(ctx.scope_layer_stack)
+        pop_layer!(ctx)
     end
     return expanded
 end

test/demo.jl

@@ -74,7 +74,7 @@ end
 
 #-------------------------------------------------------------------------------
 # Module containing macros used in the demo.
-define_macros = true
+define_macros = false
 if !define_macros
     eval(:(module M end))
 else