Support Frame construction without optimization (#503)

The main challenge is to support nested calls. Now that `FrameData.callargs` is not shared across frames, this is doable.

Co-authored-by: Kristoffer Carlsson <kcarlsson89@gmail.com>
Co-authored-by: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com>

src/interpret.jl

@@ -77,6 +77,8 @@ function lookup_or_eval(@nospecialize(recurse), frame, @nospecialize(node))
         return lookup_var(frame, node)
     elseif isa(node, SlotNumber)
         return lookup_var(frame, node)
+    elseif isa(node, GlobalRef)
+        return lookup_var(frame, node)
     elseif isa(node, Symbol)
         return getfield(moduleof(frame), node)
     elseif isa(node, QuoteNode)
@@ -130,25 +132,29 @@ function resolvefc(frame, @nospecialize(expr))
     error("unexpected ccall to ", expr)
 end
 
-function collect_args(frame::Frame, call_expr::Expr; isfc::Bool=false)
+function collect_args(@nospecialize(recurse), frame::Frame, call_expr::Expr; isfc::Bool=false)
     args = frame.framedata.callargs
     resize!(args, length(call_expr.args))
     mod = moduleof(frame)
     args[1] = isfc ? resolvefc(frame, call_expr.args[1]) : @lookup(mod, frame, call_expr.args[1])
     for i = 2:length(args)
-        args[i] = @lookup(mod, frame, call_expr.args[i])
+        if isexpr(call_expr.args[i], :call)
+            args[i] = lookup_or_eval(recurse, frame, call_expr.args[i])
+        else
+            args[i] = @lookup(mod, frame, call_expr.args[i])
+        end
     end
     return args
 end
 
 """
-    ret = evaluate_foreigncall(frame::Frame, call_expr)
+    ret = evaluate_foreigncall(recurse, frame::Frame, call_expr)
 
 Evaluate a `:foreigncall` (from a `ccall`) statement `callexpr` in the context of `frame`.
 """
-function evaluate_foreigncall(frame::Frame, call_expr::Expr)
+function evaluate_foreigncall(@nospecialize(recurse), frame::Frame, call_expr::Expr)
     head = call_expr.head
-    args = collect_args(frame, call_expr; isfc = head==:foreigncall)
+    args = collect_args(recurse, frame, call_expr; isfc = head==:foreigncall)
     for i = 2:length(args)
         arg = args[i]
         args[i] = isa(arg, Symbol) ? QuoteNode(arg) : arg
@@ -167,11 +173,11 @@ function evaluate_foreigncall(frame::Frame, call_expr::Expr)
 end
 
 # We have to intercept ccalls / llvmcalls before we try it as a builtin
-function bypass_builtins(frame, call_expr, pc)
+function bypass_builtins(@nospecialize(recurse), frame, call_expr, pc)
     if isassigned(frame.framecode.methodtables, pc)
         tme = frame.framecode.methodtables[pc]
         if isa(tme, Compiled)
-            fargs = collect_args(frame, call_expr)
+            fargs = collect_args(recurse, frame, call_expr)
             f = to_function(fargs[1])
             fmod = parentmodule(f)::Module
             if fmod === JuliaInterpreter.CompiledCalls || fmod === Core.Compiler
@@ -187,24 +193,24 @@ end
 function evaluate_call_compiled!(::Compiled, frame::Frame, call_expr::Expr; enter_generated::Bool=false)
     # @assert !enter_generated
     pc = frame.pc
-    ret = bypass_builtins(frame, call_expr, pc)
+    ret = bypass_builtins(Compiled(), frame, call_expr, pc)
     isa(ret, Some{Any}) && return ret.value
     ret = maybe_evaluate_builtin(frame, call_expr, false)
     isa(ret, Some{Any}) && return ret.value
-    fargs = collect_args(frame, call_expr)
+    fargs = collect_args(Compiled(), frame, call_expr)
     f = fargs[1]
     popfirst!(fargs)  # now it's really just `args`
     return f(fargs...)
 end
 
 function evaluate_call_recurse!(@nospecialize(recurse), frame::Frame, call_expr::Expr; enter_generated::Bool=false)
     pc = frame.pc
-    ret = bypass_builtins(frame, call_expr, pc)
+    ret = bypass_builtins(recurse, frame, call_expr, pc)
     isa(ret, Some{Any}) && return ret.value
     ret = maybe_evaluate_builtin(frame, call_expr, true)
     isa(ret, Some{Any}) && return ret.value
     call_expr = ret
-    fargs = collect_args(frame, call_expr)
+    fargs = collect_args(recurse, frame, call_expr)
     if fargs[1] === Core.eval
         return Core.eval(fargs[2], fargs[3])  # not a builtin, but worth treating specially
     elseif fargs[1] === Base.rethrow
@@ -396,7 +402,7 @@ function eval_rhs(@nospecialize(recurse), frame, node::Expr)
         isa(recurse, Compiled) && return evaluate_call_compiled!(recurse, frame, node)
         return evaluate_call_recurse!(recurse, frame, node)
     elseif head === :foreigncall || head === :cfunction
-        return evaluate_foreigncall(frame, node)
+        return evaluate_foreigncall(recurse, frame, node)
     elseif head === :copyast
         val = (node.args[1]::QuoteNode).value
         return isa(val, Expr) ? copy(val) : val

src/precompile.jl

@@ -10,7 +10,7 @@ function _precompile_()
               evaluate_primitivetype)
         @assert precompile(Tuple{typeof(f), Any, Frame, Expr})
     end
-    @assert precompile(Tuple{typeof(evaluate_foreigncall), Frame, Expr})
+    @assert precompile(Tuple{typeof(evaluate_foreigncall), Any, Frame, Expr})
     @assert precompile(Tuple{typeof(evaluate_methoddef), Frame, Expr})
     @assert precompile(Tuple{typeof(lookup_global_refs!), Expr})
     @assert precompile(Tuple{typeof(lookup_or_eval), Any, Frame, Any})

src/types.jl

@@ -250,12 +250,12 @@ function Frame(framecode::FrameCode, framedata::FrameData, pc=1, caller=nothing)
     end
 end
 """
-    frame = Frame(mod::Module, src::CodeInfo)
+    frame = Frame(mod::Module, src::CodeInfo; kwargs...)
 
 Construct a `Frame` to evaluate `src` in module `mod`.
 """
-function Frame(mod::Module, src::CodeInfo)
-    framecode = FrameCode(mod, src)
+function Frame(mod::Module, src::CodeInfo; kwargs...)
+    framecode = FrameCode(mod, src; kwargs...)
     return Frame(framecode, prepare_framedata(framecode, []))
 end
 """

test/interpret.jl

@@ -836,3 +836,41 @@ end
     frame = JuliaInterpreter.prepare_frame(framecode, frameargs, mi.sparam_vals)
     @test JuliaInterpreter.finish_and_return!(frame) === 8
 end
+
+@testset "interpretation of unoptimized frame" begin
+    let # should be able to interprete nested calls within `:foreigncall` expressions
+        # even if `JuliaInterpreter.optimize!` doesn't flatten them
+        M = Module()
+        lwr = Meta.@lower M begin
+            global foo = @ccall strlen("foo"::Cstring)::Csize_t
+            foo == 3
+        end
+        src = lwr.args[1]::Core.CodeInfo
+        frame = Frame(M, src; optimize=false)
+        @test length(frame.framecode.src.code) == length(src.code)
+        @test JuliaInterpreter.finish_and_return!(frame, true) 
+
+        M = Module()
+        lwr = Meta.@lower M begin
+            strp = Ref{Ptr{Cchar}}(0)
+            fmt = "hi+%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f\n"
+            len = @ccall asprintf(
+                strp::Ptr{Ptr{Cchar}},
+                fmt::Cstring,
+                ; # begin varargs
+                0x1::UInt8, 0x2::UInt8, 0x3::UInt8, 0x4::UInt8, 0x5::UInt8, 0x6::UInt8, 0x7::UInt8, 0x8::UInt8, 0x9::UInt8, 0xa::UInt8, 0xb::UInt8, 0xc::UInt8, 0xd::UInt8, 0xe::UInt8, 0xf::UInt8,
+                1.1::Cfloat, 2.2::Cfloat, 3.3::Cfloat, 4.4::Cfloat, 5.5::Cfloat, 6.6::Cfloat, 7.7::Cfloat, 8.8::Cfloat, 9.9::Cfloat,
+            )::Cint
+            str = unsafe_string(strp[], len)
+            @ccall free(strp[]::Cstring)::Cvoid
+            str == "hi+1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-1.1-2.2-3.3-4.4-5.5-6.6-7.7-8.8-9.9\n"
+        end
+        src = lwr.args[1]::Core.CodeInfo
+        frame = Frame(M, src; optimize=false)
+        @test length(frame.framecode.src.code) == length(src.code)
+        @test JuliaInterpreter.finish_and_return!(frame, true) 
+    end
+
+    iscallexpr(ex::Expr) = ex.head === :call
+    @test (@interpret iscallexpr(:(sin(3.14))))
+end