fix #23661, QuoteNode passed to macros instead of Expr

base/docs/Docs.jl

@@ -647,10 +647,10 @@ const BINDING_HEADS = [:typealias, :const, :global, :(=)]  # deprecation: remove
 # For the special `:@mac` / `:(Base.@mac)` syntax for documenting a macro after definition.
 isquotedmacrocall(x) =
     isexpr(x, :copyast, 1) &&
-    isa(x.args[1], QuoteNode) &&
-    isexpr(x.args[1].value, :macrocall, 2)
+    isexpr(x.args[1], :inert) &&
+    isexpr(x.args[1].args[1], :macrocall, 2)
 # Simple expressions / atoms the may be documented.
-isbasicdoc(x) = isexpr(x, :.) || isa(x, Union{QuoteNode, Symbol})
+isbasicdoc(x) = isexpr(x, :.) || isexpr(x, :quote) || isa(x, Union{QuoteNode, Symbol})
 is_signature(x) = isexpr(x, :call) || (isexpr(x, :(::), 2) && isexpr(x.args[1], :call)) || isexpr(x, :where)
 
 function docm(source::LineNumberNode, mod::Module, meta, ex, define = true)

base/docs/core.jl

@@ -7,6 +7,7 @@ import ..esc, ..push!, ..getindex, ..unsafe_load, ..Csize_t
 function doc!(source::LineNumberNode, mod::Module, str, ex)
     push!(DOCS, (mod, ex, str, source.file, source.line))
 end
+_doc!(file, line, mod, str, ex) = doc!(LineNumberNode(line, file), mod, str, ex)
 const DOCS = Array{Any, 1}()
 
 isexpr(x, h) = isa(x, Expr) && x.head === h
@@ -15,7 +16,7 @@ lazy_iterpolate(s::AbstractString) = Expr(:call, Core.svec, s)
 lazy_iterpolate(x) = isexpr(x, :string) ? Expr(:call, Core.svec, x.args...) : x
 
 function docm(source::LineNumberNode, mod::Module, str, x)
-    out = esc(Expr(:call, doc!, QuoteNode(source), mod, lazy_iterpolate(str), Expr(:quote, x)))
+    out = esc(Expr(:call, _doc!, QuoteNode(source.file), QuoteNode(source.line), mod, lazy_iterpolate(str), Expr(:quote, x)))
     isexpr(x, :module) ? Expr(:toplevel, out, esc(x)) :
     isexpr(x, :call) ? out : Expr(:block, esc(x), out)
 end

base/markdown/Markdown.jl

@@ -45,12 +45,15 @@ macro md_str(s, t...)
     mdexpr(s, t...)
 end
 
-function doc_str(md, source::LineNumberNode, mod::Module)
+function doc_str(md, source::Union{LineNumberNode,Expr}, mod::Module)
+    if source isa Expr
+        source = LineNumberNode(source.args[1], source.args[2])
+    end
     md.meta[:path] = isa(source.file, Symbol) ? String(source.file) : ""
     md.meta[:module] = mod
     md
 end
-doc_str(md::AbstractString, source::LineNumberNode, mod::Module) = doc_str(parse(md), source, mod)
+doc_str(md::AbstractString, source::Union{LineNumberNode,Expr}, mod::Module) = doc_str(parse(md), source, mod)
 
 macro doc_str(s::AbstractString, t...)
     docexpr(__source__, __module__, s, t...)

base/repl/REPLCompletions.jl

@@ -267,6 +267,9 @@ end
 # will show it consist of Expr, QuoteNode's and Symbol's which all needs to
 # be handled differently to iterate down to get the value of whitespace_chars.
 function get_value(sym::Expr, fn)
+    if sym.head === :quote
+        return get_value(QuoteNode(sym.args[1]), fn)
+    end
     sym.head != :. && return (nothing, false)
     for ex in sym.args
         fn, found = get_value(ex, fn)

src/ast.c

@@ -489,68 +489,75 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, int eo, jl_mod
             e = cdr_(e);
         else
             n++;
-        if (!eo) {
+        // IR-only forms, not part of ASTs
+        jl_value_t *scmv = NULL, *temp = NULL;
+        if (sym == label_sym) {
+            JL_GC_PUSH1(&scmv);
+            scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
+            temp = jl_new_struct(jl_labelnode_type, scmv);
+            JL_GC_POP();
+            return temp;
+        }
+        else if (sym == goto_sym) {
+            JL_GC_PUSH1(&scmv);
+            scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
+            temp = jl_new_struct(jl_gotonode_type, scmv);
+            JL_GC_POP();
+            return temp;
+        }
+        else if (sym == newvar_sym) {
+            JL_GC_PUSH1(&scmv);
+            scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
+            temp = jl_new_struct(jl_newvarnode_type, scmv);
+            JL_GC_POP();
+            return temp;
+        }
+        else if (sym == globalref_sym) {
+            JL_GC_PUSH1(&scmv);
+            scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
+            temp = scm_to_julia_(fl_ctx, car_(cdr_(e)), 0, mod);
+            assert(jl_is_module(scmv));
+            assert(jl_is_symbol(temp));
+            temp = jl_module_globalref((jl_module_t*)scmv, (jl_sym_t*)temp);
+            JL_GC_POP();
+            return temp;
+        }
+        else if (sym == top_sym) {
+            assert(mod && "top should not be generated by the parser");
+            JL_GC_PUSH1(&scmv);
+            scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
+            assert(jl_is_symbol(scmv));
+            temp = jl_module_globalref(jl_base_relative_to(mod), (jl_sym_t*)scmv);
+            JL_GC_POP();
+            return temp;
+        }
+        else if (sym == core_sym) {
+            JL_GC_PUSH1(&scmv);
+            scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
+            assert(jl_is_symbol(scmv));
+            temp = jl_module_globalref(jl_core_module, (jl_sym_t*)scmv);
+            JL_GC_POP();
+            return temp;
+        }
+        else if (!eo) {
             if (sym == line_sym && (n == 1 || n == 2)) {
                 jl_value_t *linenum = scm_to_julia_(fl_ctx, car_(e), 0, mod);
                 jl_value_t *file = jl_nothing;
                 JL_GC_PUSH2(&linenum, &file);
                 if (n == 2)
                     file = scm_to_julia_(fl_ctx, car_(cdr_(e)), 0, mod);
-                jl_value_t *temp = jl_new_struct(jl_linenumbernode_type, linenum, file);
-                JL_GC_POP();
-                return temp;
-            }
-            jl_value_t *scmv = NULL, *temp = NULL;
-            JL_GC_PUSH1(&scmv);
-            if (sym == label_sym) {
-                scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
-                temp = jl_new_struct(jl_labelnode_type, scmv);
-                JL_GC_POP();
-                return temp;
-            }
-            if (sym == goto_sym) {
-                scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
-                temp = jl_new_struct(jl_gotonode_type, scmv);
+                temp = jl_new_struct(jl_linenumbernode_type, linenum, file);
                 JL_GC_POP();
                 return temp;
             }
-            if (sym == inert_sym || (sym == quote_sym && (!iscons(car_(e))))) {
-                scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
+            else if (sym == inert_sym || (sym == quote_sym && (!iscons(car_(e))))) {
+                // `quote` expressions differ before and after lowering
+                JL_GC_PUSH1(&scmv);
+                scmv = scm_to_julia_(fl_ctx, car_(e), 1, mod);
                 temp = jl_new_struct(jl_quotenode_type, scmv);
                 JL_GC_POP();
                 return temp;
             }
-            if (sym == top_sym) {
-                assert(mod && "top should not be generated by the parser");
-                scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
-                assert(jl_is_symbol(scmv));
-                temp = jl_module_globalref(jl_base_relative_to(mod), (jl_sym_t*)scmv);
-                JL_GC_POP();
-                return temp;
-            }
-            if (sym == core_sym) {
-                scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
-                assert(jl_is_symbol(scmv));
-                temp = jl_module_globalref(jl_core_module, (jl_sym_t*)scmv);
-                JL_GC_POP();
-                return temp;
-            }
-            if (sym == globalref_sym) {
-                scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
-                temp = scm_to_julia_(fl_ctx, car_(cdr_(e)), 0, mod);
-                assert(jl_is_module(scmv));
-                assert(jl_is_symbol(temp));
-                temp = jl_module_globalref((jl_module_t*)scmv, (jl_sym_t*)temp);
-                JL_GC_POP();
-                return temp;
-            }
-            if (sym == newvar_sym) {
-                scmv = scm_to_julia_(fl_ctx, car_(e), 0, mod);
-                temp = jl_new_struct(jl_newvarnode_type, scmv);
-                JL_GC_POP();
-                return temp;
-            }
-            JL_GC_POP();
         }
         else if (sym == inert_sym && !iscons(car_(e))) {
             sym = quote_sym;
@@ -705,7 +712,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const
     value_t s = cvalue_static_cstrn(fl_ctx, str, len);
     value_t files = cvalue_static_cstrn(fl_ctx, filename, filename_len);
     value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string")), s, files);
-    jl_value_t *res = e == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, e, 0, NULL);
+    jl_value_t *res = e == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, e, 1, NULL);
     jl_ast_ctx_leave(ctx);
     return res;
 }
@@ -734,7 +741,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
     if (e == fl_ctx->FL_EOF)
         expr = jl_nothing;
     else
-        expr = scm_to_julia(fl_ctx, e, 0, NULL);
+        expr = scm_to_julia(fl_ctx, e, 1, NULL);
 
     pos1 = jl_box_long(tosize(fl_ctx, cdr_(p), "parse"));
     jl_ast_ctx_leave(ctx);
@@ -796,7 +803,7 @@ jl_value_t *jl_parse_eval_all(const char *fname,
             {
                 JL_TIMING(LOWERING);
                 if (fl_ctx->T == fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "contains-macrocall")), expression)) {
-                    form = scm_to_julia(fl_ctx, expression, 0, inmodule);
+                    form = scm_to_julia(fl_ctx, expression, 1, inmodule);
                     form = jl_expand_macros(form, inmodule, NULL, 0);
                     expression = julia_to_scm(fl_ctx, form);
                 }
@@ -874,14 +881,14 @@ static int jl_parse_deperror(fl_context_t *fl_ctx, int err)
 }
 
 // returns either an expression or a thunk
-jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule)
+jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule, int expronly)
 {
     jl_ast_context_t *ctx = jl_ast_ctx_enter();
     fl_context_t *fl_ctx = &ctx->fl;
     JL_AST_PRESERVE_PUSH(ctx, old_roots, inmodule);
     value_t arg = julia_to_scm(fl_ctx, expr);
     value_t e = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, funcname)), arg);
-    jl_value_t *result = scm_to_julia(fl_ctx, e, 0, inmodule);
+    jl_value_t *result = scm_to_julia(fl_ctx, e, expronly, inmodule);
     JL_AST_PRESERVE_POP(ctx, old_roots);
     jl_ast_ctx_leave(ctx);
     return result;
@@ -918,11 +925,23 @@ jl_code_info_t *jl_wrap_expr(jl_value_t *expr)
 
 // syntax tree accessors
 
-JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
+// if to_ir is true, convert exprs that correspond to IR forms to appropriate node types
+static jl_value_t *copy_ast(jl_value_t *expr, int to_ir, jl_module_t *mod)
 {
     if (expr && jl_is_expr(expr)) {
         jl_expr_t *e = (jl_expr_t*)expr;
         size_t i, l = jl_array_len(e->args);
+        if (to_ir) {
+            if (e->head == line_sym) {
+                if (l == 1)
+                    return jl_new_struct(jl_linenumbernode_type, jl_exprarg(e, 0), jl_nothing);
+                if (l == 2)
+                    return jl_new_struct(jl_linenumbernode_type, jl_exprarg(e, 0), jl_exprarg(e, 1));
+            }
+            else if (e->head == inert_sym || (e->head == quote_sym && !jl_is_expr(jl_exprarg(e, 0)))) {
+                return jl_new_struct(jl_quotenode_type, jl_exprarg(e, 0));
+            }
+        }
         jl_expr_t *ne = jl_exprn(e->head, l);
         JL_GC_PUSH2(&ne, &expr);
         if (l == 0) {
@@ -932,7 +951,7 @@ JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
         else {
             for (i = 0; i < l; i++) {
                 jl_value_t *a = jl_exprarg(e, i);
-                jl_exprargset(ne, i, jl_copy_ast(a));
+                jl_exprargset(ne, i, copy_ast(a, to_ir, mod));
             }
         }
         JL_GC_POP();
@@ -941,6 +960,16 @@ JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
     return expr;
 }
 
+JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
+{
+    return copy_ast(expr, 0, NULL);
+}
+
+jl_value_t *jl_copy_ast_to_ir(jl_value_t *expr, jl_module_t *mod)
+{
+    return copy_ast(expr, 1, mod);
+}
+
 JL_DLLEXPORT int jl_is_operator(char *sym)
 {
     jl_ast_context_t *ctx = jl_ast_ctx_enter();
@@ -1075,7 +1104,9 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str
         return expr;
     }
     if (e->head == quote_sym && jl_expr_nargs(e) == 1) {
-        expr = jl_call_scm_on_ast("julia-bq-macro", jl_exprarg(e, 0), inmodule);
+        if (jl_is_symbol(jl_exprarg(e, 0)))
+            return expr;
+        expr = jl_call_scm_on_ast("julia-bq-macro", jl_exprarg(e, 0), inmodule, 1);
         JL_GC_PUSH1(&expr);
         if (macroctx) {
             // in a macro, `quote` also implies `escape`
@@ -1141,7 +1172,7 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand(jl_value_t *expr, jl_module_t *inmodule)
     JL_GC_PUSH1(&expr);
     expr = jl_copy_ast(expr);
     expr = jl_expand_macros(expr, inmodule, NULL, 0);
-    expr = jl_call_scm_on_ast("julia-expand-macroscope", expr, inmodule);
+    expr = jl_call_scm_on_ast("julia-expand-macroscope", expr, inmodule, 1);
     JL_GC_POP();
     return expr;
 }
@@ -1152,7 +1183,7 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
     JL_GC_PUSH1(&expr);
     expr = jl_copy_ast(expr);
     expr = jl_expand_macros(expr, inmodule, NULL, 1);
-    expr = jl_call_scm_on_ast("julia-expand-macroscope", expr, inmodule);
+    expr = jl_call_scm_on_ast("julia-expand-macroscope", expr, inmodule, 1);
     JL_GC_POP();
     return expr;
 }
@@ -1163,7 +1194,7 @@ JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
     JL_GC_PUSH1(&expr);
     expr = jl_copy_ast(expr);
     expr = jl_expand_macros(expr, inmodule, NULL, 0);
-    expr = jl_call_scm_on_ast("jl-expand-to-thunk", expr, inmodule);
+    expr = jl_call_scm_on_ast("jl-expand-to-thunk", expr, inmodule, 0);
     JL_GC_POP();
     return expr;
 }

src/julia_internal.h

@@ -499,7 +499,8 @@ jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e,
                                           jl_code_info_t *src,
                                           jl_svec_t *sparam_vals);
 int jl_is_toplevel_only_expr(jl_value_t *e);
-jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule);
+jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule, int expronly);
+jl_value_t *jl_copy_ast_to_ir(jl_value_t *expr, jl_module_t *mod);
 
 jl_method_instance_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t nargs, int cache, size_t world);
 jl_value_t *jl_gf_invoke(jl_tupletype_t *types, jl_value_t **args, size_t nargs);

src/method.c

@@ -721,9 +721,13 @@ JL_DLLEXPORT void jl_method_def(jl_svec_t *argdata,
     mt = ftype->name->mt;
     name = mt->name;
     if (!jl_is_code_info(f)) {
-        // this occurs when there is a closure being added to an out-of-scope function
-        // the user should only do this at the toplevel
-        // the result is that the closure variables get interpolated directly into the AST
+        // This occurs when there is a closure being added to an out-of-scope function.
+        // The user should only do this at the toplevel.
+        // The result is that the closure variables get interpolated directly into the AST.
+        // This is a strange case because the function body is produced by backquote, but
+        // needs to be IR instead of the ASTs backquote normally returns. So an extra pass
+        // is needed to convert the expression to IR form.
+        f = (jl_code_info_t*)jl_copy_ast_to_ir((jl_value_t*)f, module);
         f = jl_new_code_info_from_ast((jl_expr_t*)f);
     }
     m = jl_new_method(f, name, module, (jl_tupletype_t*)argtype, nargs, isva, tvars, isstaged == jl_true);

src/toplevel.c

@@ -212,7 +212,7 @@ jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex)
     JL_TRY {
         if (std_imports) {
             // add `eval` function
-            defaultdefs = jl_call_scm_on_ast("module-default-defs", (jl_value_t*)ex, newm);
+            defaultdefs = jl_call_scm_on_ast("module-default-defs", (jl_value_t*)ex, newm, 0);
             ptls->world_age = jl_world_counter;
             jl_toplevel_eval_flex(newm, defaultdefs, 0, 1);
             defaultdefs = NULL;

test/cartesian.jl

@@ -2,6 +2,6 @@
 
 @test Base.Cartesian.exprresolve(:(1 + 3)) == 4
 ex = Base.Cartesian.exprresolve(:(if 5 > 4; :x; else :y; end))
-@test ex.args[2] == QuoteNode(:x)
+@test ex.args[2] == Expr(:quote, :x)
 
 @test Base.Cartesian.lreplace!("val_col", Base.Cartesian.LReplace{String}(:col, "col", 1)) == "val_1"

test/core.jl

@@ -5195,7 +5195,7 @@ let err = try; @macroexpand @isdefined :x; false; catch ex; ex; end,
     @test err.line === __source__.line
     e = err.error::MethodError
     @test e.f === getfield(@__MODULE__, Symbol("@isdefined"))
-    @test e.args === (__source__, @__MODULE__, :(:x))
+    @test e.args == (__source__, @__MODULE__, :(:x))
 end
 f_isdefined_cl_1(y) = (local x; for i = 1:y; x = 2; end; () -> x; @isdefined x)
 f_isdefined_cl_2(y) = (local x; for i = 1:y; x = 2; end; () -> @isdefined x)