keep track of filenames in IR with metadata push/pop

- add location info to empty blocks; fixes #15280
- remove file names from non-initial `line` nodes
- emit `(meta push_loc filename)` and `(meta pop_loc)` around nested
  blocks from different files, to handle macros
- fix line number of `catch` blocks

base/boot.jl

@@ -82,7 +82,6 @@
 #end
 
 #immutable LineNumberNode
-# file::Symbol
 # line::Int
 #end
 
@@ -276,7 +275,7 @@ _new(:GotoNode, :Int)
 _new(:NewvarNode, :SlotNumber)
 _new(:QuoteNode, :ANY)
 _new(:SSAValue, :Int)
-eval(:((::Type{LineNumberNode})(f::Symbol, l::Int) = $(Expr(:new, :LineNumberNode, :f, :l))))
+eval(:((::Type{LineNumberNode})(l::Int) = $(Expr(:new, :LineNumberNode, :l))))
 eval(:((::Type{GlobalRef})(m::Module, s::Symbol) = $(Expr(:new, :GlobalRef, :m, :s))))
 eval(:((::Type{SlotNumber})(n::Int) = $(Expr(:new, :SlotNumber, :n))))
 eval(:((::Type{TypedSlot})(n::Int, t::ANY) = $(Expr(:new, :TypedSlot, :n, :t))))

base/show.jl

@@ -531,7 +531,7 @@ end
 ## AST printing ##
 
 show_unquoted(io::IO, sym::Symbol, ::Int, ::Int) = print(io, sym)
-show_unquoted(io::IO, ex::LineNumberNode, ::Int, ::Int) = show_linenumber(io, ex.line, ex.file)
+show_unquoted(io::IO, ex::LineNumberNode, ::Int, ::Int) = show_linenumber(io, ex.line)
 show_unquoted(io::IO, ex::LabelNode, ::Int, ::Int) = print(io, ex.label, ": ")
 show_unquoted(io::IO, ex::GotoNode, ::Int, ::Int) = print(io, "goto ", ex.label)
 show_unquoted(io::IO, ex::GlobalRef, ::Int, ::Int) = print(io, ex.mod, '.', ex.name)
@@ -597,7 +597,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int)
  head, args, nargs = ex.head, ex.args, length(ex.args)
  emphstate = typeemphasize(io)
  show_type = true
- if (ex.head == :(=) ||
+ if (ex.head == :(=) || ex.head == :line ||
  ex.head == :boundscheck ||
  ex.head == :gotoifnot ||
  ex.head == :return)

src/alloc.c

@@ -404,7 +404,7 @@ static jl_lambda_info_t *jl_instantiate_staged(jl_method_t *generator, jl_tuplet
  jl_expr_t *body = jl_exprn(jl_symbol("block"), 2);
  jl_cellset(((jl_expr_t*)jl_exprarg(ex,1))->args, 0, body);
  linenum = jl_box_long(generator->line);
- jl_value_t *linenode = jl_new_struct(jl_linenumbernode_type, generator->file, linenum);
+ jl_value_t *linenode = jl_new_struct(jl_linenumbernode_type, linenum);
  jl_cellset(body->args, 0, linenode);
 
  // invoke code generator
@@ -505,7 +505,6 @@ JL_DLLEXPORT void jl_method_init_properties(jl_method_t *m)
  jl_lambda_info_t *li = m->lambda_template;
  jl_value_t *body1 = skip_meta(li->code);
  if (jl_is_linenode(body1)) {
- m->file = jl_linenode_file(body1);
  m->line = jl_linenode_line(body1);
  }
  else if (jl_is_expr(body1) && ((jl_expr_t*)body1)->head == line_sym) {

src/ast.c

@@ -498,16 +498,10 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, int eo)
  else
  n++;
  if (!eo) {
- if (sym == line_sym && n==2) {
- // NOTE: n==3 case exists: '(line, linenum, filename, funcname) passes
- // the original name through to keyword-arg specializations.
- // See 'line handling in julia-syntax.scm:keywords-method-def-expr
- jl_value_t *filename = NULL, *linenum = NULL;
- JL_GC_PUSH2(&filename, &linenum);
- filename = scm_to_julia_(fl_ctx, car_(cdr_(e)), 0);
- linenum = scm_to_julia_(fl_ctx, car_(e), 0);
- jl_value_t *temp = jl_new_struct(jl_linenumbernode_type,
- filename, linenum);
+ if (sym == line_sym && n==1) {
+ jl_value_t *linenum = scm_to_julia_(fl_ctx, car_(e), 0);
+ JL_GC_PUSH1(&linenum);
+ jl_value_t *temp = jl_new_struct(jl_linenumbernode_type, linenum);
  JL_GC_POP();
  return temp;
  }
@@ -658,21 +652,13 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v)
  fl_free_gc_handles(fl_ctx, 1);
  return scmv;
  }
- if (jl_typeis(v, jl_linenumbernode_type)) {
- // GC Note: jl_fieldref(v, 1) allocates but neither jl_fieldref(v, 0)
- // or julia_to_list2 should allocate here
- value_t args = julia_to_list2(fl_ctx, jl_fieldref(v,1), jl_fieldref(v,0));
- fl_gc_handle(fl_ctx, &args);
- value_t hd = julia_to_scm_(fl_ctx, (jl_value_t*)line_sym);
- value_t scmv = fl_cons(fl_ctx, hd, args);
- fl_free_gc_handles(fl_ctx, 1);
- return scmv;
- }
  // GC Note: jl_fieldref(v, 0) allocate for LabelNode, GotoNode
  // but we don't need a GC root here because julia_to_list2
  // shouldn't allocate in this case.
  if (jl_typeis(v, jl_labelnode_type))
  return julia_to_list2(fl_ctx, (jl_value_t*)label_sym, jl_fieldref(v,0));
+ if (jl_typeis(v, jl_linenumbernode_type))
+ return julia_to_list2(fl_ctx, (jl_value_t*)line_sym, jl_fieldref(v,0));
  if (jl_typeis(v, jl_gotonode_type))
  return julia_to_list2(fl_ctx, (jl_value_t*)goto_sym, jl_fieldref(v,0));
  if (jl_typeis(v, jl_quotenode_type))

src/cgutils.cpp

@@ -312,6 +312,8 @@ JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed)
  return PointerType::get(lt, 0);
  }
  if (jl_is_bitstype(jt)) {
+ if (jt == (jl_value_t*)jl_long_type)
+ return T_size;
  int nb = jl_datatype_size(jt);
  if (jl_is_floattype(jt)) {
 #ifndef DISABLE_FLOAT16

src/jltypes.c

@@ -3461,8 +3461,8 @@ void jl_init_types(void)
 
  jl_linenumbernode_type =
  jl_new_datatype(jl_symbol("LineNumberNode"), jl_any_type, jl_emptysvec,
- jl_svec(2, jl_symbol("file"), jl_symbol("line")),
- jl_svec(2, jl_symbol_type, jl_long_type), 0, 0, 2);
+ jl_svec(1, jl_symbol("line")),
+ jl_svec(1, jl_long_type), 0, 0, 1);
 
  jl_labelnode_type =
  jl_new_datatype(jl_symbol("LabelNode"), jl_any_type, jl_emptysvec,

src/julia-parser.scm

@@ -593,7 +593,9 @@
  (if (invalid-initial-token? t)
  (error (string "unexpected \"" t "\"")))
  (if (closer? t)
- (list head) ; empty block
+ (if add-linenums ;; empty block
+ (list head (line-number-node s))
+ (list head))
  (let loop ((ex
  ;; in allow-empty mode skip leading runs of operator
  (if (and allow-empty (memv t ops))
@@ -1138,7 +1140,10 @@
  (error "let variables should end in \";\" or newline"))
  (let ((ex (parse-block s)))
  (expect-end s word)
- `(let ,ex ,@binds))))
+ ;; don't need line info in an empty let block
+ (if (and (length= ex 2) (pair? (cadr ex)) (eq? (caadr ex) 'line))
+ `(let (block) ,@binds)
+ `(let ,ex ,@binds)))))
  ((global local)
  (let* ((lno (input-port-line (ts:port s)))
  (const (and (eq? (peek-token s) 'const)
@@ -1170,26 +1175,18 @@
  (eq? (car sig) 'tuple))))
  (error (string "expected \"(\" in " word " definition"))
  sig)))
- (loc (begin (if (not (eq? (peek-token s) 'end))
- ;; if ends on same line, don't skip the following newline
- (skip-ws-and-comments (ts:port s)))
- (line-number-node s)))
  (body (parse-block s)))
  (expect-end s word)
- (add-filename-to-block! body loc)
  (list word def body)))))
  ((abstract)
  (list 'abstract (parse-subtype-spec s)))
  ((type immutable)
  (let ((immu? (eq? word 'immutable)))
  (if (reserved-word? (peek-token s))
  (error (string "invalid type name \"" (take-token s) "\"")))
- (let ((sig (parse-subtype-spec s))
- (loc (begin (if (newline? (peek-token s))
- (skip-ws-and-comments (ts:port s)))
- (line-number-node s))))
+ (let ((sig (parse-subtype-spec s)))
  (begin0 (list 'type (if (eq? word 'type) 'true 'false)
- sig (add-filename-to-block! (parse-block s) loc))
+ sig (parse-block s))
  (expect-end s word)))))
  ((bitstype)
  (list 'bitstype (with-space-sensitive (parse-cond s))
@@ -1225,15 +1222,13 @@
  '(block)
  #f
  finalb)
- (let* ((var (if nl (parse-eq s) (parse-eq* s)))
+ (let* ((var (if nl #f (parse-eq* s)))
  (var? (and (not nl) (or (symbol? var) (and (length= var 2) (eq? (car var) '$)))))
  (catch-block (if (eq? (require-token s) 'finally)
  '(block)
  (parse-block s))))
  (loop (require-token s)
- (if var?
- catch-block
- `(block ,var ,@(cdr catch-block)))
+ catch-block
  (if var? var 'false)
  finalb)))))
  ((and (eq? nxt 'finally)
@@ -1299,23 +1294,15 @@
  (error "invalid \"do\" syntax"))
  (else (error "unhandled reserved word")))))))
 
-(define (add-filename-to-block! body loc)
- (if (and (length> body 1)
- (pair? (cadr body))
- (eq? (caadr body) 'line))
- (set-car! (cdr body) loc))
- body)
-
 (define (parse-do s)
  (with-bindings
  ((expect-end-current-line (input-port-line (ts:port s))))
  (without-whitespace-newline
- (let* ((doargs (if (memv (peek-token s) '(#\newline #\;))
- '()
- (parse-comma-separated s parse-range)))
- (loc (line-number-node s)))
+ (let ((doargs (if (memv (peek-token s) '(#\newline #\;))
+ '()
+ (parse-comma-separated s parse-range))))
  `(-> (tuple ,@doargs)
- ,(begin0 (add-filename-to-block! (parse-block s) loc)
+ ,(begin0 (parse-block s)
  (expect-end s 'do)))))))
 
 (define (macrocall-to-atsym e)

src/julia-syntax.scm

@@ -406,9 +406,7 @@
  ,(method-def-expr-
  name positional-sparams (append pargl vararg)
  `(block
- ,(if (null? lno)
- `(line 0 || ||)
- (append (car lno) '(||)))
+ ,@lno
  ,@(if (not ordered-defaults)
  '()
  (map make-assignment keynames vals))
@@ -436,11 +434,7 @@
  (car not-optional))
  ,@(cdr not-optional) ,@vararg)
  `(block
- ,@(cond ((null? lno) '())
- ((not name) lno)
- (else
- ;; TODO jb/functions get a better `name` for functions specified by type
- (list (append (car lno) (list (undot-name name))))))
+ ,@lno
  ,@stmts) isstaged)
 
  ;; call with unsorted keyword args. this sorts and re-dispatches.
@@ -455,7 +449,6 @@
  ,(if (any kwarg? pargl) (gensy) UNUSED)
  (call (core kwftype) ,ftype)) (:: ,kw (core Array)) ,@pargl ,@vararg)
  `(block
- (line 0 || ||)
  ;; initialize keyword args to their defaults, or set a flag telling
  ;; whether this keyword needs to be set.
  ,@(map (lambda (name dflt flag)
@@ -1534,17 +1527,19 @@
 
  'block
  (lambda (e)
- (let ((e (flatten-blocks e)))
- (cond ((null? (cdr e)) '(null))
- ((null? (cddr e)) (expand-forms (cadr e)))
- (else
- `(block
- ,.(map (lambda (x)
- (if (decl? x)
- `(decl ,@(map expand-forms (cdr x)))
- (expand-forms x)))
- (butlast (cdr e)))
- ,(expand-forms (last e)))))))
+ (cond ((null? (cdr e)) '(null))
+ ((and (null? (cddr e))
+ (not (and (pair? (cadr e))
+ (eq? (car (cadr e)) 'line))))
+ (expand-forms (cadr e)))
+ (else
+ `(block
+ ,.(map (lambda (x)
+ (if (decl? x)
+ `(decl ,@(map expand-forms (cdr x)))
+ (expand-forms x)))
+ (butlast (cdr e)))
+ ,(expand-forms (last (cdr e)))))))
 
  '|.|
  (lambda (e) ; e = (|.| f x)
@@ -2596,6 +2591,15 @@ f(x) = yt(x)
  (cons (last e2) (append tl (butlast (cdr e2))))
  (cons e2 tl)))))
 
+(define (first-non-meta blk)
+ (let loop ((xs (cdr blk)))
+ (if (null? xs)
+ #f
+ (let ((elt (car xs)))
+ (if (and (pair? elt) (eq? (car elt) 'meta))
+ (loop (cdr xs))
+ elt)))))
+
 ;; return `body` with `stmts` inserted after any meta nodes
 (define (insert-after-meta body stmts)
  (let ((meta (take-while (lambda (x) (and (pair? x)
@@ -2620,11 +2624,25 @@ f(x) = yt(x)
  '())))
  args)))))
 
+(define (take-statements-while pred body)
+ (let ((acc '()))
+ (define (take expr)
+ ;; returns #t as long as exprs match and we should continue
+ (cond ((and (pair? expr) (memq (car expr) '(block body)))
+ (let loop ((xs (cdr expr)))
+ (cond ((null? xs) #t)
+ ((take (car xs)) (loop (cdr xs)))
+ (else #f))))
+ ((pred expr)
+ (set! acc (cons expr acc))
+ #t)
+ (else #f)))
+ (take body)
+ (reverse! acc)))
+
 ;; clear capture bit for vars assigned once at the top, to avoid allocating
 ;; some unnecessary Boxes.
 (define (lambda-optimize-vars! lam)
- ;; flattening blocks helps us find more dominating statements
- (set-car! (cdddr lam) (flatten-blocks (lam:body lam)))
  (define (expr-uses-var ex v)
  (cond ((assignment? ex) (expr-contains-eq v (caddr ex)))
  ((eq? (car ex) 'method)
@@ -2640,13 +2658,14 @@ f(x) = yt(x)
  (or (and (eq? (car x) 'method)
  (length> (car x) 2))
  (eq? (car x) '=))))
- (take-while (lambda (e)
- (or (atom? e)
- (memq (car e) '(quote top core line inert local
- meta inbounds boundscheck simdloop
- implicit-global global globalref
- const newvar = null method))))
- (lam:body lam))))
+ (take-statements-while
+ (lambda (e)
+ (or (atom? e)
+ (memq (car e) '(quote top core line inert local
+ meta inbounds boundscheck simdloop
+ implicit-global global globalref
+ const newvar = null method))))
+ (lam:body lam))))
  (unused (map cadr leading))
  (def (table)))
  ;; TODO: reorder leading statements to put assignments where the RHS is
@@ -2926,6 +2945,8 @@ f(x) = yt(x)
 ;; only possible returned values.
 (define (compile-body e vi lam)
  (let ((code '())
+ (filename #f)
+ (first-line #t)
  (label-counter 0) ;; counter for generating label addresses
  (label-map (table)) ;; maps label names to generated addresses
  (label-level (table)) ;; exception handler level of each label
@@ -3035,11 +3056,25 @@ f(x) = yt(x)
  rr)
  (emit `(= ,(cadr e) ,rhs)))))
  ((block body)
- (let loop ((xs (cdr e)))
- (if (null? (cdr xs))
- (compile (car xs) break-labels value tail)
- (begin (compile (car xs) break-labels #f #f)
- (loop (cdr xs))))))
+ (let* ((last-fname filename)
+ (fnm (first-non-meta e))
+ (fname (if (and (length> e 1) (pair? fnm) (eq? (car fnm) 'line)
+ (length> fnm 2))
+ (caddr fnm)
+ filename)))
+ (if (not (eq? fname last-fname))
+ (begin (set! filename fname)
+ ;; don't need a filename node for start of function
+ (if (not (eq? e (lam:body lam))) (emit `(meta push_loc ,fname)))))
+ (begin0
+ (let loop ((xs (cdr e)))
+ (if (null? (cdr xs))
+ (compile (car xs) break-labels value tail)
+ (begin (compile (car xs) break-labels #f #f)
+ (loop (cdr xs)))))
+ (if (not (eq? fname last-fname))
+ (begin (if (not tail) (emit `(meta pop_loc)))
+ (set! filename last-fname))))))
  ((return)
  (compile (cadr e) break-labels #t #t)
  '(null))
@@ -3188,7 +3223,13 @@ f(x) = yt(x)
  ;; other top level expressions and metadata
  ((import importall using export line meta inbounds boundscheck simdloop)
  (let ((have-ret? (and (pair? code) (pair? (car code)) (eq? (caar code) 'return))))
- (emit e)
+ (if (eq? (car e) 'line)
+ (if first-line
+ (begin (set! first-line #f)
+ (emit e))
+ ;; strip filenames out of non-initial line nodes
+ (emit `(line ,(cadr e))))
+ (emit e))
  (if (and tail (not have-ret?))
  (emit-return '(null)))
  '(null)))