From b43ca9d652ccf94e64694f30475d0db630349955 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 24 Jul 2017 17:24:22 -0400 Subject: [PATCH] fix #18650, parsing generator expressions containing macro calls --- NEWS.md | 10 ++++++++++ src/julia-parser.scm | 43 ++++++++++++++++++++++++------------------- test/parse.jl | 6 ++++++ 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1c15124b166a1..0f113d9cdd38f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -42,6 +42,16 @@ This section lists changes that do not have deprecation warnings. * Juxtaposing string literals (e.g. `"x"y`) is now a syntax error ([#20575]). + * Macro calls with `for` expressions are now parsed as generators inside + function argument lists ([#18650]). Examples: + + + `sum(@inbounds a[i] for i = 1:n)` used to give a syntax error, but is now + parsed as `sum(@inbounds(a[i]) for i = 1:n)`. + + + `sum(@m x for i = 1:n end)` used to parse the argument to `sum` as a 2-argument + call to macro `@m`, but now parses it as a generator plus a syntax error + for the dangling `end`. + * `@__DIR__` returns the current working directory rather than `nothing` when not run from a file ([#21759]). diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 55b04612d5bb5..fd14e9a418349 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -144,7 +144,8 @@ (define range-colon-enabled #t) ; in space-sensitive mode "x -y" is 2 expressions, not a subtraction (define space-sensitive #f) -(define inside-vec #f) +; seeing `for` stops parsing macro arguments and makes a generator +(define for-generator #f) ; treat 'end' like a normal symbol instead of a reserved word (define end-symbol #f) ; treat newline like ordinary whitespace instead of as a potential separator @@ -164,7 +165,7 @@ `(with-bindings ((range-colon-enabled #t) (space-sensitive #f) (where-enabled #t) - (inside-vec #f) + (for-generator #f) (end-symbol #f) (whitespace-newline #f)) ,@body)) @@ -178,12 +179,6 @@ (whitespace-newline #f)) ,@body)) -(define-macro (with-inside-vec . body) - `(with-bindings ((space-sensitive #t) - (inside-vec #t) - (whitespace-newline #f)) - ,@body)) - (define-macro (with-end-symbol . body) `(with-bindings ((end-symbol #t)) ,@body)) @@ -1129,7 +1124,7 @@ ((#\( ) (if (ts:space? s) (disallowed-space ex t)) (take-token s) - (let ((c (let ((al (parse-arglist s #\) ))) + (let ((c (let ((al (parse-call-arglist s #\) ))) (receive (params args) (separate (lambda (x) (and (pair? x) @@ -1172,7 +1167,7 @@ (cond ((eqv? (peek-token s) #\() (begin (take-token s) - `(|.| ,ex (tuple ,@(parse-arglist s #\) ))))) + `(|.| ,ex (tuple ,@(parse-call-arglist s #\) ))))) ((eqv? (peek-token s) ':) (begin (take-token s) @@ -1195,7 +1190,7 @@ ((#\{ ) (if (ts:space? s) (disallowed-space ex t)) (take-token s) - (loop (list* 'curly ex (parse-arglist s #\} )))) + (loop (list* 'curly ex (parse-call-arglist s #\} )))) ((#\" #\`) (if (and (or (symbol? ex) (valid-modref? ex)) (not (operator? ex)) @@ -1629,7 +1624,7 @@ (let loop ((exprs '())) (if (or (closing-token? (peek-token s)) (newline? (peek-token s)) - (and inside-vec (eq? (peek-token s) 'for))) + (and for-generator (eq? (peek-token s) 'for))) (reverse! exprs) (let ((e (parse-eq s))) (case (peek-token s) @@ -1645,13 +1640,20 @@ x)) lst)) +;; like parse-arglist, but with `for` parsed as a generator +(define (parse-call-arglist s closer) + (with-bindings ((for-generator #t)) + (parse-arglist s closer))) + ;; handle function call argument list, or any comma-delimited list. ;; . an extra comma at the end is allowed ;; . expressions after a ; are enclosed in (parameters ...) ;; . an expression followed by ... becomes (... x) (define (parse-arglist s closer) -(with-normal-ops - (with-whitespace-newline +(with-bindings ((range-colon-enabled #t) + (space-sensitive #f) + (where-enabled #t) + (whitespace-newline #t)) (let loop ((lst '())) (let ((t (require-token s))) (if (eqv? t closer) @@ -1689,7 +1691,7 @@ (error (string "unexpected \"" c "\" in argument list"))) (else (error (string "missing comma or " closer - " in argument list")))))))))))) + " in argument list"))))))))))) (define (parse-vect s first closer) (let loop ((lst '()) @@ -1709,7 +1711,7 @@ ((#\;) (if (eqv? (require-token s) closer) (loop lst nxt) - (let ((params (parse-arglist s closer))) + (let ((params (parse-call-arglist s closer))) `(vcat ,@params ,@(reverse lst) ,nxt)))) ((#\] #\}) (error (string "unexpected \"" t "\""))) @@ -1792,8 +1794,11 @@ (error (string "expected space before \"" t "\"")))) (define (parse-cat s closer last-end-symbol) - (with-normal-ops - (with-inside-vec + (with-bindings ((range-colon-enabled #t) + (space-sensitive #t) + (where-enabled #t) + (whitespace-newline #f) + (for-generator #t)) (if (eqv? (require-token s) closer) (begin (take-token s) '()) @@ -1811,7 +1816,7 @@ (parse-vect s first closer) (parse-matrix s first closer #t last-end-symbol))) (else - (parse-matrix s first closer #f last-end-symbol)))))))) + (parse-matrix s first closer #f last-end-symbol))))))) (define (kw-to-= e) (if (kwarg? e) (cons '= (cdr e)) e)) (define (=-to-kw e) (if (assignment? e) (cons 'kw (cdr e)) e)) diff --git a/test/parse.jl b/test/parse.jl index 7aff2e7e7f7ff..37be9009cbdca 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -1276,3 +1276,9 @@ end @test parse("(::A)") == Expr(Symbol("::"), :A) @test_throws ParseError parse("(::, 1)") @test_throws ParseError parse("(1, ::)") + +# issue #18650 +let ex = parse("maximum(@elapsed sleep(1) for k = 1:10)") + @test isa(ex, Expr) && ex.head === :call && ex.args[2].head === :generator && + ex.args[2].args[1].head === :macrocall +end