Auto merge of #24155 - chris-chambers:stmt_macros, r=sfackler

Statement macros are now treated somewhat like item macros, in that a statement macro can now expand into a series of statements, rather than just a single statement.

This allows statement macros to be nested inside other kinds of macros and expand properly, where previously the expansion would only work when no nesting was present.

See:
- `src/test/run-pass/macro-stmt_macro_in_expr_macro.rs`
- `src/test/run-pass/macro-nested_stmt_macro.rs`

This changes the interface of the MacResult trait.  make_stmt has become make_stmts and now returns a vector, rather than a single item.  Plugin writers who were implementing MacResult will have breakage, as well as anyone using MacEager::stmt.

See:
- `src/libsyntax/ext/base.rs`

This also causes a minor difference in behavior to the diagnostics produced by certain malformed macros.

See:
- `src/test/compile-fail/macro-incomplete-parse.rs`

Author: bors

src/libsyntax/ext/base.rs

@@ -208,10 +208,11 @@ impl<F> IdentMacroExpander for F
 }
 
 // Use a macro because forwarding to a simple function has type system issues
-macro_rules! make_stmt_default {
+macro_rules! make_stmts_default {
     ($me:expr) => {
         $me.make_expr().map(|e| {
-            P(codemap::respan(e.span, ast::StmtExpr(e, ast::DUMMY_NODE_ID)))
+            SmallVector::one(P(codemap::respan(
+                e.span, ast::StmtExpr(e, ast::DUMMY_NODE_ID))))
         })
     }
 }
@@ -238,12 +239,12 @@ pub trait MacResult {
         None
     }
 
-    /// Create a statement.
+    /// Create zero or more statements.
     ///
     /// By default this attempts to create an expression statement,
     /// returning None if that fails.
-    fn make_stmt(self: Box<Self>) -> Option<P<ast::Stmt>> {
-        make_stmt_default!(self)
+    fn make_stmts(self: Box<Self>) -> Option<SmallVector<P<ast::Stmt>>> {
+        make_stmts_default!(self)
     }
 }
 
@@ -276,7 +277,7 @@ make_MacEager! {
     pat: P<ast::Pat>,
     items: SmallVector<P<ast::Item>>,
     impl_items: SmallVector<P<ast::ImplItem>>,
-    stmt: P<ast::Stmt>,
+    stmts: SmallVector<P<ast::Stmt>>,
 }
 
 impl MacResult for MacEager {
@@ -292,10 +293,10 @@ impl MacResult for MacEager {
         self.impl_items
     }
 
-    fn make_stmt(self: Box<Self>) -> Option<P<ast::Stmt>> {
-        match self.stmt {
-            None => make_stmt_default!(self),
-            s => s,
+    fn make_stmts(self: Box<Self>) -> Option<SmallVector<P<ast::Stmt>>> {
+        match self.stmts.as_ref().map_or(0, |s| s.len()) {
+            0 => make_stmts_default!(self),
+            _ => self.stmts,
         }
     }
 
@@ -384,10 +385,11 @@ impl MacResult for DummyResult {
             Some(SmallVector::zero())
         }
     }
-    fn make_stmt(self: Box<DummyResult>) -> Option<P<ast::Stmt>> {
-        Some(P(codemap::respan(self.span,
-                               ast::StmtExpr(DummyResult::raw_expr(self.span),
-                                             ast::DUMMY_NODE_ID))))
+    fn make_stmts(self: Box<DummyResult>) -> Option<SmallVector<P<ast::Stmt>>> {
+        Some(SmallVector::one(P(
+            codemap::respan(self.span,
+                            ast::StmtExpr(DummyResult::raw_expr(self.span),
+                                          ast::DUMMY_NODE_ID)))))
     }
 }
 

src/libsyntax/ext/expand.rs

@@ -720,37 +720,49 @@ pub fn expand_item_mac(it: P<ast::Item>,
 }
 
 /// Expand a stmt
-fn expand_stmt(s: Stmt, fld: &mut MacroExpander) -> SmallVector<P<Stmt>> {
-    let (mac, style) = match s.node {
+fn expand_stmt(stmt: P<Stmt>, fld: &mut MacroExpander) -> SmallVector<P<Stmt>> {
+    let stmt = stmt.and_then(|stmt| stmt);
+    let (mac, style) = match stmt.node {
         StmtMac(mac, style) => (mac, style),
-        _ => return expand_non_macro_stmt(s, fld)
+        _ => return expand_non_macro_stmt(stmt, fld)
     };
-    let expanded_stmt = match expand_mac_invoc(mac.and_then(|m| m), s.span,
-                                                |r| r.make_stmt(),
-                                                mark_stmt, fld) {
-        Some(stmt) => stmt,
-        None => {
-            return SmallVector::zero();
+
+    let maybe_new_items =
+        expand_mac_invoc(mac.and_then(|m| m), stmt.span,
+                         |r| r.make_stmts(),
+                         |stmts, mark| stmts.move_map(|m| mark_stmt(m, mark)),
+                         fld);
+
+    let mut fully_expanded = match maybe_new_items {
+        Some(stmts) => {
+            // Keep going, outside-in.
+            let new_items = stmts.into_iter().flat_map(|s| {
+                fld.fold_stmt(s).into_iter()
+            }).collect();
+            fld.cx.bt_pop();
+            new_items
         }
+        None => SmallVector::zero()
     };
 
-    // Keep going, outside-in.
-    let fully_expanded = fld.fold_stmt(expanded_stmt);
-    fld.cx.bt_pop();
-
+    // If this is a macro invocation with a semicolon, then apply that
+    // semicolon to the final statement produced by expansion.
     if style == MacStmtWithSemicolon {
-        fully_expanded.into_iter().map(|s| s.map(|Spanned {node, span}| {
-            Spanned {
-                node: match node {
-                    StmtExpr(e, stmt_id) => StmtSemi(e, stmt_id),
-                    _ => node /* might already have a semi */
-                },
-                span: span
-            }
-        })).collect()
-    } else {
-        fully_expanded
+        if let Some(stmt) = fully_expanded.pop() {
+            let new_stmt = stmt.map(|Spanned {node, span}| {
+                Spanned {
+                    node: match node {
+                        StmtExpr(e, stmt_id) => StmtSemi(e, stmt_id),
+                        _ => node /* might already have a semi */
+                    },
+                    span: span
+                }
+            });
+            fully_expanded.push(new_stmt);
+        }
     }
+
+    fully_expanded
 }
 
 // expand a non-macro stmt. this is essentially the fallthrough for
@@ -1355,7 +1367,7 @@ impl<'a, 'b> Folder for MacroExpander<'a, 'b> {
     }
 
     fn fold_stmt(&mut self, stmt: P<ast::Stmt>) -> SmallVector<P<ast::Stmt>> {
-        stmt.and_then(|stmt| expand_stmt(stmt, self))
+        expand_stmt(stmt, self)
     }
 
     fn fold_block(&mut self, block: P<Block>) -> P<Block> {
@@ -1502,8 +1514,8 @@ fn mark_pat(pat: P<ast::Pat>, m: Mrk) -> P<ast::Pat> {
 }
 
 // apply a given mark to the given stmt. Used following the expansion of a macro.
-fn mark_stmt(expr: P<ast::Stmt>, m: Mrk) -> P<ast::Stmt> {
-    Marker{mark:m}.fold_stmt(expr)
+fn mark_stmt(stmt: P<ast::Stmt>, m: Mrk) -> P<ast::Stmt> {
+    Marker{mark:m}.fold_stmt(stmt)
         .expect_one("marking a stmt didn't return exactly one stmt")
 }
 

src/libsyntax/ext/tt/macro_rules.rs

@@ -99,10 +99,24 @@ impl<'a> MacResult for ParserAnyMacro<'a> {
         Some(ret)
     }
 
-    fn make_stmt(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Stmt>> {
-        let ret = self.parser.borrow_mut().parse_stmt();
-        self.ensure_complete_parse(true);
-        ret
+    fn make_stmts(self: Box<ParserAnyMacro<'a>>)
+                 -> Option<SmallVector<P<ast::Stmt>>> {
+        let mut ret = SmallVector::zero();
+        loop {
+            let mut parser = self.parser.borrow_mut();
+            match parser.token {
+                token::Eof => break,
+                _ => match parser.parse_stmt_nopanic() {
+                    Ok(maybe_stmt) => match maybe_stmt {
+                        Some(stmt) => ret.push(stmt),
+                        None => (),
+                    },
+                    Err(_) => break,
+                }
+            }
+        }
+        self.ensure_complete_parse(false);
+        Some(ret)
     }
 }
 

src/libsyntax/util/small_vector.rs

@@ -69,6 +69,20 @@ impl<T> SmallVector<T> {
         }
     }
 
+    pub fn pop(&mut self) -> Option<T> {
+        match self.repr {
+            Zero => None,
+            One(..) => {
+                let one = mem::replace(&mut self.repr, Zero);
+                match one {
+                    One(v1) => Some(v1),
+                    _ => unreachable!()
+                }
+            }
+            Many(ref mut vs) => vs.pop(),
+        }
+    }
+
     pub fn push(&mut self, v: T) {
         match self.repr {
             Zero => self.repr = One(v),

src/test/compile-fail/macro-incomplete-parse.rs

@@ -17,7 +17,8 @@ macro_rules! ignored_item {
 }
 
 macro_rules! ignored_expr {
-    () => ( 1, 2 ) //~ ERROR macro expansion ignores token `,`
+    () => ( 1,  //~ ERROR unexpected token: `,`
+            2 ) //~ ERROR macro expansion ignores token `2`
 }
 
 macro_rules! ignored_pat {

src/test/run-pass/macro-nested_stmt_macros.rs

@@ -0,0 +1,32 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+macro_rules! foo {
+    () => {
+        struct Bar;
+        struct Baz;
+    }
+}
+
+macro_rules! grault {
+    () => {
+        foo!();
+        struct Xyzzy;
+    }
+}
+
+fn static_assert_exists<T>() { }
+
+fn main() {
+    grault!();
+    static_assert_exists::<Bar>();
+    static_assert_exists::<Baz>();
+    static_assert_exists::<Xyzzy>();
+}

src/test/run-pass/macro-stmt_macro_in_expr_macro.rs

@@ -0,0 +1,29 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+macro_rules! foo {
+    () => {
+        struct Bar;
+        struct Baz;
+    }
+}
+
+macro_rules! grault {
+    () => {{
+        foo!();
+        struct Xyzzy;
+        0
+    }}
+}
+
+fn main() {
+    let x = grault!();
+    assert_eq!(x, 0);
+}