Auto merge of #36214 - jseyfried:stackless_expansion, r=nrc

macros: stackless expansion

After this PR, macro expansion cannot overflow the stack unless the expanded crate is too deep to fold.
Everything but the stackless placeholder expansion commit is also groundwork for macro modularization.

r? @nrc or @eddyb

Author: bors

Diff for: src/librustc_driver/driver.rs

@@ -555,7 +555,7 @@ pub struct ExpansionResult<'a> {
 /// Returns `None` if we're aborting after handling -W help.
 pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
                                            cstore: &CStore,
-                                           mut krate: ast::Crate,
+                                           krate: ast::Crate,
                                            registry: Option<Registry>,
                                            crate_name: &'a str,
                                            addl_plugins: Option<Vec<String>>,
@@ -566,21 +566,9 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
 {
     let time_passes = sess.time_passes();
 
-    // strip before anything else because crate metadata may use #[cfg_attr]
-    // and so macros can depend on configuration variables, such as
-    //
-    //   #[macro_use] #[cfg(foo)]
-    //   mod bar { macro_rules! baz!(() => {{}}) }
-    //
-    // baz! should not use this definition unless foo is enabled.
-
-    krate = time(time_passes, "configuration", || {
-        let (krate, features) =
-            syntax::config::strip_unconfigured_items(krate, &sess.parse_sess, sess.opts.test);
-        // these need to be set "early" so that expansion sees `quote` if enabled.
-        *sess.features.borrow_mut() = features;
-        krate
-    });
+    let (mut krate, features) = syntax::config::features(krate, &sess.parse_sess, sess.opts.test);
+    // these need to be set "early" so that expansion sees `quote` if enabled.
+    *sess.features.borrow_mut() = features;
 
     *sess.crate_types.borrow_mut() = collect_crate_types(sess, &krate.attrs);
     *sess.crate_disambiguator.borrow_mut() =

Diff for: src/libsyntax/config.rs

@@ -10,11 +10,10 @@
 
 use attr::HasAttrs;
 use feature_gate::{emit_feature_err, EXPLAIN_STMT_ATTR_SYNTAX, Features, get_features, GateIssue};
-use fold::Folder;
 use {fold, attr};
 use ast;
 use codemap::{Spanned, respan};
-use parse::{ParseSess, token};
+use parse::ParseSess;
 use ptr::P;
 
 use util::small_vector::SmallVector;
@@ -27,8 +26,51 @@ pub struct StripUnconfigured<'a> {
     pub features: Option<&'a Features>,
 }
 
+// `cfg_attr`-process the crate's attributes and compute the crate's features.
+pub fn features(mut krate: ast::Crate, sess: &ParseSess, should_test: bool)
+                -> (ast::Crate, Features) {
+    let features;
+    {
+        let mut strip_unconfigured = StripUnconfigured {
+            config: &krate.config.clone(),
+            should_test: should_test,
+            sess: sess,
+            features: None,
+        };
+
+        let unconfigured_attrs = krate.attrs.clone();
+        let err_count = sess.span_diagnostic.err_count();
+        if let Some(attrs) = strip_unconfigured.configure(krate.attrs) {
+            krate.attrs = attrs;
+        } else { // the entire crate is unconfigured
+            krate.attrs = Vec::new();
+            krate.module.items = Vec::new();
+            return (krate, Features::new());
+        }
+
+        features = get_features(&sess.span_diagnostic, &krate.attrs);
+
+        // Avoid reconfiguring malformed `cfg_attr`s
+        if err_count == sess.span_diagnostic.err_count() {
+            strip_unconfigured.features = Some(&features);
+            strip_unconfigured.configure(unconfigured_attrs);
+        }
+    }
+
+    (krate, features)
+}
+
+macro_rules! configure {
+    ($this:ident, $node:ident) => {
+        match $this.configure($node) {
+            Some(node) => node,
+            None => return Default::default(),
+        }
+    }
+}
+
 impl<'a> StripUnconfigured<'a> {
-    fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
+    pub fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
         let node = self.process_cfg_attrs(node);
         if self.in_cfg(node.attrs()) { Some(node) } else { None }
     }
@@ -123,65 +165,35 @@ impl<'a> StripUnconfigured<'a> {
             }
         }
     }
-}
-
-// Support conditional compilation by transforming the AST, stripping out
-// any items that do not belong in the current configuration
-pub fn strip_unconfigured_items(mut krate: ast::Crate, sess: &ParseSess, should_test: bool)
-                                -> (ast::Crate, Features) {
-    let features;
-    {
-        let mut strip_unconfigured = StripUnconfigured {
-            config: &krate.config.clone(),
-            should_test: should_test,
-            sess: sess,
-            features: None,
-        };
-
-        let err_count = sess.span_diagnostic.err_count();
-        let krate_attrs = strip_unconfigured.configure(krate.attrs.clone()).unwrap_or_default();
-        features = get_features(&sess.span_diagnostic, &krate_attrs);
-        if err_count < sess.span_diagnostic.err_count() {
-            krate.attrs = krate_attrs.clone(); // Avoid reconfiguring malformed `cfg_attr`s
-        }
-
-        strip_unconfigured.features = Some(&features);
-        krate = strip_unconfigured.fold_crate(krate);
-        krate.attrs = krate_attrs;
-    }
 
-    (krate, features)
-}
-
-impl<'a> fold::Folder for StripUnconfigured<'a> {
-    fn fold_foreign_mod(&mut self, foreign_mod: ast::ForeignMod) -> ast::ForeignMod {
+    pub fn configure_foreign_mod(&mut self, foreign_mod: ast::ForeignMod) -> ast::ForeignMod {
         ast::ForeignMod {
             abi: foreign_mod.abi,
-            items: foreign_mod.items.into_iter().filter_map(|item| {
-                self.configure(item).map(|item| fold::noop_fold_foreign_item(item, self))
-            }).collect(),
+            items: foreign_mod.items.into_iter().filter_map(|item| self.configure(item)).collect(),
         }
     }
 
-    fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
-        let fold_struct = |this: &mut Self, vdata| match vdata {
+    fn configure_variant_data(&mut self, vdata: ast::VariantData) -> ast::VariantData {
+        match vdata {
             ast::VariantData::Struct(fields, id) => {
-                let fields = fields.into_iter().filter_map(|field| this.configure(field));
+                let fields = fields.into_iter().filter_map(|field| self.configure(field));
                 ast::VariantData::Struct(fields.collect(), id)
             }
             ast::VariantData::Tuple(fields, id) => {
-                let fields = fields.into_iter().filter_map(|field| this.configure(field));
+                let fields = fields.into_iter().filter_map(|field| self.configure(field));
                 ast::VariantData::Tuple(fields.collect(), id)
             }
             ast::VariantData::Unit(id) => ast::VariantData::Unit(id)
-        };
+        }
+    }
 
-        let item = match item {
+    pub fn configure_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
+        match item {
             ast::ItemKind::Struct(def, generics) => {
-                ast::ItemKind::Struct(fold_struct(self, def), generics)
+                ast::ItemKind::Struct(self.configure_variant_data(def), generics)
             }
             ast::ItemKind::Union(def, generics) => {
-                ast::ItemKind::Union(fold_struct(self, def), generics)
+                ast::ItemKind::Union(self.configure_variant_data(def), generics)
             }
             ast::ItemKind::Enum(def, generics) => {
                 let variants = def.variants.into_iter().filter_map(|v| {
@@ -190,7 +202,7 @@ impl<'a> fold::Folder for StripUnconfigured<'a> {
                             node: ast::Variant_ {
                                 name: v.node.name,
                                 attrs: v.node.attrs,
-                                data: fold_struct(self, v.node.data),
+                                data: self.configure_variant_data(v.node.data),
                                 disr_expr: v.node.disr_expr,
                             },
                             span: v.span
@@ -202,12 +214,19 @@ impl<'a> fold::Folder for StripUnconfigured<'a> {
                 }, generics)
             }
             item => item,
-        };
+        }
+    }
 
-        fold::noop_fold_item_kind(item, self)
+    pub fn configure_expr_kind(&mut self, expr_kind: ast::ExprKind) -> ast::ExprKind {
+        if let ast::ExprKind::Match(m, arms) = expr_kind {
+            let arms = arms.into_iter().filter_map(|a| self.configure(a)).collect();
+            ast::ExprKind::Match(m, arms)
+        } else {
+            expr_kind
+        }
     }
 
-    fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
+    pub fn configure_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
         self.visit_stmt_or_expr_attrs(expr.attrs());
 
         // If an expr is valid to cfg away it will have been removed by the
@@ -222,64 +241,64 @@ impl<'a> fold::Folder for StripUnconfigured<'a> {
             self.sess.span_diagnostic.span_err(attr.span, msg);
         }
 
-        let expr = self.process_cfg_attrs(expr);
-        fold_expr(self, expr)
+        self.process_cfg_attrs(expr)
     }
 
-    fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
-        self.configure(expr).map(|expr| fold_expr(self, expr))
+    pub fn configure_stmt(&mut self, stmt: ast::Stmt) -> Option<ast::Stmt> {
+        self.visit_stmt_or_expr_attrs(stmt.attrs());
+        self.configure(stmt)
     }
+}
 
-    fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
-        self.visit_stmt_or_expr_attrs(stmt.attrs());
-        self.configure(stmt).map(|stmt| fold::noop_fold_stmt(stmt, self))
-                            .unwrap_or(SmallVector::zero())
+impl<'a> fold::Folder for StripUnconfigured<'a> {
+    fn fold_foreign_mod(&mut self, foreign_mod: ast::ForeignMod) -> ast::ForeignMod {
+        let foreign_mod = self.configure_foreign_mod(foreign_mod);
+        fold::noop_fold_foreign_mod(foreign_mod, self)
     }
 
-    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
-        fold::noop_fold_mac(mac, self)
+    fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
+        let item = self.configure_item_kind(item);
+        fold::noop_fold_item_kind(item, self)
+    }
+
+    fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
+        let mut expr = self.configure_expr(expr).unwrap();
+        expr.node = self.configure_expr_kind(expr.node);
+        P(fold::noop_fold_expr(expr, self))
+    }
+
+    fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
+        let mut expr = configure!(self, expr).unwrap();
+        expr.node = self.configure_expr_kind(expr.node);
+        Some(P(fold::noop_fold_expr(expr, self)))
+    }
+
+    fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
+        match self.configure_stmt(stmt) {
+            Some(stmt) => fold::noop_fold_stmt(stmt, self),
+            None => return SmallVector::zero(),
+        }
     }
 
     fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> {
-        self.configure(item).map(|item| fold::noop_fold_item(item, self))
-                            .unwrap_or(SmallVector::zero())
+        fold::noop_fold_item(configure!(self, item), self)
     }
 
     fn fold_impl_item(&mut self, item: ast::ImplItem) -> SmallVector<ast::ImplItem> {
-        self.configure(item).map(|item| fold::noop_fold_impl_item(item, self))
-                            .unwrap_or(SmallVector::zero())
+        fold::noop_fold_impl_item(configure!(self, item), self)
     }
 
     fn fold_trait_item(&mut self, item: ast::TraitItem) -> SmallVector<ast::TraitItem> {
-        self.configure(item).map(|item| fold::noop_fold_trait_item(item, self))
-                            .unwrap_or(SmallVector::zero())
+        fold::noop_fold_trait_item(configure!(self, item), self)
     }
 
-    fn fold_interpolated(&mut self, nt: token::Nonterminal) -> token::Nonterminal {
+    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
         // Don't configure interpolated AST (c.f. #34171).
         // Interpolated AST will get configured once the surrounding tokens are parsed.
-        nt
+        mac
     }
 }
 
-fn fold_expr(folder: &mut StripUnconfigured, expr: P<ast::Expr>) -> P<ast::Expr> {
-    expr.map(|ast::Expr {id, span, node, attrs}| {
-        fold::noop_fold_expr(ast::Expr {
-            id: id,
-            node: match node {
-                ast::ExprKind::Match(m, arms) => {
-                    ast::ExprKind::Match(m, arms.into_iter()
-                                        .filter_map(|a| folder.configure(a))
-                                        .collect())
-                }
-                _ => node
-            },
-            span: span,
-            attrs: attrs,
-        }, folder)
-    })
-}
-
 fn is_cfg(attr: &ast::Attribute) -> bool {
     attr.check_name("cfg")
 }