Rollup merge of rust-lang#49350 - abonander:macros-in-extern, r=petro…

…chenkov

Expand macros in `extern {}` blocks

This permits macro and proc-macro and attribute invocations (the latter only with the `proc_macro` feature of course) in `extern {}` blocks, gated behind a new `macros_in_extern` feature.

A tracking issue is now open at rust-lang#49476

closes rust-lang#48747

src/librustc/hir/lowering.rs

@@ -2724,6 +2724,7 @@ impl<'a> LoweringContext<'a> {
                         hir::ForeignItemStatic(this.lower_ty(t, ImplTraitContext::Disallowed), m)
                     }
                     ForeignItemKind::Ty => hir::ForeignItemType,
+                    ForeignItemKind::Macro(_) => panic!("shouldn't exist here"),
                 },
                 vis: this.lower_visibility(&i.vis, None),
                 span: i.span,

src/librustc/hir/map/def_collector.rs

@@ -181,6 +181,10 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> {
     }
 
     fn visit_foreign_item(&mut self, foreign_item: &'a ForeignItem) {
+        if let ForeignItemKind::Macro(_) = foreign_item.node {
+            return self.visit_macro_invoc(foreign_item.id, false);
+        }
+
         let def = self.create_def(foreign_item.id,
                                   DefPathData::ValueNs(foreign_item.ident.name.as_str()),
                                   REGULAR_SPACE,

src/librustc_passes/ast_validation.rs

@@ -381,7 +381,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         .span_label(span, "pattern not allowed in foreign function").emit();
                 });
             }
-            ForeignItemKind::Static(..) | ForeignItemKind::Ty => {}
+            ForeignItemKind::Static(..) | ForeignItemKind::Ty | ForeignItemKind::Macro(..) => {}
         }
 
         visit::walk_foreign_item(self, fi)
@@ -460,6 +460,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         self.check_late_bound_lifetime_defs(&t.bound_generic_params);
         visit::walk_poly_trait_ref(self, t, m);
     }
+
+    fn visit_mac(&mut self, mac: &Spanned<Mac_>) {
+        // when a new macro kind is added but the author forgets to set it up for expansion
+        // because that's the only part that won't cause a compiler error
+        self.session.diagnostic()
+            .span_bug(mac.span, "macro invocation missed in expansion; did you forget to override \
+                                 the relevant `fold_*()` method in `PlaceholderExpander`?");
+    }
 }
 
 // Bans nested `impl Trait`, e.g. `impl Into<impl Debug>`.
@@ -522,6 +530,10 @@ impl<'a> Visitor<'a> for NestedImplTraitVisitor<'a> {
             }
         }
     }
+
+    fn visit_mac(&mut self, _mac: &Spanned<Mac_>) {
+        // covered in AstValidator
+    }
 }
 
 // Bans `impl Trait` in path projections like `<impl Iterator>::Item` or `Foo::Bar<impl Trait>`.
@@ -583,6 +595,10 @@ impl<'a> Visitor<'a> for ImplTraitProjectionVisitor<'a> {
             _ => visit::walk_ty(self, t),
         }
     }
+
+    fn visit_mac(&mut self, _mac: &Spanned<Mac_>) {
+        // covered in AstValidator
+    }
 }
 
 pub fn check_crate(session: &Session, krate: &Crate) {

src/librustc_resolve/build_reduced_graph.rs

@@ -456,6 +456,7 @@ impl<'a> Resolver<'a> {
             ForeignItemKind::Ty => {
                 (Def::TyForeign(self.definitions.local_def_id(item.id)), TypeNS)
             }
+            ForeignItemKind::Macro(_) => unreachable!(),
         };
         let parent = self.current_module;
         let vis = self.resolve_visibility(&item.vis);
@@ -816,6 +817,11 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> {
     }
 
     fn visit_foreign_item(&mut self, foreign_item: &'a ForeignItem) {
+        if let ForeignItemKind::Macro(_) = foreign_item.node {
+            self.visit_invoc(foreign_item.id);
+            return;
+        }
+
         self.resolver.build_reduced_graph_for_foreign_item(foreign_item, self.expansion);
         visit::walk_foreign_item(self, foreign_item);
     }

src/librustc_resolve/lib.rs

@@ -863,6 +863,7 @@ impl<'a, 'tcx> Visitor<'tcx> for Resolver<'a> {
             }
             ForeignItemKind::Static(..) => NoTypeParameters,
             ForeignItemKind::Ty => NoTypeParameters,
+            ForeignItemKind::Macro(..) => NoTypeParameters,
         };
         self.with_type_parameter_rib(type_parameters, |this| {
             visit::walk_foreign_item(this, foreign_item);

src/librustc_save_analysis/dump_visitor.rs

@@ -1812,6 +1812,7 @@ impl<'l, 'tcx: 'l, 'll, O: DumpOutput + 'll> Visitor<'l> for DumpVisitor<'l, 'tc
                     self.dumper.dump_def(&access, var_data);
                 }
             }
+            ast::ForeignItemKind::Macro(..) => {}
         }
     }
 }

src/librustc_save_analysis/lib.rs

@@ -182,6 +182,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
             }
             // FIXME(plietar): needs a new DefKind in rls-data
             ast::ForeignItemKind::Ty => None,
+            ast::ForeignItemKind::Macro(..) => None,
         }
     }
 

src/librustc_save_analysis/sig.rs

@@ -822,6 +822,7 @@ impl Sig for ast::ForeignItem {
                     refs: vec![],
                 })
             }
+            ast::ForeignItemKind::Macro(..) => Err("macro"),
         }
     }
 }

src/libsyntax/ast.rs

@@ -2192,6 +2192,8 @@ pub enum ForeignItemKind {
     Static(P<Ty>, bool),
     /// A foreign type
     Ty,
+    /// A macro invocation
+    Macro(Mac),
 }
 
 impl ForeignItemKind {
@@ -2200,6 +2202,7 @@ impl ForeignItemKind {
             ForeignItemKind::Fn(..) => "foreign function",
             ForeignItemKind::Static(..) => "foreign static item",
             ForeignItemKind::Ty => "foreign type",
+            ForeignItemKind::Macro(..) => "macro in foreign module",
         }
     }
 }

src/libsyntax/ext/base.rs

@@ -38,6 +38,7 @@ pub enum Annotatable {
     Item(P<ast::Item>),
     TraitItem(P<ast::TraitItem>),
     ImplItem(P<ast::ImplItem>),
+    ForeignItem(P<ast::ForeignItem>),
     Stmt(P<ast::Stmt>),
     Expr(P<ast::Expr>),
 }
@@ -48,6 +49,7 @@ impl HasAttrs for Annotatable {
             Annotatable::Item(ref item) => &item.attrs,
             Annotatable::TraitItem(ref trait_item) => &trait_item.attrs,
             Annotatable::ImplItem(ref impl_item) => &impl_item.attrs,
+            Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs,
             Annotatable::Stmt(ref stmt) => stmt.attrs(),
             Annotatable::Expr(ref expr) => &expr.attrs,
         }
@@ -58,6 +60,8 @@ impl HasAttrs for Annotatable {
             Annotatable::Item(item) => Annotatable::Item(item.map_attrs(f)),
             Annotatable::TraitItem(trait_item) => Annotatable::TraitItem(trait_item.map_attrs(f)),
             Annotatable::ImplItem(impl_item) => Annotatable::ImplItem(impl_item.map_attrs(f)),
+            Annotatable::ForeignItem(foreign_item) =>
+                Annotatable::ForeignItem(foreign_item.map_attrs(f)),
             Annotatable::Stmt(stmt) => Annotatable::Stmt(stmt.map_attrs(f)),
             Annotatable::Expr(expr) => Annotatable::Expr(expr.map_attrs(f)),
         }
@@ -70,6 +74,7 @@ impl Annotatable {
             Annotatable::Item(ref item) => item.span,
             Annotatable::TraitItem(ref trait_item) => trait_item.span,
             Annotatable::ImplItem(ref impl_item) => impl_item.span,
+            Annotatable::ForeignItem(ref foreign_item) => foreign_item.span,
             Annotatable::Stmt(ref stmt) => stmt.span,
             Annotatable::Expr(ref expr) => expr.span,
         }
@@ -106,6 +111,13 @@ impl Annotatable {
         }
     }
 
+    pub fn expect_foreign_item(self) -> ast::ForeignItem {
+        match self {
+            Annotatable::ForeignItem(i) => i.into_inner(),
+            _ => panic!("expected foreign item")
+        }
+    }
+
     pub fn derive_allowed(&self) -> bool {
         match *self {
             Annotatable::Item(ref item) => match item.node {
@@ -317,6 +329,9 @@ pub trait MacResult {
         None
     }
 
+    /// Create zero or more items in an `extern {}` block
+    fn make_foreign_items(self: Box<Self>) -> Option<SmallVector<ast::ForeignItem>> { None }
+
     /// Create a pattern.
     fn make_pat(self: Box<Self>) -> Option<P<ast::Pat>> {
         None
@@ -365,6 +380,7 @@ make_MacEager! {
     items: SmallVector<P<ast::Item>>,
     impl_items: SmallVector<ast::ImplItem>,
     trait_items: SmallVector<ast::TraitItem>,
+    foreign_items: SmallVector<ast::ForeignItem>,
     stmts: SmallVector<ast::Stmt>,
     ty: P<ast::Ty>,
 }
@@ -386,6 +402,10 @@ impl MacResult for MacEager {
         self.trait_items
     }
 
+    fn make_foreign_items(self: Box<Self>) -> Option<SmallVector<ast::ForeignItem>> {
+        self.foreign_items
+    }
+
     fn make_stmts(self: Box<Self>) -> Option<SmallVector<ast::Stmt>> {
         match self.stmts.as_ref().map_or(0, |s| s.len()) {
             0 => make_stmts_default!(self),
@@ -502,6 +522,14 @@ impl MacResult for DummyResult {
         }
     }
 
+    fn make_foreign_items(self: Box<Self>) -> Option<SmallVector<ast::ForeignItem>> {
+        if self.expr_only {
+            None
+        } else {
+            Some(SmallVector::new())
+        }
+    }
+
     fn make_stmts(self: Box<DummyResult>) -> Option<SmallVector<ast::Stmt>> {
         Some(SmallVector::one(ast::Stmt {
             id: ast::DUMMY_NODE_ID,

src/libsyntax/ext/expand.rs

@@ -133,6 +133,8 @@ expansions! {
         "trait item", .make_trait_items, lift .fold_trait_item, lift .visit_trait_item;
     ImplItems: SmallVector<ast::ImplItem> [SmallVector, ast::ImplItem],
         "impl item",  .make_impl_items,  lift .fold_impl_item,  lift .visit_impl_item;
+    ForeignItems: SmallVector<ast::ForeignItem> [SmallVector, ast::ForeignItem],
+        "foreign item", .make_foreign_items, lift .fold_foreign_item, lift .visit_foreign_item;
 }
 
 impl ExpansionKind {
@@ -149,6 +151,8 @@ impl ExpansionKind {
                 Expansion::ImplItems(items.map(Annotatable::expect_impl_item).collect()),
             ExpansionKind::TraitItems =>
                 Expansion::TraitItems(items.map(Annotatable::expect_trait_item).collect()),
+            ExpansionKind::ForeignItems =>
+                Expansion::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()),
             _ => unreachable!(),
         }
     }
@@ -435,6 +439,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             Annotatable::ImplItem(item) => {
                 Annotatable::ImplItem(item.map(|item| cfg.fold_impl_item(item).pop().unwrap()))
             }
+            Annotatable::ForeignItem(item) => {
+                Annotatable::ForeignItem(
+                    item.map(|item| cfg.fold_foreign_item(item).pop().unwrap())
+                )
+            }
             Annotatable::Stmt(stmt) => {
                 Annotatable::Stmt(stmt.map(|stmt| cfg.fold_stmt(stmt).pop().unwrap()))
             }
@@ -509,6 +518,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     Annotatable::Item(item) => token::NtItem(item),
                     Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
                     Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
+                    Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
                     Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
                     Annotatable::Expr(expr) => token::NtExpr(expr),
                 })).into();
@@ -793,6 +803,15 @@ impl<'a> Parser<'a> {
                 }
                 Expansion::ImplItems(items)
             }
+            ExpansionKind::ForeignItems => {
+                let mut items = SmallVector::new();
+                while self.token != token::Eof {
+                    if let Some(item) = self.parse_foreign_item()? {
+                        items.push(item);
+                    }
+                }
+                Expansion::ForeignItems(items)
+            }
             ExpansionKind::Stmts => {
                 let mut stmts = SmallVector::new();
                 while self.token != token::Eof &&
@@ -1166,6 +1185,44 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
         noop_fold_foreign_mod(self.cfg.configure_foreign_mod(foreign_mod), self)
     }
 
+    fn fold_foreign_item(&mut self,
+                         foreign_item: ast::ForeignItem) -> SmallVector<ast::ForeignItem> {
+        let (attr, traits, foreign_item) = self.classify_item(foreign_item);
+
+        let explain = if self.cx.ecfg.proc_macro_enabled() {
+            feature_gate::EXPLAIN_PROC_MACROS_IN_EXTERN
+        } else {
+            feature_gate::EXPLAIN_MACROS_IN_EXTERN
+        };
+
+        if attr.is_some() || !traits.is_empty()  {
+            if !self.cx.ecfg.macros_in_extern_enabled() {
+                if let Some(ref attr) = attr {
+                    emit_feature_err(&self.cx.parse_sess, "macros_in_extern", attr.span,
+                                     GateIssue::Language, explain);
+                }
+            }
+
+            let item = Annotatable::ForeignItem(P(foreign_item));
+            return self.collect_attr(attr, traits, item, ExpansionKind::ForeignItems)
+                .make_foreign_items();
+        }
+
+        if let ast::ForeignItemKind::Macro(mac) = foreign_item.node {
+            self.check_attributes(&foreign_item.attrs);
+
+            if !self.cx.ecfg.macros_in_extern_enabled() {
+                emit_feature_err(&self.cx.parse_sess, "macros_in_extern", foreign_item.span,
+                                 GateIssue::Language, explain);
+            }
+
+            return self.collect_bang(mac, foreign_item.span, ExpansionKind::ForeignItems)
+                .make_foreign_items();
+        }
+
+        noop_fold_foreign_item(foreign_item, self)
+    }
+
     fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
         match item {
             ast::ItemKind::MacroDef(..) => item,
@@ -1311,6 +1368,7 @@ impl<'feat> ExpansionConfig<'feat> {
         fn enable_allow_internal_unstable = allow_internal_unstable,
         fn enable_custom_derive = custom_derive,
         fn proc_macro_enabled = proc_macro,
+        fn macros_in_extern_enabled = macros_in_extern,
     }
 }
 

src/libsyntax/ext/placeholders.rs

@@ -60,6 +60,10 @@ pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion {
             defaultness: ast::Defaultness::Final,
             tokens: None,
         })),
+        ExpansionKind::ForeignItems => Expansion::ForeignItems(SmallVector::one(ast::ForeignItem {
+            id, span, ident, vis, attrs,
+            node: ast::ForeignItemKind::Macro(mac_placeholder()),
+        })),
         ExpansionKind::Pat => Expansion::Pat(P(ast::Pat {
             id, span, node: ast::PatKind::Mac(mac_placeholder()),
         })),
@@ -132,6 +136,13 @@ impl<'a, 'b> Folder for PlaceholderExpander<'a, 'b> {
         }
     }
 
+    fn fold_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVector<ast::ForeignItem> {
+        match item.node {
+            ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(),
+            _ => noop_fold_foreign_item(item, self),
+        }
+    }
+
     fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
         match expr.node {
             ast::ExprKind::Mac(_) => self.remove(expr.id).make_expr(),

src/libsyntax/feature_gate.rs

@@ -443,6 +443,9 @@ declare_features! (
 
     // Allows keywords to be escaped for use as identifiers
     (active, raw_identifiers, "1.26.0", Some(48589), None),
+
+    // Allows macro invocations in `extern {}` blocks
+    (active, macros_in_extern, "1.27.0", Some(49476), None),
 );
 
 declare_features! (
@@ -1291,6 +1294,13 @@ pub const EXPLAIN_UNSIZED_TUPLE_COERCION: &'static str =
 pub const EXPLAIN_MACRO_AT_MOST_ONCE_REP: &'static str =
     "Using the `?` macro Kleene operator for \"at most one\" repetition is unstable";
 
+pub const EXPLAIN_MACROS_IN_EXTERN: &'static str =
+    "Macro invocations in `extern {}` blocks are experimental.";
+
+// mention proc-macros when enabled
+pub const EXPLAIN_PROC_MACROS_IN_EXTERN: &'static str =
+    "Macro and proc-macro invocations in `extern {}` blocks are experimental.";
+
 struct PostExpansionVisitor<'a> {
     context: &'a Context<'a>,
 }
@@ -1595,6 +1605,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                     gate_feature_post!(&self, extern_types, i.span,
                                        "extern types are experimental");
             }
+            ast::ForeignItemKind::Macro(..) => {}
         }
 
         visit::walk_foreign_item(self, i)