Rollup merge of rust-lang#63761 - petrochenkov:procattrs, r=eddyb

Propagate spans and attributes from proc macro definitions

Thanks to rust-lang#63269 we now have spans and attributes from proc macro definitions available in metadata.

However, that PR didn't actually put them into use! This PR finishes that work.

Attributes `rustc_macro_transparency`, `allow_internal_unstable`, `allow_internal_unsafe`, `local_inner_macros`, `rustc_builtin_macro`, `stable`, `unstable`, `rustc_deprecated`, `deprecated` now have effect when applied to proc macro definition functions.
From those attributes only `deprecated` is both stable and supposed to be used in new code.
(`#![staged_api]` still cannot be used in proc macro crates for unrelated reasons though.)

`Span::def_site` from the proc macro API now returns the correct location of the proc macro definition.

Also, I made a mistake in rust-lang#63269 (comment), loaded proc macros didn't actually use the resolver cache.
This PR fixes the caching issue, now proc macros go through the `Resolver::macro_map` cache as well.

(Also, the first commit turns `proc_macro::quote` into a regular built-in macro to reduce the number of places where `SyntaxExtension`s need to be manually created.)

Author: Centril

src/libproc_macro/lib.rs

@@ -19,12 +19,15 @@
 
 #![feature(nll)]
 #![feature(staged_api)]
+#![feature(allow_internal_unstable)]
 #![feature(const_fn)]
+#![feature(decl_macro)]
 #![feature(extern_types)]
 #![feature(in_band_lifetimes)]
 #![feature(optin_builtin_traits)]
 #![feature(mem_take)]
 #![feature(non_exhaustive)]
+#![feature(rustc_attrs)]
 #![feature(specialization)]
 
 #![recursion_limit="256"]
@@ -222,11 +225,10 @@ pub mod token_stream {
 ///
 /// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
 /// To quote `$` itself, use `$$`.
-///
-/// This is a dummy macro, the actual implementation is in `quote::quote`.`
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
-#[macro_export]
-macro_rules! quote { () => {} }
+#[allow_internal_unstable(proc_macro_def_site)]
+#[cfg_attr(not(bootstrap), rustc_builtin_macro)]
+pub macro quote ($($t:tt)*) { /* compiler built-in */ }
 
 #[unstable(feature = "proc_macro_internals", issue = "27812")]
 #[doc(hidden)]

src/libproc_macro/quote.rs

@@ -57,9 +57,9 @@ macro_rules! quote {
 }
 
 /// Quote a `TokenStream` into a `TokenStream`.
-/// This is the actual `quote!()` proc macro.
+/// This is the actual implementation of the `quote!()` proc macro.
 ///
-/// It is manually loaded in `CStore::load_macro_untracked`.
+/// It is loaded by the compiler in `register_builtin_macros`.
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
 pub fn quote(stream: TokenStream) -> TokenStream {
     if stream.is_empty() {

src/librustc_metadata/cstore.rs

@@ -95,11 +95,6 @@ pub struct CrateMetadata {
     pub raw_proc_macros: Option<&'static [ProcMacro]>,
 }
 
-pub struct FullProcMacro {
-    pub name: ast::Name,
-    pub ext: Lrc<SyntaxExtension>
-}
-
 pub struct CStore {
     metas: RwLock<IndexVec<CrateNum, Option<Lrc<CrateMetadata>>>>,
     /// Map from NodeId's of local extern crate statements to crate numbers
@@ -109,7 +104,7 @@ pub struct CStore {
 
 pub enum LoadedMacro {
     MacroDef(ast::Item),
-    ProcMacro(Lrc<SyntaxExtension>),
+    ProcMacro(SyntaxExtension),
 }
 
 impl CStore {

src/librustc_metadata/cstore_impl.rs

@@ -30,11 +30,9 @@ use syntax::ast;
 use syntax::attr;
 use syntax::source_map;
 use syntax::edition::Edition;
-use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind};
-use syntax::ext::proc_macro::BangProcMacro;
 use syntax::parse::source_file_to_stream;
 use syntax::parse::parser::emit_unclosed_delims;
-use syntax::symbol::{Symbol, sym};
+use syntax::symbol::Symbol;
 use syntax_pos::{Span, FileName};
 use rustc_data_structures::bit_set::BitSet;
 
@@ -436,15 +434,7 @@ impl cstore::CStore {
     pub fn load_macro_untracked(&self, id: DefId, sess: &Session) -> LoadedMacro {
         let data = self.get_crate_data(id.krate);
         if data.is_proc_macro_crate() {
-            return LoadedMacro::ProcMacro(data.get_proc_macro(id.index, sess).ext);
-        } else if data.name == sym::proc_macro && data.item_name(id.index) == sym::quote {
-            let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
-            let kind = SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client }));
-            let ext = SyntaxExtension {
-                allow_internal_unstable: Some([sym::proc_macro_def_site][..].into()),
-                ..SyntaxExtension::default(kind, data.root.edition)
-            };
-            return LoadedMacro::ProcMacro(Lrc::new(ext));
+            return LoadedMacro::ProcMacro(data.load_proc_macro(id.index, sess));
         }
 
         let def = data.get_macro(id.index);

src/librustc_metadata/decoder.rs

@@ -1,6 +1,6 @@
 // Decoding metadata from a single crate's metadata
 
-use crate::cstore::{self, CrateMetadata, MetadataBlob, NativeLibrary, ForeignModule, FullProcMacro};
+use crate::cstore::{self, CrateMetadata, MetadataBlob, NativeLibrary, ForeignModule};
 use crate::schema::*;
 
 use rustc_data_structures::indexed_vec::IndexVec;
@@ -512,27 +512,8 @@ impl<'a, 'tcx> CrateMetadata {
         self.entry(index).span.decode((self, sess))
     }
 
-
-    pub fn get_proc_macro(&self, id: DefIndex, sess: &Session) -> FullProcMacro {
-        if sess.opts.debugging_opts.dual_proc_macros {
-            let host_lib = self.host_lib.as_ref().unwrap();
-            self.load_proc_macro(
-                &host_lib.metadata.get_root(),
-                id,
-                sess
-            )
-        } else {
-            self.load_proc_macro(&self.root, id, sess)
-        }
-    }
-
-    fn load_proc_macro(&self, root: &CrateRoot<'_>,
-                        id: DefIndex,
-                        sess: &Session)
-                        -> FullProcMacro {
-
-        let raw_macro = self.raw_proc_macro(id);
-        let (name, kind, helper_attrs) = match *raw_macro {
+    crate fn load_proc_macro(&self, id: DefIndex, sess: &Session) -> SyntaxExtension {
+        let (name, kind, helper_attrs) = match *self.raw_proc_macro(id) {
             ProcMacro::CustomDerive { trait_name, attributes, client } => {
                 let helper_attrs =
                     attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
@@ -551,17 +532,21 @@ impl<'a, 'tcx> CrateMetadata {
                 name, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client })), Vec::new()
             )
         };
+        let edition = if sess.opts.debugging_opts.dual_proc_macros {
+            self.host_lib.as_ref().unwrap().metadata.get_root().edition
+        } else {
+            self.root.edition
+        };
 
-        let span = self.get_span(id, sess);
-
-        FullProcMacro {
-            name: Symbol::intern(name),
-            ext: Lrc::new(SyntaxExtension {
-                span,
-                helper_attrs,
-                ..SyntaxExtension::default(kind, root.edition)
-            })
-        }
+        SyntaxExtension::new(
+            &sess.parse_sess,
+            kind,
+            self.get_span(id, sess),
+            helper_attrs,
+            edition,
+            Symbol::intern(name),
+            &self.get_attributes(&self.entry(id), sess),
+        )
     }
 
     pub fn get_trait_def(&self, item_id: DefIndex, sess: &Session) -> ty::TraitDef {

src/librustc_resolve/build_reduced_graph.rs

@@ -150,12 +150,12 @@ impl<'a> Resolver<'a> {
             return Some(ext.clone());
         }
 
-        let macro_def = match self.cstore.load_macro_untracked(def_id, &self.session) {
-            LoadedMacro::MacroDef(macro_def) => macro_def,
-            LoadedMacro::ProcMacro(ext) => return Some(ext),
-        };
+        let ext = Lrc::new(match self.cstore.load_macro_untracked(def_id, &self.session) {
+            LoadedMacro::MacroDef(item) =>
+                self.compile_macro(&item, self.cstore.crate_edition_untracked(def_id.krate)),
+            LoadedMacro::ProcMacro(ext) => ext,
+        });
 
-        let ext = self.compile_macro(&macro_def, self.cstore.crate_edition_untracked(def_id.krate));
         self.macro_map.insert(def_id, ext.clone());
         Some(ext)
     }
@@ -1104,7 +1104,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         let expansion = parent_scope.expansion;
         let (ext, ident, span, is_legacy) = match &item.node {
             ItemKind::MacroDef(def) => {
-                let ext = self.r.compile_macro(item, self.r.session.edition());
+                let ext = Lrc::new(self.r.compile_macro(item, self.r.session.edition()));
                 (ext, item.ident, item.span, def.legacy)
             }
             ItemKind::Fn(..) => match Self::proc_macro_stub(item) {

src/librustc_resolve/macros.rs

@@ -800,7 +800,7 @@ impl<'a> Resolver<'a> {
 
     /// Compile the macro into a `SyntaxExtension` and possibly replace it with a pre-defined
     /// extension partially or entirely for built-in macros and legacy plugin macros.
-    crate fn compile_macro(&mut self, item: &ast::Item, edition: Edition) -> Lrc<SyntaxExtension> {
+    crate fn compile_macro(&mut self, item: &ast::Item, edition: Edition) -> SyntaxExtension {
         let mut result = macro_rules::compile(
             &self.session.parse_sess, self.session.features_untracked(), item, edition
         );
@@ -822,6 +822,6 @@ impl<'a> Resolver<'a> {
             }
         }
 
-        Lrc::new(result)
+        result
     }
 }

src/libsyntax/ext/base.rs

@@ -1,11 +1,11 @@
 use crate::ast::{self, NodeId, Attribute, Name, PatKind};
-use crate::attr::{HasAttrs, Stability, Deprecation};
+use crate::attr::{self, HasAttrs, Stability, Deprecation};
 use crate::source_map::SourceMap;
 use crate::edition::Edition;
 use crate::ext::expand::{self, AstFragment, Invocation};
 use crate::ext::hygiene::{ExpnId, Transparency};
 use crate::mut_visit::{self, MutVisitor};
-use crate::parse::{self, parser, DirectoryOwnership};
+use crate::parse::{self, parser, ParseSess, DirectoryOwnership};
 use crate::parse::token;
 use crate::ptr::P;
 use crate::symbol::{kw, sym, Ident, Symbol};
@@ -601,6 +601,69 @@ impl SyntaxExtension {
         }
     }
 
+    /// Constructs a syntax extension with the given properties
+    /// and other properties converted from attributes.
+    pub fn new(
+        sess: &ParseSess,
+        kind: SyntaxExtensionKind,
+        span: Span,
+        helper_attrs: Vec<Symbol>,
+        edition: Edition,
+        name: Name,
+        attrs: &[ast::Attribute],
+    ) -> SyntaxExtension {
+        let allow_internal_unstable =
+            attr::find_by_name(attrs, sym::allow_internal_unstable).map(|attr| {
+                attr.meta_item_list()
+                    .map(|list| {
+                        list.iter()
+                            .filter_map(|it| {
+                                let name = it.ident().map(|ident| ident.name);
+                                if name.is_none() {
+                                    sess.span_diagnostic.span_err(
+                                        it.span(), "allow internal unstable expects feature names"
+                                    )
+                                }
+                                name
+                            })
+                            .collect::<Vec<Symbol>>()
+                            .into()
+                    })
+                    .unwrap_or_else(|| {
+                        sess.span_diagnostic.span_warn(
+                            attr.span,
+                            "allow_internal_unstable expects list of feature names. In the future \
+                             this will become a hard error. Please use `allow_internal_unstable(\
+                             foo, bar)` to only allow the `foo` and `bar` features",
+                        );
+                        vec![sym::allow_internal_unstable_backcompat_hack].into()
+                    })
+            });
+
+        let mut local_inner_macros = false;
+        if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) {
+            if let Some(l) = macro_export.meta_item_list() {
+                local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros);
+            }
+        }
+
+        let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro);
+
+        SyntaxExtension {
+            kind,
+            span,
+            allow_internal_unstable,
+            allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe),
+            local_inner_macros,
+            stability: attr::find_stability(&sess, attrs, span),
+            deprecation: attr::find_deprecation(&sess, attrs, span),
+            helper_attrs,
+            edition,
+            is_builtin,
+            is_derive_copy: is_builtin && name == sym::Copy,
+        }
+    }
+
     pub fn dummy_bang(edition: Edition) -> SyntaxExtension {
         fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: &[TokenTree])
                          -> Box<dyn MacResult + 'cx> {

src/libsyntax/ext/proc_macro_server.rs

@@ -360,12 +360,11 @@ pub(crate) struct Rustc<'a> {
 
 impl<'a> Rustc<'a> {
     pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
-        // No way to determine def location for a proc macro right now, so use call location.
-        let location = cx.current_expansion.id.expn_data().call_site;
+        let expn_data = cx.current_expansion.id.expn_data();
         Rustc {
             sess: cx.parse_sess,
-            def_site: cx.with_def_site_ctxt(location),
-            call_site: cx.with_call_site_ctxt(location),
+            def_site: cx.with_def_site_ctxt(expn_data.def_site),
+            call_site: cx.with_call_site_ctxt(expn_data.call_site),
         }
     }