Reformat metadata for exported macros

Instead of copy-pasting the whole macro_rules! item from the original .rs file,
we serialize a separate name, attributes list, and body, the latter as
pretty-printed TTs.  The compilation of macro_rules! macros is decoupled
somewhat from the expansion of macros in item position.

This filters out comments, and facilitates selective imports.

Author: Keegan McAllister

src/librustc/metadata/common.rs

@@ -206,8 +206,8 @@ pub const tag_native_libraries_name: uint = 0x89;
 pub const tag_native_libraries_kind: uint = 0x8a;
 
 pub const tag_plugin_registrar_fn: uint = 0x8b;
-pub const tag_exported_macros: uint = 0x8c;
-pub const tag_macro_def: uint = 0x8d;
+
+// GAP 0x8c, 0x8d
 
 pub const tag_method_argument_names: uint = 0x8e;
 pub const tag_method_argument_name: uint = 0x8f;
@@ -261,3 +261,7 @@ pub const tag_associated_type_names: uint = 0xb2;
 pub const tag_associated_type_name: uint = 0xb3;
 
 pub const tag_polarity: uint = 0xb4;
+
+pub const tag_macro_defs: uint = 0xb5;
+pub const tag_macro_def: uint = 0xb6;
+pub const tag_macro_def_body: uint = 0xb7;

src/librustc/metadata/creader.rs

@@ -29,8 +29,9 @@ use syntax::ast;
 use syntax::abi;
 use syntax::attr;
 use syntax::attr::AttrMetaMethods;
-use syntax::codemap::{Span};
+use syntax::codemap::{Span, mk_sp};
 use syntax::diagnostic::SpanHandler;
+use syntax::parse;
 use syntax::parse::token::InternedString;
 use syntax::parse::token;
 use syntax::visit;
@@ -491,7 +492,36 @@ impl<'a> CrateReader<'a> {
             }
             None => { load_ctxt.report_load_errs(); unreachable!() },
         };
-        let macros = decoder::get_exported_macros(library.metadata.as_slice());
+
+        // Read exported macros
+        let imported_from = Some(token::intern(info.ident[]).ident());
+        let source_name = format!("<{} macros>", info.ident[]);
+        let mut macros = vec![];
+        decoder::each_exported_macro(library.metadata.as_slice(), &*self.sess.cstore.intr,
+            |name, attrs, body| {
+                // NB: Don't use parse::parse_tts_from_source_str because it parses with
+                // quote_depth > 0.
+                let mut p = parse::new_parser_from_source_str(&self.sess.parse_sess,
+                                                              self.sess.opts.cfg.clone(),
+                                                              source_name.clone(),
+                                                              body);
+                let lo = p.span.lo;
+                let body = p.parse_all_token_trees();
+                let span = mk_sp(lo, p.last_span.hi);
+                p.abort_if_errors();
+                macros.push(ast::MacroDef {
+                    ident: name.ident(),
+                    attrs: attrs,
+                    id: ast::DUMMY_NODE_ID,
+                    span: span,
+                    imported_from: imported_from,
+                    body: body,
+                });
+                true
+            }
+        );
+
+        // Look for a plugin registrar
         let registrar = decoder::get_plugin_registrar_fn(library.metadata.as_slice()).map(|id| {
             decoder::get_symbol(library.metadata.as_slice(), id)
         });
@@ -504,9 +534,11 @@ impl<'a> CrateReader<'a> {
             // empty dylib.
         }
         let pc = PluginMetadata {
-            lib: library.dylib.clone(),
             macros: macros,
-            registrar_symbol: registrar,
+            registrar: match (library.dylib.as_ref(), registrar) {
+                (Some(dylib), Some(reg)) => Some((dylib.clone(), reg)),
+                _ => None,
+            },
         };
         if should_link && self.existing_match(info.name[], None).is_none() {
             // register crate now to avoid double-reading metadata

src/librustc/metadata/decoder.rs

@@ -1353,15 +1353,16 @@ pub fn get_plugin_registrar_fn(data: &[u8]) -> Option<ast::NodeId> {
         .map(|doc| FromPrimitive::from_u32(reader::doc_as_u32(doc)).unwrap())
 }
 
-pub fn get_exported_macros(data: &[u8]) -> Vec<String> {
-    let macros = reader::get_doc(rbml::Doc::new(data),
-                                 tag_exported_macros);
-    let mut result = Vec::new();
+pub fn each_exported_macro<F>(data: &[u8], intr: &IdentInterner, mut f: F) where
+    F: FnMut(ast::Name, Vec<ast::Attribute>, String) -> bool,
+{
+    let macros = reader::get_doc(rbml::Doc::new(data), tag_macro_defs);
     reader::tagged_docs(macros, tag_macro_def, |macro_doc| {
-        result.push(macro_doc.as_str().to_string());
-        true
+        let name = item_name(intr, macro_doc);
+        let attrs = get_attributes(macro_doc);
+        let body = reader::get_doc(macro_doc, tag_macro_def_body);
+        f(name, attrs, body.as_str().to_string())
     });
-    result
 }
 
 pub fn get_dylib_dependency_formats(cdata: Cmd)

src/librustc/metadata/encoder.rs

@@ -42,6 +42,7 @@ use syntax::attr::AttrMetaMethods;
 use syntax::diagnostic::SpanHandler;
 use syntax::parse::token::special_idents;
 use syntax::parse::token;
+use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::visit::Visitor;
 use syntax::visit;
@@ -1818,25 +1819,21 @@ fn encode_plugin_registrar_fn(ecx: &EncodeContext, rbml_w: &mut Encoder) {
     }
 }
 
-/// Given a span, write the text of that span into the output stream
-/// as an exported macro
-fn encode_macro_def(ecx: &EncodeContext,
-                    rbml_w: &mut Encoder,
-                    span: &syntax::codemap::Span) {
-    let def = ecx.tcx.sess.codemap().span_to_snippet(*span)
-        .expect("Unable to find source for macro");
-    rbml_w.start_tag(tag_macro_def);
-    rbml_w.wr_str(def[]);
-    rbml_w.end_tag();
-}
-
 /// Serialize the text of the exported macros
-fn encode_macro_defs(ecx: &EncodeContext,
-                     krate: &ast::Crate,
-                     rbml_w: &mut Encoder) {
-    rbml_w.start_tag(tag_exported_macros);
-    for item in krate.exported_macros.iter() {
-        encode_macro_def(ecx, rbml_w, &item.span);
+fn encode_macro_defs(rbml_w: &mut Encoder,
+                     krate: &ast::Crate) {
+    rbml_w.start_tag(tag_macro_defs);
+    for def in krate.exported_macros.iter() {
+        rbml_w.start_tag(tag_macro_def);
+
+        encode_name(rbml_w, def.ident.name);
+        encode_attributes(rbml_w, def.attrs[]);
+
+        rbml_w.start_tag(tag_macro_def_body);
+        rbml_w.wr_str(pprust::tts_to_string(def.body[])[]);
+        rbml_w.end_tag();
+
+        rbml_w.end_tag();
     }
     rbml_w.end_tag();
 }
@@ -2154,7 +2151,7 @@ fn encode_metadata_inner(wr: &mut SeekableMemWriter,
 
     // Encode macro definitions
     i = rbml_w.writer.tell().unwrap();
-    encode_macro_defs(&ecx, krate, &mut rbml_w);
+    encode_macro_defs(&mut rbml_w, krate);
     stats.macro_defs_bytes = rbml_w.writer.tell().unwrap() - i;
 
     // Encode the types of all unboxed closures in this crate.

src/librustc/plugin/load.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-//! Used by `rustc` when loading a plugin.
+//! Used by `rustc` when loading a plugin, or a crate with exported macros.
 
 use session::Session;
 use metadata::creader::CrateReader;
@@ -21,17 +21,14 @@ use syntax::ast;
 use syntax::attr;
 use syntax::visit;
 use syntax::visit::Visitor;
-use syntax::ext::expand::ExportedMacros;
 use syntax::attr::AttrMetaMethods;
 
-/// Plugin-related crate metadata.
+/// Metadata for a single plugin crate.
 pub struct PluginMetadata {
-    /// Source code of macros exported by the crate.
-    pub macros: Vec<String>,
-    /// Path to the shared library file.
-    pub lib: Option<Path>,
-    /// Symbol name of the plugin registrar function.
-    pub registrar_symbol: Option<String>,
+    /// Macros exported by the crate.
+    pub macros: Vec<ast::MacroDef>,
+    /// Path to the shared library file, and registrar function symbol.
+    pub registrar: Option<(Path, String)>,
 }
 
 /// Pointer to a registrar function.
@@ -40,8 +37,8 @@ pub type PluginRegistrarFun =
 
 /// Information about loaded plugins.
 pub struct Plugins {
-    /// Source code of exported macros.
-    pub macros: Vec<ExportedMacros>,
+    /// Imported macros.
+    pub macros: Vec<ast::MacroDef>,
     /// Registrars, as function pointers.
     pub registrars: Vec<PluginRegistrarFun>,
 }
@@ -90,7 +87,7 @@ pub fn load_plugins(sess: &Session, krate: &ast::Crate,
 impl<'a, 'v> Visitor<'v> for PluginLoader<'a> {
     fn visit_view_item(&mut self, vi: &ast::ViewItem) {
         match vi.node {
-            ast::ViewItemExternCrate(name, _, _) => {
+            ast::ViewItemExternCrate(_, _, _) => {
                 let mut plugin_phase = false;
 
                 for attr in vi.attrs.iter().filter(|a| a.check_name("phase")) {
@@ -107,17 +104,13 @@ impl<'a, 'v> Visitor<'v> for PluginLoader<'a> {
 
                 if !plugin_phase { return; }
 
-                let PluginMetadata { macros, lib, registrar_symbol } =
+                let PluginMetadata { macros, registrar } =
                     self.reader.read_plugin_metadata(vi);
 
-                self.plugins.macros.push(ExportedMacros {
-                    crate_name: name,
-                    macros: macros,
-                });
+                self.plugins.macros.extend(macros.into_iter());
 
-                match (lib, registrar_symbol) {
-                    (Some(lib), Some(symbol))
-                        => self.dylink_registrar(vi, lib, symbol),
+                match registrar {
+                    Some((lib, symbol)) => self.dylink_registrar(vi, lib, symbol),
                     _ => (),
                 }
             }

src/librustdoc/visit_ast.rs

@@ -73,7 +73,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
                                               None);
         // attach the crate's exported macros to the top-level module:
         self.module.macros = krate.exported_macros.iter()
-            .map(|it| self.visit_macro(&**it)).collect();
+            .map(|def| self.visit_macro(def)).collect();
         self.module.is_crate = true;
     }
 
@@ -363,13 +363,13 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
     }
 
     // convert each exported_macro into a doc item
-    fn visit_macro(&self, item: &ast::Item) -> Macro {
+    fn visit_macro(&self, def: &ast::MacroDef) -> Macro {
         Macro {
-            id: item.id,
-            attrs: item.attrs.clone(),
-            name: item.ident,
-            whence: item.span,
-            stab: self.stability(item.id),
+            id: def.id,
+            attrs: def.attrs.clone(),
+            name: def.ident,
+            whence: def.span,
+            stab: self.stability(def.id),
         }
     }
 }

src/libsyntax/ast.rs

@@ -476,7 +476,7 @@ pub struct Crate {
     pub attrs: Vec<Attribute>,
     pub config: CrateConfig,
     pub span: Span,
-    pub exported_macros: Vec<P<Item>>
+    pub exported_macros: Vec<MacroDef>,
 }
 
 pub type MetaItem = Spanned<MetaItem_>;
@@ -1698,6 +1698,19 @@ pub enum InlinedItem {
     IIForeign(P<ForeignItem>),
 }
 
+/// A macro definition, in this crate or imported from another.
+///
+/// Not parsed directly, but created on macro import or `macro_rules!` expansion.
+#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Show)]
+pub struct MacroDef {
+    pub ident: Ident,
+    pub attrs: Vec<Attribute>,
+    pub id: NodeId,
+    pub span: Span,
+    pub imported_from: Option<Ident>,
+    pub body: Vec<TokenTree>,
+}
+
 #[cfg(test)]
 mod test {
     use serialize::json;

src/libsyntax/ext/base.rs

@@ -28,19 +28,6 @@ use fold::Folder;
 use std::collections::HashMap;
 use std::rc::Rc;
 
-// new-style macro! tt code:
-//
-//    MacResult, NormalTT, IdentTT
-//
-// also note that ast::Mac used to have a bunch of extraneous cases and
-// is now probably a redundant AST node, can be merged with
-// ast::MacInvocTT.
-
-pub struct MacroDef {
-    pub name: String,
-    pub ext: SyntaxExtension
-}
-
 pub trait ItemDecorator {
     fn expand(&self,
               ecx: &mut ExtCtxt,
@@ -140,13 +127,6 @@ impl<F> IdentMacroExpander for F
 /// methods are spliced into the AST at the callsite of the macro (or
 /// just into the compiler's internal macro table, for `make_def`).
 pub trait MacResult {
-    /// Attempt to define a new macro.
-    // this should go away; the idea that a macro might expand into
-    // either a macro definition or an expression, depending on what
-    // the context wants, is kind of silly.
-    fn make_def(&mut self) -> Option<MacroDef> {
-        None
-    }
     /// Create an expression.
     fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> {
         None
@@ -469,7 +449,7 @@ pub struct ExtCtxt<'a> {
 
     pub mod_path: Vec<ast::Ident> ,
     pub trace_mac: bool,
-    pub exported_macros: Vec<P<ast::Item>>,
+    pub exported_macros: Vec<ast::MacroDef>,
 
     pub syntax_env: SyntaxEnv,
     pub recursion_count: uint,