Rollup merge of rust-lang#36438 - jseyfried:node_ids_in_expansion, r=nrc

Assign node ids during macro expansion

After this PR,
 - The `ExtCtxt` can access `resolve`'s `Resolver` through the trait object `ext::base::Resolver`.
  - The `Resolver` trait object can load macros and replaces today's `MacroLoader` trait object.
  - The macro expander uses the `Resolver` trait object to resolve macro invocations.
 - The macro expander assigns node ids and builds the `Resolver`'s `macros_at_scope` map.
   - This is groundwork for merging import resolution and expansion.
 - Performance of expansion together with node id assignment improves by ~5%.

**EDIT:** Since Github is reordering the commits, here is `git log`:
 - b54e1e3: Differentiate between monotonic and non-monotonic expansion and only assign node ids during monotonic expansion.
 - 78c0039: Expand generated test harnesses and macro registries.
 - f3c2dca: Remove scope placeholders from the crate root.
 - c86c8d4: Perform node id assignment and `macros_at_scope` construction during the `InvocationCollector` and `PlaceholderExpander` folds.
 - 72a6369: Move macro resolution into `librustc_resolve`.
 - 20b43b2: Rewrite the unit tests in `ext/expand.rs` as a `compile-fail` test.
 - a9821e1: Refactor `ExtCtxt` to use a `Resolver` instead of a `MacroLoader`.
 - 60440b2: Refactor `noop_fold_stmt_kind` out of `noop_fold_stmt`.
 - 50f94f6: Avoid needless reexpansions.

r? @nrc

src/librustc/middle/cstore.rs

@@ -39,6 +39,7 @@ use std::rc::Rc;
 use std::path::PathBuf;
 use syntax::ast;
 use syntax::attr;
+use syntax::ext::base::LoadedMacro;
 use syntax::ptr::P;
 use syntax::parse::token::InternedString;
 use syntax_pos::Span;
@@ -492,6 +493,9 @@ impl<'tcx> CrateStore<'tcx> for DummyCrateStore {
     fn metadata_encoding_version(&self) -> &[u8] { bug!("metadata_encoding_version") }
 }
 
+pub trait MacroLoader {
+     fn load_crate(&mut self, extern_crate: &ast::Item, allows_macros: bool) -> Vec<LoadedMacro>;
+}
 
 /// Metadata encoding and decoding can make use of thread-local encoding and
 /// decoding contexts. These allow implementers of serialize::Encodable and

src/librustc/session/mod.rs

@@ -21,7 +21,7 @@ use util::nodemap::{NodeMap, FnvHashMap};
 use util::common::duration_to_secs_str;
 use mir::transform as mir_pass;
 
-use syntax::ast::{NodeId, Name};
+use syntax::ast::NodeId;
 use errors::{self, DiagnosticBuilder};
 use errors::emitter::{Emitter, EmitterWriter};
 use syntax::json::JsonEmitter;
@@ -39,7 +39,7 @@ use llvm;
 
 use std::path::{Path, PathBuf};
 use std::cell::{self, Cell, RefCell};
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::env;
 use std::ffi::CString;
 use std::rc::Rc;
@@ -96,10 +96,6 @@ pub struct Session {
     pub injected_allocator: Cell<Option<ast::CrateNum>>,
     pub injected_panic_runtime: Cell<Option<ast::CrateNum>>,
 
-    /// Names of all bang-style macros and syntax extensions
-    /// available in this crate
-    pub available_macros: RefCell<HashSet<Name>>,
-
     /// Map from imported macro spans (which consist of
     /// the localized span for the macro body) to the
     /// macro name and defintion span in the source crate.
@@ -552,7 +548,6 @@ pub fn build_session_(sopts: config::Options,
         next_node_id: Cell::new(1),
         injected_allocator: Cell::new(None),
         injected_panic_runtime: Cell::new(None),
-        available_macros: RefCell::new(HashSet::new()),
         imported_macro_spans: RefCell::new(HashMap::new()),
         incr_comp_session: RefCell::new(IncrCompSession::NotInitialized),
         perf_stats: PerfStats {

src/librustc_driver/driver.rs

@@ -50,6 +50,7 @@ use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 use syntax::{ast, diagnostics, visit};
 use syntax::attr;
+use syntax::ext::base::ExtCtxt;
 use syntax::parse::{self, PResult, token};
 use syntax::util::node_count::NodeCounter;
 use syntax;
@@ -638,6 +639,13 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
     }
     sess.track_errors(|| sess.lint_store.borrow_mut().process_command_line(sess))?;
 
+    let mut macro_loader =
+        macro_import::MacroLoader::new(sess, &cstore, crate_name, krate.config.clone());
+
+    let resolver_arenas = Resolver::arenas();
+    let mut resolver = Resolver::new(sess, make_glob_map, &mut macro_loader, &resolver_arenas);
+    syntax_ext::register_builtins(&mut resolver, sess.features.borrow().quote);
+
     krate = time(time_passes, "expansion", || {
         // Windows dlls do not have rpaths, so they don't know how to find their
         // dependencies. It's up to us to tell the system where to find all the
@@ -672,25 +680,17 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
             trace_mac: sess.opts.debugging_opts.trace_macros,
             should_test: sess.opts.test,
         };
-        let mut loader = macro_import::MacroLoader::new(sess,
-                                                        &cstore,
-                                                        crate_name,
-                                                        krate.config.clone());
-        let mut ecx = syntax::ext::base::ExtCtxt::new(&sess.parse_sess,
-                                                      krate.config.clone(),
-                                                      cfg,
-                                                      &mut loader);
-        syntax_ext::register_builtins(&mut ecx.syntax_env);
+        let mut ecx = ExtCtxt::new(&sess.parse_sess, krate.config.clone(), cfg, &mut resolver);
         let ret = syntax::ext::expand::expand_crate(&mut ecx, syntax_exts, krate);
         if cfg!(windows) {
             env::set_var("PATH", &old_path);
         }
-        *sess.available_macros.borrow_mut() = ecx.syntax_env.names;
         ret
     });
 
     krate = time(time_passes, "maybe building test harness", || {
         syntax::test::modify_for_testing(&sess.parse_sess,
+                                         &mut resolver,
                                          sess.opts.test,
                                          krate,
                                          sess.diagnostic())
@@ -701,18 +701,14 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
         let is_rustc_macro_crate = crate_types.contains(&config::CrateTypeRustcMacro);
         let num_crate_types = crate_types.len();
         syntax_ext::rustc_macro_registrar::modify(&sess.parse_sess,
+                                                  &mut resolver,
                                                   krate,
                                                   is_rustc_macro_crate,
                                                   num_crate_types,
                                                   sess.diagnostic(),
                                                   &sess.features.borrow())
     });
 
-    let resolver_arenas = Resolver::arenas();
-    let mut resolver = Resolver::new(sess, make_glob_map, &resolver_arenas);
-
-    let krate = time(sess.time_passes(), "assigning node ids", || resolver.assign_node_ids(krate));
-
     if sess.opts.debugging_opts.input_stats {
         println!("Post-expansion node count: {}", count_nodes(&krate));
     }

src/librustc_metadata/macro_import.rs

@@ -18,6 +18,7 @@ use creader::{CrateReader, Macros};
 use cstore::CStore;
 
 use rustc::hir::def_id::DefIndex;
+use rustc::middle;
 use rustc::session::Session;
 use rustc::util::nodemap::FnvHashMap;
 use rustc_back::dynamic_lib::DynamicLibrary;
@@ -26,7 +27,6 @@ use rustc_macro::__internal::Registry;
 use syntax::ast;
 use syntax::attr;
 use syntax::ext::base::LoadedMacro;
-use syntax::ext;
 use syntax::parse::token;
 use syntax_ext::deriving::custom::CustomDerive;
 use syntax_pos::Span;
@@ -55,7 +55,7 @@ pub fn call_bad_macro_reexport(a: &Session, b: Span) {
 
 pub type MacroSelection = FnvHashMap<token::InternedString, Span>;
 
-impl<'a> ext::base::MacroLoader for MacroLoader<'a> {
+impl<'a> middle::cstore::MacroLoader for MacroLoader<'a> {
     fn load_crate(&mut self,
                   extern_crate: &ast::Item,
                   allows_macros: bool) -> Vec<LoadedMacro> {

src/librustc_resolve/lib.rs

@@ -45,6 +45,7 @@ use self::ParentLink::*;
 
 use rustc::hir::map::Definitions;
 use rustc::hir::{self, PrimTy, TyBool, TyChar, TyFloat, TyInt, TyUint, TyStr};
+use rustc::middle::cstore::MacroLoader;
 use rustc::session::Session;
 use rustc::lint;
 use rustc::hir::def::*;
@@ -79,10 +80,10 @@ use resolve_imports::{ImportDirective, NameResolution};
 // registered before they are used.
 mod diagnostics;
 
+mod macros;
 mod check_unused;
 mod build_reduced_graph;
 mod resolve_imports;
-mod assign_ids;
 
 enum SuggestionType {
     Macro(String),
@@ -1068,6 +1069,12 @@ pub struct Resolver<'a> {
     arenas: &'a ResolverArenas<'a>,
     dummy_binding: &'a NameBinding<'a>,
     new_import_semantics: bool, // true if `#![feature(item_like_imports)]`
+
+    macro_loader: &'a mut MacroLoader,
+    macro_names: FnvHashSet<Name>,
+
+    // Maps the `Mark` of an expansion to its containing module or block.
+    expansion_data: Vec<macros::ExpansionData>,
 }
 
 pub struct ResolverArenas<'a> {
@@ -1166,7 +1173,10 @@ impl Named for hir::PathSegment {
 }
 
 impl<'a> Resolver<'a> {
-    pub fn new(session: &'a Session, make_glob_map: MakeGlobMap, arenas: &'a ResolverArenas<'a>)
+    pub fn new(session: &'a Session,
+               make_glob_map: MakeGlobMap,
+               macro_loader: &'a mut MacroLoader,
+               arenas: &'a ResolverArenas<'a>)
                -> Resolver<'a> {
         let root_def_id = DefId::local(CRATE_DEF_INDEX);
         let graph_root =
@@ -1227,6 +1237,10 @@ impl<'a> Resolver<'a> {
                 vis: ty::Visibility::Public,
             }),
             new_import_semantics: session.features.borrow().item_like_imports,
+
+            macro_loader: macro_loader,
+            macro_names: FnvHashSet(),
+            expansion_data: vec![macros::ExpansionData::default()],
         }
     }
 
@@ -2768,8 +2782,7 @@ impl<'a> Resolver<'a> {
     }
 
     fn find_best_match(&mut self, name: &str) -> SuggestionType {
-        if let Some(macro_name) = self.session.available_macros
-                                  .borrow().iter().find(|n| n.as_str() == name) {
+        if let Some(macro_name) = self.macro_names.iter().find(|n| n.as_str() == name) {
             return SuggestionType::Macro(format!("{}!", macro_name));
         }