Auto merge of #86489 - cjgillot:lower, r=petrochenkov

Simplify early compilation interface

* separate resolver creation and AST configuration.
* bundle lowering with global_ctxt creation.

Author: bors

compiler/rustc_ast_lowering/src/lib.rs

@@ -296,7 +296,7 @@ pub fn lower_crate<'a, 'hir>(
     resolver: &'a mut dyn ResolverAstLowering,
     nt_to_tokenstream: NtToTokenstream,
     arena: &'hir Arena<'hir>,
-) -> hir::Crate<'hir> {
+) -> &'hir hir::Crate<'hir> {
     let _prof_timer = sess.prof.verbose_generic_activity("hir_lowering");
 
     LoweringContext {
@@ -403,7 +403,7 @@ enum AnonymousLifetimeMode {
 }
 
 impl<'a, 'hir> LoweringContext<'a, 'hir> {
-    fn lower_crate(mut self, c: &Crate) -> hir::Crate<'hir> {
+    fn lower_crate(mut self, c: &Crate) -> &'hir hir::Crate<'hir> {
         /// Full-crate AST visitor that inserts into a fresh
         /// `LoweringContext` any information that may be
         /// needed from arbitrary locations in the crate,
@@ -530,7 +530,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
         }
 
-        hir::Crate {
+        let krate = hir::Crate {
             item: module,
             exported_macros: self.arena.alloc_from_iter(self.exported_macros),
             non_exported_macro_attrs: self.arena.alloc_from_iter(self.non_exported_macro_attrs),
@@ -545,7 +545,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             proc_macros,
             trait_map,
             attrs: self.attrs,
-        }
+        };
+        self.arena.alloc(krate)
     }
 
     fn insert_item(&mut self, item: hir::Item<'hir>) -> hir::ItemId {

compiler/rustc_driver/src/lib.rs

@@ -46,7 +46,6 @@ use std::ffi::OsString;
 use std::fs;
 use std::io::{self, Read, Write};
 use std::lazy::SyncLazy;
-use std::mem;
 use std::panic::{self, catch_unwind};
 use std::path::PathBuf;
 use std::process::{self, Command, Stdio};
@@ -316,12 +315,12 @@ fn run_compiler(
 
             if let Some(ppm) = &sess.opts.pretty {
                 if ppm.needs_ast_map() {
+                    let expanded_crate = queries.expansion()?.peek().0.clone();
                     queries.global_ctxt()?.peek_mut().enter(|tcx| {
-                        let expanded_crate = queries.expansion()?.take().0;
                         pretty::print_after_hir_lowering(
                             tcx,
                             compiler.input(),
-                            &expanded_crate,
+                            &*expanded_crate,
                             *ppm,
                             compiler.output_file().as_ref().map(|p| &**p),
                         );
@@ -377,12 +376,6 @@ fn run_compiler(
 
             queries.global_ctxt()?;
 
-            // Drop AST after creating GlobalCtxt to free memory
-            {
-                let _timer = sess.prof.generic_activity("drop_ast");
-                mem::drop(queries.expansion()?.take());
-            }
-
             if sess.opts.debugging_opts.no_analysis || sess.opts.debugging_opts.ast_json {
                 return early_exit();
             }

compiler/rustc_incremental/src/persist/load.rs

@@ -21,8 +21,8 @@ pub enum LoadResult<T> {
     Error { message: String },
 }
 
-impl LoadResult<(SerializedDepGraph, WorkProductMap)> {
-    pub fn open(self, sess: &Session) -> (SerializedDepGraph, WorkProductMap) {
+impl<T: Default> LoadResult<T> {
+    pub fn open(self, sess: &Session) -> T {
         match self {
             LoadResult::Error { message } => {
                 sess.warn(&message);
@@ -74,11 +74,14 @@ pub enum MaybeAsync<T> {
     Sync(T),
     Async(std::thread::JoinHandle<T>),
 }
-impl<T> MaybeAsync<T> {
-    pub fn open(self) -> std::thread::Result<T> {
+
+impl<T> MaybeAsync<LoadResult<T>> {
+    pub fn open(self) -> LoadResult<T> {
         match self {
-            MaybeAsync::Sync(result) => Ok(result),
-            MaybeAsync::Async(handle) => handle.join(),
+            MaybeAsync::Sync(result) => result,
+            MaybeAsync::Async(handle) => handle.join().unwrap_or_else(|e| LoadResult::Error {
+                message: format!("could not decode incremental cache: {:?}", e),
+            }),
         }
     }
 }

compiler/rustc_interface/src/passes.rs

@@ -7,7 +7,6 @@ use rustc_ast::{self as ast, visit};
 use rustc_codegen_ssa::back::link::emit_metadata;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_data_structures::parallel;
-use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{par_iter, Lrc, OnceCell, ParallelIterator, WorkerLocal};
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::{ErrorReported, PResult};
@@ -101,7 +100,7 @@ mod boxed_resolver {
     }
 
     // Note: Drop order is important to prevent dangling references. Resolver must be dropped first,
-    // then resolver_arenas and finally session.
+    // then resolver_arenas and session.
     impl Drop for BoxedResolverInner {
         fn drop(&mut self) {
             self.resolver.take();
@@ -110,13 +109,10 @@ mod boxed_resolver {
     }
 
     impl BoxedResolver {
-        pub(super) fn new<F>(session: Lrc<Session>, make_resolver: F) -> Result<(ast::Crate, Self)>
-        where
-            F: for<'a> FnOnce(
-                &'a Session,
-                &'a ResolverArenas<'a>,
-            ) -> Result<(ast::Crate, Resolver<'a>)>,
-        {
+        pub(super) fn new(
+            session: Lrc<Session>,
+            make_resolver: impl for<'a> FnOnce(&'a Session, &'a ResolverArenas<'a>) -> Resolver<'a>,
+        ) -> BoxedResolver {
             let mut boxed_resolver = Box::new(BoxedResolverInner {
                 session,
                 resolver_arenas: Some(Resolver::arenas()),
@@ -127,14 +123,14 @@ mod boxed_resolver {
             // returns a resolver with the same lifetime as the arena. We ensure that the arena
             // outlives the resolver in the drop impl and elsewhere so these transmutes are sound.
             unsafe {
-                let (crate_, resolver) = make_resolver(
+                let resolver = make_resolver(
                     std::mem::transmute::<&Session, &Session>(&boxed_resolver.session),
                     std::mem::transmute::<&ResolverArenas<'_>, &ResolverArenas<'_>>(
                         boxed_resolver.resolver_arenas.as_ref().unwrap(),
                     ),
-                )?;
+                );
                 boxed_resolver.resolver = Some(resolver);
-                Ok((crate_, BoxedResolver(Pin::new_unchecked(boxed_resolver))))
+                BoxedResolver(Pin::new_unchecked(boxed_resolver))
             }
         }
 
@@ -165,35 +161,15 @@ mod boxed_resolver {
     }
 }
 
-/// Runs the "early phases" of the compiler: initial `cfg` processing, loading compiler plugins,
-/// syntax expansion, secondary `cfg` expansion, synthesis of a test
-/// harness if one is to be provided, injection of a dependency on the
-/// standard library and prelude, and name resolution.
-///
-/// Returns [`None`] if we're aborting after handling -W help.
-pub fn configure_and_expand(
+pub fn create_resolver(
     sess: Lrc<Session>,
-    lint_store: Lrc<LintStore>,
     metadata_loader: Box<MetadataLoaderDyn>,
-    krate: ast::Crate,
+    krate: &ast::Crate,
     crate_name: &str,
-) -> Result<(ast::Crate, BoxedResolver)> {
-    tracing::trace!("configure_and_expand");
-    // Currently, we ignore the name resolution data structures for the purposes of dependency
-    // tracking. Instead we will run name resolution and include its output in the hash of each
-    // item, much like we do for macro expansion. In other words, the hash reflects not just
-    // its contents but the results of name resolution on those contents. Hopefully we'll push
-    // this back at some point.
-    let crate_name = crate_name.to_string();
+) -> BoxedResolver {
+    tracing::trace!("create_resolver");
     BoxedResolver::new(sess, move |sess, resolver_arenas| {
-        configure_and_expand_inner(
-            sess,
-            &lint_store,
-            krate,
-            &crate_name,
-            &resolver_arenas,
-            metadata_loader,
-        )
+        Resolver::new(sess, &krate, &crate_name, metadata_loader, &resolver_arenas)
     })
 }
 
@@ -278,28 +254,24 @@ fn pre_expansion_lint(
     });
 }
 
-fn configure_and_expand_inner<'a>(
-    sess: &'a Session,
+/// Runs the "early phases" of the compiler: initial `cfg` processing, loading compiler plugins,
+/// syntax expansion, secondary `cfg` expansion, synthesis of a test
+/// harness if one is to be provided, injection of a dependency on the
+/// standard library and prelude, and name resolution.
+pub fn configure_and_expand(
+    sess: &Session,
     lint_store: &LintStore,
     mut krate: ast::Crate,
     crate_name: &str,
-    resolver_arenas: &'a ResolverArenas<'a>,
-    metadata_loader: Box<MetadataLoaderDyn>,
-) -> Result<(ast::Crate, Resolver<'a>)> {
-    tracing::trace!("configure_and_expand_inner");
+    resolver: &mut Resolver<'_>,
+) -> Result<ast::Crate> {
+    tracing::trace!("configure_and_expand");
     pre_expansion_lint(sess, lint_store, &krate, crate_name);
-
-    let mut resolver = Resolver::new(sess, &krate, crate_name, metadata_loader, &resolver_arenas);
-    rustc_builtin_macros::register_builtin_macros(&mut resolver);
+    rustc_builtin_macros::register_builtin_macros(resolver);
 
     krate = sess.time("crate_injection", || {
         let alt_std_name = sess.opts.alt_std_name.as_ref().map(|s| Symbol::intern(s));
-        rustc_builtin_macros::standard_library_imports::inject(
-            krate,
-            &mut resolver,
-            &sess,
-            alt_std_name,
-        )
+        rustc_builtin_macros::standard_library_imports::inject(krate, resolver, &sess, alt_std_name)
     });
 
     util::check_attr_crate_type(&sess, &krate.attrs, &mut resolver.lint_buffer());
@@ -354,7 +326,7 @@ fn configure_and_expand_inner<'a>(
             pre_expansion_lint(sess, lint_store, &krate, &ident.name.as_str());
             (krate.attrs, krate.items)
         };
-        let mut ecx = ExtCtxt::new(&sess, cfg, &mut resolver, Some(&extern_mod_loaded));
+        let mut ecx = ExtCtxt::new(&sess, cfg, resolver, Some(&extern_mod_loaded));
 
         // Expand macros now!
         let krate = sess.time("expand_crate", || ecx.monotonic_expander().expand_crate(krate));
@@ -396,16 +368,16 @@ fn configure_and_expand_inner<'a>(
     })?;
 
     sess.time("maybe_building_test_harness", || {
-        rustc_builtin_macros::test_harness::inject(&sess, &mut resolver, &mut krate)
+        rustc_builtin_macros::test_harness::inject(&sess, resolver, &mut krate)
     });
 
     if let Some(PpMode::Source(PpSourceMode::EveryBodyLoops)) = sess.opts.pretty {
         tracing::debug!("replacing bodies with loop {{}}");
-        util::ReplaceBodyWithLoop::new(&mut resolver).visit_crate(&mut krate);
+        util::ReplaceBodyWithLoop::new(resolver).visit_crate(&mut krate);
     }
 
     let has_proc_macro_decls = sess.time("AST_validation", || {
-        rustc_ast_passes::ast_validation::check_crate(sess, &krate, &mut resolver.lint_buffer())
+        rustc_ast_passes::ast_validation::check_crate(sess, &krate, resolver.lint_buffer())
     });
 
     let crate_types = sess.crate_types();
@@ -431,7 +403,7 @@ fn configure_and_expand_inner<'a>(
             let is_test_crate = sess.opts.test;
             rustc_builtin_macros::proc_macro_harness::inject(
                 &sess,
-                &mut resolver,
+                resolver,
                 krate,
                 is_proc_macro_crate,
                 has_proc_macro_decls,
@@ -471,26 +443,20 @@ fn configure_and_expand_inner<'a>(
         }
     });
 
-    Ok((krate, resolver))
+    Ok(krate)
 }
 
 pub fn lower_to_hir<'res, 'tcx>(
     sess: &'tcx Session,
     lint_store: &LintStore,
     resolver: &'res mut Resolver<'_>,
-    dep_graph: &'res DepGraph,
-    krate: &'res ast::Crate,
+    krate: Rc<ast::Crate>,
     arena: &'tcx rustc_ast_lowering::Arena<'tcx>,
-) -> Crate<'tcx> {
-    // We're constructing the HIR here; we don't care what we will
-    // read, since we haven't even constructed the *input* to
-    // incr. comp. yet.
-    dep_graph.assert_ignored();
-
+) -> &'tcx Crate<'tcx> {
     // Lower AST to HIR.
     let hir_crate = rustc_ast_lowering::lower_crate(
         sess,
-        &krate,
+        &*krate,
         resolver,
         rustc_parse::nt_to_tokenstream,
         arena,
@@ -511,6 +477,9 @@ pub fn lower_to_hir<'res, 'tcx>(
         )
     });
 
+    // Drop AST to free memory
+    sess.time("drop_ast", || std::mem::drop(krate));
+
     // Discard hygiene data, which isn't required after lowering to HIR.
     if !sess.opts.debugging_opts.keep_hygiene_data {
         rustc_span::hygiene::clear_syntax_context_map();
@@ -603,7 +572,7 @@ fn escape_dep_env(symbol: Symbol) -> String {
 
 fn write_out_deps(
     sess: &Session,
-    boxed_resolver: &Steal<Rc<RefCell<BoxedResolver>>>,
+    boxed_resolver: &RefCell<BoxedResolver>,
     outputs: &OutputFilenames,
     out_filenames: &[PathBuf],
 ) {
@@ -630,7 +599,7 @@ fn write_out_deps(
         }
 
         if sess.binary_dep_depinfo() {
-            boxed_resolver.borrow().borrow_mut().access(|resolver| {
+            boxed_resolver.borrow_mut().access(|resolver| {
                 for cnum in resolver.cstore().crates_untracked() {
                     let source = resolver.cstore().crate_source_untracked(cnum);
                     if let Some((path, _)) = source.dylib {
@@ -699,7 +668,7 @@ pub fn prepare_outputs(
     sess: &Session,
     compiler: &Compiler,
     krate: &ast::Crate,
-    boxed_resolver: &Steal<Rc<RefCell<BoxedResolver>>>,
+    boxed_resolver: &RefCell<BoxedResolver>,
     crate_name: &str,
 ) -> Result<OutputFilenames> {
     let _timer = sess.timer("prepare_outputs");
@@ -803,16 +772,26 @@ impl<'tcx> QueryContext<'tcx> {
 pub fn create_global_ctxt<'tcx>(
     compiler: &'tcx Compiler,
     lint_store: Lrc<LintStore>,
-    krate: &'tcx Crate<'tcx>,
+    krate: Rc<ast::Crate>,
     dep_graph: DepGraph,
-    resolver_outputs: ResolverOutputs,
+    resolver: Rc<RefCell<BoxedResolver>>,
     outputs: OutputFilenames,
     crate_name: &str,
     queries: &'tcx OnceCell<TcxQueries<'tcx>>,
     global_ctxt: &'tcx OnceCell<GlobalCtxt<'tcx>>,
     arena: &'tcx WorkerLocal<Arena<'tcx>>,
+    hir_arena: &'tcx WorkerLocal<rustc_ast_lowering::Arena<'tcx>>,
 ) -> QueryContext<'tcx> {
+    // We're constructing the HIR here; we don't care what we will
+    // read, since we haven't even constructed the *input* to
+    // incr. comp. yet.
+    dep_graph.assert_ignored();
+
     let sess = &compiler.session();
+    let krate = resolver
+        .borrow_mut()
+        .access(|resolver| lower_to_hir(sess, &lint_store, resolver, krate, hir_arena));
+    let resolver_outputs = BoxedResolver::to_resolver_outputs(resolver);
 
     let query_result_on_disk_cache = rustc_incremental::load_query_result_cache(sess);
 
@@ -831,7 +810,7 @@ pub fn create_global_ctxt<'tcx>(
     let queries = queries.get_or_init(|| TcxQueries::new(local_providers, extern_providers));
 
     let gcx = sess.time("setup_global_ctxt", || {
-        global_ctxt.get_or_init(|| {
+        global_ctxt.get_or_init(move || {
             TyCtxt::create_global_ctxt(
                 sess,
                 lint_store,