Auto merge of #114669 - cjgillot:metadata-wp, r=<try>

Make metadata a workproduct and reuse it

This PR aims to skip the generation of metadata by reusing the infrastructure that already exists for compiled codegen-units, namely "workproducts".

This can yield substantial gains (~10%) when we can demonstrate that metadata does not change between an incremental session and the next. This is the case if the crate is unchanged, or if all the changes are in upstream crates and have no effect on it. This latter case is most interesting, as it arises regularly for users with several crates in their workspace.

TODO:
- [ ] Materialize the fact that metadata encoding relies on the relative order of definitions;
- [ ] Refactor the handling of doc links.

Author: bors

Cargo.lock

@@ -4067,6 +4067,7 @@ dependencies = [
  "rustc_fs_util",
  "rustc_hir",
  "rustc_hir_pretty",
+ "rustc_incremental",
  "rustc_index",
  "rustc_macros",
  "rustc_middle",

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -56,6 +56,21 @@ impl OngoingCodegen {
         backend_config: &BackendConfig,
     ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
         let mut work_products = FxIndexMap::default();
+
+        if !backend_config.disable_incr_cache {
+            if let Some(path) = self.metadata.path() {
+                if let Some((id, product)) =
+                    rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+                        sess,
+                        "metadata",
+                        &[("rmeta", path)],
+                    )
+                {
+                    work_products.insert(id, product);
+                }
+            }
+        }
+
         let mut modules = vec![];
 
         for module_codegen in self.modules {

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -507,6 +507,7 @@ pub fn start_async_codegen<B: ExtraBackendMethods>(
 
 fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
     sess: &Session,
+    metadata: &EncodedMetadata,
     compiled_modules: &CompiledModules,
 ) -> FxIndexMap<WorkProductId, WorkProduct> {
     let mut work_products = FxIndexMap::default();
@@ -517,6 +518,14 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
 
     let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
 
+    if let Some(path) = metadata.path() {
+        if let Some((id, product)) =
+            copy_cgu_workproduct_to_incr_comp_cache_dir(sess, "metadata", &[("rmeta", path)])
+        {
+            work_products.insert(id, product);
+        }
+    }
+
     for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
         let mut files = Vec::new();
         if let Some(object_file_path) = &module.object {
@@ -1973,8 +1982,11 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> {
 
         sess.abort_if_errors();
 
-        let work_products =
-            copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
+        let work_products = copy_all_cgu_workproducts_to_incr_comp_cache_dir(
+            sess,
+            &self.metadata,
+            &compiled_modules,
+        );
         produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
 
         // FIXME: time_llvm_passes support - does this use a global context or

compiler/rustc_metadata/Cargo.toml

@@ -21,6 +21,7 @@ rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
+rustc_incremental = { path = "../rustc_incremental" }
 rustc_target = { path = "../rustc_target" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1468,6 +1468,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 let table = tcx.associated_types_for_impl_traits_in_associated_fn(def_id);
                 record_defaulted_array!(self.tables.associated_types_for_impl_traits_in_associated_fn[def_id] <- table);
             }
+            if let DefKind::Mod = tcx.def_kind(def_id) {
+                record!(self.tables.doc_link_resolutions[def_id] <- tcx.doc_link_resolutions(def_id));
+                record_array!(self.tables.doc_link_traits_in_scope[def_id] <- tcx.doc_link_traits_in_scope(def_id));
+            }
         }
 
         let inherent_impls = tcx.with_stable_hashing_context(|hcx| {
@@ -1479,14 +1483,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 def_id.index
             }));
         }
-
-        for (def_id, res_map) in &tcx.resolutions(()).doc_link_resolutions {
-            record!(self.tables.doc_link_resolutions[def_id.to_def_id()] <- res_map);
-        }
-
-        for (def_id, traits) in &tcx.resolutions(()).doc_link_traits_in_scope {
-            record_array!(self.tables.doc_link_traits_in_scope[def_id.to_def_id()] <- traits);
-        }
     }
 
     #[instrument(level = "trace", skip(self))]
@@ -2126,6 +2122,8 @@ fn prefetch_mir(tcx: TyCtxt<'_>) {
 pub struct EncodedMetadata {
     // The declaration order matters because `mmap` should be dropped before `_temp_dir`.
     mmap: Option<Mmap>,
+    // The path containing the metadata, to record as work product.
+    path: Option<Box<Path>>,
     // We need to carry MaybeTempDir to avoid deleting the temporary
     // directory while accessing the Mmap.
     _temp_dir: Option<MaybeTempDir>,
@@ -2137,16 +2135,21 @@ impl EncodedMetadata {
         let file = std::fs::File::open(&path)?;
         let file_metadata = file.metadata()?;
         if file_metadata.len() == 0 {
-            return Ok(Self { mmap: None, _temp_dir: None });
+            return Ok(Self { mmap: None, path: None, _temp_dir: None });
         }
         let mmap = unsafe { Some(Mmap::map(file)?) };
-        Ok(Self { mmap, _temp_dir: temp_dir })
+        Ok(Self { mmap, path: Some(path.into()), _temp_dir: temp_dir })
     }
 
     #[inline]
     pub fn raw_data(&self) -> &[u8] {
         self.mmap.as_deref().unwrap_or_default()
     }
+
+    #[inline]
+    pub fn path(&self) -> Option<&Path> {
+        self.path.as_deref()
+    }
 }
 
 impl<S: Encoder> Encodable<S> for EncodedMetadata {
@@ -2170,19 +2173,40 @@ impl<D: Decoder> Decodable<D> for EncodedMetadata {
             None
         };
 
-        Self { mmap, _temp_dir: None }
+        Self { mmap, path: None, _temp_dir: None }
     }
 }
 
+#[instrument(level = "trace", skip(tcx))]
 pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path) {
+    let dep_node = tcx.metadata_dep_node();
+
+    if tcx.dep_graph.is_fully_enabled()
+        && let work_product_id = &rustc_middle::dep_graph::WorkProductId::from_cgu_name("metadata")
+        && let Some(work_product) = tcx.dep_graph.previous_work_product(work_product_id)
+        && tcx.try_mark_green(&dep_node)
+    {
+        let saved_path = &work_product.saved_files["rmeta"];
+        let incr_comp_session_dir = tcx.sess.incr_comp_session_dir_opt().unwrap();
+        let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, saved_path);
+        debug!("copying preexisting metadata from {source_file:?} to {path:?}");
+        match rustc_fs_util::link_or_copy(&source_file, path) {
+            Ok(_) => {}
+            Err(err) => {
+                tcx.sess.emit_fatal(FailCreateFileEncoder { err });
+            }
+        };
+        return;
+    };
+
     let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata");
 
     // Since encoding metadata is not in a query, and nothing is cached,
     // there's no need to do dep-graph tracking for any of it.
     tcx.dep_graph.assert_ignored();
 
     join(
-        || encode_metadata_impl(tcx, path),
+        || tcx.dep_graph.with_task(dep_node, tcx, path, encode_metadata_impl, None),
         || {
             if tcx.sess.threads() == 1 {
                 return;

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -140,6 +140,7 @@ rustc_query_append!(define_dep_nodes![
     [] fn TraitSelect() -> (),
     [] fn CompileCodegenUnit() -> (),
     [] fn CompileMonoItem() -> (),
+    [] fn Metadata() -> (),
 ]);
 
 // WARNING: `construct` is generic and does not know that `CompileCodegenUnit` takes `Symbol`s as keys.
@@ -157,6 +158,12 @@ pub(crate) fn make_compile_mono_item<'tcx>(
     DepNode::construct(tcx, dep_kinds::CompileMonoItem, mono_item)
 }
 
+// WARNING: `construct` is generic and does not know that `Metadata` takes `()`s as keys.
+// Be very careful changing this type signature!
+pub(crate) fn make_metadata(tcx: TyCtxt<'_>) -> DepNode {
+    DepNode::construct(tcx, dep_kinds::Metadata, &())
+}
+
 pub trait DepNodeExt: Sized {
     /// Extracts the DefId corresponding to this DepNode. This will work
     /// if two conditions are met:

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -14,7 +14,7 @@ pub use rustc_query_system::dep_graph::{
 };
 
 pub use dep_node::{dep_kinds, label_strs, DepKind, DepNode, DepNodeExt};
-pub(crate) use dep_node::{make_compile_codegen_unit, make_compile_mono_item};
+pub(crate) use dep_node::{make_compile_codegen_unit, make_compile_mono_item, make_metadata};
 
 pub type DepGraph = rustc_query_system::dep_graph::DepGraph<DepsType>;
 

compiler/rustc_middle/src/query/mod.rs

@@ -132,6 +132,16 @@ rustc_queries! {
         desc { "getting the resolver for lowering" }
     }
 
+    /// Named module children from all kinds of items, including imports.
+    /// In addition to regular items this list also includes struct and variant constructors, and
+    /// items inside `extern {}` blocks because all of them introduce names into parent module.
+    ///
+    /// Module here is understood in name resolution sense - it can be a `mod` item,
+    /// or a crate root, or an enum, or a trait.
+    query module_children_local(key: LocalDefId) -> &'tcx [ModChild] {
+        desc { |tcx| "module exports for `{}`", tcx.def_path_str(key) }
+    }
+
     /// Return the span for a definition.
     /// Contrary to `def_span` below, this query returns the full absolute span of the definition.
     /// This span is meant for dep-tracking rather than diagnostics. It should not be used outside

compiler/rustc_middle/src/ty/context.rs

@@ -8,7 +8,6 @@ use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
 use crate::infer::canonical::CanonicalVarInfo;
 use crate::lint::struct_lint_level;
-use crate::metadata::ModChild;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
 use crate::middle::resolve_bound_vars;
 use crate::middle::stability;
@@ -955,9 +954,8 @@ impl<'tcx> TyCtxtAt<'tcx> {
 
 impl<'tcx> TyCtxt<'tcx> {
     pub fn iter_local_def_id(self) -> impl Iterator<Item = LocalDefId> + 'tcx {
-        // Create a dependency to the red node to be sure we re-execute this when the amount of
-        // definitions change.
-        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+        // Depend on the `analysis` query to ensure compilation if finished.
+        self.ensure().analysis(());
 
         let definitions = &self.untracked.definitions;
         std::iter::from_generator(|| {
@@ -977,9 +975,8 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn def_path_table(self) -> &'tcx rustc_hir::definitions::DefPathTable {
-        // Create a dependency to the crate to be sure we re-execute this when the amount of
-        // definitions change.
-        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+        // Depend on the `analysis` query to ensure compilation if finished.
+        self.ensure().analysis(());
 
         // Freeze definitions once we start iterating on them, to prevent adding new ones
         // while iterating. If some query needs to add definitions, it should be `ensure`d above.
@@ -2137,17 +2134,8 @@ impl<'tcx> TyCtxt<'tcx> {
         self.opt_rpitit_info(def_id).is_some()
     }
 
-    /// Named module children from all kinds of items, including imports.
-    /// In addition to regular items this list also includes struct and variant constructors, and
-    /// items inside `extern {}` blocks because all of them introduce names into parent module.
-    ///
-    /// Module here is understood in name resolution sense - it can be a `mod` item,
-    /// or a crate root, or an enum, or a trait.
-    ///
-    /// This is not a query, making it a query causes perf regressions
-    /// (probably due to hashing spans in `ModChild`ren).
-    pub fn module_children_local(self, def_id: LocalDefId) -> &'tcx [ModChild] {
-        self.resolutions(()).module_children.get(&def_id).map_or(&[], |v| &v[..])
+    pub fn metadata_dep_node(self) -> crate::dep_graph::DepNode {
+        crate::dep_graph::make_metadata(self)
     }
 }
 
@@ -2167,6 +2155,8 @@ pub struct DeducedParamAttrs {
 }
 
 pub fn provide(providers: &mut Providers) {
+    providers.module_children_local =
+        |tcx, def_id| tcx.resolutions(()).module_children.get(&def_id).map_or(&[], |v| &v[..]);
     providers.maybe_unused_trait_imports =
         |tcx, ()| &tcx.resolutions(()).maybe_unused_trait_imports;
     providers.names_imported_by_glob_use = |tcx, id| {

compiler/rustc_query_impl/src/plumbing.rs

@@ -792,6 +792,17 @@ macro_rules! define_queries {
                 }
             }
 
+            pub fn Metadata<'tcx>() -> DepKindStruct<'tcx> {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                    name: &"Metadata",
+                }
+            }
+
             $(pub(crate) fn $name<'tcx>()-> DepKindStruct<'tcx> {
                 $crate::plumbing::query_callback::<query_impl::$name::QueryType<'tcx>>(
                     is_anon!([$($modifiers)*]),

compiler/rustc_resolve/src/late.rs

@@ -1360,13 +1360,16 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
     }
 
     fn with_scope<T>(&mut self, id: NodeId, f: impl FnOnce(&mut Self) -> T) -> T {
-        if let Some(module) = self.r.get_module(self.r.local_def_id(id).to_def_id()) {
+        let def_id = self.r.local_def_id(id);
+        if let Some(module) = self.r.get_module(def_id.to_def_id()) {
             // Move down in the graph.
             let orig_module = replace(&mut self.parent_scope.module, module);
             self.with_rib(ValueNS, RibKind::Module(module), |this| {
                 this.with_rib(TypeNS, RibKind::Module(module), |this| {
                     let ret = f(this);
                     this.parent_scope.module = orig_module;
+                    this.r.doc_link_resolutions.entry(def_id).or_default();
+                    this.r.doc_link_traits_in_scope.entry(def_id).or_default();
                     ret
                 })
             })
@@ -4561,5 +4564,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         for (id, span) in late_resolution_visitor.diagnostic_metadata.unused_labels.iter() {
             self.lint_buffer.buffer_lint(lint::builtin::UNUSED_LABELS, *id, *span, "unused label");
         }
+        self.doc_link_resolutions.entry(CRATE_DEF_ID).or_default();
+        self.doc_link_traits_in_scope.entry(CRATE_DEF_ID).or_default();
     }
 }