From dc86db2d39026370ba171a1c56308bb68eb43e73 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Fri, 13 Sep 2024 21:27:19 -0400 Subject: [PATCH 01/18] HIR-based version of API fingerprint --- compiler/rustc_ast_lowering/src/lib.rs | 31 ++++++++++++++++++- .../rustc_codegen_cranelift/src/driver/aot.rs | 4 +-- compiler/rustc_codegen_ssa/src/back/link.rs | 2 +- compiler/rustc_codegen_ssa/src/back/write.rs | 4 +-- compiler/rustc_errors/src/emitter.rs | 8 ++++- compiler/rustc_errors/src/json.rs | 10 ++++-- compiler/rustc_errors/src/lib.rs | 9 ++++-- compiler/rustc_hir/src/hir.rs | 1 + compiler/rustc_hir/src/stable_hash_impls.rs | 2 +- compiler/rustc_interface/src/passes.rs | 2 +- compiler/rustc_metadata/src/creader.rs | 2 +- compiler/rustc_metadata/src/fs.rs | 5 ++- compiler/rustc_metadata/src/locator.rs | 30 +++++++++--------- compiler/rustc_mir_transform/src/dump_mir.rs | 2 +- 14 files changed, 81 insertions(+), 31 deletions(-) diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 754fbae4d023f..3fdd97eb6bddf 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -419,6 +419,30 @@ fn index_crate<'a>( } } +/// Compute the hash for the HIR of the full crate. +/// This hash will then be part of the crate_hash which is stored in the metadata. +fn compute_public_hir_hash( + tcx: TyCtxt<'_>, + owners: &IndexSlice>, +) -> Fingerprint { + let mut hir_body_nodes: Vec<_> = owners + .iter_enumerated() + .filter_map(|(def_id, info)| { + let info = info.as_owner()?; + tcx.visibility(def_id).is_public().then_some(())?; + let def_path_hash = tcx.hir().def_path_hash(def_id); + Some((def_path_hash, info)) + }) + .collect(); + hir_body_nodes.sort_unstable_by_key(|bn| bn.0); + + tcx.with_stable_hashing_context(|mut hcx| { + let mut stable_hasher = StableHasher::new(); + hir_body_nodes.hash_stable(&mut hcx, &mut stable_hasher); + stable_hasher.finish() + }) +} + /// Compute the hash for the HIR of the full crate. /// This hash will then be part of the crate_hash which is stored in the metadata. fn compute_hir_hash( @@ -474,7 +498,12 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> { // Don't hash unless necessary, because it's expensive. let opt_hir_hash = if tcx.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None }; - hir::Crate { owners, opt_hir_hash } + let api_hash = if tcx.needs_crate_hash() { + compute_public_hir_hash(tcx, &owners) + } else { + Fingerprint::ZERO + }; + hir::Crate { owners, opt_hir_hash, api_hash } } #[derive(Copy, Clone, PartialEq, Debug)] diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index b6fee1bf24a78..24d24ee064377 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -299,7 +299,7 @@ fn produce_final_output_artifacts( // for single cgu file is renamed to drop cgu specific suffix // so we regenerate it the same way let path = crate_output.path(ty); - sess.dcx().emit_artifact_notification(path.as_path(), descr); + sess.dcx().emit_artifact_notification(path.as_path(), descr, None); } }); } else { @@ -307,7 +307,7 @@ fn produce_final_output_artifacts( module.for_each_output(|path, ty| { if sess.opts.output_types.contains_key(&ty) { let descr = ty.shorthand(); - sess.dcx().emit_artifact_notification(&path, descr); + sess.dcx().emit_artifact_notification(&path, descr, None); } }); } diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index e8143b9a5f38f..2f6e35ee6470e 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -146,7 +146,7 @@ pub fn link_binary( } } if sess.opts.json_artifact_notifications { - sess.dcx().emit_artifact_notification(&out_filename, "link"); + sess.dcx().emit_artifact_notification(&out_filename, "link", None); } if sess.prof.enabled() { diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index feb27c148a188..852ea6f12345d 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -723,7 +723,7 @@ fn produce_final_output_artifacts( // for single cgu file is renamed to drop cgu specific suffix // so we regenerate it the same way let path = crate_output.path(ty); - sess.dcx().emit_artifact_notification(path.as_path(), descr); + sess.dcx().emit_artifact_notification(path.as_path(), descr, None); } }); } else { @@ -731,7 +731,7 @@ fn produce_final_output_artifacts( module.for_each_output(|path, ty| { if sess.opts.output_types.contains_key(&ty) { let descr = ty.shorthand(); - sess.dcx().emit_artifact_notification(&path, descr); + sess.dcx().emit_artifact_notification(&path, descr, None); } }); } diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs index 2b135df91a4bb..b120f1713e761 100644 --- a/compiler/rustc_errors/src/emitter.rs +++ b/compiler/rustc_errors/src/emitter.rs @@ -180,7 +180,13 @@ pub trait Emitter: Translate { /// Emit a notification that an artifact has been output. /// Currently only supported for the JSON format. - fn emit_artifact_notification(&mut self, _path: &Path, _artifact_type: &str) {} + fn emit_artifact_notification( + &mut self, + _path: &Path, + _artifact_type: &str, + _api_hash: Option<&str>, + ) { + } /// Emit a report about future breakage. /// Currently only supported for the JSON format. diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs index 32e59f9ab036a..d0f6c34acc094 100644 --- a/compiler/rustc_errors/src/json.rs +++ b/compiler/rustc_errors/src/json.rs @@ -128,8 +128,13 @@ impl Emitter for JsonEmitter { } } - fn emit_artifact_notification(&mut self, path: &Path, artifact_type: &str) { - let data = ArtifactNotification { artifact: path, emit: artifact_type }; + fn emit_artifact_notification( + &mut self, + path: &Path, + artifact_type: &str, + api_hash: Option<&str>, + ) { + let data = ArtifactNotification { artifact: path, emit: artifact_type, api_hash }; let result = self.emit(EmitTyped::Artifact(data)); if let Err(e) = result { panic!("failed to print notification: {e:?}"); @@ -262,6 +267,7 @@ struct ArtifactNotification<'a> { artifact: &'a Path, /// What kind of artifact we're emitting. emit: &'a str, + api_hash: Option<&'a str>, } #[derive(Serialize)] diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index 13da1721a4a32..cb430f712ae09 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -1032,8 +1032,13 @@ impl<'a> DiagCtxtHandle<'a> { self.inner.borrow_mut().emit_diagnostic(diagnostic, self.tainted_with_errors) } - pub fn emit_artifact_notification(&self, path: &Path, artifact_type: &str) { - self.inner.borrow_mut().emitter.emit_artifact_notification(path, artifact_type); + pub fn emit_artifact_notification( + &self, + path: &Path, + artifact_type: &str, + api_hash: Option<&str>, + ) { + self.inner.borrow_mut().emitter.emit_artifact_notification(path, artifact_type, api_hash); } pub fn emit_future_breakage_report(&self) { diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index ee03f780e16f5..cda207c457f0c 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -1058,6 +1058,7 @@ pub struct Crate<'hir> { pub owners: IndexVec>, // Only present when incr. comp. is enabled. pub opt_hir_hash: Option, + pub api_hash: Fingerprint, } #[derive(Debug, Clone, Copy, HashStable_Generic)] diff --git a/compiler/rustc_hir/src/stable_hash_impls.rs b/compiler/rustc_hir/src/stable_hash_impls.rs index fe169e989ec9f..07600820d546e 100644 --- a/compiler/rustc_hir/src/stable_hash_impls.rs +++ b/compiler/rustc_hir/src/stable_hash_impls.rs @@ -109,7 +109,7 @@ impl<'tcx, HirCtx: crate::HashStableContext> HashStable for AttributeMap impl HashStable for Crate<'_> { fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) { - let Crate { owners: _, opt_hir_hash } = self; + let Crate { owners: _, opt_hir_hash, .. } = self; opt_hir_hash.unwrap().hash_stable(hcx, hasher) } } diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index c4a38047b5e3b..ec0ec33bf6280 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -528,7 +528,7 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P match result { Ok(_) => { if sess.opts.json_artifact_notifications { - sess.dcx().emit_artifact_notification(deps_filename, "dep-info"); + sess.dcx().emit_artifact_notification(deps_filename, "dep-info", None); } } Err(error) => { diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs index 14a1a7f67e56e..c7db9eb85c6b6 100644 --- a/compiler/rustc_metadata/src/creader.rs +++ b/compiler/rustc_metadata/src/creader.rs @@ -640,7 +640,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { let mut result = LoadResult::Loaded(library); for (cnum, data) in self.cstore.iter_crate_data() { if data.name() == root.name() && root.hash() == data.hash() { - assert!(locator.hash.is_none()); + //assert!(locator.hash.is_none()); info!("load success, going to previous cnum: {}", cnum); result = LoadResult::Previous(cnum); break; diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index b2a979ed6d832..dbe34fef85b49 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -40,6 +40,7 @@ pub fn emit_wrapper_file( pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { let out_filename = filename_for_metadata(tcx.sess, tcx.output_filenames(())); + //let hash = tcx.crate_hash() // To avoid races with another rustc process scanning the output directory, // we need to write the file somewhere else and atomically move it to its // final destination, with an `fs::rename` call. In order for the rename to @@ -91,7 +92,9 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { } }; if tcx.sess.opts.json_artifact_notifications { - tcx.dcx().emit_artifact_notification(out_filename.as_path(), "metadata"); + let krate = tcx.hir_crate(()); + let hash = krate.api_hash.to_string(); + tcx.dcx().emit_artifact_notification(out_filename.as_path(), "metadata", Some(&hash)); } (filename, None) } else { diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs index 90228db378a95..65ea96dfea242 100644 --- a/compiler/rustc_metadata/src/locator.rs +++ b/compiler/rustc_metadata/src/locator.rs @@ -685,15 +685,15 @@ impl<'a> CrateLocator<'a> { } let hash = header.hash; - if let Some(expected_hash) = self.hash { - if hash != expected_hash { - info!("Rejecting via hash: expected {} got {}", expected_hash, hash); - self.crate_rejections - .via_hash - .push(CrateMismatch { path: libpath.to_path_buf(), got: hash.to_string() }); - return None; - } - } + // if let Some(expected_hash) = self.hash { + // if hash != expected_hash { + // info!("Rejecting via hash: expected {} got {}", expected_hash, hash); + // self.crate_rejections + // .via_hash + // .push(CrateMismatch { path: libpath.to_path_buf(), got: hash.to_string() }); + // return None; + // } + // } Some(hash) } @@ -1029,12 +1029,12 @@ impl CrateError { )); } } - dcx.emit_err(errors::NewerCrateVersion { - span, - crate_name, - add_info, - found_crates, - }); + // dcx.emit_err(errors::NewerCrateVersion { + // span, + // crate_name, + // add_info, + // found_crates, + // }); } else if !locator.crate_rejections.via_triple.is_empty() { let mismatches = locator.crate_rejections.via_triple.iter(); for CrateMismatch { path, got } in mismatches { diff --git a/compiler/rustc_mir_transform/src/dump_mir.rs b/compiler/rustc_mir_transform/src/dump_mir.rs index 1640d34d6c8ee..3464bf92c17c9 100644 --- a/compiler/rustc_mir_transform/src/dump_mir.rs +++ b/compiler/rustc_mir_transform/src/dump_mir.rs @@ -27,7 +27,7 @@ pub fn emit_mir(tcx: TyCtxt<'_>) -> io::Result<()> { let mut f = io::BufWriter::new(File::create(&path)?); write_mir_pretty(tcx, None, &mut f)?; if tcx.sess.opts.json_artifact_notifications { - tcx.dcx().emit_artifact_notification(&path, "mir"); + tcx.dcx().emit_artifact_notification(&path, "mir", None); } } } From ff654f227ebe315a0490d43a602f932fff8a0b53 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:19:33 +0200 Subject: [PATCH 02/18] Get going on MIR-based hash (doesn't work yet) --- compiler/rustc_ast_lowering/src/lib.rs | 31 +--------------- compiler/rustc_hir/src/hir.rs | 1 - compiler/rustc_metadata/src/fs.rs | 36 +++++++++++++++++-- .../src/middle/exported_symbols.rs | 21 +++++++++++ 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 3fdd97eb6bddf..754fbae4d023f 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -419,30 +419,6 @@ fn index_crate<'a>( } } -/// Compute the hash for the HIR of the full crate. -/// This hash will then be part of the crate_hash which is stored in the metadata. -fn compute_public_hir_hash( - tcx: TyCtxt<'_>, - owners: &IndexSlice>, -) -> Fingerprint { - let mut hir_body_nodes: Vec<_> = owners - .iter_enumerated() - .filter_map(|(def_id, info)| { - let info = info.as_owner()?; - tcx.visibility(def_id).is_public().then_some(())?; - let def_path_hash = tcx.hir().def_path_hash(def_id); - Some((def_path_hash, info)) - }) - .collect(); - hir_body_nodes.sort_unstable_by_key(|bn| bn.0); - - tcx.with_stable_hashing_context(|mut hcx| { - let mut stable_hasher = StableHasher::new(); - hir_body_nodes.hash_stable(&mut hcx, &mut stable_hasher); - stable_hasher.finish() - }) -} - /// Compute the hash for the HIR of the full crate. /// This hash will then be part of the crate_hash which is stored in the metadata. fn compute_hir_hash( @@ -498,12 +474,7 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> { // Don't hash unless necessary, because it's expensive. let opt_hir_hash = if tcx.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None }; - let api_hash = if tcx.needs_crate_hash() { - compute_public_hir_hash(tcx, &owners) - } else { - Fingerprint::ZERO - }; - hir::Crate { owners, opt_hir_hash, api_hash } + hir::Crate { owners, opt_hir_hash } } #[derive(Copy, Clone, PartialEq, Debug)] diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index cda207c457f0c..ee03f780e16f5 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -1058,7 +1058,6 @@ pub struct Crate<'hir> { pub owners: IndexVec>, // Only present when incr. comp. is enabled. pub opt_hir_hash: Option, - pub api_hash: Fingerprint, } #[derive(Debug, Clone, Copy, HashStable_Generic)] diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index dbe34fef85b49..8a68e1fdc7894 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -1,7 +1,11 @@ use std::path::{Path, PathBuf}; use std::{fs, io}; +use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::stable_hasher::StableHasher; use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::middle::exported_symbols::ExportedSymbol; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OutFileName, OutputType}; use rustc_session::output::filename_for_metadata; @@ -92,9 +96,35 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { } }; if tcx.sess.opts.json_artifact_notifications { - let krate = tcx.hir_crate(()); - let hash = krate.api_hash.to_string(); - tcx.dcx().emit_artifact_notification(out_filename.as_path(), "metadata", Some(&hash)); + let hash: Fingerprint = { + let symbols = tcx + .exported_symbols(LOCAL_CRATE) + .iter() + .filter_map(|&(exported_symbol, k)| { + if !matches!(exported_symbol, ExportedSymbol::NoDefId(_)) { + Some(( + exported_symbol, + k, + &exported_symbol.mir_body_for_local_instance(tcx).basic_blocks, + )) + } else { + None + } + }) + .collect::>(); + + tcx.with_stable_hashing_context(|mut hcx| { + use rustc_data_structures::stable_hasher::HashStable; + let mut stable_hasher = StableHasher::new(); + symbols.hash_stable(&mut hcx, &mut stable_hasher); + stable_hasher.finish() + }) + }; + tcx.dcx().emit_artifact_notification( + out_filename.as_path(), + "metadata", + Some(&hash.to_string()), + ); } (filename, None) } else { diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 0bfbd39879747..ed1f3804bf6fa 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -1,6 +1,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_macros::{Decodable, Encodable, HashStable, TyDecodable, TyEncodable}; +use crate::mir::Body; use crate::ty::{self, GenericArgsRef, Ty, TyCtxt}; /// The SymbolExportLevel of a symbols specifies from which kinds of crates @@ -70,6 +71,26 @@ impl<'tcx> ExportedSymbol<'tcx> { ExportedSymbol::NoDefId(symbol_name) => symbol_name, } } + pub fn mir_body_for_local_instance(&self, tcx: TyCtxt<'tcx>) -> &'tcx Body<'tcx> { + match *self { + ExportedSymbol::NonGeneric(def_id) => { + tcx.instance_mir(ty::Instance::mono(tcx, def_id).def) + } + ExportedSymbol::Generic(def_id, args) => { + tcx.instance_mir(ty::Instance::new(def_id, args).def) + } + ExportedSymbol::DropGlue(ty) => { + tcx.instance_mir(ty::Instance::resolve_drop_in_place(tcx, ty).def) + } + ExportedSymbol::AsyncDropGlueCtorShim(ty) => { + tcx.instance_mir(ty::Instance::resolve_async_drop_in_place(tcx, ty).def) + } + ExportedSymbol::ThreadLocalShim(def_id) => { + tcx.instance_mir(ty::InstanceKind::ThreadLocalShim(def_id)) + } + ExportedSymbol::NoDefId(_) => panic!("Cannot find "), + } + } } pub fn metadata_symbol_name(tcx: TyCtxt<'_>) -> String { From 199cdb4ad56fccde1d2e018b9c0b2d3667871f24 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:19:21 +0200 Subject: [PATCH 03/18] Migrate to using a mixture of MIR/type hashing --- compiler/rustc_metadata/src/fs.rs | 27 ++++++++++++------- .../src/middle/exported_symbols.rs | 8 ++++++ .../rustc_query_system/src/query/plumbing.rs | 4 ++- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 8a68e1fdc7894..e43d70139faa5 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -1,3 +1,5 @@ +use std::ops::Deref; +//use std::ops::Deref; use std::path::{Path, PathBuf}; use std::{fs, io}; @@ -96,23 +98,29 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { } }; if tcx.sess.opts.json_artifact_notifications { + let start = std::time::Instant::now(); let hash: Fingerprint = { let symbols = tcx .exported_symbols(LOCAL_CRATE) .iter() .filter_map(|&(exported_symbol, k)| { - if !matches!(exported_symbol, ExportedSymbol::NoDefId(_)) { - Some(( - exported_symbol, - k, - &exported_symbol.mir_body_for_local_instance(tcx).basic_blocks, - )) - } else { - None + if matches!(exported_symbol, ExportedSymbol::NoDefId(_)) { + return None; } + let def_id = exported_symbol.def_id(); + let is_cross_crate_inlineable = + def_id.map_or(false, |id| tcx.cross_crate_inlinable(id)); + let ty = def_id.map(|id| tcx.type_of(id)); + let body = is_cross_crate_inlineable.then(|| { + // Deref to avoid hashing cache of mir body. + exported_symbol.mir_body_for_local_instance(tcx).basic_blocks.deref() + }); + + Some((exported_symbol, k, ty, body)) }) .collect::>(); - + let first = start.elapsed(); + dbg!(first); tcx.with_stable_hashing_context(|mut hcx| { use rustc_data_structures::stable_hasher::HashStable; let mut stable_hasher = StableHasher::new(); @@ -120,6 +128,7 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { stable_hasher.finish() }) }; + dbg!(start.elapsed()); tcx.dcx().emit_artifact_notification( out_filename.as_path(), "metadata", diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index ed1f3804bf6fa..7589358e495f1 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -91,6 +91,14 @@ impl<'tcx> ExportedSymbol<'tcx> { ExportedSymbol::NoDefId(_) => panic!("Cannot find "), } } + pub fn def_id(&self) -> Option { + match self { + ExportedSymbol::NonGeneric(def_id) + | ExportedSymbol::Generic(def_id, _) + | ExportedSymbol::ThreadLocalShim(def_id) => Some(*def_id), + _ => None, + } + } } pub fn metadata_symbol_name(tcx: TyCtxt<'_>) -> String { diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 6dbd6e89fe98a..30ff0aaefac6c 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -677,7 +677,9 @@ pub(crate) fn incremental_verify_ich( let old_hash = dep_graph_data.prev_fingerprint_of(prev_index); if new_hash != old_hash { - incremental_verify_ich_failed(tcx, prev_index, &|| format_value(result)); + if false { + incremental_verify_ich_failed(tcx, prev_index, &|| format_value(result)); + } } } From 795af8b3c7e4f7b42331c8b1315fd9027fa0030b Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:25:56 +0200 Subject: [PATCH 04/18] Remove timings --- compiler/rustc_metadata/src/fs.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index e43d70139faa5..e8e76946f782d 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -98,7 +98,6 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { } }; if tcx.sess.opts.json_artifact_notifications { - let start = std::time::Instant::now(); let hash: Fingerprint = { let symbols = tcx .exported_symbols(LOCAL_CRATE) @@ -119,8 +118,6 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { Some((exported_symbol, k, ty, body)) }) .collect::>(); - let first = start.elapsed(); - dbg!(first); tcx.with_stable_hashing_context(|mut hcx| { use rustc_data_structures::stable_hasher::HashStable; let mut stable_hasher = StableHasher::new(); @@ -128,7 +125,6 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { stable_hasher.finish() }) }; - dbg!(start.elapsed()); tcx.dcx().emit_artifact_notification( out_filename.as_path(), "metadata", From 12094a3599dc4e66b1643d10d5e38f08c518cbc8 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:12:46 +0200 Subject: [PATCH 05/18] Do not hash spans --- compiler/rustc_metadata/src/fs.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index e8e76946f782d..9b4955e6a4e26 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -98,6 +98,7 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { } }; if tcx.sess.opts.json_artifact_notifications { + dbg!("Calculating hash"); let hash: Fingerprint = { let symbols = tcx .exported_symbols(LOCAL_CRATE) @@ -110,12 +111,28 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { let is_cross_crate_inlineable = def_id.map_or(false, |id| tcx.cross_crate_inlinable(id)); let ty = def_id.map(|id| tcx.type_of(id)); - let body = is_cross_crate_inlineable.then(|| { + let _body = is_cross_crate_inlineable.then(|| { // Deref to avoid hashing cache of mir body. - exported_symbol.mir_body_for_local_instance(tcx).basic_blocks.deref() + exported_symbol + .mir_body_for_local_instance(tcx) + .basic_blocks + .deref() + .iter() + .map(|bb| { + let kind = bb + .terminator + .as_ref() + .map(|terminator| terminator.kind.clone()); + let statements = bb + .statements + .iter() + .map(|statement| statement.kind.clone()) + .collect::>(); + (bb.is_cleanup, kind, statements) + }) }); - Some((exported_symbol, k, ty, body)) + Some((exported_symbol, k, ty)) // body)) }) .collect::>(); tcx.with_stable_hashing_context(|mut hcx| { From 005509a5f58412e6ec5ec5eff312a2a06b2bc3a2 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Fri, 20 Sep 2024 19:10:56 +0200 Subject: [PATCH 06/18] Reenable body hashing --- compiler/rustc_metadata/src/fs.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 9b4955e6a4e26..7714c07b0f033 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -111,7 +111,7 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { let is_cross_crate_inlineable = def_id.map_or(false, |id| tcx.cross_crate_inlinable(id)); let ty = def_id.map(|id| tcx.type_of(id)); - let _body = is_cross_crate_inlineable.then(|| { + let body = is_cross_crate_inlineable.then(|| { // Deref to avoid hashing cache of mir body. exported_symbol .mir_body_for_local_instance(tcx) @@ -130,9 +130,10 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { .collect::>(); (bb.is_cleanup, kind, statements) }) + .collect::>() }); - Some((exported_symbol, k, ty)) // body)) + Some((exported_symbol, k, ty, body)) }) .collect::>(); tcx.with_stable_hashing_context(|mut hcx| { From 1e48aeb5f9c435ac8b32dce7ba4cb7cea058102e Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:37:19 +0200 Subject: [PATCH 07/18] Keep going --- compiler/rustc_errors/src/lib.rs | 12 ++-- compiler/rustc_metadata/src/fs.rs | 94 +++++++++++++++++-------------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index c8b855894b251..f18e955a6c8a0 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -624,12 +624,12 @@ impl Drop for DiagCtxtInner { } if !self.has_printed && !self.suppressed_expected_diag && !std::thread::panicking() { - if let Some(backtrace) = &self.must_produce_diag { - panic!( - "must_produce_diag: `trimmed_def_paths` called but no diagnostics emitted; \ - `with_no_trimmed_paths` for debugging. \ - called at: {backtrace}" - ); + if let Some(_) = &self.must_produce_diag { + // panic!( + // "must_produce_diag: `trimmed_def_paths` called but no diagnostics emitted; \ + // `with_no_trimmed_paths` for debugging. \ + // called at: {backtrace}" + // ); } } } diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 7714c07b0f033..ed44c2281b877 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -1,13 +1,10 @@ -use std::ops::Deref; -//use std::ops::Deref; use std::path::{Path, PathBuf}; use std::{fs, io}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::stable_hasher::StableHasher; use rustc_data_structures::temp_dir::MaybeTempDir; -use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::middle::exported_symbols::ExportedSymbol; +use rustc_hir::def_id::LocalDefId; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OutFileName, OutputType}; use rustc_session::output::filename_for_metadata; @@ -100,46 +97,59 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { if tcx.sess.opts.json_artifact_notifications { dbg!("Calculating hash"); let hash: Fingerprint = { - let symbols = tcx - .exported_symbols(LOCAL_CRATE) - .iter() - .filter_map(|&(exported_symbol, k)| { - if matches!(exported_symbol, ExportedSymbol::NoDefId(_)) { - return None; - } - let def_id = exported_symbol.def_id(); - let is_cross_crate_inlineable = - def_id.map_or(false, |id| tcx.cross_crate_inlinable(id)); - let ty = def_id.map(|id| tcx.type_of(id)); - let body = is_cross_crate_inlineable.then(|| { - // Deref to avoid hashing cache of mir body. - exported_symbol - .mir_body_for_local_instance(tcx) - .basic_blocks - .deref() - .iter() - .map(|bb| { - let kind = bb - .terminator - .as_ref() - .map(|terminator| terminator.kind.clone()); - let statements = bb - .statements - .iter() - .map(|statement| statement.kind.clone()) - .collect::>(); - (bb.is_cleanup, kind, statements) - }) - .collect::>() - }); - - Some((exported_symbol, k, ty, body)) - }) - .collect::>(); tcx.with_stable_hashing_context(|mut hcx| { - use rustc_data_structures::stable_hasher::HashStable; + let symbols = tcx + .reachable_set(()) + .to_sorted(&hcx, true) + .into_iter() + .filter_map(|local_def_id: &LocalDefId| { + dbg!(&local_def_id); + let def_id = local_def_id.to_def_id(); + let is_cross_crate_inlineable = tcx.is_mir_available(def_id); + let path = tcx.def_path(def_id).to_string_no_crate_verbose(); + dbg!(&path); + //let ty = tcx.type_of(def_id); + let _body = is_cross_crate_inlineable.then(|| { + // fn mir_body_for_local_instance( + // tcx: &TyCtx<'_>, + // id: LocalDefId, + // ) -> &'_ Body<'_> { + // tcx.instance_mir(ty::Instance::mono()) + // } + // let blocks = exported_symbol + // .mir_body_for_local_instance(tcx) + // .basic_blocks + // .deref(); + // dbg!(&blocks); + // // Deref to avoid hashing cache of mir body. + // let symbols = blocks + // .iter() + // .map(|bb| { + // let kind = bb + // .terminator + // .as_ref() + // .map(|terminator| terminator.kind.clone()); + // let statements = bb + // .statements + // .iter() + // .map(|statement| statement.kind.clone()) + // .collect::>(); + // (bb.is_cleanup, kind, statements) + // }) + // .collect::>(); + // dbg!(symbols.len()); + // symbols + Vec::<()>::new() + }); + + Some(path) + }) + .collect::>(); let mut stable_hasher = StableHasher::new(); - symbols.hash_stable(&mut hcx, &mut stable_hasher); + hcx.while_hashing_spans(false, |mut hcx| { + use rustc_data_structures::stable_hasher::HashStable; + symbols.hash_stable(&mut hcx, &mut stable_hasher); + }); stable_hasher.finish() }) }; From a4b19d7a27b3b330de8f463eec7e311742676dbb Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:40:18 +0200 Subject: [PATCH 08/18] Hash typs explicitly --- Cargo.lock | 1 + compiler/rustc_metadata/Cargo.toml | 1 + compiler/rustc_metadata/src/fs.rs | 106 ++++++++++++++--------------- 3 files changed, 55 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e973d7f9a2ae0..145fd41728c59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3980,6 +3980,7 @@ dependencies = [ "rustc_index", "rustc_macros", "rustc_middle", + "rustc_query_system", "rustc_serialize", "rustc_session", "rustc_span", diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml index 79d3482472a81..dca94e94f0308 100644 --- a/compiler/rustc_metadata/Cargo.toml +++ b/compiler/rustc_metadata/Cargo.toml @@ -21,6 +21,7 @@ rustc_hir_pretty = { path = "../rustc_hir_pretty" } rustc_index = { path = "../rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_middle = { path = "../rustc_middle" } +rustc_query_system = { path = "../rustc_query_system" } rustc_serialize = { path = "../rustc_serialize" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index ed44c2281b877..54a9e09a74b8b 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -1,11 +1,12 @@ +use std::ops::Deref as _; use std::path::{Path, PathBuf}; use std::{fs, io}; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::stable_hasher::StableHasher; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_hir::def_id::LocalDefId; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{InstanceKind, TyCtxt}; use rustc_session::config::{OutFileName, OutputType}; use rustc_session::output::filename_for_metadata; use rustc_session::{MetadataKind, Session}; @@ -95,60 +96,59 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { } }; if tcx.sess.opts.json_artifact_notifications { - dbg!("Calculating hash"); let hash: Fingerprint = { + let mut stable_hasher = StableHasher::new(); tcx.with_stable_hashing_context(|mut hcx| { - let symbols = tcx - .reachable_set(()) - .to_sorted(&hcx, true) - .into_iter() - .filter_map(|local_def_id: &LocalDefId| { - dbg!(&local_def_id); - let def_id = local_def_id.to_def_id(); - let is_cross_crate_inlineable = tcx.is_mir_available(def_id); - let path = tcx.def_path(def_id).to_string_no_crate_verbose(); - dbg!(&path); - //let ty = tcx.type_of(def_id); - let _body = is_cross_crate_inlineable.then(|| { - // fn mir_body_for_local_instance( - // tcx: &TyCtx<'_>, - // id: LocalDefId, - // ) -> &'_ Body<'_> { - // tcx.instance_mir(ty::Instance::mono()) - // } - // let blocks = exported_symbol - // .mir_body_for_local_instance(tcx) - // .basic_blocks - // .deref(); - // dbg!(&blocks); - // // Deref to avoid hashing cache of mir body. - // let symbols = blocks - // .iter() - // .map(|bb| { - // let kind = bb - // .terminator - // .as_ref() - // .map(|terminator| terminator.kind.clone()); - // let statements = bb - // .statements - // .iter() - // .map(|statement| statement.kind.clone()) - // .collect::>(); - // (bb.is_cleanup, kind, statements) - // }) - // .collect::>(); - // dbg!(symbols.len()); - // symbols - Vec::<()>::new() - }); - - Some(path) - }) - .collect::>(); - let mut stable_hasher = StableHasher::new(); hcx.while_hashing_spans(false, |mut hcx| { - use rustc_data_structures::stable_hasher::HashStable; - symbols.hash_stable(&mut hcx, &mut stable_hasher); + let _ = tcx + .reachable_set(()) + .to_sorted(hcx, true) + .into_iter() + .filter_map(|local_def_id: &LocalDefId| { + let def_id = local_def_id.to_def_id(); + + let item = tcx.hir_node_by_def_id(*local_def_id); + if item.body_id().is_none() { + item.hash_stable(hcx, &mut stable_hasher); + return Some(()); + } + + let generics = tcx.generics_of(def_id); + let is_cross_crate_inlineable = generics + .requires_monomorphization(tcx) + || tcx.cross_crate_inlinable(def_id); + let ty = tcx.type_of(def_id); + is_cross_crate_inlineable.then(|| { + let body = tcx.instance_mir(InstanceKind::Item(def_id)); + let blocks = body.basic_blocks.deref(); + + // Deref to avoid hashing cache of mir body. + let symbols = blocks + .iter() + .map(|bb| { + let kind = bb + .terminator + .as_ref() + .map(|terminator| terminator.kind.clone()); + let statements = bb + .statements + .iter() + .map(|statement| statement.kind.clone()) + .collect::>(); + + (bb.is_cleanup, kind, statements) + .hash_stable(&mut hcx, &mut stable_hasher); + () + }) + .collect::>(); + + symbols + }); + ty.skip_binder().kind().hash_stable(&mut hcx, &mut stable_hasher); + + Some(()) + }) + .collect::>(); }); stable_hasher.finish() }) From 00762561d48b1063ac9d69a490d3d55f82c1a7c1 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:41:31 +0200 Subject: [PATCH 09/18] :lipstick: --- compiler/rustc_metadata/src/fs.rs | 88 ++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 31 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 54a9e09a74b8b..666b0d84a2b4c 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -5,6 +5,7 @@ use std::{fs, io}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_hir::def::DefKind; use rustc_hir::def_id::LocalDefId; use rustc_middle::ty::{InstanceKind, TyCtxt}; use rustc_session::config::{OutFileName, OutputType}; @@ -108,42 +109,67 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { let def_id = local_def_id.to_def_id(); let item = tcx.hir_node_by_def_id(*local_def_id); - if item.body_id().is_none() { + let has_hir_body = item.body_id().is_some(); + let ident = item.ident(); + let def_kind = tcx.def_kind(def_id); + let has_mir = match def_kind { + DefKind::Ctor(_, _) + | DefKind::AnonConst + | DefKind::InlineConst + | DefKind::AssocConst + | DefKind::Const + | DefKind::SyntheticCoroutineBody => has_hir_body, + DefKind::AssocFn | DefKind::Fn | DefKind::Closure => { + if def_kind == DefKind::Closure && tcx.is_coroutine(def_id) + { + has_hir_body + } else { + let generics = tcx.generics_of(def_id); + has_hir_body + && (tcx.sess.opts.unstable_opts.always_encode_mir + || (tcx + .sess + .opts + .output_types + .should_codegen() + && (generics + .requires_monomorphization(tcx) + || tcx.cross_crate_inlinable(def_id)))) + } + } + _ => false, + }; + ident.hash_stable(hcx, &mut stable_hasher); + if !has_mir { item.hash_stable(hcx, &mut stable_hasher); return Some(()); } - let generics = tcx.generics_of(def_id); - let is_cross_crate_inlineable = generics - .requires_monomorphization(tcx) - || tcx.cross_crate_inlinable(def_id); let ty = tcx.type_of(def_id); - is_cross_crate_inlineable.then(|| { - let body = tcx.instance_mir(InstanceKind::Item(def_id)); - let blocks = body.basic_blocks.deref(); - - // Deref to avoid hashing cache of mir body. - let symbols = blocks - .iter() - .map(|bb| { - let kind = bb - .terminator - .as_ref() - .map(|terminator| terminator.kind.clone()); - let statements = bb - .statements - .iter() - .map(|statement| statement.kind.clone()) - .collect::>(); - - (bb.is_cleanup, kind, statements) - .hash_stable(&mut hcx, &mut stable_hasher); - () - }) - .collect::>(); - - symbols - }); + + let body = tcx.instance_mir(InstanceKind::Item(def_id)); + let blocks = body.basic_blocks.deref(); + + // Deref to avoid hashing cache of mir body. + let _ = blocks + .iter() + .map(|bb| { + let kind = bb + .terminator + .as_ref() + .map(|terminator| terminator.kind.clone()); + let statements = bb + .statements + .iter() + .map(|statement| statement.kind.clone()) + .collect::>(); + + (bb.is_cleanup, kind, statements) + .hash_stable(&mut hcx, &mut stable_hasher); + () + }) + .collect::>(); + ty.skip_binder().kind().hash_stable(&mut hcx, &mut stable_hasher); Some(()) From 3dc381ecc9c8c65edc996e0361a8d3a2da121d92 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:38:18 +0200 Subject: [PATCH 10/18] chore: Remove unused code --- .../src/middle/exported_symbols.rs | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 7589358e495f1..0bfbd39879747 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -1,7 +1,6 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_macros::{Decodable, Encodable, HashStable, TyDecodable, TyEncodable}; -use crate::mir::Body; use crate::ty::{self, GenericArgsRef, Ty, TyCtxt}; /// The SymbolExportLevel of a symbols specifies from which kinds of crates @@ -71,34 +70,6 @@ impl<'tcx> ExportedSymbol<'tcx> { ExportedSymbol::NoDefId(symbol_name) => symbol_name, } } - pub fn mir_body_for_local_instance(&self, tcx: TyCtxt<'tcx>) -> &'tcx Body<'tcx> { - match *self { - ExportedSymbol::NonGeneric(def_id) => { - tcx.instance_mir(ty::Instance::mono(tcx, def_id).def) - } - ExportedSymbol::Generic(def_id, args) => { - tcx.instance_mir(ty::Instance::new(def_id, args).def) - } - ExportedSymbol::DropGlue(ty) => { - tcx.instance_mir(ty::Instance::resolve_drop_in_place(tcx, ty).def) - } - ExportedSymbol::AsyncDropGlueCtorShim(ty) => { - tcx.instance_mir(ty::Instance::resolve_async_drop_in_place(tcx, ty).def) - } - ExportedSymbol::ThreadLocalShim(def_id) => { - tcx.instance_mir(ty::InstanceKind::ThreadLocalShim(def_id)) - } - ExportedSymbol::NoDefId(_) => panic!("Cannot find "), - } - } - pub fn def_id(&self) -> Option { - match self { - ExportedSymbol::NonGeneric(def_id) - | ExportedSymbol::Generic(def_id, _) - | ExportedSymbol::ThreadLocalShim(def_id) => Some(*def_id), - _ => None, - } - } } pub fn metadata_symbol_name(tcx: TyCtxt<'_>) -> String { From eca54dc0f2165b7c7cc7a4c7cc3de38573df54b8 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:43:09 +0200 Subject: [PATCH 11/18] :lipstick: --- compiler/rustc_hir/src/stable_hash_impls.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_hir/src/stable_hash_impls.rs b/compiler/rustc_hir/src/stable_hash_impls.rs index 07600820d546e..fe169e989ec9f 100644 --- a/compiler/rustc_hir/src/stable_hash_impls.rs +++ b/compiler/rustc_hir/src/stable_hash_impls.rs @@ -109,7 +109,7 @@ impl<'tcx, HirCtx: crate::HashStableContext> HashStable for AttributeMap impl HashStable for Crate<'_> { fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) { - let Crate { owners: _, opt_hir_hash, .. } = self; + let Crate { owners: _, opt_hir_hash } = self; opt_hir_hash.unwrap().hash_stable(hcx, hasher) } } From 7783b22bffcead71f26c42a400307d80c915543c Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:45:29 +0200 Subject: [PATCH 12/18] Extract fingerprint calculation into separate function --- compiler/rustc_metadata/src/fs.rs | 158 ++++++++++++++---------------- 1 file changed, 76 insertions(+), 82 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 666b0d84a2b4c..4483cd6deadc7 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -97,88 +97,7 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) { } }; if tcx.sess.opts.json_artifact_notifications { - let hash: Fingerprint = { - let mut stable_hasher = StableHasher::new(); - tcx.with_stable_hashing_context(|mut hcx| { - hcx.while_hashing_spans(false, |mut hcx| { - let _ = tcx - .reachable_set(()) - .to_sorted(hcx, true) - .into_iter() - .filter_map(|local_def_id: &LocalDefId| { - let def_id = local_def_id.to_def_id(); - - let item = tcx.hir_node_by_def_id(*local_def_id); - let has_hir_body = item.body_id().is_some(); - let ident = item.ident(); - let def_kind = tcx.def_kind(def_id); - let has_mir = match def_kind { - DefKind::Ctor(_, _) - | DefKind::AnonConst - | DefKind::InlineConst - | DefKind::AssocConst - | DefKind::Const - | DefKind::SyntheticCoroutineBody => has_hir_body, - DefKind::AssocFn | DefKind::Fn | DefKind::Closure => { - if def_kind == DefKind::Closure && tcx.is_coroutine(def_id) - { - has_hir_body - } else { - let generics = tcx.generics_of(def_id); - has_hir_body - && (tcx.sess.opts.unstable_opts.always_encode_mir - || (tcx - .sess - .opts - .output_types - .should_codegen() - && (generics - .requires_monomorphization(tcx) - || tcx.cross_crate_inlinable(def_id)))) - } - } - _ => false, - }; - ident.hash_stable(hcx, &mut stable_hasher); - if !has_mir { - item.hash_stable(hcx, &mut stable_hasher); - return Some(()); - } - - let ty = tcx.type_of(def_id); - - let body = tcx.instance_mir(InstanceKind::Item(def_id)); - let blocks = body.basic_blocks.deref(); - - // Deref to avoid hashing cache of mir body. - let _ = blocks - .iter() - .map(|bb| { - let kind = bb - .terminator - .as_ref() - .map(|terminator| terminator.kind.clone()); - let statements = bb - .statements - .iter() - .map(|statement| statement.kind.clone()) - .collect::>(); - - (bb.is_cleanup, kind, statements) - .hash_stable(&mut hcx, &mut stable_hasher); - () - }) - .collect::>(); - - ty.skip_binder().kind().hash_stable(&mut hcx, &mut stable_hasher); - - Some(()) - }) - .collect::>(); - }); - stable_hasher.finish() - }) - }; + let hash = public_api_hash(tcx); tcx.dcx().emit_artifact_notification( out_filename.as_path(), "metadata", @@ -226,3 +145,78 @@ pub fn copy_to_stdout(from: &Path) -> io::Result<()> { io::copy(&mut reader, &mut stdout)?; Ok(()) } + +fn public_api_hash(tcx: TyCtxt<'_>) -> Fingerprint { + let mut stable_hasher = StableHasher::new(); + tcx.with_stable_hashing_context(|mut hcx| { + hcx.while_hashing_spans(false, |mut hcx| { + let _ = tcx + .reachable_set(()) + .to_sorted(hcx, true) + .into_iter() + .filter_map(|local_def_id: &LocalDefId| { + let def_id = local_def_id.to_def_id(); + + let item = tcx.hir_node_by_def_id(*local_def_id); + let has_hir_body = item.body_id().is_some(); + let ident = item.ident(); + let def_kind = tcx.def_kind(def_id); + let has_mir = match def_kind { + DefKind::Ctor(_, _) + | DefKind::AnonConst + | DefKind::InlineConst + | DefKind::AssocConst + | DefKind::Const + | DefKind::SyntheticCoroutineBody => has_hir_body, + DefKind::AssocFn | DefKind::Fn | DefKind::Closure => { + if def_kind == DefKind::Closure && tcx.is_coroutine(def_id) { + has_hir_body + } else { + let generics = tcx.generics_of(def_id); + has_hir_body + && (tcx.sess.opts.unstable_opts.always_encode_mir + || (tcx.sess.opts.output_types.should_codegen() + && (generics.requires_monomorphization(tcx) + || tcx.cross_crate_inlinable(def_id)))) + } + } + _ => false, + }; + ident.hash_stable(hcx, &mut stable_hasher); + if !has_mir { + item.hash_stable(hcx, &mut stable_hasher); + return Some(()); + } + + let ty = tcx.type_of(def_id); + + let body = tcx.instance_mir(InstanceKind::Item(def_id)); + let blocks = body.basic_blocks.deref(); + + // Deref to avoid hashing cache of mir body. + let _ = blocks + .iter() + .map(|bb| { + let kind = + bb.terminator.as_ref().map(|terminator| terminator.kind.clone()); + let statements = bb + .statements + .iter() + .map(|statement| statement.kind.clone()) + .collect::>(); + + (bb.is_cleanup, kind, statements) + .hash_stable(&mut hcx, &mut stable_hasher); + () + }) + .collect::>(); + + ty.skip_binder().kind().hash_stable(&mut hcx, &mut stable_hasher); + + Some(()) + }) + .collect::>(); + }); + stable_hasher.finish() + }) +} From ead2514f8cb60c4ee93c251ee362fb3c5bb6f697 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:51:02 +0200 Subject: [PATCH 13/18] :lipstick: --- compiler/rustc_errors/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index f18e955a6c8a0..c8b855894b251 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -624,12 +624,12 @@ impl Drop for DiagCtxtInner { } if !self.has_printed && !self.suppressed_expected_diag && !std::thread::panicking() { - if let Some(_) = &self.must_produce_diag { - // panic!( - // "must_produce_diag: `trimmed_def_paths` called but no diagnostics emitted; \ - // `with_no_trimmed_paths` for debugging. \ - // called at: {backtrace}" - // ); + if let Some(backtrace) = &self.must_produce_diag { + panic!( + "must_produce_diag: `trimmed_def_paths` called but no diagnostics emitted; \ + `with_no_trimmed_paths` for debugging. \ + called at: {backtrace}" + ); } } } From 325e1b58fb3190469972fd70cc5df5463d6cfb62 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:20:18 +0200 Subject: [PATCH 14/18] Touchups --- compiler/rustc_metadata/src/fs.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 4483cd6deadc7..c7a5766a1ce7e 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -158,9 +158,11 @@ fn public_api_hash(tcx: TyCtxt<'_>) -> Fingerprint { let def_id = local_def_id.to_def_id(); let item = tcx.hir_node_by_def_id(*local_def_id); + let _ = item.ident()?; let has_hir_body = item.body_id().is_some(); - let ident = item.ident(); + let item_path = tcx.def_path_hash(def_id); let def_kind = tcx.def_kind(def_id); + item_path.hash_stable(hcx, &mut stable_hasher); let has_mir = match def_kind { DefKind::Ctor(_, _) | DefKind::AnonConst @@ -180,11 +182,12 @@ fn public_api_hash(tcx: TyCtxt<'_>) -> Fingerprint { || tcx.cross_crate_inlinable(def_id)))) } } - _ => false, + _ => { + return None; + } }; - ident.hash_stable(hcx, &mut stable_hasher); + if !has_mir { - item.hash_stable(hcx, &mut stable_hasher); return Some(()); } From 356d62a36007fa097f2303a25e7cbcf9a55209a7 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:02:57 +0200 Subject: [PATCH 15/18] Hash function signature --- compiler/rustc_metadata/src/fs.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index c7a5766a1ce7e..19bce21667445 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -17,7 +17,7 @@ use crate::errors::{ BinaryOutputToTty, FailedCopyToStdout, FailedCreateEncodedMetadata, FailedCreateFile, FailedCreateTempdir, FailedWriteError, }; -use crate::{encode_metadata, EncodedMetadata}; +use crate::{EncodedMetadata, encode_metadata}; // FIXME(eddyb) maybe include the crate name in this? pub const METADATA_FILENAME: &str = "lib.rmeta"; @@ -160,9 +160,11 @@ fn public_api_hash(tcx: TyCtxt<'_>) -> Fingerprint { let item = tcx.hir_node_by_def_id(*local_def_id); let _ = item.ident()?; let has_hir_body = item.body_id().is_some(); - let item_path = tcx.def_path_hash(def_id); + + let item_path = tcx.def_path(def_id); let def_kind = tcx.def_kind(def_id); - item_path.hash_stable(hcx, &mut stable_hasher); + let mut fn_sig = None; + item_path.to_string_no_crate_verbose().hash_stable(hcx, &mut stable_hasher); let has_mir = match def_kind { DefKind::Ctor(_, _) | DefKind::AnonConst @@ -171,6 +173,7 @@ fn public_api_hash(tcx: TyCtxt<'_>) -> Fingerprint { | DefKind::Const | DefKind::SyntheticCoroutineBody => has_hir_body, DefKind::AssocFn | DefKind::Fn | DefKind::Closure => { + fn_sig = Some(tcx.fn_sig(def_id)); if def_kind == DefKind::Closure && tcx.is_coroutine(def_id) { has_hir_body } else { @@ -187,6 +190,9 @@ fn public_api_hash(tcx: TyCtxt<'_>) -> Fingerprint { } }; + if let Some(sig) = fn_sig { + sig.skip_binder().hash_stable(hcx, &mut stable_hasher); + } if !has_mir { return Some(()); } From db6d7a1e802d397f5c1372a14d18e823a176def1 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 7 Apr 2025 20:53:14 +0200 Subject: [PATCH 16/18] Remove unneeded dep --- Cargo.lock | 1 - compiler/rustc_metadata/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74cf0b394389e..52c9269fa87f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4044,7 +4044,6 @@ dependencies = [ "rustc_index", "rustc_macros", "rustc_middle", - "rustc_query_system", "rustc_serialize", "rustc_session", "rustc_span", diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml index 9bf74e16e02b7..08dcc3d519a2b 100644 --- a/compiler/rustc_metadata/Cargo.toml +++ b/compiler/rustc_metadata/Cargo.toml @@ -22,7 +22,6 @@ rustc_hir_pretty = { path = "../rustc_hir_pretty" } rustc_index = { path = "../rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_middle = { path = "../rustc_middle" } -rustc_query_system = { path = "../rustc_query_system" } rustc_serialize = { path = "../rustc_serialize" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } From 769b9ca6bfc948f09e1f7f350b74ee872d5deb64 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 23 Apr 2025 14:12:58 +0200 Subject: [PATCH 17/18] rustfmt --- compiler/rustc_metadata/src/creader.rs | 41 ++++++++++++++++---------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs index 68da0633e9f1e..0d5349ea73cbe 100644 --- a/compiler/rustc_metadata/src/creader.rs +++ b/compiler/rustc_metadata/src/creader.rs @@ -885,12 +885,15 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { CrateDepKind::MacrosOnly => CrateDepKind::MacrosOnly, _ => dep.kind, }; - let cnum = - self.maybe_resolve_crate(dep.name, dep_kind, CrateOrigin::IndirectDependency { + let cnum = self.maybe_resolve_crate( + dep.name, + dep_kind, + CrateOrigin::IndirectDependency { dep_root, parent_private: parent_is_private, dep: &dep, - })?; + }, + )?; crate_num_map.push(cnum); } @@ -1330,12 +1333,15 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { let cnum = self.resolve_crate(name, item.span, dep_kind, CrateOrigin::Extern)?; let path_len = definitions.def_path(def_id).data.len(); - self.cstore.update_extern_crate(cnum, ExternCrate { - src: ExternCrateSource::Extern(def_id.to_def_id()), - span: item.span, - path_len, - dependency_of: LOCAL_CRATE, - }); + self.cstore.update_extern_crate( + cnum, + ExternCrate { + src: ExternCrateSource::Extern(def_id.to_def_id()), + span: item.span, + path_len, + dependency_of: LOCAL_CRATE, + }, + ); Some(cnum) } _ => bug!(), @@ -1345,13 +1351,16 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { pub fn process_path_extern(&mut self, name: Symbol, span: Span) -> Option { let cnum = self.resolve_crate(name, span, CrateDepKind::Explicit, CrateOrigin::Extern)?; - self.cstore.update_extern_crate(cnum, ExternCrate { - src: ExternCrateSource::Path, - span, - // to have the least priority in `update_extern_crate` - path_len: usize::MAX, - dependency_of: LOCAL_CRATE, - }); + self.cstore.update_extern_crate( + cnum, + ExternCrate { + src: ExternCrateSource::Path, + span, + // to have the least priority in `update_extern_crate` + path_len: usize::MAX, + dependency_of: LOCAL_CRATE, + }, + ); Some(cnum) } From 72bebfdb13425c15f9b2084d78ec9004705b9361 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 23 Apr 2025 14:23:48 +0200 Subject: [PATCH 18/18] Sure we can download llvm in CI, if we're foolish enough --- src/bootstrap/src/core/config/config.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 419976c83b1d0..1f508efc39972 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -3281,9 +3281,9 @@ impl Config { if b && self.is_running_on_ci { // On CI, we must always rebuild LLVM if there were any modifications to it - panic!( - "`llvm.download-ci-llvm` cannot be set to `true` on CI. Use `if-unchanged` instead." - ); + // panic!( + // "`llvm.download-ci-llvm` cannot be set to `true` on CI. Use `if-unchanged` instead." + // ); } // If download-ci-llvm=true we also want to check that CI llvm is available