diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 3dbcc4d2e8aab..80618fd1abed2 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -535,16 +535,22 @@ impl DepGraph { // value to an existing node. // // For sanity, we still check that the loaded stable hash and the new one match. - if let Some(dep_node_index) = data.dep_node_index_of_opt(&node) { - let _current_fingerprint = - crate::query::incremental_verify_ich(cx, data, result, &node, hash_result); + if let Some(prev_index) = data.previous.node_to_index_opt(&node) { + let dep_node_index = data.current.prev_index_to_index.lock()[prev_index]; + if let Some(dep_node_index) = dep_node_index { + crate::query::incremental_verify_ich(cx, data, result, prev_index, hash_result); - #[cfg(debug_assertions)] - if hash_result.is_some() { - data.current.record_edge(dep_node_index, node, _current_fingerprint); - } + #[cfg(debug_assertions)] + if hash_result.is_some() { + data.current.record_edge( + dep_node_index, + node, + data.prev_fingerprint_of(prev_index), + ); + } - return dep_node_index; + return dep_node_index; + } } let mut edges = SmallVec::new(); @@ -626,13 +632,19 @@ impl DepGraphData { /// Returns true if the given node has been marked as green during the /// current compilation session. Used in various assertions - pub fn is_green(&self, dep_node: &DepNode) -> bool { - self.node_color(dep_node).map_or(false, |c| c.is_green()) + #[inline] + pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool { + self.colors.get(prev_index).map_or(false, |c| c.is_green()) + } + + #[inline] + pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint { + self.previous.fingerprint_by_index(prev_index) } #[inline] - pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { - self.previous.fingerprint_of(dep_node) + pub fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode { + self.previous.index_to_node(prev_index) } pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) { @@ -643,7 +655,7 @@ impl DepGraphData { impl DepGraph { #[inline] pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool { - self.data.as_ref().and_then(|data| data.dep_node_index_of_opt(dep_node)).is_some() + self.data.as_ref().map_or(false, |data| data.dep_node_exists(dep_node)) } /// Checks whether a previous work product exists for `v` and, if diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 29513df460fff..3d19a84915aec 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -79,11 +79,6 @@ impl SerializedDepGraph { self.index.get(dep_node).cloned() } - #[inline] - pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { - self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index]) - } - #[inline] pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { self.fingerprints[dep_node_index] diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index ba2f859ff0f8e..186417e862a23 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -7,6 +7,7 @@ use crate::dep_graph::{DepGraphData, HasDepContext}; use crate::ich::StableHashingContext; use crate::query::caches::QueryCache; use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; +use crate::query::SerializedDepNodeIndex; use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame}; use crate::values::Value; use crate::HandleCycleError; @@ -19,7 +20,6 @@ use rustc_data_structures::sharded::Sharded; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError}; -use rustc_session::Session; use rustc_span::{Span, DUMMY_SP}; use std::cell::Cell; use std::collections::hash_map::Entry; @@ -537,7 +537,7 @@ where let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?; - debug_assert!(dep_graph_data.is_green(dep_node)); + debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index)); // First we try to load the result from the on-disk cache. // Some things are never cached on disk. @@ -561,8 +561,7 @@ where dep_graph_data.mark_debug_loaded_from_disk(*dep_node) } - let prev_fingerprint = - dep_graph_data.prev_fingerprint_of(dep_node).unwrap_or(Fingerprint::ZERO); + let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index); // If `-Zincremental-verify-ich` is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. // @@ -578,7 +577,7 @@ where *qcx.dep_context(), dep_graph_data, &result, - dep_node, + prev_dep_node_index, query.hash_result(), ); } @@ -623,7 +622,7 @@ where *qcx.dep_context(), dep_graph_data, &result, - dep_node, + prev_dep_node_index, query.hash_result(), ); @@ -636,77 +635,50 @@ pub(crate) fn incremental_verify_ich( tcx: Tcx, dep_graph_data: &DepGraphData, result: &V, - dep_node: &DepNode, + prev_index: SerializedDepNodeIndex, hash_result: Option, &V) -> Fingerprint>, -) -> Fingerprint -where +) where Tcx: DepContext, { - assert!( - dep_graph_data.is_green(dep_node), - "fingerprint for green query instance not loaded from cache: {dep_node:?}", - ); + if !dep_graph_data.is_index_green(prev_index) { + incremental_verify_ich_not_green(tcx, prev_index) + } let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| { tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result)) }); - let old_hash = dep_graph_data.prev_fingerprint_of(dep_node); + let old_hash = dep_graph_data.prev_fingerprint_of(prev_index); - if Some(new_hash) != old_hash { - incremental_verify_ich_failed( - tcx.sess(), - DebugArg::from(&dep_node), - DebugArg::from(&result), - ); + if new_hash != old_hash { + incremental_verify_ich_failed(tcx, prev_index, result); } - - new_hash } -// This DebugArg business is largely a mirror of std::fmt::ArgumentV1, which is -// currently not exposed publicly. -// -// The PR which added this attempted to use `&dyn Debug` instead, but that -// showed statistically significant worse compiler performance. It's not -// actually clear what the cause there was -- the code should be cold. If this -// can be replaced with `&dyn Debug` with on perf impact, then it probably -// should be. -extern "C" { - type Opaque; -} - -struct DebugArg<'a> { - value: &'a Opaque, - fmt: fn(&Opaque, &mut std::fmt::Formatter<'_>) -> std::fmt::Result, -} - -impl<'a, T> From<&'a T> for DebugArg<'a> +#[cold] +#[inline(never)] +fn incremental_verify_ich_not_green(tcx: Tcx, prev_index: SerializedDepNodeIndex) where - T: std::fmt::Debug, + Tcx: DepContext, { - fn from(value: &'a T) -> DebugArg<'a> { - DebugArg { - value: unsafe { std::mem::transmute(value) }, - fmt: unsafe { - std::mem::transmute(::fmt as fn(_, _) -> std::fmt::Result) - }, - } - } -} - -impl std::fmt::Debug for DebugArg<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - (self.fmt)(self.value, f) - } + panic!( + "fingerprint for green query instance not loaded from cache: {:?}", + tcx.dep_graph().data().unwrap().prev_node_of(prev_index) + ) } -// Note that this is marked #[cold] and intentionally takes the equivalent of -// `dyn Debug` for its arguments, as we want to avoid generating a bunch of -// different implementations for LLVM to chew on (and filling up the final -// binary, too). +// Note that this is marked #[cold] and intentionally takes `dyn Debug` for `result`, +// as we want to avoid generating a bunch of different implementations for LLVM to +// chew on (and filling up the final binary, too). #[cold] -fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>) { +#[inline(never)] +fn incremental_verify_ich_failed( + tcx: Tcx, + prev_index: SerializedDepNodeIndex, + result: &dyn Debug, +) where + Tcx: DepContext, +{ // When we emit an error message and panic, we try to debug-print the `DepNode` // and query result. Unfortunately, this can cause us to run additional queries, // which may result in another fingerprint mismatch while we're in the middle @@ -720,15 +692,17 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true)); if old_in_panic { - sess.emit_err(crate::error::Reentrant); + tcx.sess().emit_err(crate::error::Reentrant); } else { - let run_cmd = if let Some(crate_name) = &sess.opts.crate_name { + let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name { format!("`cargo clean -p {crate_name}` or `cargo clean`") } else { "`cargo clean`".to_string() }; - sess.emit_err(crate::error::IncrementCompilation { + let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index); + + let dep_node = tcx.sess().emit_err(crate::error::IncrementCompilation { run_cmd, dep_node: format!("{dep_node:?}"), });