diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs new file mode 100644 index 0000000000000..fe4cce60e1bba --- /dev/null +++ b/compiler/rustc_middle/src/query/erase.rs @@ -0,0 +1,272 @@ +use crate::mir; +use crate::traits; +use crate::ty; +use rustc_ast::expand::allocator::AllocatorKind; +use rustc_hir::MaybeOwner; +pub use rustc_middle::traits::query::type_op; +use std::intrinsics::type_name; +use std::{ + fmt, + mem::{size_of, transmute, transmute_copy, MaybeUninit}, +}; + +#[derive(Copy, Clone)] +struct Erased { + data: MaybeUninit, + formatter: fn(&Self, &mut std::fmt::Formatter<'_>) -> std::fmt::Result, + #[cfg(debug_assertions)] + type_id: &'static str, +} + +impl fmt::Debug for Erased { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (self.formatter)(self, f) + } +} + +pub trait EraseType: Copy { + type Result: Copy; +} + +pub type Erase = impl Copy + fmt::Debug; + +fn formatter( + this: &Erase, + f: &mut std::fmt::Formatter<'_>, +) -> fmt::Result { + fmt::Debug::fmt(restore_ref(this), f) +} + +#[inline(always)] +pub fn erase(src: T) -> Erase { + assert_eq!( + size_of::(), + size_of::(), + "size of {} must match erased type {}", + type_name::(), + type_name::() + ); + Erased::<::Result> { + // SAFETY:: Is it safe to transmute to MaybeUninit + data: unsafe { transmute_copy(&src) }, + #[cfg(debug_assertions)] + type_id: type_name::(), + formatter: formatter::, + } +} + +/// Restores an erased value. +/// +/// This is only safe if `value` is a valid instance of `T`. +/// For example if `T` was erased with `erase` previously. +#[inline(always)] +pub fn restore_ref(value: &Erase) -> &T { + let value: &Erased<::Result> = &value; + #[cfg(debug_assertions)] + assert_eq!(value.type_id, type_name::()); + assert_eq!( + size_of::(), + size_of::(), + "size of {} must match erased type {}", + type_name::(), + type_name::() + ); + // SAFETY: Thanks to the TAIT, this function can only be called with the result of `erase`. + unsafe { transmute(&value.data) } +} + +/// Restores an erased value. +/// +/// This is only safe if `value` is a valid instance of `T`. +/// For example if `T` was erased with `erase` previously. +#[inline(always)] +pub fn restore(value: Erase) -> T { + let value: Erased<::Result> = value; + #[cfg(debug_assertions)] + assert_eq!(value.type_id, type_name::()); + assert_eq!( + size_of::(), + size_of::(), + "size of {} must match erased type {}", + type_name::(), + type_name::() + ); + // SAFETY: Thanks to the TAIT, this function can only be called with the result of `erase`. + unsafe { transmute_copy(&value.data) } +} + +impl EraseType for &'_ T { + type Result = [u8; size_of::<*const ()>()]; +} + +impl EraseType for &'_ [T] { + type Result = [u8; size_of::<*const [()]>()]; +} + +impl EraseType for &'_ ty::List { + type Result = [u8; size_of::<*const ()>()]; +} + +impl EraseType for Result { + type Result = Self; +} + +impl EraseType for Option { + type Result = Self; +} + +impl EraseType for MaybeOwner { + type Result = Self; +} + +impl EraseType for ty::Visibility { + type Result = Self; +} + +impl EraseType for ty::Binder<'_, T> { + type Result = Self; +} + +impl EraseType for ty::EarlyBinder { + type Result = Self; +} + +impl EraseType for (T0, T1) { + type Result = Self; +} + +macro_rules! trivial { + ($($ty:ty),+ $(,)?) => { + $( + impl EraseType for $ty { + type Result = [u8; size_of::<$ty>()]; + } + )* + } +} + +trivial! { + (), + AllocatorKind, + bool, + crate::metadata::ModChild, + crate::middle::exported_symbols::SymbolExportInfo, + crate::middle::resolve_bound_vars::ObjectLifetimeDefault, + crate::middle::resolve_bound_vars::ResolvedArg, + crate::middle::stability::DeprecationEntry, + crate::mir::ConstQualifs, + crate::mir::interpret::ErrorHandled, + crate::mir::interpret::LitToConstError, + crate::thir::ExprId, + crate::traits::CodegenObligationError, + mir::Field, + mir::interpret::AllocId, + rustc_attr::ConstStability, + rustc_attr::DefaultBodyStability, + rustc_attr::Deprecation, + rustc_attr::Stability, + rustc_data_structures::svh::Svh, + rustc_errors::ErrorGuaranteed, + rustc_hir::Constness, + rustc_hir::def_id::DefId, + rustc_hir::def_id::DefIndex, + rustc_hir::def_id::LocalDefId, + rustc_hir::def::DefKind, + rustc_hir::Defaultness, + rustc_hir::definitions::DefKey, + rustc_hir::GeneratorKind, + rustc_hir::HirId, + rustc_hir::IsAsync, + rustc_hir::ItemLocalId, + rustc_hir::LangItem, + rustc_hir::OwnerId, + rustc_hir::Upvar, + rustc_index::bit_set::FiniteBitSet, + rustc_middle::middle::dependency_format::Linkage, + rustc_session::config::CrateType, + rustc_session::config::EntryFnType, + rustc_session::config::OptLevel, + rustc_session::config::SymbolManglingVersion, + rustc_session::cstore::CrateDepKind, + rustc_session::cstore::ExternCrate, + rustc_session::cstore::LinkagePreference, + rustc_session::Limits, + rustc_session::lint::LintExpectationId, + rustc_span::def_id::CrateNum, + rustc_span::def_id::DefPathHash, + rustc_span::ExpnHash, + rustc_span::ExpnId, + rustc_span::Span, + rustc_span::Symbol, + rustc_span::symbol::Ident, + rustc_target::spec::PanicStrategy, + rustc_type_ir::Variance, + traits::EvaluationResult, + traits::OverflowError, + traits::query::NoSolution, + traits::WellFormedLoc, + ty::adjustment::CoerceUnsizedInfo, + ty::AssocItem, + ty::AssocItemContainer, + ty::BoundVariableKind, + ty::DeducedParamAttrs, + ty::Destructor, + ty::fast_reject::SimplifiedType, + ty::ImplPolarity, + ty::Representability, + ty::ReprOptions, + ty::UnusedGenericParams, + ty::util::AlwaysRequiresDrop, + u32, + usize, +} + +macro_rules! tcx_lifetime { + ($($($fake_path:ident)::+),+ $(,)?) => { + $( + impl<'tcx> EraseType for $($fake_path)::+<'tcx> { + type Result = [u8; size_of::<$($fake_path)::+<'static>>()]; + } + )* + } +} + +tcx_lifetime! { + crate::middle::exported_symbols::ExportedSymbol, + crate::mir::DestructuredConstant, + crate::mir::interpret::ConstValue, + mir::ConstantKind, + mir::interpret::ConstAlloc, + mir::interpret::GlobalId, + mir::interpret::LitToConstInput, + rustc_middle::hir::Owner, + traits::ChalkEnvironmentAndGoal, + traits::query::MethodAutoderefStepsResult, + ty::AdtDef, + ty::AliasTy, + ty::Clause, + ty::ClosureTypeInfo, + ty::Const, + ty::DestructuredConst, + ty::ExistentialTraitRef, + ty::FnSig, + ty::GenericArg, + ty::GenericPredicates, + ty::inhabitedness::InhabitedPredicate, + ty::Instance, + ty::InstanceDef, + ty::layout::FnAbiError, + ty::layout::LayoutError, + ty::ParamEnv, + ty::Predicate, + ty::SymbolName, + ty::TraitRef, + ty::Ty, + ty::UnevaluatedConst, + ty::ValTree, + ty::VtblEntry, + type_op::AscribeUserType, + type_op::Eq, + type_op::ProvePredicate, + type_op::Subtype, +} diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 5133da3429a9c..bf01c194cfb20 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -7,6 +7,7 @@ use crate::ty::{self, print::describe_as_module, TyCtxt}; use rustc_span::def_id::LOCAL_CRATE; +pub mod erase; mod keys; pub use keys::Key; diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index 2bc51baf87905..ae0f535d6a86a 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -69,6 +69,7 @@ use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; +use crate::query::erase::{erase, restore, Erase}; pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; @@ -246,7 +247,30 @@ macro_rules! define_callbacks { use super::*; $( - pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache; + pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>>::Cache; + )* + } + #[allow(nonstandard_style, unused_lifetimes)] + pub mod query_cache { + $( + #[derive(Default)] + pub struct $name<'tcx> { + _phantom_lt: std::marker::PhantomData<&'tcx ()>, + _phantom_key: std::marker::PhantomData>, + _phantom_value: std::marker::PhantomData>, + cache: super::query_storage::$name<'static>, + } + + impl<'tcx> $name<'tcx> { + pub fn cache(&'tcx self) -> &'tcx super::query_storage::$name<'tcx> { + unsafe { + std::mem::transmute::< + &super::query_storage::$name<'_>, + &super::query_storage::$name<'_>, + >(&self.cache) + } + } + } )* } @@ -270,7 +294,7 @@ macro_rules! define_callbacks { #[derive(Default)] pub struct QueryCaches<'tcx> { - $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)* + $($(#[$attr])* pub $name: query_cache::$name<'tcx>,)* } impl<'tcx> TyCtxtEnsure<'tcx> { @@ -280,7 +304,7 @@ macro_rules! define_callbacks { let key = key.into_query_param(); opt_remap_env_constness!([$($modifiers)*][key]); - match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) { + match try_get_cached(self.tcx, self.tcx.query_system.caches.$name.cache(), &key) { Some(_) => return, None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure), }; @@ -305,10 +329,11 @@ macro_rules! define_callbacks { let key = key.into_query_param(); opt_remap_env_constness!([$($modifiers)*][key]); - match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) { + let value = match try_get_cached(self.tcx, self.tcx.query_system.caches.$name.cache(), &key) { Some(value) => value, None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(), - } + }; + restore(value) })* } @@ -372,7 +397,7 @@ macro_rules! define_callbacks { span: Span, key: query_keys::$name<'tcx>, mode: QueryMode, - ) -> Option<$V>;)* + ) -> Option>;)* } }; } @@ -400,10 +425,11 @@ macro_rules! define_feedable { let tcx = self.tcx; let value = query_provided_to_value::$name(tcx, value); - let cache = &tcx.query_system.caches.$name; + let cache = tcx.query_system.caches.$name.cache(); match try_get_cached(tcx, cache, &key) { Some(old) => { + let old = restore(old); bug!( "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}", stringify!($name), @@ -418,7 +444,7 @@ macro_rules! define_feedable { &value, hash_result!([$($modifiers)*]), ); - cache.complete(key, value, dep_node_index); + cache.complete(key, erase(value), dep_node_index); value } } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index d7708a3bc3f42..d6fa68b1b6ecf 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -20,6 +20,7 @@ extern crate rustc_middle; use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::{self, DepKindStruct}; +use rustc_middle::query::erase::{erase, restore, Erase}; use rustc_middle::query::Key; use rustc_middle::ty::query::{ query_keys, query_provided, query_provided_to_value, query_storage, query_values, diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs index 46e34462cf225..535d22a5b5df2 100644 --- a/compiler/rustc_query_impl/src/on_disk_cache.rs +++ b/compiler/rustc_query_impl/src/on_disk_cache.rs @@ -10,6 +10,7 @@ use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState}; use rustc_middle::mir::{self, interpret}; +use rustc_middle::query::erase::{restore_ref, Erase, EraseType}; use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_query_system::dep_graph::DepContext; @@ -1056,14 +1057,14 @@ impl<'a, 'tcx> Encodable> for [u8] { } } -pub fn encode_query_results<'a, 'tcx, CTX, Q>( +pub fn encode_query_results<'a, 'tcx, CTX, Q, V>( tcx: CTX, encoder: &mut CacheEncoder<'a, 'tcx>, query_result_index: &mut EncodedDepNodeIndex, ) where CTX: QueryContext + 'tcx, - Q: super::QueryConfig, - Q::Value: Encodable>, + Q: super::QueryConfig>, + V: EraseType + std::fmt::Debug + Encodable>, { let _timer = tcx .dep_context() @@ -1081,6 +1082,7 @@ pub fn encode_query_results<'a, 'tcx, CTX, Q>( // Encode the type check tables with the `SerializedDepNodeIndex` // as tag. + let value = restore_ref(value); encoder.encode_tagged(dep_node, value); } }); diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index a8592bd70862c..029dfdf67b701 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -264,7 +264,9 @@ macro_rules! feedable { macro_rules! hash_result { ([]) => {{ - Some(dep_graph::hash_result) + Some(|hcx, result| { + dep_graph::hash_result(hcx, &restore(*result)) + }) }}; ([(no_hash) $($rest:tt)*]) => {{ None @@ -469,7 +471,7 @@ macro_rules! define_queries { $(impl<'tcx> QueryConfig> for queries::$name<'tcx> { type Key = query_keys::$name<'tcx>; - type Value = query_values::$name<'tcx>; + type Value = Erase>; const NAME: &'static str = stringify!($name); #[inline] @@ -490,20 +492,20 @@ macro_rules! define_queries { fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache where 'tcx:'a { - &tcx.query_system.caches.$name + tcx.query_system.caches.$name.cache() } fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { - tcx.$name(key) + erase(tcx.$name(key)) } #[inline] #[allow(unused_variables)] fn compute(qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { - query_provided_to_value::$name( + erase(query_provided_to_value::$name( qcx.tcx, get_provider!([$($modifiers)*][qcx, $name, key])(qcx.tcx, key) - ) + )) } #[inline] @@ -516,6 +518,7 @@ macro_rules! define_queries { dep_node ); value.map(|value| query_provided_to_value::$name(qcx.tcx, value)) + .map(erase) }) } else { None @@ -645,12 +648,12 @@ macro_rules! define_queries { $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( tcx, stringify!($name), - &tcx.query_system.caches.$name, + tcx.query_system.caches.$name.cache(), string_cache, ) }, encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index| - $crate::on_disk_cache::encode_query_results::<_, super::queries::$name<'_>>(tcx, encoder, query_result_index) + $crate::on_disk_cache::encode_query_results::<_, super::queries::$name<'_>, _>(tcx, encoder, query_result_index) ), }})* } @@ -695,7 +698,7 @@ macro_rules! define_queries_struct { $( $(#[$attr])* $name: QueryState< - as QueryConfig>>::Key, + query_keys::$name<'tcx>, rustc_middle::dep_graph::DepKind, >, )* @@ -737,7 +740,7 @@ macro_rules! define_queries_struct { span: Span, key: as QueryConfig>>::Key, mode: QueryMode, - ) -> Option> { + ) -> Option>> { let qcx = QueryCtxt { tcx, queries: self }; get_query::, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode) })* diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 4b3cd16c29f25..ca374f7534ace 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -55,14 +55,14 @@ impl Default for DefaultCache { } } -impl QueryStorage for DefaultCache { +impl QueryStorage for DefaultCache { type Value = V; } impl QueryCache for DefaultCache where K: Eq + Hash + Copy + Debug, - V: Copy + Debug, + V: Copy, { type Key = K; @@ -127,13 +127,13 @@ impl Default for SingleCache { } } -impl QueryStorage for SingleCache { +impl QueryStorage for SingleCache { type Value = V; } impl QueryCache for SingleCache where - V: Copy + Debug, + V: Copy, { type Key = (); @@ -173,14 +173,14 @@ impl Default for VecCache { } } -impl QueryStorage for VecCache { +impl QueryStorage for VecCache { type Value = V; } impl QueryCache for VecCache where K: Eq + Idx + Copy + Debug, - V: Copy + Debug, + V: Copy, { type Key = K;