diff --git a/Cargo.lock b/Cargo.lock index 60686c87359d9..be95e63eb7698 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -854,7 +854,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.7.1", "scopeguard", ] @@ -1241,6 +1241,16 @@ dependencies = [ "instant", ] +[[package]] +name = "field-offset" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf3a800ff6e860c863ca6d4b16fd999db8b752819c1606884047b73e468535" +dependencies = [ + "memoffset 0.8.0", + "rustc_version", +] + [[package]] name = "filetime" version = "0.2.20" @@ -2188,6 +2198,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -2197,6 +2216,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -3782,6 +3810,7 @@ dependencies = [ "chalk-ir", "derive_more", "either", + "field-offset", "gsgdt", "measureme", "polonius-engine", @@ -3996,7 +4025,9 @@ dependencies = [ name = "rustc_query_impl" version = "0.0.0" dependencies = [ + "field-offset", "measureme", + "memoffset 0.6.5", "rustc-rayon-core", "rustc_ast", "rustc_data_structures", diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 3af3012649da6..4a2ca85060a84 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -352,7 +352,7 @@ pub fn try_print_query_stack(handler: &Handler, num_frames: Option) { // state if it was responsible for triggering the panic. let i = ty::tls::with_context_opt(|icx| { if let Some(icx) = icx { - print_query_stack(QueryCtxt { tcx: icx.tcx }, icx.query, handler, num_frames) + print_query_stack(QueryCtxt::new(icx.tcx), icx.query, handler, num_frames) } else { 0 } diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 48401eabd1ed1..d3af01474b85f 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -700,9 +700,12 @@ pub fn create_global_ctxt<'tcx>( hir_arena, untracked, dep_graph, - query_result_on_disk_cache, rustc_query_impl::query_callbacks(arena), - rustc_query_impl::query_system_fns(local_providers, extern_providers), + rustc_query_impl::query_system( + local_providers, + extern_providers, + query_result_on_disk_cache, + ), ) }) }) diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml index a7d97bd3cf5ee..7c56af1da41a5 100644 --- a/compiler/rustc_middle/Cargo.toml +++ b/compiler/rustc_middle/Cargo.toml @@ -11,6 +11,7 @@ chalk-ir = "0.87.0" derive_more = "0.99.17" either = "1.5.0" gsgdt = "0.1.2" +field-offset = "0.3.5" measureme = "10.0.0" polonius-engine = "0.13.0" rustc_apfloat = { path = "../rustc_apfloat" } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 2ecd014acdfbc..15d697d8636b5 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -14,14 +14,12 @@ use crate::middle::resolve_bound_vars; use crate::middle::stability; use crate::mir::interpret::{self, Allocation, ConstAllocation}; use crate::mir::{Body, Local, Place, PlaceElem, ProjectionKind, Promoted}; -use crate::query::on_disk_cache::OnDiskCache; use crate::query::LocalCrate; use crate::thir::Thir; use crate::traits; use crate::traits::solve; use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData}; use crate::ty::query::QuerySystem; -use crate::ty::query::QuerySystemFns; use crate::ty::query::{self, TyCtxtAt}; use crate::ty::{ self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, FloatTy, FloatVar, FloatVid, @@ -653,9 +651,8 @@ impl<'tcx> TyCtxt<'tcx> { hir_arena: &'tcx WorkerLocal>, untracked: Untracked, dep_graph: DepGraph, - on_disk_cache: Option>, query_kinds: &'tcx [DepKindStruct<'tcx>], - query_system_fns: QuerySystemFns<'tcx>, + query_system: QuerySystem<'tcx>, ) -> GlobalCtxt<'tcx> { let data_layout = s.target.parse_data_layout().unwrap_or_else(|err| { s.emit_fatal(err); @@ -677,7 +674,7 @@ impl<'tcx> TyCtxt<'tcx> { lifetimes: common_lifetimes, consts: common_consts, untracked, - query_system: QuerySystem::new(query_system_fns, on_disk_cache), + query_system, query_kinds, ty_rcache: Default::default(), pred_rcache: Default::default(), diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index 07d47cae5ee93..fcdf33f9de471 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -43,6 +43,7 @@ use crate::ty::subst::{GenericArg, SubstsRef}; use crate::ty::util::AlwaysRequiresDrop; use crate::ty::GeneratorDiagnosticData; use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams}; +use field_offset::FieldOffset; use measureme::StringId; use rustc_arena::TypedArena; use rustc_ast as ast; @@ -66,9 +67,12 @@ use rustc_hir::hir_id::OwnerId; use rustc_hir::lang_items::{LangItem, LanguageItems}; use rustc_hir::{Crate, ItemLocalId, TraitCandidate}; use rustc_index::IndexVec; +use rustc_query_system::dep_graph::DepNodeIndex; +use rustc_query_system::dep_graph::SerializedDepNodeIndex; use rustc_query_system::ich::StableHashingContext; pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; +use rustc_query_system::HandleCycleError; use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion}; use rustc_session::cstore::{CrateDepKind, CrateSource}; use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib}; @@ -78,8 +82,6 @@ use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi; use rustc_target::spec::PanicStrategy; - -use std::marker::PhantomData; use std::mem; use std::ops::Deref; use std::path::PathBuf; @@ -103,6 +105,31 @@ pub struct QueryStruct<'tcx> { Option, &mut CacheEncoder<'_, 'tcx>, &mut EncodedDepNodeIndex)>, } +pub struct DynamicQuery<'tcx, C: QueryCache> { + pub name: &'static str, + pub eval_always: bool, + pub dep_kind: rustc_middle::dep_graph::DepKind, + pub handle_cycle_error: HandleCycleError, + pub query_state: FieldOffset, QueryState>, + pub query_cache: FieldOffset, C>, + pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool, + pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value, + pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value, + pub can_load_from_disk: bool, + pub try_load_from_disk: fn( + tcx: TyCtxt<'tcx>, + key: &C::Key, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, + ) -> Option, + pub loadable_from_disk: + fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool, + pub hash_result: HashResult, + pub value_from_cycle_error: + fn(tcx: TyCtxt<'tcx>, cycle: &[QueryInfo]) -> C::Value, + pub format_value: fn(&C::Value) -> String, +} + pub struct QuerySystemFns<'tcx> { pub engine: QueryEngine, pub local_providers: Providers, @@ -120,6 +147,7 @@ pub struct QuerySystem<'tcx> { pub states: QueryStates<'tcx>, pub arenas: QueryArenas<'tcx>, pub caches: QueryCaches<'tcx>, + pub dynamic_queries: DynamicQueries<'tcx>, /// This provides access to the incremental compilation on-disk cache for query results. /// Do not access this directly. It is only meant to be used by @@ -130,23 +158,6 @@ pub struct QuerySystem<'tcx> { pub fns: QuerySystemFns<'tcx>, pub jobs: AtomicU64, - - // Since we erase query value types we tell the typesystem about them with `PhantomData`. - _phantom_values: QueryPhantomValues<'tcx>, -} - -impl<'tcx> QuerySystem<'tcx> { - pub fn new(fns: QuerySystemFns<'tcx>, on_disk_cache: Option>) -> Self { - QuerySystem { - states: Default::default(), - arenas: Default::default(), - caches: Default::default(), - on_disk_cache, - fns, - jobs: AtomicU64::new(1), - _phantom_values: Default::default(), - } - } } #[derive(Copy, Clone)] @@ -427,11 +438,6 @@ macro_rules! define_callbacks { } } - #[derive(Default)] - pub struct QueryPhantomValues<'tcx> { - $($(#[$attr])* pub $name: PhantomData>,)* - } - #[derive(Default)] pub struct QueryCaches<'tcx> { $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)* @@ -490,6 +496,12 @@ macro_rules! define_callbacks { })* } + pub struct DynamicQueries<'tcx> { + $( + pub $name: DynamicQuery<'tcx, query_storage::$name<'tcx>>, + )* + } + #[derive(Default)] pub struct QueryStates<'tcx> { $( diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml index b107a3f03fe59..e596993465c1b 100644 --- a/compiler/rustc_query_impl/Cargo.toml +++ b/compiler/rustc_query_impl/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" [dependencies] +memoffset = { version = "0.6.0", features = ["unstable_const"] } +field-offset = "0.3.5" measureme = "10.0.0" rustc_ast = { path = "../rustc_ast" } rustc_data_structures = { path = "../rustc_data_structures" } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 82b335f4b4b57..39ca465954175 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -3,11 +3,12 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] // this shouldn't be necessary, but the check for `&mut _` is too naive and denies returning a function pointer that takes a mut ref #![feature(const_mut_refs)] +#![feature(const_refs_to_cell)] #![feature(min_specialization)] #![feature(never_type)] #![feature(rustc_attrs)] #![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] +#![allow(rustc::potential_query_instability, unused_parens)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] @@ -15,16 +16,27 @@ extern crate rustc_middle; use crate::plumbing::{encode_all_query_results, try_mark_green}; +use field_offset::offset_of; +use rustc_data_structures::stable_hasher::HashStable; +use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; +use rustc_middle::dep_graph::DepNodeIndex; use rustc_middle::dep_graph::{self, DepKind, DepKindStruct}; use rustc_middle::query::erase::{erase, restore, Erase}; +use rustc_middle::query::on_disk_cache::OnDiskCache; use rustc_middle::query::AsLocalKey; use rustc_middle::ty::query::{ query_keys, query_provided, query_provided_to_value, query_storage, query_values, + DynamicQueries, DynamicQuery, ExternProviders, Providers, QueryCaches, QueryEngine, + QueryStates, QuerySystem, QuerySystemFns, }; -use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine, QuerySystemFns}; use rustc_middle::ty::TyCtxt; use rustc_query_system::dep_graph::SerializedDepNodeIndex; +use rustc_query_system::ich::StableHashingContext; +use rustc_query_system::query::{ + get_query, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, QueryMode, QueryState, +}; +use rustc_query_system::HandleCycleError; use rustc_query_system::Value; use rustc_span::Span; @@ -32,31 +44,182 @@ use rustc_span::Span; mod plumbing; pub use crate::plumbing::QueryCtxt; -pub use rustc_query_system::query::QueryConfig; -use rustc_query_system::query::*; - mod profiling_support; pub use self::profiling_support::alloc_self_profile_query_strings; -/// This is implemented per query and restoring query values from their erased state. -trait QueryConfigRestored<'tcx>: QueryConfig> + Default { - type RestoredValue; +struct DynamicConfig< + 'tcx, + C: QueryCache, + const ANON: bool, + const DEPTH_LIMIT: bool, + const FEEDABLE: bool, +> { + dynamic: &'tcx DynamicQuery<'tcx, C>, +} - fn restore(value: >>::Value) -> Self::RestoredValue; +impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy + for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +{ +} +impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone + for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +{ + fn clone(&self) -> Self { + DynamicConfig { dynamic: self.dynamic } + } } -rustc_query_append! { define_queries! } +impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> + QueryConfig> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +where + for<'a> C::Key: HashStable>, +{ + type Key = C::Key; + type Value = C::Value; + type Cache = C; + + #[inline(always)] + fn name(self) -> &'static str { + self.dynamic.name + } + + #[inline(always)] + fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool { + (self.dynamic.cache_on_disk)(tcx, key) + } + + #[inline(always)] + fn query_state<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a QueryState + where + QueryCtxt<'tcx>: 'a, + { + self.dynamic.query_state.apply(&qcx.tcx.query_system.states) + } + + #[inline(always)] + fn query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::Cache + where + 'tcx: 'a, + { + self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches) + } + + #[inline(always)] + fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { + (self.dynamic.execute_query)(tcx, key) + } + + #[inline(always)] + fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { + (self.dynamic.compute)(qcx.tcx, key) + } + + #[inline(always)] + fn try_load_from_disk( + self, + qcx: QueryCtxt<'tcx>, + key: &Self::Key, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, + ) -> Option { + if self.dynamic.can_load_from_disk { + (self.dynamic.try_load_from_disk)(qcx.tcx, key, prev_index, index) + } else { + None + } + } + + #[inline] + fn loadable_from_disk( + self, + qcx: QueryCtxt<'tcx>, + key: &Self::Key, + index: SerializedDepNodeIndex, + ) -> bool { + (self.dynamic.loadable_from_disk)(qcx.tcx, key, index) + } + + fn value_from_cycle_error( + self, + tcx: TyCtxt<'tcx>, + cycle: &[QueryInfo], + ) -> Self::Value { + (self.dynamic.value_from_cycle_error)(tcx, cycle) + } + + #[inline(always)] + fn format_value(self) -> fn(&Self::Value) -> String { + self.dynamic.format_value + } + + #[inline(always)] + fn anon(self) -> bool { + ANON + } + + #[inline(always)] + fn eval_always(self) -> bool { + self.dynamic.eval_always + } + + #[inline(always)] + fn depth_limit(self) -> bool { + DEPTH_LIMIT + } + + #[inline(always)] + fn feedable(self) -> bool { + FEEDABLE + } + + #[inline(always)] + fn dep_kind(self) -> DepKind { + self.dynamic.dep_kind + } + + #[inline(always)] + fn handle_cycle_error(self) -> HandleCycleError { + self.dynamic.handle_cycle_error + } + + #[inline(always)] + fn hash_result(self) -> HashResult { + self.dynamic.hash_result + } +} -pub fn query_system_fns<'tcx>( +/// This is implemented per query. It allows restoring query values from their erased state +/// and constructing a QueryConfig. +trait QueryConfigRestored<'tcx> { + type RestoredValue; + type Config: QueryConfig>; + + fn config(tcx: TyCtxt<'tcx>) -> Self::Config; + fn restore(value: >>::Value) + -> Self::RestoredValue; +} + +pub fn query_system<'tcx>( local_providers: Providers, extern_providers: ExternProviders, -) -> QuerySystemFns<'tcx> { - QuerySystemFns { - engine: engine(), - local_providers, - extern_providers, - query_structs: make_dep_kind_array!(query_structs).to_vec(), - encode_query_results: encode_all_query_results, - try_mark_green: try_mark_green, + on_disk_cache: Option>, +) -> QuerySystem<'tcx> { + QuerySystem { + states: Default::default(), + arenas: Default::default(), + caches: Default::default(), + dynamic_queries: dynamic_queries(), + on_disk_cache, + fns: QuerySystemFns { + engine: engine(), + local_providers, + extern_providers, + query_structs: make_dep_kind_array!(query_structs).to_vec(), + encode_query_results: encode_all_query_results, + try_mark_green: try_mark_green, + }, + jobs: AtomicU64::new(1), } } + +rustc_query_append! { define_queries! } diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 9f8ac7ccd0b8e..74924e8113e6f 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -4,6 +4,7 @@ use crate::rustc_middle::dep_graph::DepContext; use crate::rustc_middle::ty::TyEncoder; +use crate::QueryConfigRestored; use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher}; use rustc_data_structures::sync::Lock; use rustc_errors::Diagnostic; @@ -265,14 +266,14 @@ macro_rules! hash_result { } macro_rules! call_provider { - ([][$qcx:expr, $name:ident, $key:expr]) => {{ - ($qcx.query_system.fns.local_providers.$name)($qcx, $key) + ([][$tcx:expr, $name:ident, $key:expr]) => {{ + ($tcx.query_system.fns.local_providers.$name)($tcx, $key) }}; - ([(separate_provide_extern) $($rest:tt)*][$qcx:expr, $name:ident, $key:expr]) => {{ + ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{ if let Some(key) = $key.as_local_key() { - ($qcx.query_system.fns.local_providers.$name)($qcx, key) + ($tcx.query_system.fns.local_providers.$name)($tcx, key) } else { - ($qcx.query_system.fns.extern_providers.$name)($qcx, $key) + ($tcx.query_system.fns.extern_providers.$name)($tcx, $key) } }}; ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { @@ -341,7 +342,7 @@ pub(crate) fn create_query_frame< } pub(crate) fn encode_query_results<'a, 'tcx, Q>( - query: Q, + query: Q::Config, qcx: QueryCtxt<'tcx>, encoder: &mut CacheEncoder<'a, 'tcx>, query_result_index: &mut EncodedDepNodeIndex, @@ -392,12 +393,26 @@ pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeI pub(crate) fn try_load_from_disk<'tcx, V>( tcx: TyCtxt<'tcx>, - id: SerializedDepNodeIndex, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, ) -> Option where V: for<'a> Decodable>, { - tcx.query_system.on_disk_cache.as_ref()?.try_load_query_result(tcx, id) + let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?; + + let prof_timer = tcx.prof.incr_cache_loading(); + + // The call to `with_query_deserialization` enforces that no new `DepNodes` + // are created during deserialization. See the docs of that method for more + // details. + let value = tcx + .dep_graph + .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index)); + + prof_timer.finish_with_query_invocation_id(index.into()); + + value } fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool @@ -434,10 +449,9 @@ where pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx> where - Q: QueryConfig> + Default, - Q::Key: DepNodeParams>, + Q: QueryConfigRestored<'tcx>, { - let fingerprint_style = Q::Key::fingerprint_style(); + let fingerprint_style = >>::Key::fingerprint_style(); if is_anon || !fingerprint_style.reconstructible() { return DepKindStruct { @@ -453,9 +467,11 @@ where is_anon, is_eval_always, fingerprint_style, - force_from_dep_node: Some(|tcx, dep_node| force_from_dep_node(Q::default(), tcx, dep_node)), + force_from_dep_node: Some(|tcx, dep_node| { + force_from_dep_node(Q::config(tcx), tcx, dep_node) + }), try_load_from_on_disk_cache: Some(|tcx, dep_node| { - try_load_from_on_disk_cache(Q::default(), tcx, dep_node) + try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node) }), } } @@ -491,7 +507,7 @@ macro_rules! define_queries { mode: QueryMode, ) -> Option>> { get_query( - queries::$name::default(), + queries::$name::config(tcx), QueryCtxt::new(tcx), span, key, @@ -519,146 +535,91 @@ macro_rules! define_queries { )* } - $(impl<'tcx> QueryConfig> for queries::$name<'tcx> { - type Key = query_keys::$name<'tcx>; - type Value = Erase>; - - #[inline(always)] - fn name(self) -> &'static str { - stringify!($name) - } - - #[inline] - fn format_value(self) -> fn(&Self::Value) -> String { - |value| format!("{:?}", restore::>(*value)) - } - - #[inline] - fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool { - ::rustc_middle::query::cached::$name(tcx, key) - } - - type Cache = query_storage::$name<'tcx>; - - #[inline(always)] - fn query_state<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a QueryState - where QueryCtxt<'tcx>: 'a - { - &tcx.query_system.states.$name - } - - #[inline(always)] - fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache - where 'tcx:'a - { - &tcx.query_system.caches.$name - } - - fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { - erase(tcx.$name(key)) - } - - #[inline] - #[allow(unused_variables)] - fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { - query_provided_to_value::$name( - qcx.tcx, - call_provider!([$($modifiers)*][qcx.tcx, $name, key]) - ) - } + #[allow(nonstandard_style)] + mod dynamic_query { + use super::*; - #[inline] - fn try_load_from_disk( - self, - _qcx: QueryCtxt<'tcx>, - _key: &Self::Key - ) -> rustc_query_system::query::TryLoadFromDisk, Self::Value> { - should_ever_cache_on_disk!([$($modifiers)*] { - if ::rustc_middle::query::cached::$name(_qcx.tcx, _key) { - Some(|qcx: QueryCtxt<'tcx>, dep_node| { - let value = $crate::plumbing::try_load_from_disk::>( - qcx.tcx, - dep_node - ); - value.map(|value| query_provided_to_value::$name(qcx.tcx, value)) - }) - } else { - None + $( + pub(super) fn $name<'tcx>() -> DynamicQuery<'tcx, query_storage::$name<'tcx>> { + DynamicQuery { + name: stringify!($name), + eval_always: is_eval_always!([$($modifiers)*]), + dep_kind: dep_graph::DepKind::$name, + handle_cycle_error: handle_cycle_error!([$($modifiers)*]), + query_state: offset_of!(QueryStates<'tcx> => $name), + query_cache: offset_of!(QueryCaches<'tcx> => $name), + cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key), + execute_query: |tcx, key| erase(tcx.$name(key)), + compute: |tcx, key| query_provided_to_value::$name( + tcx, + call_provider!([$($modifiers)*][tcx, $name, key]) + ), + can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false), + try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] { + |tcx, key, prev_index, index| { + if ::rustc_middle::query::cached::$name(tcx, key) { + let value = $crate::plumbing::try_load_from_disk::>( + tcx, + prev_index, + index, + ); + value.map(|value| query_provided_to_value::$name(tcx, value)) + } else { + None + } + } + } { + |_tcx, _key, _prev_index, _index| None + }), + value_from_cycle_error: |tcx, cycle| { + let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle); + erase(result) + }, + loadable_from_disk: |_tcx, _key, _index| { + should_ever_cache_on_disk!([$($modifiers)*] { + ::rustc_middle::query::cached::$name(_tcx, _key) && + $crate::plumbing::loadable_from_disk(_tcx, _index) + } { + false + }) + }, + hash_result: hash_result!([$($modifiers)*][query_values::$name<'tcx>]), + format_value: |value| format!("{:?}", restore::>(*value)), } - } { - None - }) - } - - #[inline] - fn loadable_from_disk( - self, - _qcx: QueryCtxt<'tcx>, - _key: &Self::Key, - _index: SerializedDepNodeIndex, - ) -> bool { - should_ever_cache_on_disk!([$($modifiers)*] { - self.cache_on_disk(_qcx.tcx, _key) && - $crate::plumbing::loadable_from_disk(_qcx.tcx, _index) - } { - false - }) - } - - #[inline] - fn value_from_cycle_error( - self, - tcx: TyCtxt<'tcx>, - cycle: &[QueryInfo], - ) -> Self::Value { - let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle); - erase(result) - } - - #[inline(always)] - fn anon(self) -> bool { - is_anon!([$($modifiers)*]) - } - - #[inline(always)] - fn eval_always(self) -> bool { - is_eval_always!([$($modifiers)*]) - } - - #[inline(always)] - fn depth_limit(self) -> bool { - depth_limit!([$($modifiers)*]) - } - - #[inline(always)] - fn feedable(self) -> bool { - feedable!([$($modifiers)*]) - } + } + )* + } - #[inline(always)] - fn dep_kind(self) -> rustc_middle::dep_graph::DepKind { - dep_graph::DepKind::$name - } + $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> { + type RestoredValue = query_values::$name<'tcx>; + type Config = DynamicConfig< + 'tcx, + query_storage::$name<'tcx>, + { is_anon!([$($modifiers)*]) }, + { depth_limit!([$($modifiers)*]) }, + { feedable!([$($modifiers)*]) }, + >; #[inline(always)] - fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError { - handle_cycle_error!([$($modifiers)*]) + fn config(tcx: TyCtxt<'tcx>) -> Self::Config { + DynamicConfig { + dynamic: &tcx.query_system.dynamic_queries.$name, + } } #[inline(always)] - fn hash_result(self) -> rustc_query_system::query::HashResult { - hash_result!([$($modifiers)*][query_values::$name<'tcx>]) + fn restore(value: >>::Value) -> Self::RestoredValue { + restore::>(value) } })* - $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> { - type RestoredValue = query_values::$name<'tcx>; - - #[inline(always)] - fn restore(value: >>::Value) -> Self::RestoredValue { - restore::>(value) + pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> { + DynamicQueries { + $( + $name: dynamic_query::$name(), + )* } - })* + } #[allow(nonstandard_style)] mod query_callbacks { @@ -730,6 +691,7 @@ macro_rules! define_queries { use rustc_middle::ty::query::QueryStruct; use rustc_middle::ty::query::QueryKeyStringCache; use rustc_middle::dep_graph::DepKind; + use crate::QueryConfigRestored; pub(super) const fn dummy_query_struct<'tcx>() -> QueryStruct<'tcx> { fn noop_try_collect_active_jobs(_: TyCtxt<'_>, _: &mut QueryMap) -> Option<()> { @@ -774,7 +736,7 @@ macro_rules! define_queries { }, encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index| $crate::plumbing::encode_query_results::>( - super::queries::$name::default(), + super::queries::$name::config(tcx), QueryCtxt::new(tcx), encoder, query_result_index, diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index bb9ea50a1ea56..7e47d70120544 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -4,6 +4,7 @@ use crate::dep_graph::{DepNode, DepNodeParams, SerializedDepNodeIndex}; use crate::error::HandleCycleError; use crate::ich::StableHashingContext; use crate::query::caches::QueryCache; +use crate::query::DepNodeIndex; use crate::query::{QueryContext, QueryInfo, QueryState}; use rustc_data_structures::fingerprint::Fingerprint; @@ -12,8 +13,6 @@ use std::hash::Hash; pub type HashResult = Option, &V) -> Fingerprint>; -pub type TryLoadFromDisk = Option Option>; - pub trait QueryConfig: Copy { fn name(self) -> &'static str; @@ -43,7 +42,13 @@ pub trait QueryConfig: Copy { fn compute(self, tcx: Qcx, key: Self::Key) -> Self::Value; - fn try_load_from_disk(self, qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk; + fn try_load_from_disk( + self, + tcx: Qcx, + key: &Self::Key, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, + ) -> Option; fn loadable_from_disk(self, qcx: Qcx, key: &Self::Key, idx: SerializedDepNodeIndex) -> bool; diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index fa1f51b04da77..f7619d75be768 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -12,7 +12,7 @@ pub use self::caches::{ }; mod config; -pub use self::config::{HashResult, QueryConfig, TryLoadFromDisk}; +pub use self::config::{HashResult, QueryConfig}; use crate::dep_graph::DepKind; use crate::dep_graph::{DepNodeIndex, HasDepContext, SerializedDepNodeIndex}; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 3b17c665fb7e9..4aaedc7a6c15e 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -564,59 +564,44 @@ where // First we try to load the result from the on-disk cache. // Some things are never cached on disk. - if let Some(try_load_from_disk) = query.try_load_from_disk(qcx, &key) { - let prof_timer = qcx.dep_context().profiler().incr_cache_loading(); - - // The call to `with_query_deserialization` enforces that no new `DepNodes` - // are created during deserialization. See the docs of that method for more - // details. - let result = qcx - .dep_context() - .dep_graph() - .with_query_deserialization(|| try_load_from_disk(qcx, prev_dep_node_index)); - - prof_timer.finish_with_query_invocation_id(dep_node_index.into()); - - if let Some(result) = result { - if std::intrinsics::unlikely( - qcx.dep_context().sess().opts.unstable_opts.query_dep_graph, - ) { - dep_graph_data.mark_debug_loaded_from_disk(*dep_node) - } - - let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index); - // If `-Zincremental-verify-ich` is specified, re-hash results from - // the cache and make sure that they have the expected fingerprint. - // - // If not, we still seek to verify a subset of fingerprints loaded - // from disk. Re-hashing results is fairly expensive, so we can't - // currently afford to verify every hash. This subset should still - // give us some coverage of potential bugs though. - let try_verify = prev_fingerprint.split().1.as_u64() % 32 == 0; - if std::intrinsics::unlikely( - try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich, - ) { - incremental_verify_ich( - *qcx.dep_context(), - dep_graph_data, - &result, - prev_dep_node_index, - query.hash_result(), - query.format_value(), - ); - } + if let Some(result) = query.try_load_from_disk(qcx, key, prev_dep_node_index, dep_node_index) { + if std::intrinsics::unlikely(qcx.dep_context().sess().opts.unstable_opts.query_dep_graph) { + dep_graph_data.mark_debug_loaded_from_disk(*dep_node) + } - return Some((result, dep_node_index)); + let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index); + // If `-Zincremental-verify-ich` is specified, re-hash results from + // the cache and make sure that they have the expected fingerprint. + // + // If not, we still seek to verify a subset of fingerprints loaded + // from disk. Re-hashing results is fairly expensive, so we can't + // currently afford to verify every hash. This subset should still + // give us some coverage of potential bugs though. + let try_verify = prev_fingerprint.split().1.as_u64() % 32 == 0; + if std::intrinsics::unlikely( + try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich, + ) { + incremental_verify_ich( + *qcx.dep_context(), + dep_graph_data, + &result, + prev_dep_node_index, + query.hash_result(), + query.format_value(), + ); } - // We always expect to find a cached result for things that - // can be forced from `DepNode`. - debug_assert!( - !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(), - "missing on-disk cache entry for reconstructible {dep_node:?}" - ); + return Some((result, dep_node_index)); } + // We always expect to find a cached result for things that + // can be forced from `DepNode`. + debug_assert!( + !query.cache_on_disk(*qcx.dep_context(), key) + || !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(), + "missing on-disk cache entry for {dep_node:?}" + ); + // Sanity check for the logic in `ensure`: if the node is green and the result loadable, // we should actually be able to load it. debug_assert!( diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index afa6bce943f34..e1004c796c586 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -145,6 +145,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "expect-test", "fallible-iterator", // dependency of `thorin` "fastrand", + "field-offset", "fixedbitset", "flate2", "fluent-bundle",