diff --git a/Cargo.lock b/Cargo.lock index a9e65ffc39107..b441c5dd95745 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3878,6 +3878,7 @@ version = "0.0.0" dependencies = [ "libc", "rustc-rayon", + "rustc-rayon-core", "rustc_ast", "rustc_ast_lowering", "rustc_ast_passes", @@ -3890,6 +3891,7 @@ dependencies = [ "rustc_expand", "rustc_hir", "rustc_incremental", + "rustc_index", "rustc_lint", "rustc_metadata", "rustc_middle", diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index df89883f0bbb7..ffedf1fdf060d 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -465,9 +465,5 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR cgu.name() ); - if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { - CguReuse::PreLto - } else { - CguReuse::No - } + if tcx.try_mark_green(&dep_node) { CguReuse::PreLto } else { CguReuse::No } } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 658ad3c375d29..08e31c3b37f34 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -867,7 +867,7 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR cgu.name() ); - if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { + if tcx.try_mark_green(&dep_node) { // We can re-use either the pre- or the post-thinlto state. If no LTO is // being performed then we can use post-LTO artifacts, otherwise we must // reuse pre-LTO artifacts diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 485719c517564..14db71cb8f070 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -63,23 +63,9 @@ impl Sharded { if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) } } - /// Get a shard with a pre-computed hash value. If `get_shard_by_value` is - /// ever used in combination with `get_shard_by_hash` on a single `Sharded` - /// instance, then `hash` must be computed with `FxHasher`. Otherwise, - /// `hash` can be computed with any hasher, so long as that hasher is used - /// consistently for each `Sharded` instance. - #[inline] - pub fn get_shard_index_by_hash(&self, hash: u64) -> usize { - let hash_len = mem::size_of::(); - // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. - // hashbrown also uses the lowest bits, so we can't use those - let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; - bits % SHARDS - } - #[inline] pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { - &self.shards[self.get_shard_index_by_hash(hash)].0 + &self.shards[get_shard_index_by_hash(hash)].0 } #[inline] @@ -166,3 +152,17 @@ fn make_hash(val: &K) -> u64 { val.hash(&mut state); state.finish() } + +/// Get a shard with a pre-computed hash value. If `get_shard_by_value` is +/// ever used in combination with `get_shard_by_hash` on a single `Sharded` +/// instance, then `hash` must be computed with `FxHasher`. Otherwise, +/// `hash` can be computed with any hasher, so long as that hasher is used +/// consistently for each `Sharded` instance. +#[inline] +pub fn get_shard_index_by_hash(hash: u64) -> usize { + let hash_len = mem::size_of::(); + // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. + // hashbrown also uses the lowest bits, so we can't use those + let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; + bits % SHARDS +} diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs index 8295e88f75ac7..a80b6090cafac 100644 --- a/compiler/rustc_driver/src/lib.rs +++ b/compiler/rustc_driver/src/lib.rs @@ -27,7 +27,6 @@ use rustc_interface::{interface, Queries}; use rustc_lint::LintStore; use rustc_metadata::locator; use rustc_middle::middle::cstore::MetadataLoader; -use rustc_middle::ty::TyCtxt; use rustc_save_analysis as save; use rustc_save_analysis::DumpHandler; use rustc_serialize::json::{self, ToJson}; @@ -1240,7 +1239,7 @@ pub fn report_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) { let num_frames = if backtrace { None } else { Some(2) }; - TyCtxt::try_print_query_stack(&handler, num_frames); + interface::try_print_query_stack(&handler, num_frames); #[cfg(windows)] unsafe { diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index 2481a27dee795..2be78e8d3b6b8 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -10,6 +10,7 @@ doctest = false [dependencies] libc = "0.2" tracing = "0.1" +rustc-rayon-core = "0.3.0" rayon = { version = "0.3.0", package = "rustc-rayon" } smallvec = { version = "1.0", features = ["union", "may_dangle"] } rustc_ast = { path = "../rustc_ast" } @@ -30,6 +31,7 @@ rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } rustc_codegen_llvm = { path = "../rustc_codegen_llvm", optional = true } rustc_hir = { path = "../rustc_hir" } +rustc_index = { path = "../rustc_index" } rustc_metadata = { path = "../rustc_metadata" } rustc_mir = { path = "../rustc_mir" } rustc_mir_build = { path = "../rustc_mir_build" } diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 28eb1fed6a0ab..502e7155c2e0d 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -8,7 +8,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::sync::Lrc; use rustc_data_structures::OnDrop; use rustc_errors::registry::Registry; -use rustc_errors::ErrorReported; +use rustc_errors::{ErrorReported, Handler}; use rustc_lint::LintStore; use rustc_middle::ty; use rustc_parse::new_parser_from_source_str; @@ -213,3 +213,24 @@ pub fn run_compiler(mut config: Config, f: impl FnOnce(&Compiler) -> R || create_compiler_and_run(config, f), ) } + +pub fn try_print_query_stack(handler: &Handler, num_frames: Option) { + eprintln!("query stack during panic:"); + + // Be careful relying on global state here: this code is called from + // a panic hook, which means that the global `Handler` may be in a weird + // state if it was responsible for triggering the panic. + let i = ty::tls::with_context_opt(|icx| { + if let Some(icx) = icx { + icx.tcx.queries.try_print_query_stack(icx.tcx, icx.query, handler, num_frames) + } else { + 0 + } + }); + + if num_frames == None || num_frames >= Some(i) { + eprintln!("end of query stack"); + } else { + eprintln!("we're just showing a limited slice of the query stack"); + } +} diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 56aa3939b22dc..2728a0e8b59ea 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -15,11 +15,13 @@ use rustc_expand::base::ExtCtxt; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_hir::definitions::Definitions; use rustc_hir::Crate; +use rustc_index::vec::IndexVec; use rustc_lint::LintStore; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; use rustc_middle::middle; use rustc_middle::middle::cstore::{CrateStore, MetadataLoader, MetadataLoaderDyn}; +use rustc_middle::ty::query; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, GlobalCtxt, ResolverOutputs, TyCtxt}; use rustc_mir as mir; @@ -738,20 +740,18 @@ pub static DEFAULT_EXTERN_QUERY_PROVIDERS: SyncLazy = SyncLazy::new(| extern_providers }); -pub struct QueryContext<'tcx>(&'tcx GlobalCtxt<'tcx>); +pub struct QueryContext<'tcx> { + gcx: &'tcx GlobalCtxt<'tcx>, +} impl<'tcx> QueryContext<'tcx> { pub fn enter(&mut self, f: F) -> R where F: FnOnce(TyCtxt<'tcx>) -> R, { - let icx = ty::tls::ImplicitCtxt::new(self.0); + let icx = ty::tls::ImplicitCtxt::new(self.gcx); ty::tls::enter_context(&icx, |_| f(icx.tcx)) } - - pub fn print_stats(&mut self) { - self.enter(ty::query::print_stats) - } } pub fn create_global_ctxt<'tcx>( @@ -762,6 +762,7 @@ pub fn create_global_ctxt<'tcx>( mut resolver_outputs: ResolverOutputs, outputs: OutputFilenames, crate_name: &str, + queries: &'tcx OnceCell>, global_ctxt: &'tcx OnceCell>, arena: &'tcx WorkerLocal>, ) -> QueryContext<'tcx> { @@ -785,26 +786,33 @@ pub fn create_global_ctxt<'tcx>( callback(sess, &mut local_providers, &mut extern_providers); } + let queries = { + let crates = resolver_outputs.cstore.crates_untracked(); + let max_cnum = crates.iter().map(|c| c.as_usize()).max().unwrap_or(0); + let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1); + providers[LOCAL_CRATE] = local_providers; + queries.get_or_init(|| query::Queries::new(providers, extern_providers)) + }; + let gcx = sess.time("setup_global_ctxt", || { global_ctxt.get_or_init(|| { TyCtxt::create_global_ctxt( sess, lint_store, - local_providers, - extern_providers, arena, resolver_outputs, krate, defs, dep_graph, query_result_on_disk_cache, + queries, &crate_name, &outputs, ) }) }); - QueryContext(gcx) + QueryContext { gcx } } /// Runs the resolution, type-checking, region checking and other diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs index ac6b6d0311545..f70701257e242 100644 --- a/compiler/rustc_interface/src/queries.rs +++ b/compiler/rustc_interface/src/queries.rs @@ -13,6 +13,7 @@ use rustc_incremental::DepGraphFuture; use rustc_lint::LintStore; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; +use rustc_middle::ty::query; use rustc_middle::ty::{GlobalCtxt, ResolverOutputs, TyCtxt}; use rustc_serialize::json; use rustc_session::config::{self, OutputFilenames, OutputType}; @@ -71,6 +72,7 @@ impl Default for Query { pub struct Queries<'tcx> { compiler: &'tcx Compiler, gcx: OnceCell>, + queries: OnceCell>, arena: WorkerLocal>, hir_arena: WorkerLocal>, @@ -92,6 +94,7 @@ impl<'tcx> Queries<'tcx> { Queries { compiler, gcx: OnceCell::new(), + queries: OnceCell::new(), arena: WorkerLocal::new(|_| Arena::default()), hir_arena: WorkerLocal::new(|_| rustc_ast_lowering::Arena::default()), dep_graph_future: Default::default(), @@ -265,6 +268,7 @@ impl<'tcx> Queries<'tcx> { resolver_outputs.steal(), outputs, &crate_name, + &self.queries, &self.gcx, &self.arena, )) @@ -425,11 +429,11 @@ impl Compiler { { let _prof_timer = queries.session().prof.generic_activity("self_profile_alloc_query_strings"); - gcx.enter(|tcx| tcx.alloc_self_profile_query_strings()); + gcx.enter(query::alloc_self_profile_query_strings); } if self.session().opts.debugging_opts.query_stats { - gcx.print_stats(); + gcx.enter(query::print_stats); } } diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index b7dc539c6d606..798996263c700 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -10,6 +10,8 @@ use rustc_data_structures::stable_hasher::StableHasher; use rustc_data_structures::sync::Lrc; use rustc_errors::registry::Registry; use rustc_metadata::dynamic_lib::DynamicLibrary; +#[cfg(parallel_compiler)] +use rustc_middle::ty::tls; use rustc_resolve::{self, Resolver}; use rustc_session as session; use rustc_session::config::{self, CrateType}; @@ -29,11 +31,12 @@ use std::io; use std::lazy::SyncOnceCell; use std::mem; use std::ops::DerefMut; +#[cfg(not(parallel_compiler))] +use std::panic; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, Once}; -#[cfg(not(parallel_compiler))] -use std::{panic, thread}; +use std::thread; use tracing::info; /// Adds `target_feature = "..."` cfgs for a variety of platform @@ -156,6 +159,28 @@ pub fn setup_callbacks_and_run_in_thread_pool_with_globals R + Se scoped_thread(cfg, main_handler) } +/// Creates a new thread and forwards information in thread locals to it. +/// The new thread runs the deadlock handler. +/// Must only be called when a deadlock is about to happen. +#[cfg(parallel_compiler)] +unsafe fn handle_deadlock() { + let registry = rustc_rayon_core::Registry::current(); + + let context = tls::get_tlv(); + assert!(context != 0); + rustc_data_structures::sync::assert_sync::>(); + let icx: &tls::ImplicitCtxt<'_, '_> = &*(context as *const tls::ImplicitCtxt<'_, '_>); + + let session_globals = rustc_span::SESSION_GLOBALS.with(|sg| sg as *const _); + let session_globals = &*session_globals; + thread::spawn(move || { + tls::enter_context(icx, |_| { + rustc_span::SESSION_GLOBALS + .set(session_globals, || tls::with(|tcx| tcx.queries.deadlock(tcx, ®istry))) + }); + }); +} + #[cfg(parallel_compiler)] pub fn setup_callbacks_and_run_in_thread_pool_with_globals R + Send, R: Send>( edition: Edition, @@ -163,7 +188,6 @@ pub fn setup_callbacks_and_run_in_thread_pool_with_globals R + Se stderr: &Option>>>, f: F, ) -> R { - use rustc_middle::ty; crate::callbacks::setup_callbacks(); let mut config = rayon::ThreadPoolBuilder::new() @@ -171,7 +195,7 @@ pub fn setup_callbacks_and_run_in_thread_pool_with_globals R + Se .acquire_thread_handler(jobserver::acquire_thread) .release_thread_handler(jobserver::release_thread) .num_threads(threads) - .deadlock_handler(|| unsafe { ty::query::handle_deadlock() }); + .deadlock_handler(|| unsafe { handle_deadlock() }); if let Some(size) = get_stack_size() { config = config.stack_size(size); diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs index bd20c7689ea2e..5326f0ae26966 100644 --- a/compiler/rustc_macros/src/query.rs +++ b/compiler/rustc_macros/src/query.rs @@ -97,7 +97,7 @@ impl Parse for QueryModifier { Ok(QueryModifier::Cache(args, block)) } else if modifier == "load_cached" { // Parse a load_cached modifier like: - // `load_cached(tcx, id) { tcx.queries.on_disk_cache.try_load_query_result(tcx, id) }` + // `load_cached(tcx, id) { tcx.on_disk_cache.try_load_query_result(tcx, id) }` let args; parenthesized!(args in input); let tcx = args.parse()?; @@ -344,7 +344,6 @@ fn add_query_description_impl( impls: &mut proc_macro2::TokenStream, ) { let name = &query.name; - let arg = &query.arg; let key = &query.key.0; // Find out if we should cache the query on disk @@ -354,7 +353,7 @@ fn add_query_description_impl( quote! { #[inline] fn try_load_from_disk( - #tcx: TyCtxt<'tcx>, + #tcx: QueryCtxt<'tcx>, #id: SerializedDepNodeIndex ) -> Option { #block @@ -365,10 +364,10 @@ fn add_query_description_impl( quote! { #[inline] fn try_load_from_disk( - tcx: TyCtxt<'tcx>, + tcx: QueryCtxt<'tcx>, id: SerializedDepNodeIndex ) -> Option { - tcx.queries.on_disk_cache.as_ref().and_then(|c| c.try_load_query_result(tcx, id)) + tcx.on_disk_cache.as_ref()?.try_load_query_result(*tcx, id) } } }; @@ -393,7 +392,7 @@ fn add_query_description_impl( #[inline] #[allow(unused_variables, unused_braces)] fn cache_on_disk( - #tcx: TyCtxt<'tcx>, + #tcx: QueryCtxt<'tcx>, #key: &Self::Key, #value: Option<&Self::Value> ) -> bool { @@ -414,16 +413,14 @@ fn add_query_description_impl( let desc = quote! { #[allow(unused_variables)] - fn describe( - #tcx: TyCtxt<'tcx>, - #key: #arg, - ) -> String { - ::rustc_middle::ty::print::with_no_trimmed_paths(|| format!(#desc)) + fn describe(tcx: QueryCtxt<'tcx>, key: Self::Key) -> String { + let (#tcx, #key) = (*tcx, key); + ::rustc_middle::ty::print::with_no_trimmed_paths(|| format!(#desc).into()) } }; impls.extend(quote! { - impl<'tcx> QueryDescription> for queries::#name<'tcx> { + impl<'tcx> QueryDescription> for queries::#name<'tcx> { #desc #cache } @@ -522,7 +519,8 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { $($macro)*(#cached_queries); } } - - #query_description_stream + macro_rules! rustc_query_description { + () => { #query_description_stream } + } }) } diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index 1cb75757379c9..ba9d0a40732e6 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -62,7 +62,6 @@ use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX}; use rustc_hir::definitions::DefPathHash; use rustc_hir::HirId; use rustc_span::symbol::Symbol; -use rustc_span::DUMMY_SP; use std::hash::Hash; pub use rustc_query_system::dep_graph::{DepContext, DepNodeParams}; @@ -91,53 +90,6 @@ pub struct DepKindStruct { // FIXME: Make this a simple boolean once DepNodeParams::can_reconstruct_query_key // can be made a specialized associated const. can_reconstruct_query_key: fn() -> bool, - - /// The red/green evaluation system will try to mark a specific DepNode in the - /// dependency graph as green by recursively trying to mark the dependencies of - /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode` - /// where we don't know if it is red or green and we therefore actually have - /// to recompute its value in order to find out. Since the only piece of - /// information that we have at that point is the `DepNode` we are trying to - /// re-evaluate, we need some way to re-run a query from just that. This is what - /// `force_from_dep_node()` implements. - /// - /// In the general case, a `DepNode` consists of a `DepKind` and an opaque - /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint - /// is usually constructed by computing a stable hash of the query-key that the - /// `DepNode` corresponds to. Consequently, it is not in general possible to go - /// back from hash to query-key (since hash functions are not reversible). For - /// this reason `force_from_dep_node()` is expected to fail from time to time - /// because we just cannot find out, from the `DepNode` alone, what the - /// corresponding query-key is and therefore cannot re-run the query. - /// - /// The system deals with this case letting `try_mark_green` fail which forces - /// the root query to be re-evaluated. - /// - /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless. - /// Fortunately, we can use some contextual information that will allow us to - /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we - /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a - /// valid `DefPathHash`. Since we also always build a huge table that maps every - /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have - /// everything we need to re-run the query. - /// - /// Take the `mir_promoted` query as an example. Like many other queries, it - /// just has a single parameter: the `DefId` of the item it will compute the - /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode` - /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode` - /// is actually a `DefPathHash`, and can therefore just look up the corresponding - /// `DefId` in `tcx.def_path_hash_to_def_id`. - /// - /// When you implement a new query, it will likely have a corresponding new - /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As - /// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter, - /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just - /// add it to the "We don't have enough information to reconstruct..." group in - /// the match below. - pub(super) force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool, - - /// Invoke a query to put the on-disk cached value in memory. - pub(super) try_load_from_on_disk_cache: fn(TyCtxt<'_>, &DepNode), } impl std::ops::Deref for DepKind { @@ -196,8 +148,7 @@ macro_rules! contains_eval_always_attr { #[allow(non_upper_case_globals)] pub mod dep_kind { use super::*; - use crate::ty::query::{queries, query_keys}; - use rustc_query_system::query::{force_query, QueryDescription}; + use crate::ty::query::query_keys; // We use this for most things when incr. comp. is turned off. pub const Null: DepKindStruct = DepKindStruct { @@ -206,8 +157,6 @@ pub mod dep_kind { is_eval_always: false, can_reconstruct_query_key: || true, - force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node), - try_load_from_on_disk_cache: |_, _| {}, }; pub const TraitSelect: DepKindStruct = DepKindStruct { @@ -216,8 +165,6 @@ pub mod dep_kind { is_eval_always: false, can_reconstruct_query_key: || true, - force_from_dep_node: |_, _| false, - try_load_from_on_disk_cache: |_, _| {}, }; pub const CompileCodegenUnit: DepKindStruct = DepKindStruct { @@ -226,8 +173,6 @@ pub mod dep_kind { is_eval_always: false, can_reconstruct_query_key: || false, - force_from_dep_node: |_, _| false, - try_load_from_on_disk_cache: |_, _| {}, }; macro_rules! define_query_dep_kinds { @@ -246,59 +191,11 @@ pub mod dep_kind { ::can_reconstruct_query_key() } - fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option> { - as DepNodeParams>>::recover(tcx, dep_node) - } - - fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool { - if is_anon { - return false; - } - - if !can_reconstruct_query_key() { - return false; - } - - if let Some(key) = recover(tcx, dep_node) { - force_query::, _>( - tcx, - key, - DUMMY_SP, - *dep_node - ); - return true; - } - - false - } - - fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: &DepNode) { - if is_anon { - return - } - - if !can_reconstruct_query_key() { - return - } - - debug_assert!(tcx.dep_graph - .node_color(dep_node) - .map(|c| c.is_green()) - .unwrap_or(false)); - - let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)); - if queries::$variant::cache_on_disk(tcx, &key, None) { - let _ = tcx.$variant(key); - } - } - DepKindStruct { has_params, is_anon, is_eval_always, can_reconstruct_query_key, - force_from_dep_node, - try_load_from_on_disk_cache, } };)* ); @@ -314,7 +211,12 @@ macro_rules! define_dep_nodes { $variant:ident $(( $tuple_arg_ty:ty $(,)? ))* ,)* ) => ( - static DEP_KINDS: &[DepKindStruct] = &[ $(dep_kind::$variant),* ]; + #[macro_export] + macro_rules! make_dep_kind_array { + ($mod:ident) => {[ $(($mod::$variant),)* ]}; + } + + static DEP_KINDS: &[DepKindStruct] = &make_dep_kind_array!(dep_kind); /// This enum serves as an index into the `DEP_KINDS` array. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encodable, Decodable)] @@ -414,10 +316,7 @@ impl DepNodeExt for DepNode { /// has been removed. fn extract_def_id(&self, tcx: TyCtxt<'tcx>) -> Option { if self.kind.can_reconstruct_query_key() { - tcx.queries - .on_disk_cache - .as_ref()? - .def_path_hash_to_def_id(tcx, DefPathHash(self.hash.into())) + tcx.on_disk_cache.as_ref()?.def_path_hash_to_def_id(tcx, DefPathHash(self.hash.into())) } else { None } @@ -472,7 +371,7 @@ impl<'tcx> DepNodeParams> for DefId { // we will use the old DefIndex as an initial guess for // a lookup into the crate metadata. if !self.is_local() { - if let Some(cache) = &tcx.queries.on_disk_cache { + if let Some(cache) = &tcx.on_disk_cache { cache.store_foreign_def_id_hash(*self, hash); } } diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs index b88ffa2bb7347..d862db2674ebd 100644 --- a/compiler/rustc_middle/src/dep_graph/mod.rs +++ b/compiler/rustc_middle/src/dep_graph/mod.rs @@ -2,10 +2,8 @@ use crate::ich::StableHashingContext; use crate::ty::{self, TyCtxt}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::Lock; -use rustc_data_structures::thin_vec::ThinVec; -use rustc_errors::Diagnostic; -use rustc_hir::def_id::LocalDefId; +#[macro_use] mod dep_node; pub use rustc_query_system::dep_graph::{ @@ -94,7 +92,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { type StableHashingContext = StableHashingContext<'tcx>; fn register_reused_dep_node(&self, dep_node: &DepNode) { - if let Some(cache) = self.queries.on_disk_cache.as_ref() { + if let Some(cache) = self.on_disk_cache.as_ref() { cache.register_reused_dep_node(*self, dep_node) } } @@ -111,104 +109,12 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { || self.sess.opts.debugging_opts.query_dep_graph } - fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool { - // FIXME: This match is just a workaround for incremental bugs and should - // be removed. https://github.com/rust-lang/rust/issues/62649 is one such - // bug that must be fixed before removing this. - match dep_node.kind { - DepKind::hir_owner | DepKind::hir_owner_nodes => { - if let Some(def_id) = dep_node.extract_def_id(*self) { - if !def_id_corresponds_to_hir_dep_node(*self, def_id.expect_local()) { - // This `DefPath` does not have a - // corresponding `DepNode` (e.g. a - // struct field), and the ` DefPath` - // collided with the `DefPath` of a - // proper item that existed in the - // previous compilation session. - // - // Since the given `DefPath` does not - // denote the item that previously - // existed, we just fail to mark green. - return false; - } - } else { - // If the node does not exist anymore, we - // just fail to mark green. - return false; - } - } - _ => { - // For other kinds of nodes it's OK to be - // forced. - } - } - - debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node); - - // We must avoid ever having to call `force_from_dep_node()` for a - // `DepNode::codegen_unit`: - // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we - // would always end up having to evaluate the first caller of the - // `codegen_unit` query that *is* reconstructible. This might very well be - // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just - // to re-trigger calling the `codegen_unit` query with the right key. At - // that point we would already have re-done all the work we are trying to - // avoid doing in the first place. - // The solution is simple: Just explicitly call the `codegen_unit` query for - // each CGU, right after partitioning. This way `try_mark_green` will always - // hit the cache instead of having to go through `force_from_dep_node`. - // This assertion makes sure, we actually keep applying the solution above. - debug_assert!( - dep_node.kind != DepKind::codegen_unit, - "calling force_from_dep_node() on DepKind::codegen_unit" - ); - - (dep_node.kind.force_from_dep_node)(*self, dep_node) - } - - fn has_errors_or_delayed_span_bugs(&self) -> bool { - self.sess.has_errors_or_delayed_span_bugs() - } - - fn diagnostic(&self) -> &rustc_errors::Handler { - self.sess.diagnostic() - } - - // Interactions with on_disk_cache - fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) { - (dep_node.kind.try_load_from_on_disk_cache)(*self, dep_node) - } - - fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec { - self.queries - .on_disk_cache - .as_ref() - .map(|c| c.load_diagnostics(*self, prev_dep_node_index)) - .unwrap_or_default() - } - - fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec) { - if let Some(c) = self.queries.on_disk_cache.as_ref() { - c.store_diagnostics(dep_node_index, diagnostics) - } - } - - fn store_diagnostics_for_anon_node( - &self, - dep_node_index: DepNodeIndex, - diagnostics: ThinVec, - ) { - if let Some(c) = self.queries.on_disk_cache.as_ref() { - c.store_diagnostics_for_anon_node(dep_node_index, diagnostics) - } + #[inline] + fn dep_graph(&self) -> &DepGraph { + &self.dep_graph } fn profiler(&self) -> &SelfProfilerRef { &self.prof } } - -fn def_id_corresponds_to_hir_dep_node(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { - let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); - def_id == hir_id.owner -} diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index 6ae83a7f66750..d9e8826505002 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -76,6 +76,7 @@ pub mod query; #[macro_use] pub mod arena; +#[macro_use] pub mod dep_graph; pub mod hir; pub mod ich; diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index f0166ec21672b..d9abcf76ed09d 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1,27 +1,3 @@ -use crate::dep_graph::SerializedDepNodeIndex; -use crate::mir::interpret::{GlobalId, LitToConstInput}; -use crate::traits; -use crate::traits::query::{ - CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal, - CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal, - CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal, -}; -use crate::ty::query::queries; -use crate::ty::subst::{GenericArg, SubstsRef}; -use crate::ty::{self, ParamEnvAnd, Ty, TyCtxt}; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId}; -use rustc_query_system::query::QueryDescription; - -use rustc_span::symbol::Symbol; - -fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String { - if def_id.is_top_level_module() { - "top-level module".to_string() - } else { - format!("module `{}`", tcx.def_path_str(def_id.to_def_id())) - } -} - // Each of these queries corresponds to a function pointer field in the // `Providers` struct for requesting a value of that type, and a method // on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way @@ -125,11 +101,6 @@ rustc_queries! { desc { |tcx| "computing generics of `{}`", tcx.def_path_str(key) } storage(ArenaCacheSelector<'tcx>) cache_on_disk_if { key.is_local() } - load_cached(tcx, id) { - let generics: Option = tcx.queries.on_disk_cache.as_ref() - .and_then(|c| c.try_load_query_result(tcx, id)); - generics - } } /// Maps from the `DefId` of an item (trait/struct/enum/fn) to the @@ -702,8 +673,8 @@ rustc_queries! { cache_on_disk_if { true } load_cached(tcx, id) { let typeck_results: Option> = tcx - .queries.on_disk_cache.as_ref() - .and_then(|c| c.try_load_query_result(tcx, id)); + .on_disk_cache.as_ref() + .and_then(|c| c.try_load_query_result(*tcx, id)); typeck_results.map(|x| &*tcx.arena.alloc(x)) } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index f83056ebe2a45..ec680eeb30f06 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -14,7 +14,7 @@ use crate::middle::stability; use crate::mir::interpret::{self, Allocation, ConstValue, Scalar}; use crate::mir::{Body, Field, Local, Place, PlaceElem, ProjectionKind, Promoted}; use crate::traits; -use crate::ty::query::{self, TyCtxtAt}; +use crate::ty::query::{self, OnDiskCache, Queries, TyCtxtAt}; use crate::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, Subst, SubstsRef, UserSubsts}; use crate::ty::TyKind::*; use crate::ty::{ @@ -962,7 +962,14 @@ pub struct GlobalCtxt<'tcx> { pub(crate) untracked_crate: &'tcx hir::Crate<'tcx>, pub(crate) definitions: &'tcx Definitions, - pub queries: query::Queries<'tcx>, + /// This provides access to the incremental compilation on-disk cache for query results. + /// Do not access this directly. It is only meant to be used by + /// `DepGraph::try_mark_green()` and the query infrastructure. + /// This is `None` if we are not incremental compilation mode + pub(crate) on_disk_cache: Option>, + + pub queries: &'tcx query::Queries<'tcx>, + pub query_caches: query::QueryCaches<'tcx>, maybe_unused_trait_imports: FxHashSet, maybe_unused_extern_crates: Vec<(LocalDefId, Span)>, @@ -1102,14 +1109,13 @@ impl<'tcx> TyCtxt<'tcx> { pub fn create_global_ctxt( s: &'tcx Session, lint_store: Lrc, - local_providers: ty::query::Providers, - extern_providers: ty::query::Providers, arena: &'tcx WorkerLocal>, resolutions: ty::ResolverOutputs, krate: &'tcx hir::Crate<'tcx>, definitions: &'tcx Definitions, dep_graph: DepGraph, - on_disk_query_result_cache: Option>, + on_disk_cache: Option>, + queries: &'tcx Queries<'tcx>, crate_name: &str, output_filenames: &OutputFilenames, ) -> GlobalCtxt<'tcx> { @@ -1121,10 +1127,6 @@ impl<'tcx> TyCtxt<'tcx> { let common_lifetimes = CommonLifetimes::new(&interners); let common_consts = CommonConsts::new(&interners, &common_types); let cstore = resolutions.cstore; - let crates = cstore.crates_untracked(); - let max_cnum = crates.iter().map(|c| c.as_usize()).max().unwrap_or(0); - let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1); - providers[LOCAL_CRATE] = local_providers; let mut trait_map: FxHashMap<_, FxHashMap<_, _>> = FxHashMap::default(); for (hir_id, v) in krate.trait_map.iter() { @@ -1153,7 +1155,9 @@ impl<'tcx> TyCtxt<'tcx> { extern_prelude: resolutions.extern_prelude, untracked_crate: krate, definitions, - queries: query::Queries::new(providers, extern_providers, on_disk_query_result_cache), + on_disk_cache, + queries, + query_caches: query::QueryCaches::default(), ty_rcache: Default::default(), pred_rcache: Default::default(), selection_cache: Default::default(), @@ -1318,7 +1322,7 @@ impl<'tcx> TyCtxt<'tcx> { } pub fn serialize_query_result_cache(self, encoder: &mut FileEncoder) -> FileEncodeResult { - self.queries.on_disk_cache.as_ref().map_or(Ok(()), |c| c.serialize(self, encoder)) + self.on_disk_cache.as_ref().map_or(Ok(()), |c| c.serialize(self, encoder)) } /// If `true`, we should use the MIR-based borrowck, but also diff --git a/compiler/rustc_middle/src/ty/query/job.rs b/compiler/rustc_middle/src/ty/query/job.rs deleted file mode 100644 index bd2e7747b7db8..0000000000000 --- a/compiler/rustc_middle/src/ty/query/job.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::ty::tls; - -use rustc_query_system::query::deadlock; -use rustc_rayon_core as rayon_core; -use std::thread; - -/// Creates a new thread and forwards information in thread locals to it. -/// The new thread runs the deadlock handler. -/// Must only be called when a deadlock is about to happen. -pub unsafe fn handle_deadlock() { - let registry = rayon_core::Registry::current(); - - let context = tls::get_tlv(); - assert!(context != 0); - rustc_data_structures::sync::assert_sync::>(); - let icx: &tls::ImplicitCtxt<'_, '_> = &*(context as *const tls::ImplicitCtxt<'_, '_>); - - let session_globals = rustc_span::SESSION_GLOBALS.with(|sg| sg as *const _); - let session_globals = &*session_globals; - thread::spawn(move || { - tls::enter_context(icx, |_| { - rustc_span::SESSION_GLOBALS - .set(session_globals, || tls::with(|tcx| deadlock(tcx, ®istry))) - }) - }); -} diff --git a/compiler/rustc_middle/src/ty/query/keys.rs b/compiler/rustc_middle/src/ty/query/keys.rs index 6b4714b1bb8c8..2f76237e8fb79 100644 --- a/compiler/rustc_middle/src/ty/query/keys.rs +++ b/compiler/rustc_middle/src/ty/query/keys.rs @@ -6,15 +6,12 @@ use crate::ty::fast_reject::SimplifiedType; use crate::ty::subst::{GenericArg, SubstsRef}; use crate::ty::{self, Ty, TyCtxt}; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; -use rustc_query_system::query::DefaultCacheSelector; use rustc_span::symbol::{Ident, Symbol}; use rustc_span::{Span, DUMMY_SP}; /// The `Key` trait controls what types can legally be used as the key /// for a query. pub trait Key { - type CacheSelector; - /// Given an instance of this key, what crate is it referring to? /// This is used to find the provider. fn query_crate(&self) -> CrateNum; @@ -25,8 +22,6 @@ pub trait Key { } impl<'tcx> Key for ty::InstanceDef<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -37,8 +32,6 @@ impl<'tcx> Key for ty::InstanceDef<'tcx> { } impl<'tcx> Key for ty::Instance<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -49,8 +42,6 @@ impl<'tcx> Key for ty::Instance<'tcx> { } impl<'tcx> Key for mir::interpret::GlobalId<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.instance.query_crate() } @@ -61,8 +52,6 @@ impl<'tcx> Key for mir::interpret::GlobalId<'tcx> { } impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -73,8 +62,6 @@ impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> { } impl Key for CrateNum { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { *self } @@ -84,8 +71,6 @@ impl Key for CrateNum { } impl Key for LocalDefId { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.to_def_id().query_crate() } @@ -95,8 +80,6 @@ impl Key for LocalDefId { } impl Key for DefId { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.krate } @@ -106,8 +89,6 @@ impl Key for DefId { } impl Key for ty::WithOptConstParam { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.did.query_crate() } @@ -117,8 +98,6 @@ impl Key for ty::WithOptConstParam { } impl Key for (DefId, DefId) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0.krate } @@ -128,8 +107,6 @@ impl Key for (DefId, DefId) { } impl Key for (ty::Instance<'tcx>, LocalDefId) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0.query_crate() } @@ -139,8 +116,6 @@ impl Key for (ty::Instance<'tcx>, LocalDefId) { } impl Key for (DefId, LocalDefId) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0.krate } @@ -150,8 +125,6 @@ impl Key for (DefId, LocalDefId) { } impl Key for (LocalDefId, DefId) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -161,8 +134,6 @@ impl Key for (LocalDefId, DefId) { } impl Key for (DefId, Option) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0.krate } @@ -172,8 +143,6 @@ impl Key for (DefId, Option) { } impl Key for (DefId, LocalDefId, Ident) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0.krate } @@ -183,8 +152,6 @@ impl Key for (DefId, LocalDefId, Ident) { } impl Key for (CrateNum, DefId) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0 } @@ -194,8 +161,6 @@ impl Key for (CrateNum, DefId) { } impl Key for (DefId, SimplifiedType) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0.krate } @@ -205,8 +170,6 @@ impl Key for (DefId, SimplifiedType) { } impl<'tcx> Key for SubstsRef<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -216,8 +179,6 @@ impl<'tcx> Key for SubstsRef<'tcx> { } impl<'tcx> Key for (DefId, SubstsRef<'tcx>) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.0.krate } @@ -232,8 +193,6 @@ impl<'tcx> Key (ty::WithOptConstParam, SubstsRef<'tcx>), ) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { (self.0).0.did.krate } @@ -243,8 +202,6 @@ impl<'tcx> Key } impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -254,8 +211,6 @@ impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) { } impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.1.def_id().krate } @@ -265,8 +220,6 @@ impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) { } impl<'tcx> Key for (&'tcx ty::Const<'tcx>, mir::Field) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -276,8 +229,6 @@ impl<'tcx> Key for (&'tcx ty::Const<'tcx>, mir::Field) { } impl<'tcx> Key for ty::PolyTraitRef<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.def_id().krate } @@ -287,8 +238,6 @@ impl<'tcx> Key for ty::PolyTraitRef<'tcx> { } impl<'tcx> Key for GenericArg<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -298,8 +247,6 @@ impl<'tcx> Key for GenericArg<'tcx> { } impl<'tcx> Key for &'tcx ty::Const<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -309,8 +256,6 @@ impl<'tcx> Key for &'tcx ty::Const<'tcx> { } impl<'tcx> Key for Ty<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -320,8 +265,6 @@ impl<'tcx> Key for Ty<'tcx> { } impl<'tcx> Key for &'tcx ty::List> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -331,8 +274,6 @@ impl<'tcx> Key for &'tcx ty::List> { } impl<'tcx> Key for ty::ParamEnv<'tcx> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -342,8 +283,6 @@ impl<'tcx> Key for ty::ParamEnv<'tcx> { } impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { self.value.query_crate() } @@ -353,8 +292,6 @@ impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> { } impl Key for Symbol { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -366,8 +303,6 @@ impl Key for Symbol { /// Canonical query goals correspond to abstract trait operations that /// are not tied to any crate in particular. impl<'tcx, T> Key for Canonical<'tcx, T> { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -378,8 +313,6 @@ impl<'tcx, T> Key for Canonical<'tcx, T> { } impl Key for (Symbol, u32, u32) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -390,8 +323,6 @@ impl Key for (Symbol, u32, u32) { } impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) { - type CacheSelector = DefaultCacheSelector; - fn query_crate(&self) -> CrateNum { LOCAL_CRATE } diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs index 804c045a69010..faef1f048cc2b 100644 --- a/compiler/rustc_middle/src/ty/query/mod.rs +++ b/compiler/rustc_middle/src/ty/query/mod.rs @@ -44,6 +44,7 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId}; use rustc_hir::lang_items::{LangItem, LanguageItems}; use rustc_hir::{Crate, ItemLocalId, TraitCandidate}; use rustc_index::{bit_set::FiniteBitSet, vec::IndexVec}; +use rustc_serialize::opaque; use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion}; use rustc_session::utils::NativeLibKind; use rustc_session::CrateDisambiguator; @@ -60,16 +61,14 @@ use std::sync::Arc; #[macro_use] mod plumbing; +pub use plumbing::QueryCtxt; +use plumbing::QueryStruct; pub(crate) use rustc_query_system::query::CycleError; use rustc_query_system::query::*; mod stats; pub use self::stats::print_stats; -#[cfg(parallel_compiler)] -mod job; -#[cfg(parallel_compiler)] -pub use self::job::handle_deadlock; pub use rustc_query_system::query::{QueryInfo, QueryJob, QueryJobId}; mod keys; @@ -86,7 +85,163 @@ mod on_disk_cache; pub use self::on_disk_cache::OnDiskCache; mod profiling_support; -pub use self::profiling_support::{IntoSelfProfilingString, QueryKeyStringBuilder}; +pub use self::profiling_support::alloc_self_profile_query_strings; + +#[derive(Copy, Clone)] +pub struct TyCtxtAt<'tcx> { + pub tcx: TyCtxt<'tcx>, + pub span: Span, +} + +impl Deref for TyCtxtAt<'tcx> { + type Target = TyCtxt<'tcx>; + #[inline(always)] + fn deref(&self) -> &Self::Target { + &self.tcx + } +} + +#[derive(Copy, Clone)] +pub struct TyCtxtEnsure<'tcx> { + pub tcx: TyCtxt<'tcx>, +} + +impl TyCtxt<'tcx> { + /// Returns a transparent wrapper for `TyCtxt`, which ensures queries + /// are executed instead of just returning their results. + #[inline(always)] + pub fn ensure(self) -> TyCtxtEnsure<'tcx> { + TyCtxtEnsure { tcx: self } + } + + /// Returns a transparent wrapper for `TyCtxt` which uses + /// `span` as the location of queries performed through it. + #[inline(always)] + pub fn at(self, span: Span) -> TyCtxtAt<'tcx> { + TyCtxtAt { tcx: self, span } + } + + pub fn try_mark_green(self, dep_node: &dep_graph::DepNode) -> bool { + let qcx = QueryCtxt { tcx: self, queries: self.queries }; + self.dep_graph.try_mark_green(qcx, dep_node).is_some() + } +} + +macro_rules! query_helper_param_ty { + (DefId) => { impl IntoQueryParam }; + ($K:ty) => { $K }; +} + +macro_rules! define_callbacks { + (<$tcx:tt> + $($(#[$attr:meta])* + [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => { + + // HACK(eddyb) this is like the `impl QueryConfig for queries::$name` + // below, but using type aliases instead of associated types, to bypass + // the limitations around normalizing under HRTB - for example, this: + // `for<'tcx> fn(...) -> as QueryConfig>>::Value` + // doesn't currently normalize to `for<'tcx> fn(...) -> query_values::$name<'tcx>`. + // This is primarily used by the `provide!` macro in `rustc_metadata`. + #[allow(nonstandard_style, unused_lifetimes)] + pub mod query_keys { + use super::*; + + $(pub type $name<$tcx> = $($K)*;)* + } + #[allow(nonstandard_style, unused_lifetimes)] + pub mod query_values { + use super::*; + + $(pub type $name<$tcx> = $V;)* + } + #[allow(nonstandard_style, unused_lifetimes)] + pub mod query_storage { + use super::*; + + $(pub type $name<$tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)* + } + #[allow(nonstandard_style, unused_lifetimes)] + pub mod query_stored { + use super::*; + + $(pub type $name<$tcx> = as QueryStorage>::Stored;)* + } + + #[derive(Default)] + pub struct QueryCaches<$tcx> { + $($(#[$attr])* $name: QueryCacheStore>,)* + } + + impl TyCtxtEnsure<$tcx> { + $($(#[$attr])* + #[inline(always)] + pub fn $name(self, key: query_helper_param_ty!($($K)*)) { + let key = key.into_query_param(); + let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |_| {}); + + let lookup = match cached { + Ok(()) => return, + Err(lookup) => lookup, + }; + + self.tcx.queries.$name(self.tcx, DUMMY_SP, key, lookup, QueryMode::Ensure); + })* + } + + impl TyCtxt<$tcx> { + $($(#[$attr])* + #[inline(always)] + #[must_use] + pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx> + { + self.at(DUMMY_SP).$name(key) + })* + } + + impl TyCtxtAt<$tcx> { + $($(#[$attr])* + #[inline(always)] + pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx> + { + let key = key.into_query_param(); + let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |value| { + value.clone() + }); + + let lookup = match cached { + Ok(value) => return value, + Err(lookup) => lookup, + }; + + self.tcx.queries.$name(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap() + })* + } + + pub struct Providers { + $(pub $name: for<'tcx> fn( + TyCtxt<'tcx>, + query_keys::$name<'tcx>, + ) -> query_values::$name<'tcx>,)* + } + + impl Default for Providers { + fn default() -> Self { + Providers { + $($name: |_, key| bug!( + "`tcx.{}({:?})` unsupported by its crate", + stringify!($name), key + ),)* + } + } + } + + impl Copy for Providers {} + impl Clone for Providers { + fn clone(&self) -> Self { *self } + } + }; +} // Each of these queries corresponds to a function pointer field in the // `Providers` struct for requesting a value of that type, and a method @@ -101,6 +256,7 @@ pub use self::profiling_support::{IntoSelfProfilingString, QueryKeyStringBuilder // as they will raise an fatal error on query cycles instead. rustc_query_append! { [define_queries!][<'tcx>] } +rustc_query_append! { [define_callbacks!][<'tcx>] } mod sealed { use super::{DefId, LocalDefId}; diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index cfe47004e01b6..adeeb309205d2 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -14,6 +14,8 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE}; use rustc_hir::definitions::DefPathHash; use rustc_hir::definitions::Definitions; use rustc_index::vec::{Idx, IndexVec}; +use rustc_query_system::dep_graph::DepContext; +use rustc_query_system::query::QueryContext; use rustc_serialize::{ opaque::{self, FileEncodeResult, FileEncoder}, Decodable, Decoder, Encodable, Encoder, @@ -132,7 +134,7 @@ struct Footer { foreign_def_path_hashes: UnhashMap, } -type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; +pub type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; type EncodedDiagnosticsIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; type EncodedDiagnostics = Vec; @@ -140,7 +142,7 @@ type EncodedDiagnostics = Vec; struct SourceFileIndex(u32); #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)] -struct AbsoluteBytePos(u32); +pub struct AbsoluteBytePos(u32); impl AbsoluteBytePos { fn new(pos: usize) -> AbsoluteBytePos { @@ -284,7 +286,7 @@ impl<'sess> OnDiskCache<'sess> { // Do this *before* we clone 'latest_foreign_def_path_hashes', since // loading existing queries may cause us to create new DepNodes, which // may in turn end up invoking `store_foreign_def_id_hash` - tcx.dep_graph.exec_cache_promotions(tcx); + tcx.queries.exec_cache_promotions(tcx); let latest_foreign_def_path_hashes = self.latest_foreign_def_path_hashes.lock().clone(); let hygiene_encode_context = HygieneEncodeContext::default(); @@ -307,22 +309,7 @@ impl<'sess> OnDiskCache<'sess> { tcx.sess.time("encode_query_results", || -> FileEncodeResult { let enc = &mut encoder; let qri = &mut query_result_index; - - macro_rules! encode_queries { - ($($query:ident,)*) => { - $( - encode_query_results::>( - tcx, - enc, - qri - )?; - )* - } - } - - rustc_cached_queries!(encode_queries!); - - Ok(()) + tcx.queries.encode_query_results(tcx, enc, qri) })?; // Encode diagnostics. @@ -918,7 +905,6 @@ impl<'a, 'tcx> Decodable> for DefId { // which means that the definition with this hash is guaranteed to // still exist in the current compilation session. Ok(d.tcx() - .queries .on_disk_cache .as_ref() .unwrap() @@ -973,7 +959,7 @@ impl<'a, 'tcx> Decodable> for &'tcx [Span] { //- ENCODING ------------------------------------------------------------------- -trait OpaqueEncoder: Encoder { +pub trait OpaqueEncoder: Encoder { fn position(&self) -> usize; } @@ -985,7 +971,7 @@ impl OpaqueEncoder for FileEncoder { } /// An encoder that can write to the incremental compilation cache. -struct CacheEncoder<'a, 'tcx, E: OpaqueEncoder> { +pub struct CacheEncoder<'a, 'tcx, E: OpaqueEncoder> { tcx: TyCtxt<'tcx>, encoder: &'a mut E, type_shorthands: FxHashMap, usize>, @@ -1230,24 +1216,24 @@ impl<'a> Decodable> for IntEncodedWithFixedSize { } } -fn encode_query_results<'a, 'tcx, Q>( - tcx: TyCtxt<'tcx>, +pub fn encode_query_results<'a, 'tcx, CTX, Q>( + tcx: CTX, encoder: &mut CacheEncoder<'a, 'tcx, FileEncoder>, query_result_index: &mut EncodedQueryResultIndex, ) -> FileEncodeResult where - Q: super::QueryDescription> + super::QueryAccessors>, + CTX: QueryContext + 'tcx, + Q: super::QueryDescription + super::QueryAccessors, Q::Value: Encodable>, { let _timer = tcx - .sess - .prof + .dep_context() + .profiler() .extra_verbose_generic_activity("encode_query_results_for", std::any::type_name::()); - let state = Q::query_state(tcx); - assert!(state.all_inactive()); - - state.iter_results(|results| { + assert!(Q::query_state(tcx).all_inactive()); + let cache = Q::query_cache(tcx); + cache.iter_results(|results| { for (key, value, dep_node) in results { if Q::cache_on_disk(tcx, &key, Some(value)) { let dep_node = SerializedDepNodeIndex::new(dep_node.index()); diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs index 46addcdaead43..53bac4181e67b 100644 --- a/compiler/rustc_middle/src/ty/query/plumbing.rs +++ b/compiler/rustc_middle/src/ty/query/plumbing.rs @@ -2,21 +2,48 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::DepGraph; -use crate::ty::query::Query; +use crate::dep_graph::{self, DepKind, DepNode, DepNodeExt, DepNodeIndex, SerializedDepNodeIndex}; +use crate::ty::query::{on_disk_cache, queries, Queries, Query}; use crate::ty::tls::{self, ImplicitCtxt}; use crate::ty::{self, TyCtxt}; -use rustc_query_system::query::QueryContext; +use rustc_query_system::dep_graph::HasDepContext; use rustc_query_system::query::{CycleError, QueryJobId, QueryJobInfo}; +use rustc_query_system::query::{QueryContext, QueryDescription}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lock; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, Handler, Level}; -use rustc_span::def_id::DefId; +use rustc_serialize::opaque; +use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::Span; -impl QueryContext for TyCtxt<'tcx> { +#[derive(Copy, Clone)] +pub struct QueryCtxt<'tcx> { + pub tcx: TyCtxt<'tcx>, + pub queries: &'tcx super::Queries<'tcx>, +} + +impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> { + type Target = TyCtxt<'tcx>; + + fn deref(&self) -> &Self::Target { + &self.tcx + } +} + +impl HasDepContext for QueryCtxt<'tcx> { + type DepKind = crate::dep_graph::DepKind; + type StableHashingContext = crate::ich::StableHashingContext<'tcx>; + type DepContext = TyCtxt<'tcx>; + + #[inline] + fn dep_context(&self) -> &Self::DepContext { + &self.tcx + } +} + +impl QueryContext for QueryCtxt<'tcx> { type Query = Query<'tcx>; fn incremental_verify_ich(&self) -> bool { @@ -27,15 +54,11 @@ impl QueryContext for TyCtxt<'tcx> { } fn def_path_str(&self, def_id: DefId) -> String { - TyCtxt::def_path_str(*self, def_id) - } - - fn dep_graph(&self) -> &DepGraph { - &self.dep_graph + self.tcx.def_path_str(def_id) } fn current_query_job(&self) -> Option> { - tls::with_related_context(*self, |icx| icx.query) + tls::with_related_context(**self, |icx| icx.query) } fn try_collect_active_jobs( @@ -45,6 +68,101 @@ impl QueryContext for TyCtxt<'tcx> { self.queries.try_collect_active_jobs() } + fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) { + let cb = &super::QUERY_CALLBACKS[dep_node.kind as usize]; + (cb.try_load_from_on_disk_cache)(*self, dep_node) + } + + fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool { + // FIXME: This match is just a workaround for incremental bugs and should + // be removed. https://github.com/rust-lang/rust/issues/62649 is one such + // bug that must be fixed before removing this. + match dep_node.kind { + DepKind::hir_owner | DepKind::hir_owner_nodes => { + if let Some(def_id) = dep_node.extract_def_id(**self) { + let def_id = def_id.expect_local(); + let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id); + if def_id != hir_id.owner { + // This `DefPath` does not have a + // corresponding `DepNode` (e.g. a + // struct field), and the ` DefPath` + // collided with the `DefPath` of a + // proper item that existed in the + // previous compilation session. + // + // Since the given `DefPath` does not + // denote the item that previously + // existed, we just fail to mark green. + return false; + } + } else { + // If the node does not exist anymore, we + // just fail to mark green. + return false; + } + } + _ => { + // For other kinds of nodes it's OK to be + // forced. + } + } + + debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node); + + // We must avoid ever having to call `force_from_dep_node()` for a + // `DepNode::codegen_unit`: + // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we + // would always end up having to evaluate the first caller of the + // `codegen_unit` query that *is* reconstructible. This might very well be + // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just + // to re-trigger calling the `codegen_unit` query with the right key. At + // that point we would already have re-done all the work we are trying to + // avoid doing in the first place. + // The solution is simple: Just explicitly call the `codegen_unit` query for + // each CGU, right after partitioning. This way `try_mark_green` will always + // hit the cache instead of having to go through `force_from_dep_node`. + // This assertion makes sure, we actually keep applying the solution above. + debug_assert!( + dep_node.kind != DepKind::codegen_unit, + "calling force_from_dep_node() on DepKind::codegen_unit" + ); + + let cb = &super::QUERY_CALLBACKS[dep_node.kind as usize]; + (cb.force_from_dep_node)(*self, dep_node) + } + + fn has_errors_or_delayed_span_bugs(&self) -> bool { + self.sess.has_errors_or_delayed_span_bugs() + } + + fn diagnostic(&self) -> &rustc_errors::Handler { + self.sess.diagnostic() + } + + // Interactions with on_disk_cache + fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec { + self.on_disk_cache + .as_ref() + .map(|c| c.load_diagnostics(**self, prev_dep_node_index)) + .unwrap_or_default() + } + + fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec) { + if let Some(c) = self.on_disk_cache.as_ref() { + c.store_diagnostics(dep_node_index, diagnostics) + } + } + + fn store_diagnostics_for_anon_node( + &self, + dep_node_index: DepNodeIndex, + diagnostics: ThinVec, + ) { + if let Some(c) = self.on_disk_cache.as_ref() { + c.store_diagnostics_for_anon_node(dep_node_index, diagnostics) + } + } + /// Executes a job by changing the `ImplicitCtxt` to point to the /// new query job while it executes. It returns the diagnostics /// captured during execution and the actual result. @@ -53,15 +171,15 @@ impl QueryContext for TyCtxt<'tcx> { &self, token: QueryJobId, diagnostics: Option<&Lock>>, - compute: impl FnOnce(Self) -> R, + compute: impl FnOnce() -> R, ) -> R { // The `TyCtxt` stored in TLS has the same global interner lifetime // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes // when accessing the `ImplicitCtxt`. - tls::with_related_context(*self, move |current_icx| { + tls::with_related_context(**self, move |current_icx| { // Update the `ImplicitCtxt` to point to our new query job. let new_icx = ImplicitCtxt { - tcx: *self, + tcx: **self, query: Some(token), diagnostics, layout_depth: current_icx.layout_depth, @@ -70,13 +188,13 @@ impl QueryContext for TyCtxt<'tcx> { // Use the `ImplicitCtxt` while we execute the query. tls::enter_context(&new_icx, |_| { - rustc_data_structures::stack::ensure_sufficient_stack(|| compute(*self)) + rustc_data_structures::stack::ensure_sufficient_stack(compute) }) }) } } -impl<'tcx> TyCtxt<'tcx> { +impl<'tcx> QueryCtxt<'tcx> { #[inline(never)] #[cold] pub(super) fn report_cycle( @@ -86,7 +204,7 @@ impl<'tcx> TyCtxt<'tcx> { assert!(!stack.is_empty()); let fix_span = |span: Span, query: &Query<'tcx>| { - self.sess.source_map().guess_head_span(query.default_span(self, span)) + self.sess.source_map().guess_head_span(query.default_span(*self, span)) }; // Disable naming impls with types in this path, since that @@ -125,56 +243,126 @@ impl<'tcx> TyCtxt<'tcx> { }) } - pub fn try_print_query_stack(handler: &Handler, num_frames: Option) { - eprintln!("query stack during panic:"); + pub(super) fn encode_query_results( + self, + encoder: &mut on_disk_cache::CacheEncoder<'a, 'tcx, opaque::FileEncoder>, + query_result_index: &mut on_disk_cache::EncodedQueryResultIndex, + ) -> opaque::FileEncodeResult { + macro_rules! encode_queries { + ($($query:ident,)*) => { + $( + on_disk_cache::encode_query_results::<_, ty::query::queries::$query<'_>>( + self, + encoder, + query_result_index + )?; + )* + } + } - // Be careful relying on global state here: this code is called from - // a panic hook, which means that the global `Handler` may be in a weird - // state if it was responsible for triggering the panic. - let mut i = 0; - ty::tls::with_context_opt(|icx| { - if let Some(icx) = icx { - let query_map = icx.tcx.queries.try_collect_active_jobs(); + rustc_cached_queries!(encode_queries!); - let mut current_query = icx.query; + Ok(()) + } +} - while let Some(query) = current_query { - if Some(i) == num_frames { - break; - } - let query_info = - if let Some(info) = query_map.as_ref().and_then(|map| map.get(&query)) { - info - } else { - break; - }; - let mut diag = Diagnostic::new( - Level::FailureNote, - &format!( - "#{} [{}] {}", - i, - query_info.info.query.name(), - query_info.info.query.describe(icx.tcx) - ), - ); - diag.span = - icx.tcx.sess.source_map().guess_head_span(query_info.info.span).into(); - handler.force_print_diagnostic(diag); - - current_query = query_info.job.parent; - i += 1; - } +impl<'tcx> Queries<'tcx> { + pub fn try_print_query_stack( + &'tcx self, + tcx: TyCtxt<'tcx>, + query: Option>, + handler: &Handler, + num_frames: Option, + ) -> usize { + let query_map = self.try_collect_active_jobs(); + + let mut current_query = query; + let mut i = 0; + + while let Some(query) = current_query { + if Some(i) == num_frames { + break; } - }); + let query_info = if let Some(info) = query_map.as_ref().and_then(|map| map.get(&query)) + { + info + } else { + break; + }; + let mut diag = Diagnostic::new( + Level::FailureNote, + &format!( + "#{} [{}] {}", + i, + query_info.info.query.name(), + query_info.info.query.describe(QueryCtxt { tcx, queries: self }) + ), + ); + diag.span = tcx.sess.source_map().guess_head_span(query_info.info.span).into(); + handler.force_print_diagnostic(diag); - if num_frames == None || num_frames >= Some(i) { - eprintln!("end of query stack"); - } else { - eprintln!("we're just showing a limited slice of the query stack"); + current_query = query_info.job.parent; + i += 1; } + + i } } +/// This struct stores metadata about each Query. +/// +/// Information is retrieved by indexing the `QUERIES` array using the integer value +/// of the `DepKind`. Overall, this allows to implement `QueryContext` using this manual +/// jump table instead of large matches. +pub struct QueryStruct { + /// The red/green evaluation system will try to mark a specific DepNode in the + /// dependency graph as green by recursively trying to mark the dependencies of + /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode` + /// where we don't know if it is red or green and we therefore actually have + /// to recompute its value in order to find out. Since the only piece of + /// information that we have at that point is the `DepNode` we are trying to + /// re-evaluate, we need some way to re-run a query from just that. This is what + /// `force_from_dep_node()` implements. + /// + /// In the general case, a `DepNode` consists of a `DepKind` and an opaque + /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint + /// is usually constructed by computing a stable hash of the query-key that the + /// `DepNode` corresponds to. Consequently, it is not in general possible to go + /// back from hash to query-key (since hash functions are not reversible). For + /// this reason `force_from_dep_node()` is expected to fail from time to time + /// because we just cannot find out, from the `DepNode` alone, what the + /// corresponding query-key is and therefore cannot re-run the query. + /// + /// The system deals with this case letting `try_mark_green` fail which forces + /// the root query to be re-evaluated. + /// + /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless. + /// Fortunately, we can use some contextual information that will allow us to + /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we + /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a + /// valid `DefPathHash`. Since we also always build a huge table that maps every + /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have + /// everything we need to re-run the query. + /// + /// Take the `mir_promoted` query as an example. Like many other queries, it + /// just has a single parameter: the `DefId` of the item it will compute the + /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode` + /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode` + /// is actually a `DefPathHash`, and can therefore just look up the corresponding + /// `DefId` in `tcx.def_path_hash_to_def_id`. + /// + /// When you implement a new query, it will likely have a corresponding new + /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As + /// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter, + /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just + /// add it to the "We don't have enough information to reconstruct..." group in + /// the match below. + pub(crate) force_from_dep_node: fn(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool, + + /// Invoke a query to put the on-disk cached value in memory. + pub(crate) try_load_from_on_disk_cache: fn(QueryCtxt<'_>, &DepNode), +} + macro_rules! handle_cycle_error { ([][$tcx: expr, $error:expr]) => {{ $tcx.report_cycle($error).emit(); @@ -220,7 +408,7 @@ macro_rules! is_eval_always { macro_rules! query_storage { ([][$K:ty, $V:ty]) => { - <<$K as Key>::CacheSelector as CacheSelector<$K, $V>>::Cache + >::Cache }; ([storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => { <$ty as CacheSelector<$K, $V>>::Cache @@ -242,11 +430,6 @@ macro_rules! hash_result { }; } -macro_rules! query_helper_param_ty { - (DefId) => { impl IntoQueryParam }; - ($K:ty) => { $K }; -} - macro_rules! define_queries { (<$tcx:tt> $($(#[$attr:meta])* @@ -277,7 +460,7 @@ macro_rules! define_queries { } } - pub fn describe(&self, tcx: TyCtxt<$tcx>) -> String { + pub(crate) fn describe(&self, tcx: QueryCtxt<$tcx>) -> String { let (r, name) = match *self { $(Query::$name(key) => { (queries::$name::describe(tcx, key), stringify!($name)) @@ -324,49 +507,34 @@ macro_rules! define_queries { })* } - // HACK(eddyb) this is like the `impl QueryConfig for queries::$name` - // below, but using type aliases instead of associated types, to bypass - // the limitations around normalizing under HRTB - for example, this: - // `for<'tcx> fn(...) -> as QueryConfig>>::Value` - // doesn't currently normalize to `for<'tcx> fn(...) -> query_values::$name<'tcx>`. - // This is primarily used by the `provide!` macro in `rustc_metadata`. - #[allow(nonstandard_style, unused_lifetimes)] - pub mod query_keys { - use super::*; - - $(pub type $name<$tcx> = $($K)*;)* - } - #[allow(nonstandard_style, unused_lifetimes)] - pub mod query_values { - use super::*; - - $(pub type $name<$tcx> = $V;)* - } - $(impl<$tcx> QueryConfig for queries::$name<$tcx> { type Key = $($K)*; type Value = $V; - type Stored = < - query_storage!([$($modifiers)*][$($K)*, $V]) - as QueryStorage - >::Stored; + type Stored = query_stored::$name<$tcx>; const NAME: &'static str = stringify!($name); } - impl<$tcx> QueryAccessors> for queries::$name<$tcx> { + impl<$tcx> QueryAccessors> for queries::$name<$tcx> { const ANON: bool = is_anon!([$($modifiers)*]); const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]); const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$name; - type Cache = query_storage!([$($modifiers)*][$($K)*, $V]); + type Cache = query_storage::$name<$tcx>; #[inline(always)] - fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState as QueryContext>::Query, Self::Cache> { + fn query_state<'a>(tcx: QueryCtxt<$tcx>) -> &'a QueryState, Self::Key> { &tcx.queries.$name } + #[inline(always)] + fn query_cache<'a>(tcx: QueryCtxt<$tcx>) -> &'a QueryCacheStore + where 'tcx:'a + { + &tcx.query_caches.$name + } + #[inline] - fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { + fn compute(tcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { let provider = tcx.queries.providers.get(key.query_crate()) // HACK(eddyb) it's possible crates may be loaded after // the query engine is created, and because crate loading @@ -374,7 +542,7 @@ macro_rules! define_queries { // would be missing appropriate entries in `providers`. .unwrap_or(&tcx.queries.fallback_extern_providers) .$name; - provider(tcx, key) + provider(*tcx, key) } fn hash_result( @@ -385,118 +553,95 @@ macro_rules! define_queries { } fn handle_cycle_error( - tcx: TyCtxt<'tcx>, + tcx: QueryCtxt<'tcx>, error: CycleError> ) -> Self::Value { handle_cycle_error!([$($modifiers)*][tcx, error]) } })* - #[derive(Copy, Clone)] - pub struct TyCtxtEnsure<'tcx> { - pub tcx: TyCtxt<'tcx>, - } + #[allow(non_upper_case_globals)] + pub mod query_callbacks { + use super::*; + use crate::dep_graph::DepNode; + use crate::ty::query::{queries, query_keys}; + use rustc_query_system::dep_graph::DepNodeParams; + use rustc_query_system::query::{force_query, QueryDescription}; + + // We use this for most things when incr. comp. is turned off. + pub const Null: QueryStruct = QueryStruct { + force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node), + try_load_from_on_disk_cache: |_, _| {}, + }; - impl TyCtxtEnsure<$tcx> { - $($(#[$attr])* - #[inline(always)] - pub fn $name(self, key: query_helper_param_ty!($($K)*)) { - ensure_query::, _>(self.tcx, key.into_query_param()) - })* - } + pub const TraitSelect: QueryStruct = QueryStruct { + force_from_dep_node: |_, _| false, + try_load_from_on_disk_cache: |_, _| {}, + }; - #[derive(Copy, Clone)] - pub struct TyCtxtAt<'tcx> { - pub tcx: TyCtxt<'tcx>, - pub span: Span, - } + pub const CompileCodegenUnit: QueryStruct = QueryStruct { + force_from_dep_node: |_, _| false, + try_load_from_on_disk_cache: |_, _| {}, + }; - impl Deref for TyCtxtAt<'tcx> { - type Target = TyCtxt<'tcx>; - #[inline(always)] - fn deref(&self) -> &Self::Target { - &self.tcx - } - } + $(pub const $name: QueryStruct = { + const is_anon: bool = is_anon!([$($modifiers)*]); - impl TyCtxt<$tcx> { - /// Returns a transparent wrapper for `TyCtxt`, which ensures queries - /// are executed instead of just returning their results. - #[inline(always)] - pub fn ensure(self) -> TyCtxtEnsure<$tcx> { - TyCtxtEnsure { - tcx: self, + #[inline(always)] + fn can_reconstruct_query_key() -> bool { + as DepNodeParams>> + ::can_reconstruct_query_key() } - } - /// Returns a transparent wrapper for `TyCtxt` which uses - /// `span` as the location of queries performed through it. - #[inline(always)] - pub fn at(self, span: Span) -> TyCtxtAt<$tcx> { - TyCtxtAt { - tcx: self, - span + fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option> { + as DepNodeParams>>::recover(tcx, dep_node) } - } - $($(#[$attr])* - #[inline(always)] - #[must_use] - pub fn $name(self, key: query_helper_param_ty!($($K)*)) - -> as QueryConfig>::Stored - { - self.at(DUMMY_SP).$name(key.into_query_param()) - })* + fn force_from_dep_node(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool { + if is_anon { + return false; + } - /// All self-profiling events generated by the query engine use - /// virtual `StringId`s for their `event_id`. This method makes all - /// those virtual `StringId`s point to actual strings. - /// - /// If we are recording only summary data, the ids will point to - /// just the query names. If we are recording query keys too, we - /// allocate the corresponding strings here. - pub fn alloc_self_profile_query_strings(self) { - use crate::ty::query::profiling_support::{ - alloc_self_profile_query_strings_for_query_cache, - QueryKeyStringCache, - }; + if !can_reconstruct_query_key() { + return false; + } - if !self.prof.enabled() { - return; + if let Some(key) = recover(*tcx, dep_node) { + force_query::, _>(tcx, key, DUMMY_SP, *dep_node); + return true; + } + + false } - let mut string_cache = QueryKeyStringCache::new(); + fn try_load_from_on_disk_cache(tcx: QueryCtxt<'_>, dep_node: &DepNode) { + if is_anon { + return + } - $({ - alloc_self_profile_query_strings_for_query_cache( - self, - stringify!($name), - &self.queries.$name, - &mut string_cache, - ); - })* - } - } + if !can_reconstruct_query_key() { + return + } - impl TyCtxtAt<$tcx> { - $($(#[$attr])* - #[inline(always)] - pub fn $name(self, key: query_helper_param_ty!($($K)*)) - -> as QueryConfig>::Stored - { - get_query::, _>(self.tcx, self.span, key.into_query_param()) - })* - } + debug_assert!(tcx.dep_graph + .node_color(dep_node) + .map(|c| c.is_green()) + .unwrap_or(false)); - define_provider_struct! { - tcx: $tcx, - input: ($(([$($modifiers)*] [$name] [$($K)*] [$V]))*) - } + let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)); + if queries::$name::cache_on_disk(tcx, &key, None) { + let _ = tcx.$name(key); + } + } - impl Copy for Providers {} - impl Clone for Providers { - fn clone(&self) -> Self { *self } + QueryStruct { + force_from_dep_node, + try_load_from_on_disk_cache, + } + };)* } + + static QUERY_CALLBACKS: &[QueryStruct] = &make_dep_kind_array!(query_callbacks); } } @@ -507,44 +652,36 @@ macro_rules! define_queries_struct { (tcx: $tcx:tt, input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { pub struct Queries<$tcx> { - /// This provides access to the incremental compilation on-disk cache for query results. - /// Do not access this directly. It is only meant to be used by - /// `DepGraph::try_mark_green()` and the query infrastructure. - /// This is `None` if we are not incremental compilation mode - pub(crate) on_disk_cache: Option>, - providers: IndexVec, fallback_extern_providers: Box, $($(#[$attr])* $name: QueryState< crate::dep_graph::DepKind, - as QueryContext>::Query, - as QueryAccessors>>::Cache, + Query<$tcx>, + query_keys::$name<$tcx>, >,)* } impl<$tcx> Queries<$tcx> { - pub(crate) fn new( + pub fn new( providers: IndexVec, fallback_extern_providers: Providers, - on_disk_cache: Option>, ) -> Self { Queries { providers, fallback_extern_providers: Box::new(fallback_extern_providers), - on_disk_cache, $($name: Default::default()),* } } pub(crate) fn try_collect_active_jobs( &self - ) -> Option, QueryJobInfo as QueryContext>::Query>>> { + ) -> Option, QueryJobInfo>>> { let mut jobs = FxHashMap::default(); $( self.$name.try_collect_active_jobs( - as QueryAccessors>>::DEP_KIND, + as QueryAccessors>>::DEP_KIND, Query::$name, &mut jobs, )?; @@ -552,25 +689,51 @@ macro_rules! define_queries_struct { Some(jobs) } - } - }; -} -macro_rules! define_provider_struct { - (tcx: $tcx:tt, - input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => { - pub struct Providers { - $(pub $name: for<$tcx> fn(TyCtxt<$tcx>, $K) -> $R,)* - } + #[cfg(parallel_compiler)] + pub unsafe fn deadlock(&'tcx self, tcx: TyCtxt<'tcx>, registry: &rustc_rayon_core::Registry) { + let tcx = QueryCtxt { tcx, queries: self }; + rustc_query_system::query::deadlock(tcx, registry) + } + + pub(crate) fn encode_query_results( + &'tcx self, + tcx: TyCtxt<'tcx>, + encoder: &mut on_disk_cache::CacheEncoder<'a, 'tcx, opaque::FileEncoder>, + query_result_index: &mut on_disk_cache::EncodedQueryResultIndex, + ) -> opaque::FileEncodeResult { + let tcx = QueryCtxt { tcx, queries: self }; + tcx.encode_query_results(encoder, query_result_index) + } - impl Default for Providers { - fn default() -> Self { - $(fn $name<$tcx>(_: TyCtxt<$tcx>, key: $K) -> $R { - bug!("`tcx.{}({:?})` unsupported by its crate", - stringify!($name), key); - })* - Providers { $($name),* } + fn exec_cache_promotions(&'tcx self, tcx: TyCtxt<'tcx>) { + let tcx = QueryCtxt { tcx, queries: self }; + tcx.dep_graph.exec_cache_promotions(tcx) } + + $($(#[$attr])* + #[inline(always)] + fn $name( + &'tcx self, + tcx: TyCtxt<$tcx>, + span: Span, + key: query_keys::$name<$tcx>, + lookup: QueryLookup, + mode: QueryMode, + ) -> Option> { + let qcx = QueryCtxt { tcx, queries: self }; + get_query::, _>(qcx, span, key, lookup, mode) + })* } }; } + +fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String { + if def_id.is_top_level_module() { + "top-level module".to_string() + } else { + format!("module `{}`", tcx.def_path_str(def_id.to_def_id())) + } +} + +rustc_query_description! {} diff --git a/compiler/rustc_middle/src/ty/query/profiling_support.rs b/compiler/rustc_middle/src/ty/query/profiling_support.rs index cbcecb8849188..aab5fd9e8a194 100644 --- a/compiler/rustc_middle/src/ty/query/profiling_support.rs +++ b/compiler/rustc_middle/src/ty/query/profiling_support.rs @@ -5,28 +5,28 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::SelfProfiler; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::definitions::DefPathData; -use rustc_query_system::query::{QueryCache, QueryContext, QueryState}; +use rustc_query_system::query::{QueryCache, QueryCacheStore}; use std::fmt::Debug; use std::io::Write; -pub struct QueryKeyStringCache { +struct QueryKeyStringCache { def_id_cache: FxHashMap, } impl QueryKeyStringCache { - pub fn new() -> QueryKeyStringCache { + fn new() -> QueryKeyStringCache { QueryKeyStringCache { def_id_cache: Default::default() } } } -pub struct QueryKeyStringBuilder<'p, 'c, 'tcx> { +struct QueryKeyStringBuilder<'p, 'c, 'tcx> { profiler: &'p SelfProfiler, tcx: TyCtxt<'tcx>, string_cache: &'c mut QueryKeyStringCache, } impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> { - pub fn new( + fn new( profiler: &'p SelfProfiler, tcx: TyCtxt<'tcx>, string_cache: &'c mut QueryKeyStringCache, @@ -98,7 +98,7 @@ impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> { } } -pub trait IntoSelfProfilingString { +trait IntoSelfProfilingString { fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId; } @@ -123,7 +123,7 @@ impl IntoSelfProfilingString for T { } #[rustc_specialization_trait] -pub trait SpecIntoSelfProfilingString: Debug { +trait SpecIntoSelfProfilingString: Debug { fn spec_to_self_profile_string( &self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>, @@ -227,10 +227,10 @@ where /// Allocate the self-profiling query strings for a single query cache. This /// method is called from `alloc_self_profile_query_strings` which knows all /// the queries via macro magic. -pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>( +fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>( tcx: TyCtxt<'tcx>, query_name: &'static str, - query_state: &QueryState as QueryContext>::Query, C>, + query_cache: &QueryCacheStore, string_cache: &mut QueryKeyStringCache, ) where C: QueryCache, @@ -251,7 +251,7 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>( // need to invoke queries itself, we cannot keep the query caches // locked while doing so. Instead we copy out the // `(query_key, dep_node_index)` pairs and release the lock again. - let query_keys_and_indices: Vec<_> = query_state + let query_keys_and_indices: Vec<_> = query_cache .iter_results(|results| results.map(|(k, _, i)| (k.clone(), i)).collect()); // Now actually allocate the strings. If allocating the strings @@ -276,7 +276,7 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>( let query_name = profiler.get_or_alloc_cached_string(query_name); let event_id = event_id_builder.from_label(query_name).to_string_id(); - query_state.iter_results(|results| { + query_cache.iter_results(|results| { let query_invocation_ids: Vec<_> = results.map(|v| v.2.into()).collect(); profiler.bulk_map_query_invocation_id_to_single_string( @@ -287,3 +287,35 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>( } }); } + +/// All self-profiling events generated by the query engine use +/// virtual `StringId`s for their `event_id`. This method makes all +/// those virtual `StringId`s point to actual strings. +/// +/// If we are recording only summary data, the ids will point to +/// just the query names. If we are recording query keys too, we +/// allocate the corresponding strings here. +pub fn alloc_self_profile_query_strings(tcx: TyCtxt<'tcx>) { + if !tcx.prof.enabled() { + return; + } + + let mut string_cache = QueryKeyStringCache::new(); + + macro_rules! alloc_once { + (<$tcx:tt> + $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($K:ty) -> $V:ty,)* + ) => { + $({ + alloc_self_profile_query_strings_for_query_cache( + tcx, + stringify!($name), + &tcx.query_caches.$name, + &mut string_cache, + ); + })* + } + } + + rustc_query_append! { [alloc_once!][<'tcx>] } +} diff --git a/compiler/rustc_middle/src/ty/query/stats.rs b/compiler/rustc_middle/src/ty/query/stats.rs index e0b44ce23c912..f6fe4f1563490 100644 --- a/compiler/rustc_middle/src/ty/query/stats.rs +++ b/compiler/rustc_middle/src/ty/query/stats.rs @@ -1,10 +1,9 @@ -use crate::ty::query::queries; +use crate::ty::query::query_storage; use crate::ty::TyCtxt; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_query_system::query::{QueryAccessors, QueryCache, QueryContext, QueryState}; +use rustc_query_system::query::{QueryCache, QueryCacheStore}; use std::any::type_name; -use std::hash::Hash; use std::mem; #[cfg(debug_assertions)] use std::sync::atomic::Ordering; @@ -37,10 +36,8 @@ struct QueryStats { local_def_id_keys: Option, } -fn stats(name: &'static str, map: &QueryState) -> QueryStats +fn stats(name: &'static str, map: &QueryCacheStore) -> QueryStats where - D: Copy + Clone + Eq + Hash, - Q: Clone, C: QueryCache, { let mut stats = QueryStats { @@ -128,12 +125,10 @@ macro_rules! print_stats { $( queries.push(stats::< - crate::dep_graph::DepKind, - as QueryContext>::Query, - as QueryAccessors>>::Cache, + query_storage::$name<'_>, >( stringify!($name), - &tcx.queries.$name, + &tcx.query_caches.$name, )); )* diff --git a/compiler/rustc_middle/src/ty/query/values.rs b/compiler/rustc_middle/src/ty/query/values.rs index f28b0f499f082..fa15395db4e9b 100644 --- a/compiler/rustc_middle/src/ty/query/values.rs +++ b/compiler/rustc_middle/src/ty/query/values.rs @@ -1,18 +1,19 @@ -use crate::ty::{self, AdtSizedConstraint, Ty, TyCtxt, TyS}; +use crate::ty::query::QueryCtxt; +use crate::ty::{self, AdtSizedConstraint, Ty, TyS}; pub(super) trait Value<'tcx>: Sized { - fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self; + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self; } impl<'tcx, T> Value<'tcx> for T { - default fn from_cycle_error(tcx: TyCtxt<'tcx>) -> T { + default fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> T { tcx.sess.abort_if_errors(); bug!("Value::from_cycle_error called without errors"); } } impl<'tcx> Value<'tcx> for &'_ TyS<'_> { - fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self { + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self { // SAFETY: This is never called when `Self` is not `Ty<'tcx>`. // FIXME: Represent the above fact in the trait system somehow. unsafe { std::mem::transmute::, Ty<'_>>(tcx.ty_error()) } @@ -20,19 +21,19 @@ impl<'tcx> Value<'tcx> for &'_ TyS<'_> { } impl<'tcx> Value<'tcx> for ty::SymbolName<'_> { - fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self { + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self { // SAFETY: This is never called when `Self` is not `SymbolName<'tcx>`. // FIXME: Represent the above fact in the trait system somehow. unsafe { std::mem::transmute::, ty::SymbolName<'_>>(ty::SymbolName::new( - tcx, "", + *tcx, "", )) } } } impl<'tcx> Value<'tcx> for AdtSizedConstraint<'_> { - fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self { + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self { // SAFETY: This is never called when `Self` is not `AdtSizedConstraint<'tcx>`. // FIXME: Represent the above fact in the trait system somehow. unsafe { diff --git a/compiler/rustc_query_system/src/cache.rs b/compiler/rustc_query_system/src/cache.rs index be3d360772854..c6dc7b4fe2851 100644 --- a/compiler/rustc_query_system/src/cache.rs +++ b/compiler/rustc_query_system/src/cache.rs @@ -1,7 +1,6 @@ //! Cache for candidate selection. -use crate::dep_graph::DepNodeIndex; -use crate::query::QueryContext; +use crate::dep_graph::{DepContext, DepNodeIndex}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::HashMapExt; @@ -28,7 +27,7 @@ impl Cache { } impl Cache { - pub fn get(&self, key: &Key, tcx: CTX) -> Option { + pub fn get(&self, key: &Key, tcx: CTX) -> Option { Some(self.hashmap.borrow().get(key)?.get(tcx)) } @@ -55,7 +54,7 @@ impl WithDepNode { WithDepNode { dep_node, cached_value } } - pub fn get(&self, tcx: CTX) -> T { + pub fn get(&self, tcx: CTX) -> T { tcx.dep_graph().read_index(self.dep_node); self.cached_value.clone() } diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs index 64aba870502c7..1319a31b8f54d 100644 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs @@ -79,7 +79,7 @@ impl DepNode { pub fn construct(tcx: Ctxt, kind: K, arg: &Key) -> DepNode where - Ctxt: crate::query::QueryContext, + Ctxt: super::DepContext, Key: DepNodeParams, { let hash = arg.to_fingerprint(tcx); diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 4fb3a683ea225..6d7c29b229ade 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -23,7 +23,8 @@ use super::debug::EdgeFilter; use super::prev::PreviousDepGraph; use super::query::DepGraphQuery; use super::serialized::SerializedDepNodeIndex; -use super::{DepContext, DepKind, DepNode, WorkProductId}; +use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId}; +use crate::query::QueryContext; #[derive(Clone)] pub struct DepGraph { @@ -235,7 +236,7 @@ impl DepGraph { /// `arg` parameter. /// /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/incremental-compilation.html - pub fn with_task, A, R>( + pub fn with_task, A, R>( &self, key: DepNode, cx: Ctxt, @@ -261,7 +262,7 @@ impl DepGraph { ) } - fn with_task_impl, A, R>( + fn with_task_impl, A, R>( &self, key: DepNode, cx: Ctxt, @@ -271,14 +272,15 @@ impl DepGraph { hash_result: impl FnOnce(&mut Ctxt::StableHashingContext, &R) -> Option, ) -> (R, DepNodeIndex) { if let Some(ref data) = self.data { + let dcx = cx.dep_context(); let task_deps = create_task(key).map(Lock::new); let result = K::with_deps(task_deps.as_ref(), || task(cx, arg)); let edges = task_deps.map_or_else(|| smallvec![], |lock| lock.into_inner().reads); - let mut hcx = cx.create_stable_hashing_context(); + let mut hcx = dcx.create_stable_hashing_context(); let current_fingerprint = hash_result(&mut hcx, &result); - let print_status = cfg!(debug_assertions) && cx.debug_dep_tasks(); + let print_status = cfg!(debug_assertions) && dcx.debug_dep_tasks(); // Intern the new `DepNode`. let dep_node_index = if let Some(prev_index) = data.previous.node_to_index_opt(&key) { @@ -408,7 +410,7 @@ impl DepGraph { /// Executes something within an "eval-always" task which is a task /// that runs whenever anything changes. - pub fn with_eval_always_task, A, R>( + pub fn with_eval_always_task, A, R>( &self, key: DepNode, cx: Ctxt, @@ -585,7 +587,7 @@ impl DepGraph { /// A node will have an index, when it's already been marked green, or when we can mark it /// green. This function will mark the current task as a reader of the specified node, when /// a node index can be found for that node. - pub fn try_mark_green_and_read>( + pub fn try_mark_green_and_read>( &self, tcx: Ctxt, dep_node: &DepNode, @@ -597,7 +599,7 @@ impl DepGraph { }) } - pub fn try_mark_green>( + pub fn try_mark_green>( &self, tcx: Ctxt, dep_node: &DepNode, @@ -625,7 +627,7 @@ impl DepGraph { } /// Try to mark a dep-node which existed in the previous compilation session as green. - fn try_mark_previous_green>( + fn try_mark_previous_green>( &self, tcx: Ctxt, data: &DepGraphData, @@ -809,7 +811,7 @@ impl DepGraph { /// This may be called concurrently on multiple threads for the same dep node. #[cold] #[inline(never)] - fn emit_diagnostics>( + fn emit_diagnostics>( &self, tcx: Ctxt, data: &DepGraphData, @@ -874,7 +876,8 @@ impl DepGraph { // // This method will only load queries that will end up in the disk cache. // Other queries will not be executed. - pub fn exec_cache_promotions>(&self, tcx: Ctxt) { + pub fn exec_cache_promotions>(&self, qcx: Ctxt) { + let tcx = qcx.dep_context(); let _prof_timer = tcx.profiler().generic_activity("incr_comp_query_cache_promotion"); let data = self.data.as_ref().unwrap(); @@ -882,7 +885,7 @@ impl DepGraph { match data.colors.get(prev_index) { Some(DepNodeColor::Green(_)) => { let dep_node = data.previous.index_to_node(prev_index); - tcx.try_load_from_on_disk_cache(&dep_node); + qcx.try_load_from_on_disk_cache(&dep_node); } None | Some(DepNodeColor::Red) => { // We can skip red nodes because a node can only be marked diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs index b1c901633a71b..a647381fb03fa 100644 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ b/compiler/rustc_query_system/src/dep_graph/mod.rs @@ -13,8 +13,6 @@ pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::Lock; -use rustc_data_structures::thin_vec::ThinVec; -use rustc_errors::Diagnostic; use std::fmt; use std::hash::Hash; @@ -29,35 +27,34 @@ pub trait DepContext: Copy { fn debug_dep_tasks(&self) -> bool; fn debug_dep_node(&self) -> bool; - /// Try to force a dep node to execute and see if it's green. - fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool; + /// Access the DepGraph. + fn dep_graph(&self) -> &DepGraph; fn register_reused_dep_node(&self, dep_node: &DepNode); - /// Return whether the current session is tainted by errors. - fn has_errors_or_delayed_span_bugs(&self) -> bool; - - /// Return the diagnostic handler. - fn diagnostic(&self) -> &rustc_errors::Handler; - - /// Load data from the on-disk cache. - fn try_load_from_on_disk_cache(&self, dep_node: &DepNode); + /// Access the profiler. + fn profiler(&self) -> &SelfProfilerRef; +} - /// Load diagnostics associated to the node in the previous session. - fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec; +pub trait HasDepContext: Copy { + type DepKind: self::DepKind; + type StableHashingContext; + type DepContext: self::DepContext< + DepKind = Self::DepKind, + StableHashingContext = Self::StableHashingContext, + >; - /// Register diagnostics for the given node, for use in next session. - fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec); + fn dep_context(&self) -> &Self::DepContext; +} - /// Register diagnostics for the given node, for use in next session. - fn store_diagnostics_for_anon_node( - &self, - dep_node_index: DepNodeIndex, - diagnostics: ThinVec, - ); +impl HasDepContext for T { + type DepKind = T::DepKind; + type StableHashingContext = T::StableHashingContext; + type DepContext = Self; - /// Access the profiler. - fn profiler(&self) -> &SelfProfilerRef; + fn dep_context(&self) -> &Self::DepContext { + self + } } /// Describe the different families of dependency nodes. diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 1d2bc1a99a596..ec71c8685804f 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -1,5 +1,5 @@ use crate::dep_graph::DepNodeIndex; -use crate::query::plumbing::{QueryLookup, QueryState}; +use crate::query::plumbing::{QueryCacheStore, QueryLookup}; use rustc_arena::TypedArena; use rustc_data_structures::fx::FxHashMap; @@ -31,17 +31,15 @@ pub trait QueryCache: QueryStorage { /// It returns the shard index and a lock guard to the shard, /// which will be used if the query is not in the cache and we need /// to compute it. - fn lookup( + fn lookup<'s, R, OnHit>( &self, - state: &QueryState, - key: Self::Key, + state: &'s QueryCacheStore, + key: &Self::Key, // `on_hit` can be called while holding a lock to the query state shard. on_hit: OnHit, - on_miss: OnMiss, - ) -> R + ) -> Result where - OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R, - OnMiss: FnOnce(Self::Key, QueryLookup<'_, D, Q, Self::Key, Self::Sharded>) -> R; + OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R; fn complete( &self, @@ -95,23 +93,24 @@ where type Sharded = FxHashMap; #[inline(always)] - fn lookup( + fn lookup<'s, R, OnHit>( &self, - state: &QueryState, - key: K, + state: &'s QueryCacheStore, + key: &K, on_hit: OnHit, - on_miss: OnMiss, - ) -> R + ) -> Result where OnHit: FnOnce(&V, DepNodeIndex) -> R, - OnMiss: FnOnce(K, QueryLookup<'_, D, Q, K, Self::Sharded>) -> R, { - let mut lookup = state.get_lookup(&key); - let lock = &mut *lookup.lock; + let (lookup, lock) = state.get_lookup(key); + let result = lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key); - let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); - - if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) } + if let Some((_, value)) = result { + let hit_result = on_hit(&value.0, value.1); + Ok(hit_result) + } else { + Err(lookup) + } } #[inline] @@ -177,26 +176,23 @@ where type Sharded = FxHashMap; #[inline(always)] - fn lookup( + fn lookup<'s, R, OnHit>( &self, - state: &QueryState, - key: K, + state: &'s QueryCacheStore, + key: &K, on_hit: OnHit, - on_miss: OnMiss, - ) -> R + ) -> Result where OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R, - OnMiss: FnOnce(K, QueryLookup<'_, D, Q, K, Self::Sharded>) -> R, { - let mut lookup = state.get_lookup(&key); - let lock = &mut *lookup.lock; - - let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); + let (lookup, lock) = state.get_lookup(key); + let result = lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key); if let Some((_, value)) = result { - on_hit(&&value.0, value.1) + let hit_result = on_hit(&&value.0, value.1); + Ok(hit_result) } else { - on_miss(key, lookup) + Err(lookup) } } diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 94e906fc433d5..3873b47d4d40b 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -4,7 +4,7 @@ use crate::dep_graph::DepNode; use crate::dep_graph::SerializedDepNodeIndex; use crate::query::caches::QueryCache; use crate::query::plumbing::CycleError; -use crate::query::{QueryContext, QueryState}; +use crate::query::{QueryCacheStore, QueryContext, QueryState}; use rustc_data_structures::fingerprint::Fingerprint; use std::fmt::Debug; @@ -33,9 +33,9 @@ pub(crate) struct QueryVtable { } impl QueryVtable { - pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode + pub(crate) fn to_dep_node(&self, tcx: CTX::DepContext, key: &K) -> DepNode where - K: crate::dep_graph::DepNodeParams, + K: crate::dep_graph::DepNodeParams, { DepNode::construct(tcx, self.dep_kind, key) } @@ -73,14 +73,12 @@ pub trait QueryAccessors: QueryConfig { type Cache: QueryCache; // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_state<'a>(tcx: CTX) -> &'a QueryState; + fn query_state<'a>(tcx: CTX) -> &'a QueryState; - fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode + // Don't use this method to access query results, instead use the methods on TyCtxt + fn query_cache<'a>(tcx: CTX) -> &'a QueryCacheStore where - Self::Key: crate::dep_graph::DepNodeParams, - { - DepNode::construct(tcx, Self::DEP_KIND, key) - } + CTX: 'a; // Don't use this method to compute query results, instead use the methods on TyCtxt fn compute(tcx: CTX, key: Self::Key) -> Self::Value; diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index 5fed500390b68..0ecc2694a7906 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -10,7 +10,8 @@ use std::num::NonZeroU32; #[cfg(parallel_compiler)] use { - super::QueryContext, + crate::dep_graph::DepContext, + crate::query::QueryContext, parking_lot::{Condvar, Mutex}, rustc_data_structures::fx::FxHashSet, rustc_data_structures::stable_hasher::{HashStable, StableHasher}, @@ -432,7 +433,7 @@ where { // Deterministically pick an entry point // FIXME: Sort this instead - let mut hcx = tcx.create_stable_hashing_context(); + let mut hcx = tcx.dep_context().create_stable_hashing_context(); queries .iter() .min_by_key(|v| { diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index da45565dbe6bd..c935e1b9c5cd6 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -14,7 +14,7 @@ pub use self::caches::{ mod config; pub use self::config::{QueryAccessors, QueryConfig, QueryDescription}; -use crate::dep_graph::{DepContext, DepGraph}; +use crate::dep_graph::{DepNode, DepNodeIndex, HasDepContext, SerializedDepNodeIndex}; use crate::query::job::QueryMap; use rustc_data_structures::stable_hasher::HashStable; @@ -23,7 +23,7 @@ use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::Diagnostic; use rustc_span::def_id::DefId; -pub trait QueryContext: DepContext { +pub trait QueryContext: HasDepContext { type Query: Clone + HashStable; fn incremental_verify_ich(&self) -> bool; @@ -32,14 +32,36 @@ pub trait QueryContext: DepContext { /// Get string representation from DefPath. fn def_path_str(&self, def_id: DefId) -> String; - /// Access the DepGraph. - fn dep_graph(&self) -> &DepGraph; - /// Get the query information from the TLS context. fn current_query_job(&self) -> Option>; fn try_collect_active_jobs(&self) -> Option>; + /// Load data from the on-disk cache. + fn try_load_from_on_disk_cache(&self, dep_node: &DepNode); + + /// Try to force a dep node to execute and see if it's green. + fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool; + + /// Return whether the current session is tainted by errors. + fn has_errors_or_delayed_span_bugs(&self) -> bool; + + /// Return the diagnostic handler. + fn diagnostic(&self) -> &rustc_errors::Handler; + + /// Load diagnostics associated to the node in the previous session. + fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec; + + /// Register diagnostics for the given node, for use in next session. + fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec); + + /// Register diagnostics for the given node, for use in next session. + fn store_diagnostics_for_anon_node( + &self, + dep_node_index: DepNodeIndex, + diagnostics: ThinVec, + ); + /// Executes a job by changing the `ImplicitCtxt` to point to the /// new query job while it executes. It returns the diagnostics /// captured during execution and the actual result. @@ -47,6 +69,6 @@ pub trait QueryContext: DepContext { &self, token: QueryJobId, diagnostics: Option<&Lock>>, - compute: impl FnOnce(Self) -> R, + compute: impl FnOnce() -> R, ) -> R; } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index cbbb449b4f8ab..bfa62a5b045da 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -2,7 +2,7 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::{DepKind, DepNode}; +use crate::dep_graph::{DepContext, DepKind, DepNode}; use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use crate::query::caches::QueryCache; use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt}; @@ -13,11 +13,10 @@ use crate::query::{QueryContext, QueryMap}; use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHasher}; -use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::sharded::{get_shard_index_by_hash, Sharded}; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; -use rustc_span::source_map::DUMMY_SP; use rustc_span::Span; use std::collections::hash_map::Entry; use std::fmt::Debug; @@ -28,45 +27,77 @@ use std::ptr; #[cfg(debug_assertions)] use std::sync::atomic::{AtomicUsize, Ordering}; -pub(super) struct QueryStateShard { - pub(super) cache: C, - active: FxHashMap>, - - /// Used to generate unique ids for active jobs. - jobs: u32, +pub struct QueryCacheStore { + cache: C, + shards: Sharded, + #[cfg(debug_assertions)] + pub cache_hits: AtomicUsize, } -impl Default for QueryStateShard { - fn default() -> QueryStateShard { - QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 } +impl Default for QueryCacheStore { + fn default() -> Self { + Self { + cache: C::default(), + shards: Default::default(), + #[cfg(debug_assertions)] + cache_hits: AtomicUsize::new(0), + } } } -pub struct QueryState { - cache: C, - shards: Sharded>, - #[cfg(debug_assertions)] - pub cache_hits: AtomicUsize, +/// Values used when checking a query cache which can be reused on a cache-miss to execute the query. +pub struct QueryLookup { + pub(super) key_hash: u64, + shard: usize, } -impl QueryState { +// We compute the key's hash once and then use it for both the +// shard lookup and the hashmap lookup. This relies on the fact +// that both of them use `FxHasher`. +fn hash_for_shard(key: &K) -> u64 { + let mut hasher = FxHasher::default(); + key.hash(&mut hasher); + hasher.finish() +} + +impl QueryCacheStore { pub(super) fn get_lookup<'tcx>( &'tcx self, key: &C::Key, - ) -> QueryLookup<'tcx, D, Q, C::Key, C::Sharded> { - // We compute the key's hash once and then use it for both the - // shard lookup and the hashmap lookup. This relies on the fact - // that both of them use `FxHasher`. - let mut hasher = FxHasher::default(); - key.hash(&mut hasher); - let key_hash = hasher.finish(); - - let shard = self.shards.get_shard_index_by_hash(key_hash); + ) -> (QueryLookup, LockGuard<'tcx, C::Sharded>) { + let key_hash = hash_for_shard(key); + let shard = get_shard_index_by_hash(key_hash); let lock = self.shards.get_shard_by_index(shard).lock(); - QueryLookup { key_hash, shard, lock } + (QueryLookup { key_hash, shard }, lock) + } + + pub fn iter_results( + &self, + f: impl for<'a> FnOnce( + Box + 'a>, + ) -> R, + ) -> R { + self.cache.iter(&self.shards, |shard| &mut *shard, f) } } +struct QueryStateShard { + active: FxHashMap>, + + /// Used to generate unique ids for active jobs. + jobs: u32, +} + +impl Default for QueryStateShard { + fn default() -> QueryStateShard { + QueryStateShard { active: Default::default(), jobs: 0 } + } +} + +pub struct QueryState { + shards: Sharded>, +} + /// Indicates the state of a query for a given key in a query map. enum QueryResult { /// An already executing query. The query job can be used to await for its completion. @@ -77,21 +108,12 @@ enum QueryResult { Poisoned, } -impl QueryState +impl QueryState where D: Copy + Clone + Eq + Hash, Q: Clone, - C: QueryCache, + K: Eq + Hash + Clone + Debug, { - pub fn iter_results( - &self, - f: impl for<'a> FnOnce( - Box + 'a>, - ) -> R, - ) -> R { - self.cache.iter(&self.shards, |shard| &mut shard.cache, f) - } - pub fn all_inactive(&self) -> bool { let shards = self.shards.lock_shards(); shards.iter().all(|shard| shard.active.is_empty()) @@ -100,7 +122,7 @@ where pub fn try_collect_active_jobs( &self, kind: D, - make_query: fn(C::Key) -> Q, + make_query: fn(K) -> Q, jobs: &mut QueryMap, ) -> Option<()> { // We use try_lock_shards here since we are called from the @@ -123,24 +145,12 @@ where } } -impl Default for QueryState { - fn default() -> QueryState { - QueryState { - cache: C::default(), - shards: Default::default(), - #[cfg(debug_assertions)] - cache_hits: AtomicUsize::new(0), - } +impl Default for QueryState { + fn default() -> QueryState { + QueryState { shards: Default::default() } } } -/// Values used when checking a query cache which can be reused on a cache-miss to execute the query. -pub struct QueryLookup<'tcx, D, Q, K, C> { - pub(super) key_hash: u64, - shard: usize, - pub(super) lock: LockGuard<'tcx, QueryStateShard>, -} - /// A type representing the responsibility to execute the job in the `job` field. /// This will poison the relevant query if dropped. struct JobOwner<'tcx, D, Q, C> @@ -149,7 +159,8 @@ where Q: Clone, C: QueryCache, { - state: &'tcx QueryState, + state: &'tcx QueryState, + cache: &'tcx QueryCacheStore, key: C::Key, id: QueryJobId, } @@ -169,18 +180,21 @@ where /// This function is inlined because that results in a noticeable speed-up /// for some compile-time benchmarks. #[inline(always)] - fn try_start<'a, 'b, CTX>( + fn try_start<'b, CTX>( tcx: CTX, - state: &'b QueryState, + state: &'b QueryState, + cache: &'b QueryCacheStore, span: Span, key: &C::Key, - mut lookup: QueryLookup<'a, CTX::DepKind, CTX::Query, C::Key, C::Sharded>, + lookup: QueryLookup, query: &QueryVtable, ) -> TryGetJob<'b, CTX::DepKind, CTX::Query, C> where CTX: QueryContext, { - let lock = &mut *lookup.lock; + let shard = lookup.shard; + let mut state_lock = state.shards.get_shard_by_index(shard).lock(); + let lock = &mut *state_lock; let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) { Entry::Occupied(mut entry) => { @@ -190,13 +204,13 @@ where // in another thread has completed. Record how long we wait in the // self-profiler. let _query_blocked_prof_timer = if cfg!(parallel_compiler) { - Some(tcx.profiler().query_blocked()) + Some(tcx.dep_context().profiler().query_blocked()) } else { None }; // Create the id of the job we're waiting for - let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind); + let id = QueryJobId::new(job.id, shard, query.dep_kind); (job.latch(id), _query_blocked_prof_timer) } @@ -211,18 +225,18 @@ where lock.jobs = id; let id = QueryShardJobId(NonZeroU32::new(id).unwrap()); - let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind); + let global_id = QueryJobId::new(id, shard, query.dep_kind); let job = tcx.current_query_job(); let job = QueryJob::new(id, span, job); entry.insert(QueryResult::Started(job)); - let owner = JobOwner { state, id: global_id, key: (*key).clone() }; + let owner = JobOwner { state, cache, id: global_id, key: (*key).clone() }; return TryGetJob::NotYetStarted(owner); } }; - mem::drop(lookup.lock); + mem::drop(state_lock); // If we are single-threaded we know that we have cycle error, // so we just return the error. @@ -234,7 +248,7 @@ where span, ); let value = query.handle_cycle_error(tcx, error); - state.cache.store_nocache(value) + cache.cache.store_nocache(value) })); // With parallel queries we might just have to wait on some other @@ -245,17 +259,23 @@ where if let Err(cycle) = result { let value = query.handle_cycle_error(tcx, cycle); - let value = state.cache.store_nocache(value); + let value = cache.cache.store_nocache(value); return TryGetJob::Cycle(value); } - let cached = try_get_cached( - tcx, - state, - (*key).clone(), - |value, index| (value.clone(), index), - |_, _| panic!("value must be in cache after waiting"), - ); + let cached = cache + .cache + .lookup(cache, &key, |value, index| { + if unlikely!(tcx.dep_context().profiler().enabled()) { + tcx.dep_context().profiler().query_cache_hit(index.into()); + } + #[cfg(debug_assertions)] + { + cache.cache_hits.fetch_add(1, Ordering::Relaxed); + } + (value.clone(), index) + }) + .unwrap_or_else(|_| panic!("value must be in cache after waiting")); if let Some(prof_timer) = _query_blocked_prof_timer.take() { prof_timer.finish_with_query_invocation_id(cached.1.into()); @@ -271,17 +291,25 @@ where // We can move out of `self` here because we `mem::forget` it below let key = unsafe { ptr::read(&self.key) }; let state = self.state; + let cache = self.cache; // Forget ourself so our destructor won't poison the query mem::forget(self); let (job, result) = { - let mut lock = state.shards.get_shard_by_value(&key).lock(); - let job = match lock.active.remove(&key).unwrap() { - QueryResult::Started(job) => job, - QueryResult::Poisoned => panic!(), + let key_hash = hash_for_shard(&key); + let shard = get_shard_index_by_hash(key_hash); + let job = { + let mut lock = state.shards.get_shard_by_index(shard).lock(); + match lock.active.remove(&key).unwrap() { + QueryResult::Started(job) => job, + QueryResult::Poisoned => panic!(), + } + }; + let result = { + let mut lock = cache.shards.get_shard_by_index(shard).lock(); + cache.cache.complete(&mut lock, key, result, dep_node_index) }; - let result = state.cache.complete(&mut lock.cache, key, result, dep_node_index); (job, result) }; @@ -357,81 +385,78 @@ where /// It returns the shard index and a lock guard to the shard, /// which will be used if the query is not in the cache and we need /// to compute it. -fn try_get_cached( +pub fn try_get_cached<'a, CTX, C, R, OnHit>( tcx: CTX, - state: &QueryState, - key: C::Key, + cache: &'a QueryCacheStore, + key: &C::Key, // `on_hit` can be called while holding a lock to the query cache on_hit: OnHit, - on_miss: OnMiss, -) -> R +) -> Result where C: QueryCache, - CTX: QueryContext, - OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R, - OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>) -> R, + CTX: DepContext, + OnHit: FnOnce(&C::Stored) -> R, { - state.cache.lookup( - state, - key, - |value, index| { - if unlikely!(tcx.profiler().enabled()) { - tcx.profiler().query_cache_hit(index.into()); - } - #[cfg(debug_assertions)] - { - state.cache_hits.fetch_add(1, Ordering::Relaxed); - } - on_hit(value, index) - }, - on_miss, - ) + cache.cache.lookup(cache, &key, |value, index| { + if unlikely!(tcx.profiler().enabled()) { + tcx.profiler().query_cache_hit(index.into()); + } + #[cfg(debug_assertions)] + { + cache.cache_hits.fetch_add(1, Ordering::Relaxed); + } + tcx.dep_graph().read_index(index); + on_hit(value) + }) } fn try_execute_query( tcx: CTX, - state: &QueryState, + state: &QueryState, + cache: &QueryCacheStore, span: Span, key: C::Key, - lookup: QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>, + lookup: QueryLookup, query: &QueryVtable, ) -> C::Stored where C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + C::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( - tcx, state, span, &key, lookup, query, + tcx, state, cache, span, &key, lookup, query, ) { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(result) => return result, #[cfg(parallel_compiler)] TryGetJob::JobCompleted((v, index)) => { - tcx.dep_graph().read_index(index); + tcx.dep_context().dep_graph().read_index(index); return v; } }; // Fast path for when incr. comp. is off. `to_dep_node` is // expensive for some `DepKind`s. - if !tcx.dep_graph().is_fully_enabled() { + if !tcx.dep_context().dep_graph().is_fully_enabled() { let null_dep_node = DepNode::new_no_params(DepKind::NULL); return force_query_with_job(tcx, key, job, null_dep_node, query).0; } if query.anon { - let prof_timer = tcx.profiler().query_provider(); + let prof_timer = tcx.dep_context().profiler().query_provider(); let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job.id, diagnostics, |tcx| { - tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key)) + tcx.start_query(job.id, diagnostics, || { + tcx.dep_context() + .dep_graph() + .with_anon_task(query.dep_kind, || query.compute(tcx, key)) }) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); - tcx.dep_graph().read_index(dep_node_index); + tcx.dep_context().dep_graph().read_index(dep_node_index); if unlikely!(!diagnostics.is_empty()) { tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); @@ -440,14 +465,14 @@ where return job.complete(result, dep_node_index); } - let dep_node = query.to_dep_node(tcx, &key); + let dep_node = query.to_dep_node(*tcx.dep_context(), &key); if !query.eval_always { // The diagnostics for this query will be // promoted to the current session during // `try_mark_green()`, so we can ignore them here. - let loaded = tcx.start_query(job.id, None, |tcx| { - let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node); + let loaded = tcx.start_query(job.id, None, || { + let marked = tcx.dep_context().dep_graph().try_mark_green_and_read(tcx, &dep_node); marked.map(|(prev_dep_node_index, dep_node_index)| { ( load_from_disk_and_cache_in_memory( @@ -468,7 +493,7 @@ where } let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query); - tcx.dep_graph().read_index(dep_node_index); + tcx.dep_context().dep_graph().read_index(dep_node_index); result } @@ -486,11 +511,11 @@ where // Note this function can be called concurrently from the same query // We must ensure that this is handled correctly. - debug_assert!(tcx.dep_graph().is_green(dep_node)); + debug_assert!(tcx.dep_context().dep_graph().is_green(dep_node)); // First we try to load the result from the on-disk cache. let result = if query.cache_on_disk(tcx, &key, None) { - let prof_timer = tcx.profiler().incr_cache_loading(); + let prof_timer = tcx.dep_context().profiler().incr_cache_loading(); let result = query.try_load_from_disk(tcx, prev_dep_node_index); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -512,10 +537,10 @@ where } else { // We could not load a result from the on-disk cache, so // recompute. - let prof_timer = tcx.profiler().query_provider(); + let prof_timer = tcx.dep_context().profiler().query_provider(); // The dep-graph for this computation is already in-place. - let result = tcx.dep_graph().with_ignore(|| query.compute(tcx, key)); + let result = tcx.dep_context().dep_graph().with_ignore(|| query.compute(tcx, key)); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -525,7 +550,7 @@ where // If `-Zincremental-verify-ich` is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. if unlikely!(tcx.incremental_verify_ich()) { - incremental_verify_ich(tcx, &result, dep_node, dep_node_index, query); + incremental_verify_ich(*tcx.dep_context(), &result, dep_node, dep_node_index, query); } result @@ -534,7 +559,7 @@ where #[inline(never)] #[cold] fn incremental_verify_ich( - tcx: CTX, + tcx: CTX::DepContext, result: &V, dep_node: &DepNode, dep_node_index: DepNodeIndex, @@ -577,7 +602,7 @@ where // 2. Two distinct query keys get mapped to the same `DepNode` // (see for example #48923). assert!( - !tcx.dep_graph().dep_node_exists(&dep_node), + !tcx.dep_context().dep_graph().dep_node_exists(&dep_node), "forcing query with already existing `DepNode`\n\ - query-key: {:?}\n\ - dep-node: {:?}", @@ -585,12 +610,12 @@ where dep_node ); - let prof_timer = tcx.profiler().query_provider(); + let prof_timer = tcx.dep_context().profiler().query_provider(); let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job.id, diagnostics, |tcx| { + tcx.start_query(job.id, diagnostics, || { if query.eval_always { - tcx.dep_graph().with_eval_always_task( + tcx.dep_context().dep_graph().with_eval_always_task( dep_node, tcx, key, @@ -598,7 +623,13 @@ where query.hash_result, ) } else { - tcx.dep_graph().with_task(dep_node, tcx, key, query.compute, query.hash_result) + tcx.dep_context().dep_graph().with_task( + dep_node, + tcx, + key, + query.compute, + query.hash_result, + ) } }) }); @@ -617,57 +648,45 @@ where #[inline(never)] fn get_query_impl( tcx: CTX, - state: &QueryState, + state: &QueryState, + cache: &QueryCacheStore, span: Span, key: C::Key, + lookup: QueryLookup, query: &QueryVtable, ) -> C::Stored where CTX: QueryContext, C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + C::Key: crate::dep_graph::DepNodeParams, { - try_get_cached( - tcx, - state, - key, - |value, index| { - tcx.dep_graph().read_index(index); - value.clone() - }, - |key, lookup| try_execute_query(tcx, state, span, key, lookup, query), - ) + try_execute_query(tcx, state, cache, span, key, lookup, query) } /// Ensure that either this query has all green inputs or been executed. /// Executing `query::ensure(D)` is considered a read of the dep-node `D`. +/// Returns true if the query should still run. /// /// This function is particularly useful when executing passes for their /// side-effects -- e.g., in order to report errors for erroneous programs. /// /// Note: The optimization is only available during incr. comp. #[inline(never)] -fn ensure_query_impl( - tcx: CTX, - state: &QueryState, - key: C::Key, - query: &QueryVtable, -) where - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, +fn ensure_must_run(tcx: CTX, key: &K, query: &QueryVtable) -> bool +where + K: crate::dep_graph::DepNodeParams, CTX: QueryContext, { if query.eval_always { - let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); - return; + return true; } // Ensuring an anonymous query makes no sense assert!(!query.anon); - let dep_node = query.to_dep_node(tcx, &key); + let dep_node = query.to_dep_node(*tcx.dep_context(), key); - match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) { + match tcx.dep_context().dep_graph().try_mark_green_and_read(tcx, &dep_node) { None => { // A None return from `try_mark_green_and_read` means that this is either // a new dep node or that the dep node has already been marked red. @@ -675,10 +694,11 @@ fn ensure_query_impl( // DepNodeIndex. We must invoke the query itself. The performance cost // this introduces should be negligible as we'll immediately hit the // in-memory cache, or another query down the line will. - let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); + true } Some((_, dep_node_index)) => { - tcx.profiler().query_cache_hit(dep_node_index.into()); + tcx.dep_context().profiler().query_cache_hit(dep_node_index.into()); + false } } } @@ -686,65 +706,80 @@ fn ensure_query_impl( #[inline(never)] fn force_query_impl( tcx: CTX, - state: &QueryState, + state: &QueryState, + cache: &QueryCacheStore, key: C::Key, span: Span, dep_node: DepNode, query: &QueryVtable, ) where C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + C::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. + let cached = cache.cache.lookup(cache, &key, |_, index| { + if unlikely!(tcx.dep_context().profiler().enabled()) { + tcx.dep_context().profiler().query_cache_hit(index.into()); + } + #[cfg(debug_assertions)] + { + cache.cache_hits.fetch_add(1, Ordering::Relaxed); + } + }); - try_get_cached( - tcx, - state, - key, - |_, _| { - // Cache hit, do nothing - }, - |key, lookup| { - let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( - tcx, state, span, &key, lookup, query, - ) { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) => return, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted(_) => return, - }; - force_query_with_job(tcx, key, job, dep_node, query); - }, - ); -} + let lookup = match cached { + Ok(()) => return, + Err(lookup) => lookup, + }; -pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored -where - Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, - CTX: QueryContext, -{ - debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); + let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( + tcx, state, cache, span, &key, lookup, query, + ) { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(_) => return, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted(_) => return, + }; + force_query_with_job(tcx, key, job, dep_node, query); +} - get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE) +pub enum QueryMode { + Get, + Ensure, } -pub fn ensure_query(tcx: CTX, key: Q::Key) +pub fn get_query( + tcx: CTX, + span: Span, + key: Q::Key, + lookup: QueryLookup, + mode: QueryMode, +) -> Option where Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, + Q::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { - ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE) + let query = &Q::VTABLE; + if let QueryMode::Ensure = mode { + if !ensure_must_run(tcx, &key, query) { + return None; + } + } + + debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); + let value = + get_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), span, key, lookup, query); + Some(value) } pub fn force_query(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode) where Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, + Q::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { - force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE) + force_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), key, span, dep_node, &Q::VTABLE) } diff --git a/src/tools/clippy/src/driver.rs b/src/tools/clippy/src/driver.rs index f5f6c09ed8e94..d5143e1438ee0 100644 --- a/src/tools/clippy/src/driver.rs +++ b/src/tools/clippy/src/driver.rs @@ -11,10 +11,8 @@ extern crate rustc_driver; extern crate rustc_errors; extern crate rustc_interface; -extern crate rustc_middle; use rustc_interface::interface; -use rustc_middle::ty::TyCtxt; use rustc_tools_util::VersionInfo; use std::borrow::Cow; @@ -168,7 +166,7 @@ fn report_clippy_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) { let num_frames = if backtrace { None } else { Some(2) }; - TyCtxt::try_print_query_stack(&handler, num_frames); + interface::try_print_query_stack(&handler, num_frames); } fn toolchain_path(home: Option, toolchain: Option) -> Option {