diff --git a/Cargo.lock b/Cargo.lock index 325503a0e956..98ca2d85b91d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3883,6 +3883,7 @@ dependencies = [ "rustc_plugin_impl", "rustc_privacy", "rustc_query_impl", + "rustc_query_system", "rustc_resolve", "rustc_serialize", "rustc_session", diff --git a/compiler/rustc_incremental/src/assert_dep_graph.rs b/compiler/rustc_incremental/src/assert_dep_graph.rs index 60b48e9bc8a0..5689d8349aa6 100644 --- a/compiler/rustc_incremental/src/assert_dep_graph.rs +++ b/compiler/rustc_incremental/src/assert_dep_graph.rs @@ -41,7 +41,7 @@ use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_hir::intravisit::{self, Visitor}; use rustc_middle::dep_graph::{ - DepGraphQuery, DepKind, DepNode, DepNodeExt, DepNodeFilter, EdgeFilter, + dep_kind, DepGraphQuery, DepNode, DepNodeExt, DepNodeFilter, EdgeFilter, }; use rustc_middle::hir::nested_filter; use rustc_middle::ty::TyCtxt; @@ -128,7 +128,7 @@ impl<'tcx> IfThisChanged<'tcx> { let dep_node_interned = self.argument(attr); let dep_node = match dep_node_interned { None => { - DepNode::from_def_path_hash(self.tcx, def_path_hash, DepKind::hir_owner) + DepNode::from_def_path_hash(self.tcx, def_path_hash, dep_kind::hir_owner) } Some(n) => { match DepNode::from_label_string(self.tcx, n.as_str(), def_path_hash) { diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index 29d1cd0e0546..e349ad50d3f5 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -43,6 +43,7 @@ rustc_lint = { path = "../rustc_lint" } rustc_errors = { path = "../rustc_errors" } rustc_plugin_impl = { path = "../rustc_plugin_impl" } rustc_privacy = { path = "../rustc_privacy" } +rustc_query_system = { path = "../rustc_query_system" } rustc_query_impl = { path = "../rustc_query_impl" } rustc_resolve = { path = "../rustc_resolve" } rustc_trait_selection = { path = "../rustc_trait_selection" } diff --git a/compiler/rustc_interface/src/callbacks.rs b/compiler/rustc_interface/src/callbacks.rs index a18e2d1d6388..b442d999dfce 100644 --- a/compiler/rustc_interface/src/callbacks.rs +++ b/compiler/rustc_interface/src/callbacks.rs @@ -9,8 +9,9 @@ //! The functions in this file should fall back to the default set in their //! origin crate when the `TyCtxt` is not present in TLS. -use rustc_errors::{Diagnostic, TRACK_DIAGNOSTICS}; +use rustc_middle::dep_graph::DepNodeExt; use rustc_middle::ty::tls; +use rustc_query_system::dep_graph::{DepKind, DepNode}; use std::fmt; fn track_span_parent(def_id: rustc_span::def_id::LocalDefId) { @@ -23,20 +24,6 @@ fn track_span_parent(def_id: rustc_span::def_id::LocalDefId) { }) } -/// This is a callback from `rustc_ast` as it cannot access the implicit state -/// in `rustc_middle` otherwise. It is used to when diagnostic messages are -/// emitted and stores them in the current query, if there is one. -fn track_diagnostic(diagnostic: &Diagnostic) { - tls::with_context_opt(|icx| { - if let Some(icx) = icx { - if let Some(diagnostics) = icx.diagnostics { - let mut diagnostics = diagnostics.lock(); - diagnostics.extend(Some(diagnostic.clone())); - } - } - }) -} - /// This is a callback from `rustc_hir` as it cannot access the implicit state /// in `rustc_middle` otherwise. fn def_id_debug(def_id: rustc_hir::def_id::DefId, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -50,10 +37,42 @@ fn def_id_debug(def_id: rustc_hir::def_id::DefId, f: &mut fmt::Formatter<'_>) -> write!(f, ")") } +fn dep_kind_debug(kind: &DepKind, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + tls::with_opt(|opt_tcx| { + if let Some(tcx) = opt_tcx { + write!(f, "{}", tcx.query_name(*kind)) + } else { + write!(f, "DepKind({:?})", kind.index()) + } + }) +} + +fn dep_node_debug(node: &DepNode, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + tls::with_opt(|opt_tcx| { + if let Some(tcx) = opt_tcx { + write!(f, "{}(", tcx.query_name(node.kind))?; + + if let Some(def_id) = node.extract_def_id(tcx) { + write!(f, "{}", tcx.def_path_debug_str(def_id))?; + } else if let Some(ref s) = tcx.dep_graph.dep_node_debug_str(*node) { + write!(f, "{}", s)?; + } else { + write!(f, "{}", node.hash)?; + } + + write!(f, ")") + } else { + write!(f, "DepKind({:?})({})", node.kind.index(), node.hash) + } + }) +} + /// Sets up the callbacks in prior crates which we want to refer to the /// TyCtxt in. pub fn setup_callbacks() { rustc_span::SPAN_TRACK.swap(&(track_span_parent as fn(_))); rustc_hir::def_id::DEF_ID_DEBUG.swap(&(def_id_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); - TRACK_DIAGNOSTICS.swap(&(track_diagnostic as fn(&_))); + rustc_query_system::dep_graph::KIND_DEBUG.swap(&(dep_kind_debug as _)); + rustc_query_system::dep_graph::NODE_DEBUG.swap(&(dep_node_debug as _)); + rustc_errors::TRACK_DIAGNOSTICS.swap(&(rustc_query_system::tls::track_diagnostic as fn(&_))); } diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index e7563933c88a..648142a9ea34 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -344,9 +344,9 @@ pub fn try_print_query_stack(handler: &Handler, num_frames: Option) { // Be careful relying on global state here: this code is called from // a panic hook, which means that the global `Handler` may be in a weird // state if it was responsible for triggering the panic. - let i = ty::tls::with_context_opt(|icx| { - if let Some(icx) = icx { - QueryCtxt::from_tcx(icx.tcx).try_print_query_stack(icx.query, handler, num_frames) + let i = ty::tls::with_opt(|tcx| { + if let Some(tcx) = tcx { + QueryCtxt::from_tcx(tcx).try_print_query_stack(handler, num_frames) } else { 0 } diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 2a01b677e33b..5f396deb037a 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -806,13 +806,12 @@ pub struct QueryContext<'tcx> { gcx: &'tcx GlobalCtxt<'tcx>, } -impl<'tcx> QueryContext<'tcx> { +impl<'gcx> QueryContext<'gcx> { pub fn enter(&mut self, f: F) -> R where - F: FnOnce(TyCtxt<'tcx>) -> R, + F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> R, { - let icx = ty::tls::ImplicitCtxt::new(self.gcx); - ty::tls::enter_context(&icx, |_| f(icx.tcx)) + ty::tls::enter_context(self.gcx, f) } } diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 3fa8017dc93c..e2df8ddc6be4 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -7,7 +7,7 @@ use rustc_data_structures::jobserver; use rustc_data_structures::sync::Lrc; use rustc_errors::registry::Registry; #[cfg(parallel_compiler)] -use rustc_middle::ty::tls; +use rustc_middle::ty::{tls, GlobalCtxt}; use rustc_parse::validate_attr; #[cfg(parallel_compiler)] use rustc_query_impl::QueryCtxt; @@ -166,19 +166,21 @@ pub fn run_in_thread_pool_with_globals R + Send, R: Send>( unsafe fn handle_deadlock() { let registry = rustc_rayon_core::Registry::current(); - let context = tls::get_tlv(); - assert!(context != 0); - rustc_data_structures::sync::assert_sync::>(); - let icx: &tls::ImplicitCtxt<'_, '_> = &*(context as *const tls::ImplicitCtxt<'_, '_>); + // We do not need to copy the query ImplicitCtxt since this deadlock handler is not part of a + // normal query invocation. let session_globals = rustc_span::with_session_globals(|sg| sg as *const _); let session_globals = &*session_globals; + + // Extend the lifetime of the GlobalCtxt so the new thread can know of it. + // The current thread will not free it, it is deadlocked. + let tcx: &'static GlobalCtxt<'static> = + tls::with(|tcx| &*(*tcx as *const GlobalCtxt<'_>).cast()); + thread::spawn(move || { - tls::enter_context(icx, |_| { - rustc_span::set_session_globals_then(session_globals, || { - tls::with(|tcx| QueryCtxt::from_tcx(tcx).deadlock(®istry)) - }) - }); + rustc_span::set_session_globals_then(session_globals, || { + tls::enter_context(tcx, |tcx| QueryCtxt::from_tcx(tcx).deadlock(®istry)) + }) }); } diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 63bf929fb863..ca294de473b7 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -50,8 +50,8 @@ macro_rules! provide_one { // External query providers call `crate_hash` in order to register a dependency // on the crate metadata. The exception is `crate_hash` itself, which obviously // doesn't need to do this (and can't, as it would cause a query cycle). - use rustc_middle::dep_graph::DepKind; - if DepKind::$name != DepKind::crate_hash && $tcx.dep_graph.is_fully_enabled() { + use rustc_middle::dep_graph::dep_kind; + if dep_kind::$name != dep_kind::crate_hash && $tcx.dep_graph.is_fully_enabled() { $tcx.ensure().crate_hash($def_id.krate); } diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index d20be0a34d2d..fc195fa68862 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -63,11 +63,8 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX}; use rustc_hir::definitions::DefPathHash; use rustc_hir::HirId; -use rustc_query_system::dep_graph::FingerprintStyle; +use rustc_query_system::dep_graph::{DepKind, DepNode, DepNodeParams, FingerprintStyle}; use rustc_span::symbol::Symbol; -use std::hash::Hash; - -pub use rustc_query_system::dep_graph::{DepContext, DepNodeParams}; /// This struct stores metadata about each DepKind. /// @@ -89,6 +86,9 @@ pub struct DepKindStruct { /// See [DepNodeParams] trait for the behaviour of each key type. pub fingerprint_style: FingerprintStyle, + /// Name of the query for pretty-printing. + pub label: &'static str, + /// The red/green evaluation system will try to mark a specific DepNode in the /// dependency graph as green by recursively trying to mark the dependencies of /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode` @@ -130,16 +130,21 @@ pub struct DepKindStruct { pub try_load_from_on_disk_cache: Option, DepNode)>, } -impl DepKind { +impl TyCtxt<'_> { #[inline(always)] - pub fn fingerprint_style(self, tcx: TyCtxt<'_>) -> FingerprintStyle { + crate fn query_fingerprint_style(self, dep_kind: DepKind) -> FingerprintStyle { // Only fetch the DepKindStruct once. - let data = tcx.query_kind(self); + let data = self.query_kind(dep_kind); if data.is_anon { return FingerprintStyle::Opaque; } data.fingerprint_style } + + #[inline(always)] + pub fn query_name(self, dep_kind: DepKind) -> &'static str { + self.query_kind(dep_kind).label + } } macro_rules! define_dep_nodes { @@ -151,38 +156,40 @@ macro_rules! define_dep_nodes { ) => ( #[macro_export] macro_rules! make_dep_kind_array { - ($mod:ident) => {[ $($mod::$variant()),* ]}; + ($mod:ident) => {[ $mod::NULL(), $($mod::$variant()),* ]}; } - /// This enum serves as an index into arrays built by `make_dep_kind_array`. - #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encodable, Decodable)] - #[allow(non_camel_case_types)] - pub enum DepKind { - $($variant),* + /// Definition of the `DepKind`s for the known queries. + /// The constants are indices in the arrays produced by `make_dep_kind_array`, + /// and should be kept in sync. + #[allow(non_upper_case_globals)] + pub mod dep_kind { + use super::DepKind; + + /// We use this for most things when incr. comp. is turned off. + pub const NULL: DepKind = DepKind::NULL; + $(pub const $variant: DepKind = DepKind::new(1 + ${index()});)* } fn dep_kind_from_label_string(label: &str) -> Result { match label { - $(stringify!($variant) => Ok(DepKind::$variant),)* + "NULL" => Ok(dep_kind::NULL), + $(stringify!($variant) => Ok(dep_kind::$variant),)* _ => Err(()), } } /// Contains variant => str representations for constructing /// DepNode groups for tests. - #[allow(dead_code, non_upper_case_globals)] + #[allow(non_upper_case_globals)] pub mod label_strs { - $( - pub const $variant: &str = stringify!($variant); - )* + pub const NULL: &str = "NULL"; + $(pub const $variant: &str = stringify!($variant);)* } ); } rustc_dep_node_append!([define_dep_nodes!][ <'tcx> - // We use this for most things when incr. comp. is turned off. - [] Null, - [anon] TraitSelect, // WARNING: if `Symbol` is changed, make sure you update `make_compile_codegen_unit` below. @@ -196,26 +203,15 @@ rustc_dep_node_append!([define_dep_nodes!][ <'tcx> // WARNING: `construct` is generic and does not know that `CompileCodegenUnit` takes `Symbol`s as keys. // Be very careful changing this type signature! crate fn make_compile_codegen_unit(tcx: TyCtxt<'_>, name: Symbol) -> DepNode { - DepNode::construct(tcx, DepKind::CompileCodegenUnit, &name) + DepNode::construct(tcx, dep_kind::CompileCodegenUnit, &name) } // WARNING: `construct` is generic and does not know that `CompileMonoItem` takes `MonoItem`s as keys. // Be very careful changing this type signature! crate fn make_compile_mono_item<'tcx>(tcx: TyCtxt<'tcx>, mono_item: &MonoItem<'tcx>) -> DepNode { - DepNode::construct(tcx, DepKind::CompileMonoItem, mono_item) + DepNode::construct(tcx, dep_kind::CompileMonoItem, mono_item) } -pub type DepNode = rustc_query_system::dep_graph::DepNode; - -// We keep a lot of `DepNode`s in memory during compilation. It's not -// required that their size stay the same, but we don't want to change -// it inadvertently. This assert just ensures we're aware of any change. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -static_assert_size!(DepNode, 18); - -#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] -static_assert_size!(DepNode, 24); - pub trait DepNodeExt: Sized { /// Construct a DepNode from the given DepKind and DefPathHash. This /// method will assert that the given DepKind actually requires a @@ -249,8 +245,9 @@ impl DepNodeExt for DepNode { /// Construct a DepNode from the given DepKind and DefPathHash. This /// method will assert that the given DepKind actually requires a /// single DefId/DefPathHash parameter. + #[inline] fn from_def_path_hash(tcx: TyCtxt<'_>, def_path_hash: DefPathHash, kind: DepKind) -> DepNode { - debug_assert!(kind.fingerprint_style(tcx) == FingerprintStyle::DefPathHash); + debug_assert!(tcx.query_fingerprint_style(kind) == FingerprintStyle::DefPathHash); DepNode { kind, hash: def_path_hash.0.into() } } @@ -264,8 +261,9 @@ impl DepNodeExt for DepNode { /// DepNode. Condition (2) might not be fulfilled if a DepNode /// refers to something from the previous compilation session that /// has been removed. + #[inline] fn extract_def_id<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option { - if self.kind.fingerprint_style(tcx) == FingerprintStyle::DefPathHash { + if tcx.query_fingerprint_style(self.kind) == FingerprintStyle::DefPathHash { Some(tcx.def_path_hash_to_def_id(DefPathHash(self.hash.into()), &mut || { panic!("Failed to extract DefId: {:?} {}", self.kind, self.hash) })) @@ -282,7 +280,7 @@ impl DepNodeExt for DepNode { ) -> Result { let kind = dep_kind_from_label_string(label)?; - match kind.fingerprint_style(tcx) { + match tcx.query_fingerprint_style(kind) { FingerprintStyle::Opaque => Err(()), FingerprintStyle::Unit => Ok(DepNode::new_no_params(tcx, kind)), FingerprintStyle::DefPathHash => { @@ -292,6 +290,7 @@ impl DepNodeExt for DepNode { } /// Used in testing + #[inline] fn has_label_string(label: &str) -> bool { dep_kind_from_label_string(label).is_ok() } diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs index 6bfd1b7ffab2..2869c51eb293 100644 --- a/compiler/rustc_middle/src/dep_graph/mod.rs +++ b/compiler/rustc_middle/src/dep_graph/mod.rs @@ -1,4 +1,4 @@ -use crate::ty::{self, TyCtxt}; +use crate::ty::TyCtxt; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_query_system::ich::StableHashingContext; use rustc_session::Session; @@ -7,69 +7,15 @@ use rustc_session::Session; mod dep_node; pub use rustc_query_system::dep_graph::{ - debug::DepNodeFilter, hash_result, DepContext, DepNodeColor, DepNodeIndex, - SerializedDepNodeIndex, WorkProduct, WorkProductId, + debug::DepNodeFilter, debug::EdgeFilter, hash_result, DepContext, DepGraph, DepGraphQuery, + DepKind, DepNode, DepNodeColor, DepNodeIndex, SerializedDepGraph, SerializedDepNodeIndex, + TaskDeps, WorkProduct, WorkProductId, }; -pub use dep_node::{label_strs, DepKind, DepKindStruct, DepNode, DepNodeExt}; +pub use dep_node::{dep_kind, label_strs, DepKindStruct, DepNodeExt}; crate use dep_node::{make_compile_codegen_unit, make_compile_mono_item}; -pub type DepGraph = rustc_query_system::dep_graph::DepGraph; -pub type TaskDeps = rustc_query_system::dep_graph::TaskDeps; -pub type TaskDepsRef<'a> = rustc_query_system::dep_graph::TaskDepsRef<'a, DepKind>; -pub type DepGraphQuery = rustc_query_system::dep_graph::DepGraphQuery; -pub type SerializedDepGraph = rustc_query_system::dep_graph::SerializedDepGraph; -pub type EdgeFilter = rustc_query_system::dep_graph::debug::EdgeFilter; - -impl rustc_query_system::dep_graph::DepKind for DepKind { - const NULL: Self = DepKind::Null; - - fn debug_node(node: &DepNode, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}(", node.kind)?; - - ty::tls::with_opt(|opt_tcx| { - if let Some(tcx) = opt_tcx { - if let Some(def_id) = node.extract_def_id(tcx) { - write!(f, "{}", tcx.def_path_debug_str(def_id))?; - } else if let Some(ref s) = tcx.dep_graph.dep_node_debug_str(*node) { - write!(f, "{}", s)?; - } else { - write!(f, "{}", node.hash)?; - } - } else { - write!(f, "{}", node.hash)?; - } - Ok(()) - })?; - - write!(f, ")") - } - - fn with_deps(task_deps: TaskDepsRef<'_>, op: OP) -> R - where - OP: FnOnce() -> R, - { - ty::tls::with_context(|icx| { - let icx = ty::tls::ImplicitCtxt { task_deps, ..icx.clone() }; - - ty::tls::enter_context(&icx, |_| op()) - }) - } - - fn read_deps(op: OP) - where - OP: for<'a> FnOnce(TaskDepsRef<'a>), - { - ty::tls::with_context_opt(|icx| { - let Some(icx) = icx else { return }; - op(icx.task_deps) - }) - } -} - impl<'tcx> DepContext for TyCtxt<'tcx> { - type DepKind = DepKind; - #[inline] fn create_stable_hashing_context(&self) -> StableHashingContext<'_> { TyCtxt::create_stable_hashing_context(*self) @@ -92,7 +38,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { #[inline(always)] fn fingerprint_style(&self, kind: DepKind) -> rustc_query_system::dep_graph::FingerprintStyle { - kind.fingerprint_style(*self) + self.query_fingerprint_style(kind) } #[inline(always)] @@ -117,8 +63,8 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { // hit the cache instead of having to go through `force_from_dep_node`. // This assertion makes sure, we actually keep applying the solution above. debug_assert!( - dep_node.kind != DepKind::codegen_unit, - "calling force_from_dep_node() on DepKind::codegen_unit" + dep_node.kind != dep_kind::codegen_unit, + "calling force_from_dep_node() on dep_kind::codegen_unit" ); let cb = self.query_kind(dep_node.kind); diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs index 65796fbc698d..8c512b29adf9 100644 --- a/compiler/rustc_middle/src/hir/map/mod.rs +++ b/compiler/rustc_middle/src/hir/map/mod.rs @@ -1156,18 +1156,8 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String { let id_str = format!(" (hir_id={})", id); let path_str = || { - // This functionality is used for debugging, try to use `TyCtxt` to get - // the user-friendly path, otherwise fall back to stringifying `DefPath`. - crate::ty::tls::with_opt(|tcx| { - if let Some(tcx) = tcx { - let def_id = map.local_def_id(id); - tcx.def_path_str(def_id.to_def_id()) - } else if let Some(path) = map.def_path_from_hir_id(id) { - path.data.into_iter().map(|elem| elem.to_string()).collect::>().join("::") - } else { - String::from("") - } - }) + let def_id = map.local_def_id(id); + map.tcx.def_path_str(def_id.to_def_id()) }; let span_str = || map.tcx.sess.source_map().span_to_snippet(map.span(id)).unwrap_or_default(); diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index fd2b5f5335f6..e424528cc1fa 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -43,6 +43,7 @@ #![feature(once_cell)] #![feature(let_chains)] #![feature(let_else)] +#![feature(macro_metavar_expr)] #![feature(min_specialization)] #![feature(trusted_len)] #![feature(type_alias_impl_trait)] diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 4e6be84ad7f8..5926bc19e868 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1176,7 +1176,7 @@ impl<'tcx> TyCtxt<'tcx> { } crate fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct { - &self.query_kinds[k as usize] + &self.query_kinds[k.index()] } /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` to ensure it gets used. @@ -1681,174 +1681,62 @@ nop_list_lift! {substs; GenericArg<'a> => GenericArg<'tcx>} CloneLiftImpls! { for<'tcx> { Constness, traits::WellFormedLoc, } } pub mod tls { - use super::{ptr_eq, GlobalCtxt, TyCtxt}; - - use crate::dep_graph::TaskDepsRef; - use crate::ty::query; - use rustc_data_structures::sync::{self, Lock}; - use rustc_data_structures::thin_vec::ThinVec; - use rustc_errors::Diagnostic; - use std::mem; - - #[cfg(not(parallel_compiler))] + use super::{GlobalCtxt, TyCtxt}; + use rustc_data_structures::sync; use std::cell::Cell; - #[cfg(parallel_compiler)] - use rustc_rayon_core as rayon_core; - - /// This is the implicit state of rustc. It contains the current - /// `TyCtxt` and query. It is updated when creating a local interner or - /// executing a new query. Whenever there's a `TyCtxt` value available - /// you should also have access to an `ImplicitCtxt` through the functions - /// in this module. - #[derive(Clone)] - pub struct ImplicitCtxt<'a, 'tcx> { - /// The current `TyCtxt`. - pub tcx: TyCtxt<'tcx>, - - /// The current query job, if any. This is updated by `JobOwner::start` in - /// `ty::query::plumbing` when executing a query. - pub query: Option, - - /// Where to store diagnostics for the current query job, if any. - /// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query. - pub diagnostics: Option<&'a Lock>>, - - /// Used to prevent layout from recursing too deeply. - pub layout_depth: usize, - - /// The current dep graph task. This is used to add dependencies to queries - /// when executing them. - pub task_deps: TaskDepsRef<'a>, - } - - impl<'a, 'tcx> ImplicitCtxt<'a, 'tcx> { - pub fn new(gcx: &'tcx GlobalCtxt<'tcx>) -> Self { - let tcx = TyCtxt { gcx }; - ImplicitCtxt { - tcx, - query: None, - diagnostics: None, - layout_depth: 0, - task_deps: TaskDepsRef::Ignore, - } - } - } - - /// Sets Rayon's thread-local variable, which is preserved for Rayon jobs - /// to `value` during the call to `f`. It is restored to its previous value after. - /// This is used to set the pointer to the new `ImplicitCtxt`. - #[cfg(parallel_compiler)] - #[inline] - fn set_tlv R, R>(value: usize, f: F) -> R { - rayon_core::tlv::with(value, f) - } - - /// Gets Rayon's thread-local variable, which is preserved for Rayon jobs. - /// This is used to get the pointer to the current `ImplicitCtxt`. - #[cfg(parallel_compiler)] - #[inline] - pub fn get_tlv() -> usize { - rayon_core::tlv::get() - } - - #[cfg(not(parallel_compiler))] thread_local! { - /// A thread local variable that stores a pointer to the current `ImplicitCtxt`. - static TLV: Cell = const { Cell::new(0) }; + /// This is the implicit state of rustc. It contains the `TyCtxt`. Whenever there's a + /// `TyCtxt` value available you should also have access to it through the functions in this + /// module. + static TLV: Cell<*const GlobalCtxt<'static>> = const { Cell::new(std::ptr::null()) }; } - /// Sets TLV to `value` during the call to `f`. - /// It is restored to its previous value after. - /// This is used to set the pointer to the new `ImplicitCtxt`. - #[cfg(not(parallel_compiler))] + /// Sets `context` as the new current `TyCtxt` for the duration of the function `f`. #[inline] - fn set_tlv R, R>(value: usize, f: F) -> R { - let old = get_tlv(); - let _reset = rustc_data_structures::OnDrop(move || TLV.with(|tlv| tlv.set(old))); - TLV.with(|tlv| tlv.set(value)); - f() - } - - /// Gets the pointer to the current `ImplicitCtxt`. - #[cfg(not(parallel_compiler))] - #[inline] - fn get_tlv() -> usize { - TLV.with(|tlv| tlv.get()) - } - - /// Sets `context` as the new current `ImplicitCtxt` for the duration of the function `f`. - #[inline] - pub fn enter_context<'a, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'tcx>, f: F) -> R + pub fn enter_context<'gcx, F, R>(gcx: &'gcx GlobalCtxt<'gcx>, f: F) -> R where - F: FnOnce(&ImplicitCtxt<'a, 'tcx>) -> R, - { - set_tlv(context as *const _ as usize, || f(&context)) - } - - /// Allows access to the current `ImplicitCtxt` in a closure if one is available. - #[inline] - pub fn with_context_opt(f: F) -> R - where - F: for<'a, 'tcx> FnOnce(Option<&ImplicitCtxt<'a, 'tcx>>) -> R, - { - let context = get_tlv(); - if context == 0 { - f(None) - } else { - // We could get an `ImplicitCtxt` pointer from another thread. - // Ensure that `ImplicitCtxt` is `Sync`. - sync::assert_sync::>(); - - unsafe { f(Some(&*(context as *const ImplicitCtxt<'_, '_>))) } - } - } - - /// Allows access to the current `ImplicitCtxt`. - /// Panics if there is no `ImplicitCtxt` available. - #[inline] - pub fn with_context(f: F) -> R - where - F: for<'a, 'tcx> FnOnce(&ImplicitCtxt<'a, 'tcx>) -> R, + F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> R, { - with_context_opt(|opt_context| f(opt_context.expect("no ImplicitCtxt stored in tls"))) - } + // We are storing `GlobalCtxt` as a global. + // Ensure that it is `Sync`. + sync::assert_sync::>(); - /// Allows access to the current `ImplicitCtxt` whose tcx field is the same as the tcx argument - /// passed in. This means the closure is given an `ImplicitCtxt` with the same `'tcx` lifetime - /// as the `TyCtxt` passed in. - /// This will panic if you pass it a `TyCtxt` which is different from the current - /// `ImplicitCtxt`'s `tcx` field. - #[inline] - pub fn with_related_context<'tcx, F, R>(tcx: TyCtxt<'tcx>, f: F) -> R - where - F: FnOnce(&ImplicitCtxt<'_, 'tcx>) -> R, - { - with_context(|context| unsafe { - assert!(ptr_eq(context.tcx.gcx, tcx.gcx)); - let context: &ImplicitCtxt<'_, '_> = mem::transmute(context); - f(context) - }) + let address = (gcx as *const GlobalCtxt<'_>).cast(); + let old = TLV.with(|tlv| tlv.replace(address)); + let _reset = rustc_data_structures::OnDrop(move || TLV.with(|tlv| tlv.set(old))); + debug_assert!( + old == address || old == std::ptr::null(), + "There can only be one GlobalCtxt." + ); + f(TyCtxt { gcx }) } - /// Allows access to the `TyCtxt` in the current `ImplicitCtxt`. - /// Panics if there is no `ImplicitCtxt` available. + /// Allows access to the current `TyCtxt`. + /// Panics if there is no `TyCtxt` available. #[inline] pub fn with(f: F) -> R where F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> R, { - with_context(|context| f(context.tcx)) + with_opt(|opt_tcx| f(opt_tcx.expect("no TyCtxt stored in tls"))) } - /// Allows access to the `TyCtxt` in the current `ImplicitCtxt`. - /// The closure is passed None if there is no `ImplicitCtxt` available. + /// Allows access to the current `TyCtxt`. + /// The closure is passed None if there is no `TyCtxt` available. #[inline] pub fn with_opt(f: F) -> R where F: for<'tcx> FnOnce(Option>) -> R, { - with_context_opt(|opt_context| f(opt_context.map(|context| context.tcx))) + let context = TLV.with(|tlv| tlv.get()); + let tcx = if context == std::ptr::null() { + None + } else { + let gcx = unsafe { &*context.cast::>() }; + Some(TyCtxt { gcx }) + }; + f(tcx) } } @@ -2872,12 +2760,6 @@ impl InternIteratorElement for Result { } } -// We are comparing types with different invariant lifetimes, so `ptr::eq` -// won't work for us. -fn ptr_eq(t: *const T, u: *const U) -> bool { - t as *const () == u as *const () -} - pub fn provide(providers: &mut ty::query::Providers) { providers.resolutions = |tcx, ()| &tcx.untracked_resolutions; providers.module_reexports = diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index f6d139fe59d3..67fe2b930de4 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -9,6 +9,7 @@ use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_index::bit_set::BitSet; use rustc_index::vec::{Idx, IndexVec}; +use rustc_query_system::tls; use rustc_session::{config::OptLevel, DataTypeKind, FieldInfo, SizeKind, VariantInfo}; use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; @@ -225,51 +226,46 @@ fn layout_of<'tcx>( tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, ) -> Result, LayoutError<'tcx>> { - ty::tls::with_related_context(tcx, move |icx| { + tls::with_increased_layout_depth(move |layout_depth| { let (param_env, ty) = query.into_parts(); debug!(?ty); - if !tcx.recursion_limit().value_within_limit(icx.layout_depth) { + if !tcx.recursion_limit().value_within_limit(layout_depth) { tcx.sess.fatal(&format!("overflow representing the type `{}`", ty)); } - // Update the ImplicitCtxt to increase the layout_depth - let icx = ty::tls::ImplicitCtxt { layout_depth: icx.layout_depth + 1, ..icx.clone() }; - - ty::tls::enter_context(&icx, |_| { - let param_env = param_env.with_reveal_all_normalized(tcx); - let unnormalized_ty = ty; - - // FIXME: We might want to have two different versions of `layout_of`: - // One that can be called after typecheck has completed and can use - // `normalize_erasing_regions` here and another one that can be called - // before typecheck has completed and uses `try_normalize_erasing_regions`. - let ty = match tcx.try_normalize_erasing_regions(param_env, ty) { - Ok(t) => t, - Err(normalization_error) => { - return Err(LayoutError::NormalizationFailure(ty, normalization_error)); - } - }; - - if ty != unnormalized_ty { - // Ensure this layout is also cached for the normalized type. - return tcx.layout_of(param_env.and(ty)); + let param_env = param_env.with_reveal_all_normalized(tcx); + let unnormalized_ty = ty; + + // FIXME: We might want to have two different versions of `layout_of`: + // One that can be called after typecheck has completed and can use + // `normalize_erasing_regions` here and another one that can be called + // before typecheck has completed and uses `try_normalize_erasing_regions`. + let ty = match tcx.try_normalize_erasing_regions(param_env, ty) { + Ok(t) => t, + Err(normalization_error) => { + return Err(LayoutError::NormalizationFailure(ty, normalization_error)); } + }; + + if ty != unnormalized_ty { + // Ensure this layout is also cached for the normalized type. + return tcx.layout_of(param_env.and(ty)); + } - let cx = LayoutCx { tcx, param_env }; + let cx = LayoutCx { tcx, param_env }; - let layout = cx.layout_of_uncached(ty)?; - let layout = TyAndLayout { ty, layout }; + let layout = cx.layout_of_uncached(ty)?; + let layout = TyAndLayout { ty, layout }; - cx.record_layout_for_printing(layout); + cx.record_layout_for_printing(layout); - // Type-level uninhabitedness should always imply ABI uninhabitedness. - if tcx.conservative_is_privately_uninhabited(param_env.and(ty)) { - assert!(layout.abi.is_uninhabited()); - } + // Type-level uninhabitedness should always imply ABI uninhabitedness. + if tcx.conservative_is_privately_uninhabited(param_env.and(ty)) { + assert!(layout.abi.is_uninhabited()); + } - Ok(layout) - }) + Ok(layout) }) } diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index 9e48c569c253..db14fc04172f 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -60,7 +60,6 @@ use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; -pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; #[derive(Copy, Clone)] diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 6ebff5388f45..437be15514d9 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -17,7 +17,7 @@ extern crate rustc_middle; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; -use rustc_middle::dep_graph::{self, DepKindStruct, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{self, label_strs, DepKindStruct, SerializedDepNodeIndex}; use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values}; use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine}; use rustc_middle::ty::{self, TyCtxt}; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index bc82b0053b9e..1645245804c3 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -3,15 +3,11 @@ //! manage the caches, and so forth. use crate::{on_disk_cache, Queries}; +use rustc_errors::Handler; use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; -use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::ty::TyCtxt; use rustc_query_system::dep_graph::HasDepContext; use rustc_query_system::query::{QueryContext, QueryJobId, QueryMap, QuerySideEffects}; - -use rustc_data_structures::sync::Lock; -use rustc_data_structures::thin_vec::ThinVec; -use rustc_errors::{Diagnostic, Handler}; use rustc_serialize::opaque; use std::any::Any; @@ -33,7 +29,6 @@ impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> { } impl<'tcx> HasDepContext for QueryCtxt<'tcx> { - type DepKind = rustc_middle::dep_graph::DepKind; type DepContext = TyCtxt<'tcx>; #[inline] @@ -52,10 +47,6 @@ impl QueryContext for QueryCtxt<'_> { ) } - fn current_query_job(&self) -> Option { - tls::with_related_context(**self, |icx| icx.query) - } - fn try_collect_active_jobs(&self) -> Option { self.queries.try_collect_active_jobs(**self) } @@ -84,36 +75,6 @@ impl QueryContext for QueryCtxt<'_> { c.store_side_effects_for_anon_node(dep_node_index, side_effects) } } - - /// Executes a job by changing the `ImplicitCtxt` to point to the - /// new query job while it executes. It returns the diagnostics - /// captured during execution and the actual result. - #[inline(always)] - fn start_query( - &self, - token: QueryJobId, - diagnostics: Option<&Lock>>, - compute: impl FnOnce() -> R, - ) -> R { - // The `TyCtxt` stored in TLS has the same global interner lifetime - // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes - // when accessing the `ImplicitCtxt`. - tls::with_related_context(**self, move |current_icx| { - // Update the `ImplicitCtxt` to point to our new query job. - let new_icx = ImplicitCtxt { - tcx: **self, - query: Some(token), - diagnostics, - layout_depth: current_icx.layout_depth, - task_deps: current_icx.task_deps, - }; - - // Use the `ImplicitCtxt` while we execute the query. - tls::enter_context(&new_icx, |_| { - rustc_data_structures::stack::ensure_sufficient_stack(compute) - }) - }) - } } impl<'tcx> QueryCtxt<'tcx> { @@ -160,13 +121,8 @@ impl<'tcx> QueryCtxt<'tcx> { Ok(()) } - pub fn try_print_query_stack( - self, - query: Option, - handler: &Handler, - num_frames: Option, - ) -> usize { - rustc_query_system::query::print_query_stack(self, query, handler, num_frames) + pub fn try_print_query_stack(self, handler: &Handler, num_frames: Option) -> usize { + rustc_query_system::query::print_query_stack(self, handler, num_frames) } } @@ -268,7 +224,7 @@ macro_rules! define_queries { $(#[allow(nonstandard_style)] $(#[$attr])* pub fn $name<$tcx>(tcx: QueryCtxt<$tcx>, key: query_keys::$name<$tcx>) -> QueryStackFrame { opt_remap_env_constness!([$($modifiers)*][key]); - let kind = dep_graph::DepKind::$name; + let kind = dep_graph::dep_kind::$name; let name = stringify!($name); // Disable visible paths printing for performance reasons. // Showing visible path instead of any path is not that important in production. @@ -283,7 +239,7 @@ macro_rules! define_queries { } else { description }; - let span = if kind == dep_graph::DepKind::def_span { + let span = if kind == dep_graph::dep_kind::def_span { // The `def_span` query is used to calculate `default_span`, // so exit to avoid infinite recursion. None @@ -298,7 +254,7 @@ macro_rules! define_queries { let hash = || { let mut hcx = tcx.create_stable_hashing_context(); let mut hasher = StableHasher::new(); - std::mem::discriminant(&kind).hash_stable(&mut hcx, &mut hasher); + kind.index().hash_stable(&mut hcx, &mut hasher); key.hash_stable(&mut hcx, &mut hasher); hasher.finish::() }; @@ -351,7 +307,7 @@ macro_rules! define_queries { QueryVtable { anon: is_anon!([$($modifiers)*]), eval_always: is_eval_always!([$($modifiers)*]), - dep_kind: dep_graph::DepKind::$name, + dep_kind: dep_graph::dep_kind::$name, hash_result: hash_result!([$($modifiers)*]), handle_cycle_error: |tcx, mut error| handle_cycle_error!([$($modifiers)*][tcx, error]), compute, @@ -371,8 +327,9 @@ macro_rules! define_queries { use rustc_query_system::dep_graph::FingerprintStyle; // We use this for most things when incr. comp. is turned off. - pub fn Null() -> DepKindStruct { + pub fn NULL() -> DepKindStruct { DepKindStruct { + label: label_strs::NULL, is_anon: false, is_eval_always: false, fingerprint_style: FingerprintStyle::Unit, @@ -383,6 +340,7 @@ macro_rules! define_queries { pub fn TraitSelect() -> DepKindStruct { DepKindStruct { + label: label_strs::TraitSelect, is_anon: true, is_eval_always: false, fingerprint_style: FingerprintStyle::Unit, @@ -393,6 +351,7 @@ macro_rules! define_queries { pub fn CompileCodegenUnit() -> DepKindStruct { DepKindStruct { + label: label_strs::CompileCodegenUnit, is_anon: false, is_eval_always: false, fingerprint_style: FingerprintStyle::Opaque, @@ -403,6 +362,7 @@ macro_rules! define_queries { pub fn CompileMonoItem() -> DepKindStruct { DepKindStruct { + label: label_strs::CompileMonoItem, is_anon: false, is_eval_always: false, fingerprint_style: FingerprintStyle::Opaque, @@ -420,6 +380,7 @@ macro_rules! define_queries { if is_anon || !fingerprint_style.reconstructible() { return DepKindStruct { + label: label_strs::$name, is_anon, is_eval_always, fingerprint_style, @@ -453,6 +414,7 @@ macro_rules! define_queries { } DepKindStruct { + label: stringify!($name), is_anon, is_eval_always, fingerprint_style, diff --git a/compiler/rustc_query_system/src/dep_graph/debug.rs b/compiler/rustc_query_system/src/dep_graph/debug.rs index a544ac2c343a..968ca20655db 100644 --- a/compiler/rustc_query_system/src/dep_graph/debug.rs +++ b/compiler/rustc_query_system/src/dep_graph/debug.rs @@ -1,6 +1,6 @@ //! Code for debugging the dep-graph. -use super::{DepKind, DepNode, DepNodeIndex}; +use super::{DepNode, DepNodeIndex}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lock; use std::error::Error; @@ -28,7 +28,7 @@ impl DepNodeFilter { } /// Tests whether `node` meets the filter, returning true if so. - pub fn test(&self, node: &DepNode) -> bool { + pub fn test(&self, node: &DepNode) -> bool { let debug_str = format!("{:?}", node); self.text.split('&').map(|s| s.trim()).all(|f| debug_str.contains(f)) } @@ -36,14 +36,14 @@ impl DepNodeFilter { /// A filter like `F -> G` where `F` and `G` are valid dep-node /// filters. This can be used to test the source/target independently. -pub struct EdgeFilter { +pub struct EdgeFilter { pub source: DepNodeFilter, pub target: DepNodeFilter, - pub index_to_node: Lock>>, + pub index_to_node: Lock>, } -impl EdgeFilter { - pub fn new(test: &str) -> Result, Box> { +impl EdgeFilter { + pub fn new(test: &str) -> Result> { let parts: Vec<_> = test.split("->").collect(); if parts.len() != 2 { Err(format!("expected a filter like `a&b -> c&d`, not `{}`", test).into()) @@ -57,7 +57,7 @@ impl EdgeFilter { } #[cfg(debug_assertions)] - pub fn test(&self, source: &DepNode, target: &DepNode) -> bool { + pub fn test(&self, source: &DepNode, target: &DepNode) -> bool { self.source.test(source) && self.target.test(target) } } diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs index c274c2cc26c1..2f89f5328b94 100644 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs @@ -42,35 +42,55 @@ //! `DefId` it was computed from. In other cases, too much information gets //! lost during fingerprint computation. -use super::{DepContext, DepKind, FingerprintStyle}; +use super::{DepContext, FingerprintStyle}; use crate::ich::StableHashingContext; use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::AtomicRef; use std::fmt; use std::hash::Hash; -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] -pub struct DepNode { - pub kind: K, +#[derive(Clone, Copy, PartialEq, Eq, Hash, Encodable, Decodable)] +pub struct DepKind(u16); + +impl DepKind { + /// We use this for most things when incr. comp. is turned off. + pub const NULL: Self = DepKind(0); + + #[inline] + pub const fn new(index: u16) -> Self { + DepKind(index) + } + + #[inline] + pub const fn index(self) -> usize { + self.0 as usize + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Encodable, Decodable)] +pub struct DepNode { + pub kind: DepKind, pub hash: PackedFingerprint, } -impl DepNode { +impl DepNode { /// Creates a new, parameterless DepNode. This method will assert /// that the DepNode corresponding to the given DepKind actually /// does not require any parameters. - pub fn new_no_params(tcx: Ctxt, kind: K) -> DepNode + #[inline] + pub fn new_no_params(tcx: Ctxt, kind: DepKind) -> DepNode where - Ctxt: super::DepContext, + Ctxt: super::DepContext, { debug_assert_eq!(tcx.fingerprint_style(kind), FingerprintStyle::Unit); DepNode { kind, hash: Fingerprint::ZERO.into() } } - pub fn construct(tcx: Ctxt, kind: K, arg: &Key) -> DepNode + pub fn construct(tcx: Ctxt, kind: DepKind, arg: &Key) -> DepNode where - Ctxt: super::DepContext, + Ctxt: super::DepContext, Key: DepNodeParams, { let hash = arg.to_fingerprint(tcx); @@ -90,12 +110,44 @@ impl DepNode { } } -impl fmt::Debug for DepNode { +impl fmt::Debug for DepKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - K::debug_node(self, f) + (*KIND_DEBUG)(self, f) } } +pub static KIND_DEBUG: AtomicRef) -> fmt::Result> = + AtomicRef::new(&(default_kind_debug as _)); + +fn default_kind_debug(kind: &DepKind, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", kind.0) +} + +impl fmt::Debug for DepNode { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (*NODE_DEBUG)(self, f) + } +} + +pub static NODE_DEBUG: AtomicRef) -> fmt::Result> = + AtomicRef::new(&(default_node_debug as _)); + +fn default_node_debug(node: &DepNode, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}({})", node.kind.0, node.hash) +} + +// We keep a lot of `DepNode`s in memory during compilation. It's not +// required that their size stay the same, but we don't want to change +// it inadvertently. This assert just ensures we're aware of any change. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +static_assert_size!(DepNode, 18); + +#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] +static_assert_size!(DepNode, 24); + +static_assert_size!(DepKind, 2); + pub trait DepNodeParams: fmt::Debug + Sized { fn fingerprint_style() -> FingerprintStyle; @@ -117,7 +169,7 @@ pub trait DepNodeParams: fmt::Debug + Sized { /// `fingerprint_style()` is not `FingerprintStyle::Opaque`. /// It is always valid to return `None` here, in which case incremental /// compilation will treat the query as having changed instead of forcing it. - fn recover(tcx: Ctxt, dep_node: &DepNode) -> Option; + fn recover(tcx: Ctxt, dep_node: &DepNode) -> Option; } impl DepNodeParams for T @@ -145,7 +197,7 @@ where } #[inline(always)] - default fn recover(_: Ctxt, _: &DepNode) -> Option { + default fn recover(_: Ctxt, _: &DepNode) -> Option { None } } @@ -162,6 +214,7 @@ pub struct WorkProductId { } impl WorkProductId { + #[inline] pub fn from_cgu_name(cgu_name: &str) -> WorkProductId { let mut hasher = StableHasher::new(); cgu_name.len().hash(&mut hasher); diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index f7655e55d348..4027d607cac2 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -26,8 +26,8 @@ use crate::query::{QueryContext, QuerySideEffects}; use {super::debug::EdgeFilter, std::env}; #[derive(Clone)] -pub struct DepGraph { - data: Option>>, +pub struct DepGraph { + data: Option>, /// This field is used for assigning DepNodeIndices when running in /// non-incremental mode. Even in non-incremental mode we make sure that @@ -67,16 +67,16 @@ impl DepNodeColor { } } -struct DepGraphData { +struct DepGraphData { /// The new encoding of the dependency graph, optimized for red/green /// tracking. The `current` field is the dependency graph of only the /// current compilation session: We don't merge the previous dep-graph into /// current one anymore, but we do reference shared data to save space. - current: CurrentDepGraph, + current: CurrentDepGraph, /// The dep-graph from the previous compilation session. It contains all /// nodes and edges as well as all fingerprints of nodes that have them. - previous: SerializedDepGraph, + previous: SerializedDepGraph, colors: DepNodeColorMap, @@ -88,12 +88,12 @@ struct DepGraphData { /// this map. We can later look for and extract that data. previous_work_products: FxHashMap, - dep_node_debug: Lock, String>>, + dep_node_debug: Lock>, /// Used by incremental compilation tests to assert that /// a particular query result was decoded from disk /// (not just marked green) - debug_loaded_from_disk: Lock>>, + debug_loaded_from_disk: Lock>, } pub fn hash_result(hcx: &mut StableHashingContext<'_>, result: &R) -> Fingerprint @@ -105,15 +105,15 @@ where stable_hasher.finish() } -impl DepGraph { +impl DepGraph { pub fn new( profiler: &SelfProfilerRef, - prev_graph: SerializedDepGraph, + prev_graph: SerializedDepGraph, prev_work_products: FxHashMap, encoder: FileEncoder, record_graph: bool, record_stats: bool, - ) -> DepGraph { + ) -> DepGraph { let prev_graph_node_count = prev_graph.node_count(); let current = CurrentDepGraph::new( @@ -147,7 +147,8 @@ impl DepGraph { } } - pub fn new_disabled() -> DepGraph { + #[inline] + pub fn new_disabled() -> DepGraph { DepGraph { data: None, virtual_dep_node_index: Lrc::new(AtomicU32::new(0)) } } @@ -157,15 +158,16 @@ impl DepGraph { self.data.is_some() } - pub fn with_query(&self, f: impl Fn(&DepGraphQuery)) { + pub fn with_query(&self, f: impl Fn(&DepGraphQuery)) { if let Some(data) = &self.data { data.current.encoder.borrow().with_query(f) } } + #[inline] pub fn assert_ignored(&self) { if let Some(..) = self.data { - K::read_deps(|task_deps| { + crate::tls::read_deps(|task_deps| { assert_matches!( task_deps, TaskDepsRef::Ignore, @@ -179,7 +181,7 @@ impl DepGraph { where OP: FnOnce() -> R, { - K::with_deps(TaskDepsRef::Ignore, op) + crate::tls::with_deps(TaskDepsRef::Ignore, op) } /// Used to wrap the deserialization of a query result from disk, @@ -232,7 +234,7 @@ impl DepGraph { where OP: FnOnce() -> R, { - K::with_deps(TaskDepsRef::Forbid, op) + crate::tls::with_deps(TaskDepsRef::Forbid, op) } /// Starts a new dep-graph task. Dep-graph tasks are specified @@ -262,9 +264,9 @@ impl DepGraph { /// `arg` parameter. /// /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/incremental-compilation.html - pub fn with_task, A: Debug, R>( + pub fn with_task( &self, - key: DepNode, + key: DepNode, cx: Ctxt, arg: A, task: fn(Ctxt, A) -> R, @@ -281,9 +283,9 @@ impl DepGraph { } } - fn with_task_impl, A: Debug, R>( + fn with_task_impl( &self, - key: DepNode, + key: DepNode, cx: Ctxt, arg: A, task: fn(Ctxt, A) -> R, @@ -323,7 +325,7 @@ impl DepGraph { None => TaskDepsRef::Ignore, }; - let result = K::with_deps(task_deps_ref, || task(cx, arg)); + let result = crate::tls::with_deps(task_deps_ref, || task(cx, arg)); let edges = task_deps.map_or_else(|| smallvec![], |lock| lock.into_inner().reads); let dcx = cx.dep_context(); @@ -363,10 +365,10 @@ impl DepGraph { /// Executes something within an "anonymous" task, that is, a task the /// `DepNode` of which is determined by the list of inputs it read from. - pub fn with_anon_task, OP, R>( + pub fn with_anon_task( &self, cx: Ctxt, - dep_kind: K, + dep_kind: DepKind, op: OP, ) -> (R, DepNodeIndex) where @@ -376,7 +378,7 @@ impl DepGraph { if let Some(ref data) = self.data { let task_deps = Lock::new(TaskDeps::default()); - let result = K::with_deps(TaskDepsRef::Allow(&task_deps), op); + let result = crate::tls::with_deps(TaskDepsRef::Allow(&task_deps), op); let task_deps = task_deps.into_inner(); let task_deps = task_deps.reads; @@ -428,7 +430,7 @@ impl DepGraph { #[inline] pub fn read_index(&self, dep_node_index: DepNodeIndex) { if let Some(ref data) = self.data { - K::read_deps(|task_deps| { + crate::tls::read_deps(|task_deps| { let mut task_deps = match task_deps { TaskDepsRef::Allow(deps) => deps.lock(), TaskDepsRef::Ignore => return, @@ -476,12 +478,12 @@ impl DepGraph { } #[inline] - pub fn dep_node_index_of(&self, dep_node: &DepNode) -> DepNodeIndex { + pub fn dep_node_index_of(&self, dep_node: &DepNode) -> DepNodeIndex { self.dep_node_index_of_opt(dep_node).unwrap() } #[inline] - pub fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option { + pub fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option { let data = self.data.as_ref().unwrap(); let current = &data.current; @@ -493,36 +495,39 @@ impl DepGraph { } #[inline] - pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool { + pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool { self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some() } - pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { + #[inline] + pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { self.data.as_ref().unwrap().previous.fingerprint_of(dep_node) } /// Checks whether a previous work product exists for `v` and, if /// so, return the path that leads to it. Used to skip doing work. + #[inline] pub fn previous_work_product(&self, v: &WorkProductId) -> Option { self.data.as_ref().and_then(|data| data.previous_work_products.get(v).cloned()) } /// Access the map of work-products created during the cached run. Only /// used during saving of the dep-graph. + #[inline] pub fn previous_work_products(&self) -> &FxHashMap { &self.data.as_ref().unwrap().previous_work_products } - pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) { + pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) { self.data.as_ref().unwrap().debug_loaded_from_disk.lock().insert(dep_node); } - pub fn debug_was_loaded_from_disk(&self, dep_node: DepNode) -> bool { + pub fn debug_was_loaded_from_disk(&self, dep_node: DepNode) -> bool { self.data.as_ref().unwrap().debug_loaded_from_disk.lock().contains(&dep_node) } #[inline(always)] - pub fn register_dep_node_debug_str(&self, dep_node: DepNode, debug_str_gen: F) + pub fn register_dep_node_debug_str(&self, dep_node: DepNode, debug_str_gen: F) where F: FnOnce() -> String, { @@ -535,11 +540,13 @@ impl DepGraph { dep_node_debug.borrow_mut().insert(dep_node, debug_str); } - pub fn dep_node_debug_str(&self, dep_node: DepNode) -> Option { + #[inline] + pub fn dep_node_debug_str(&self, dep_node: DepNode) -> Option { self.data.as_ref()?.dep_node_debug.borrow().get(&dep_node).cloned() } - fn node_color(&self, dep_node: &DepNode) -> Option { + #[inline] + fn node_color(&self, dep_node: &DepNode) -> Option { if let Some(ref data) = self.data { if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) { return data.colors.get(prev_index); @@ -557,10 +564,10 @@ impl DepGraph { /// A node will have an index, when it's already been marked green, or when we can mark it /// green. This function will mark the current task as a reader of the specified node, when /// a node index can be found for that node. - pub fn try_mark_green>( + pub fn try_mark_green( &self, tcx: Ctxt, - dep_node: &DepNode, + dep_node: &DepNode, ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { debug_assert!(!tcx.dep_context().is_eval_always(dep_node.kind)); @@ -584,12 +591,12 @@ impl DepGraph { } } - fn try_mark_parent_green>( + fn try_mark_parent_green( &self, tcx: Ctxt, - data: &DepGraphData, + data: &DepGraphData, parent_dep_node_index: SerializedDepNodeIndex, - dep_node: &DepNode, + dep_node: &DepNode, ) -> Option<()> { let dep_dep_node_color = data.colors.get(parent_dep_node_index); let dep_dep_node = &data.previous.index_to_node(parent_dep_node_index); @@ -696,12 +703,12 @@ impl DepGraph { } /// Try to mark a dep-node which existed in the previous compilation session as green. - fn try_mark_previous_green>( + fn try_mark_previous_green( &self, tcx: Ctxt, - data: &DepGraphData, + data: &DepGraphData, prev_dep_node_index: SerializedDepNodeIndex, - dep_node: &DepNode, + dep_node: &DepNode, ) -> Option { debug!("try_mark_previous_green({:?}) - BEGIN", dep_node); @@ -766,10 +773,10 @@ impl DepGraph { /// This may be called concurrently on multiple threads for the same dep node. #[cold] #[inline(never)] - fn emit_side_effects>( + fn emit_side_effects( &self, tcx: Ctxt, - data: &DepGraphData, + data: &DepGraphData, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects, ) { @@ -792,13 +799,15 @@ impl DepGraph { // Returns true if the given node has been marked as red during the // current compilation session. Used in various assertions - pub fn is_red(&self, dep_node: &DepNode) -> bool { + #[inline] + pub fn is_red(&self, dep_node: &DepNode) -> bool { self.node_color(dep_node) == Some(DepNodeColor::Red) } // Returns true if the given node has been marked as green during the // current compilation session. Used in various assertions - pub fn is_green(&self, dep_node: &DepNode) -> bool { + #[inline] + pub fn is_green(&self, dep_node: &DepNode) -> bool { self.node_color(dep_node).map_or(false, |c| c.is_green()) } @@ -810,7 +819,7 @@ impl DepGraph { // // This method will only load queries that will end up in the disk cache. // Other queries will not be executed. - pub fn exec_cache_promotions>(&self, tcx: Ctxt) { + pub fn exec_cache_promotions(&self, tcx: Ctxt) { let _prof_timer = tcx.profiler().generic_activity("incr_comp_query_cache_promotion"); let data = self.data.as_ref().unwrap(); @@ -838,6 +847,7 @@ impl DepGraph { } } + #[inline] pub fn encode(&self, profiler: &SelfProfilerRef) -> FileEncodeResult { if let Some(data) = &self.data { data.current.encoder.steal().finish(profiler) @@ -846,6 +856,7 @@ impl DepGraph { } } + #[inline] pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex { let index = self.virtual_dep_node_index.fetch_add(1, Relaxed); DepNodeIndex::from_u32(index) @@ -918,15 +929,15 @@ rustc_index::newtype_index! { /// `new_node_to_index` and `data`, or `prev_index_to_index` and `data`. When /// manipulating both, we acquire `new_node_to_index` or `prev_index_to_index` /// first, and `data` second. -pub(super) struct CurrentDepGraph { - encoder: Steal>, - new_node_to_index: Sharded, DepNodeIndex>>, +pub(super) struct CurrentDepGraph { + encoder: Steal, + new_node_to_index: Sharded>, prev_index_to_index: Lock>>, /// Used to trap when a specific edge is added to the graph. /// This is used for debug purposes and is only active with `debug_assertions`. #[cfg(debug_assertions)] - forbidden_edge: Option>, + forbidden_edge: Option, /// Anonymous `DepNode`s are nodes whose IDs we compute from the list of /// their edges. This has the beneficial side-effect that multiple anonymous @@ -953,14 +964,14 @@ pub(super) struct CurrentDepGraph { node_intern_event_id: Option, } -impl CurrentDepGraph { +impl CurrentDepGraph { fn new( profiler: &SelfProfilerRef, prev_graph_node_count: usize, encoder: FileEncoder, record_graph: bool, record_stats: bool, - ) -> CurrentDepGraph { + ) -> CurrentDepGraph { use std::time::{SystemTime, UNIX_EPOCH}; let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); @@ -1012,7 +1023,8 @@ impl CurrentDepGraph { } #[cfg(debug_assertions)] - fn record_edge(&self, dep_node_index: DepNodeIndex, key: DepNode) { + #[inline] + fn record_edge(&self, dep_node_index: DepNodeIndex, key: DepNode) { if let Some(forbidden_edge) = &self.forbidden_edge { forbidden_edge.index_to_node.lock().insert(dep_node_index, key); } @@ -1020,10 +1032,11 @@ impl CurrentDepGraph { /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it. /// Assumes that this is a node that has no equivalent in the previous dep-graph. + #[inline] fn intern_new_node( &self, profiler: &SelfProfilerRef, - key: DepNode, + key: DepNode, edges: EdgesVec, current_fingerprint: Fingerprint, ) -> DepNodeIndex { @@ -1040,11 +1053,12 @@ impl CurrentDepGraph { } } + #[inline] fn intern_node( &self, profiler: &SelfProfilerRef, - prev_graph: &SerializedDepGraph, - key: DepNode, + prev_graph: &SerializedDepGraph, + key: DepNode, edges: EdgesVec, fingerprint: Option, print_status: bool, @@ -1142,10 +1156,11 @@ impl CurrentDepGraph { } } + #[inline] fn promote_node_and_deps_to_current( &self, profiler: &SelfProfilerRef, - prev_graph: &SerializedDepGraph, + prev_graph: &SerializedDepGraph, prev_index: SerializedDepNodeIndex, ) -> DepNodeIndex { self.debug_assert_not_in_new_nodes(prev_graph, prev_index); @@ -1177,7 +1192,7 @@ impl CurrentDepGraph { #[inline] fn debug_assert_not_in_new_nodes( &self, - prev_graph: &SerializedDepGraph, + prev_graph: &SerializedDepGraph, prev_index: SerializedDepNodeIndex, ) { let node = &prev_graph.index_to_node(prev_index); @@ -1193,11 +1208,11 @@ const TASK_DEPS_READS_CAP: usize = 8; type EdgesVec = SmallVec<[DepNodeIndex; TASK_DEPS_READS_CAP]>; #[derive(Debug, Clone, Copy)] -pub enum TaskDepsRef<'a, K: DepKind> { +pub enum TaskDepsRef<'a> { /// New dependencies can be added to the /// `TaskDeps`. This is used when executing a 'normal' query /// (no `eval_always` modifier) - Allow(&'a Lock>), + Allow(&'a Lock), /// New dependencies are ignored. This is used when /// executing an `eval_always` query, since there's no /// need to track dependencies for a query that's always @@ -1210,25 +1225,20 @@ pub enum TaskDepsRef<'a, K: DepKind> { Forbid, } -#[derive(Debug)] -pub struct TaskDeps { +impl Default for TaskDepsRef<'_> { + #[inline] + fn default() -> Self { + TaskDepsRef::Ignore + } +} + +#[derive(Default, Debug)] +pub struct TaskDeps { #[cfg(debug_assertions)] - node: Option>, + node: Option, reads: EdgesVec, read_set: FxHashSet, - phantom_data: PhantomData>, -} - -impl Default for TaskDeps { - fn default() -> Self { - Self { - #[cfg(debug_assertions)] - node: None, - reads: EdgesVec::new(), - read_set: FxHashSet::default(), - phantom_data: PhantomData, - } - } + phantom_data: PhantomData, } // A data structure that stores Option values as a contiguous @@ -1242,6 +1252,7 @@ const COMPRESSED_RED: u32 = 1; const COMPRESSED_FIRST_GREEN: u32 = 2; impl DepNodeColorMap { + #[inline] fn new(size: usize) -> DepNodeColorMap { DepNodeColorMap { values: (0..size).map(|_| AtomicU32::new(COMPRESSED_NONE)).collect() } } @@ -1257,6 +1268,7 @@ impl DepNodeColorMap { } } + #[inline] fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) { self.values[index].store( match color { diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs index 5907ae309ca3..b951c236ccc0 100644 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ b/compiler/rustc_query_system/src/dep_graph/mod.rs @@ -4,7 +4,7 @@ mod graph; mod query; mod serialized; -pub use dep_node::{DepNode, DepNodeParams, WorkProductId}; +pub use dep_node::{DepKind, DepNode, DepNodeParams, WorkProductId, KIND_DEBUG, NODE_DEBUG}; pub use graph::{ hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef, WorkProduct, }; @@ -13,20 +13,14 @@ pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; use crate::ich::StableHashingContext; use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_serialize::{opaque::FileEncoder, Encodable}; use rustc_session::Session; -use std::fmt; -use std::hash::Hash; - pub trait DepContext: Copy { - type DepKind: self::DepKind; - /// Create a hashing context for hashing new results. fn create_stable_hashing_context(&self) -> StableHashingContext<'_>; /// Access the DepGraph. - fn dep_graph(&self) -> &DepGraph; + fn dep_graph(&self) -> &DepGraph; /// Access the profiler. fn profiler(&self) -> &SelfProfilerRef; @@ -35,26 +29,24 @@ pub trait DepContext: Copy { fn sess(&self) -> &Session; /// Return whether this kind always require evaluation. - fn is_eval_always(&self, kind: Self::DepKind) -> bool; + fn is_eval_always(&self, kind: DepKind) -> bool; - fn fingerprint_style(&self, kind: Self::DepKind) -> FingerprintStyle; + fn fingerprint_style(&self, kind: DepKind) -> FingerprintStyle; /// Try to force a dep node to execute and see if it's green. - fn try_force_from_dep_node(&self, dep_node: DepNode) -> bool; + fn try_force_from_dep_node(&self, dep_node: DepNode) -> bool; /// Load data from the on-disk cache. - fn try_load_from_on_disk_cache(&self, dep_node: DepNode); + fn try_load_from_on_disk_cache(&self, dep_node: DepNode); } pub trait HasDepContext: Copy { - type DepKind: self::DepKind; - type DepContext: self::DepContext; + type DepContext: self::DepContext; fn dep_context(&self) -> &Self::DepContext; } impl HasDepContext for T { - type DepKind = T::DepKind; type DepContext = Self; fn dep_context(&self) -> &Self::DepContext { @@ -82,21 +74,3 @@ impl FingerprintStyle { } } } - -/// Describe the different families of dependency nodes. -pub trait DepKind: Copy + fmt::Debug + Eq + Hash + Send + Encodable + 'static { - const NULL: Self; - - /// Implementation of `std::fmt::Debug` for `DepNode`. - fn debug_node(node: &DepNode, f: &mut fmt::Formatter<'_>) -> fmt::Result; - - /// Execute the operation with provided dependencies. - fn with_deps(deps: TaskDepsRef<'_, Self>, op: OP) -> R - where - OP: FnOnce() -> R; - - /// Access dependencies from current implicit context. - fn read_deps(op: OP) - where - OP: for<'a> FnOnce(TaskDepsRef<'a, Self>); -} diff --git a/compiler/rustc_query_system/src/dep_graph/query.rs b/compiler/rustc_query_system/src/dep_graph/query.rs index 27b3b5e13667..380b9c8d9151 100644 --- a/compiler/rustc_query_system/src/dep_graph/query.rs +++ b/compiler/rustc_query_system/src/dep_graph/query.rs @@ -2,16 +2,16 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::implementation::{Direction, Graph, NodeIndex, INCOMING}; use rustc_index::vec::IndexVec; -use super::{DepKind, DepNode, DepNodeIndex}; +use super::{DepNode, DepNodeIndex}; -pub struct DepGraphQuery { - pub graph: Graph, ()>, - pub indices: FxHashMap, NodeIndex>, +pub struct DepGraphQuery { + pub graph: Graph, + pub indices: FxHashMap, pub dep_index_to_index: IndexVec>, } -impl DepGraphQuery { - pub fn new(prev_node_count: usize) -> DepGraphQuery { +impl DepGraphQuery { + pub fn new(prev_node_count: usize) -> DepGraphQuery { let node_count = prev_node_count + prev_node_count / 4; let edge_count = 6 * node_count; @@ -22,7 +22,7 @@ impl DepGraphQuery { DepGraphQuery { graph, indices, dep_index_to_index } } - pub fn push(&mut self, index: DepNodeIndex, node: DepNode, edges: &[DepNodeIndex]) { + pub fn push(&mut self, index: DepNodeIndex, node: DepNode, edges: &[DepNodeIndex]) { let source = self.graph.add_node(node); if index.index() >= self.dep_index_to_index.len() { self.dep_index_to_index.resize(index.index() + 1, None); @@ -40,11 +40,11 @@ impl DepGraphQuery { } } - pub fn nodes(&self) -> Vec<&DepNode> { + pub fn nodes(&self) -> Vec<&DepNode> { self.graph.all_nodes().iter().map(|n| &n.data).collect() } - pub fn edges(&self) -> Vec<(&DepNode, &DepNode)> { + pub fn edges(&self) -> Vec<(&DepNode, &DepNode)> { self.graph .all_edges() .iter() @@ -53,7 +53,7 @@ impl DepGraphQuery { .collect() } - fn reachable_nodes(&self, node: &DepNode, direction: Direction) -> Vec<&DepNode> { + fn reachable_nodes(&self, node: &DepNode, direction: Direction) -> Vec<&DepNode> { if let Some(&index) = self.indices.get(node) { self.graph.depth_traverse(index, direction).map(|s| self.graph.node_data(s)).collect() } else { @@ -62,7 +62,7 @@ impl DepGraphQuery { } /// All nodes that can reach `node`. - pub fn transitive_predecessors(&self, node: &DepNode) -> Vec<&DepNode> { + pub fn transitive_predecessors(&self, node: &DepNode) -> Vec<&DepNode> { self.reachable_nodes(node, INCOMING) } } diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 1f8d87a7e913..bd72478086cf 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -35,9 +35,9 @@ rustc_index::newtype_index! { /// Data for use when recompiling the **current crate**. #[derive(Debug)] -pub struct SerializedDepGraph { +pub struct SerializedDepGraph { /// The set of all DepNodes in the graph - nodes: IndexVec>, + nodes: IndexVec, /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to /// the DepNode at the same index in the nodes vector. fingerprints: IndexVec, @@ -49,10 +49,11 @@ pub struct SerializedDepGraph { /// implicit in edge_list_indices. edge_list_data: Vec, /// Reciprocal map to `nodes`. - index: FxHashMap, SerializedDepNodeIndex>, + index: FxHashMap, } -impl Default for SerializedDepGraph { +impl Default for SerializedDepGraph { + #[inline] fn default() -> Self { SerializedDepGraph { nodes: Default::default(), @@ -64,7 +65,7 @@ impl Default for SerializedDepGraph { } } -impl SerializedDepGraph { +impl SerializedDepGraph { #[inline] pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] { let targets = self.edge_list_indices[source]; @@ -72,17 +73,17 @@ impl SerializedDepGraph { } #[inline] - pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { + pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { self.nodes[dep_node_index] } #[inline] - pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { + pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { self.index.get(dep_node).cloned() } #[inline] - pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { + pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index]) } @@ -91,16 +92,15 @@ impl SerializedDepGraph { self.fingerprints[dep_node_index] } + #[inline] pub fn node_count(&self) -> usize { self.index.len() } } -impl<'a, K: DepKind + Decodable>> Decodable> - for SerializedDepGraph -{ +impl<'a> Decodable> for SerializedDepGraph { #[instrument(level = "debug", skip(d))] - fn decode(d: &mut opaque::Decoder<'a>) -> SerializedDepGraph { + fn decode(d: &mut opaque::Decoder<'a>) -> SerializedDepGraph { let start_position = d.position(); // The last 16 bytes are the node count and edge count. @@ -122,7 +122,7 @@ impl<'a, K: DepKind + Decodable>> Decodable = Decodable::decode(d); + let dep_node: DepNode = Decodable::decode(d); let _i: SerializedDepNodeIndex = nodes.push(dep_node); debug_assert_eq!(_i.index(), _index); @@ -149,28 +149,29 @@ impl<'a, K: DepKind + Decodable>> Decodable { - node: DepNode, +#[derive(Debug, Encodable)] +pub struct NodeInfo { + node: DepNode, fingerprint: Fingerprint, edges: SmallVec<[DepNodeIndex; 8]>, } -struct Stat { - kind: K, +struct Stat { + kind: DepKind, node_counter: u64, edge_counter: u64, } -struct EncoderState { +struct EncoderState { encoder: FileEncoder, total_node_count: usize, total_edge_count: usize, result: FileEncodeResult, - stats: Option>>, + stats: Option>, } -impl EncoderState { +impl EncoderState { + #[inline] fn new(encoder: FileEncoder, record_stats: bool) -> Self { Self { encoder, @@ -183,8 +184,8 @@ impl EncoderState { fn encode_node( &mut self, - node: &NodeInfo, - record_graph: &Option>>, + node: &NodeInfo, + record_graph: &Option>, ) -> DepNodeIndex { let index = DepNodeIndex::new(self.total_node_count); self.total_node_count += 1; @@ -235,12 +236,13 @@ impl EncoderState { } } -pub struct GraphEncoder { - status: Lock>, - record_graph: Option>>, +pub struct GraphEncoder { + status: Lock, + record_graph: Option>, } -impl> GraphEncoder { +impl GraphEncoder { + #[inline] pub fn new( encoder: FileEncoder, prev_node_count: usize, @@ -253,7 +255,7 @@ impl> GraphEncoder { GraphEncoder { status, record_graph } } - pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery)) { + pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery)) { if let Some(record_graph) = &self.record_graph { f(&record_graph.lock()) } @@ -314,10 +316,11 @@ impl> GraphEncoder { } } + #[inline] pub(crate) fn send( &self, profiler: &SelfProfilerRef, - node: DepNode, + node: DepNode, fingerprint: Fingerprint, edges: SmallVec<[DepNodeIndex; 8]>, ) -> DepNodeIndex { @@ -326,6 +329,7 @@ impl> GraphEncoder { self.status.lock().encode_node(&node, &self.record_graph) } + #[inline] pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult { let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph"); self.status.into_inner().finish(profiler) diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs index 6b70e9342fa6..dcfb35e22186 100644 --- a/compiler/rustc_query_system/src/lib.rs +++ b/compiler/rustc_query_system/src/lib.rs @@ -18,3 +18,5 @@ pub mod cache; pub mod dep_graph; pub mod ich; pub mod query; + +pub mod tls; diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 7ca668f8a1f0..e7816616ce0d 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -1,5 +1,6 @@ //! Query configuration and description traits. +use crate::dep_graph::DepKind; use crate::dep_graph::DepNode; use crate::dep_graph::SerializedDepNodeIndex; use crate::ich::StableHashingContext; @@ -21,7 +22,7 @@ pub trait QueryConfig { pub struct QueryVtable { pub anon: bool, - pub dep_kind: CTX::DepKind, + pub dep_kind: DepKind, pub eval_always: bool, pub cache_on_disk: bool, @@ -32,7 +33,7 @@ pub struct QueryVtable { } impl QueryVtable { - pub(crate) fn to_dep_node(&self, tcx: CTX::DepContext, key: &K) -> DepNode + pub(crate) fn to_dep_node(&self, tcx: CTX::DepContext, key: &K) -> DepNode where K: crate::dep_graph::DepNodeParams, { diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index 2a07d9b7f809..0f20dac3b880 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -41,21 +41,25 @@ pub type QueryMap = FxHashMap; pub struct QueryJobId(pub NonZeroU64); impl QueryJobId { + #[inline] fn query(self, map: &QueryMap) -> QueryStackFrame { map.get(&self).unwrap().query.clone() } #[cfg(parallel_compiler)] + #[inline] fn span(self, map: &QueryMap) -> Span { map.get(&self).unwrap().job.span } #[cfg(parallel_compiler)] + #[inline] fn parent(self, map: &QueryMap) -> Option { map.get(&self).unwrap().job.parent } #[cfg(parallel_compiler)] + #[inline] fn latch<'a>(self, map: &'a QueryMap) -> Option<&'a QueryLatch> { map.get(&self).unwrap().job.latch.as_ref() } @@ -84,6 +88,7 @@ pub struct QueryJob { impl QueryJob { /// Creates a new query job. + #[inline] pub fn new(id: QueryJobId, span: Span, parent: Option) -> Self { QueryJob { id, @@ -95,6 +100,7 @@ impl QueryJob { } #[cfg(parallel_compiler)] + #[inline] pub(super) fn latch(&mut self) -> QueryLatch { if self.latch.is_none() { self.latch = Some(QueryLatch::new()); @@ -106,6 +112,7 @@ impl QueryJob { /// /// This does nothing for single threaded rustc, /// as there are no concurrent jobs which could be waiting on us + #[inline] pub fn signal_complete(self) { #[cfg(parallel_compiler)] { @@ -188,6 +195,7 @@ pub(super) struct QueryLatch { #[cfg(parallel_compiler)] impl QueryLatch { + #[inline] fn new() -> Self { QueryLatch { info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })), @@ -195,6 +203,7 @@ impl QueryLatch { } /// Awaits for the query job to complete. + #[inline] pub(super) fn wait_on(&self, query: Option, span: Span) -> Result<(), CycleError> { let waiter = Lrc::new(QueryWaiter { query, span, cycle: Lock::new(None), condvar: Condvar::new() }); @@ -210,6 +219,7 @@ impl QueryLatch { } /// Awaits the caller on this latch by blocking the current thread. + #[inline] fn wait_on_inner(&self, waiter: &Lrc) { let mut info = self.info.lock(); if !info.complete { @@ -232,6 +242,7 @@ impl QueryLatch { } /// Sets the latch and resumes all waiters on it + #[inline] fn set(&self) { let mut info = self.info.lock(); debug_assert!(!info.complete); @@ -244,6 +255,7 @@ impl QueryLatch { /// Removes a single waiter from the list of waiters. /// This is used to break query cycles. + #[inline] fn extract_waiter(&self, waiter: usize) -> Lrc { let mut info = self.info.lock(); debug_assert!(!info.complete); @@ -584,10 +596,11 @@ pub(crate) fn report_cycle<'a>( pub fn print_query_stack( tcx: CTX, - mut current_query: Option, handler: &Handler, num_frames: Option, ) -> usize { + let mut current_query = crate::tls::current_query_job(); + // Be careful relying on global state here: this code is called from // a panic hook, which means that the global `Handler` may be in a weird // state if it was responsible for triggering the panic. diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index de64ebb62030..9341e9c2ca16 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -16,7 +16,6 @@ pub use self::config::{QueryConfig, QueryDescription, QueryVtable}; use crate::dep_graph::{DepNodeIndex, HasDepContext, SerializedDepNodeIndex}; -use rustc_data_structures::sync::Lock; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::Diagnostic; use rustc_hir::def::DefKind; @@ -94,9 +93,6 @@ impl QuerySideEffects { pub trait QueryContext: HasDepContext { fn next_job_id(&self) -> QueryJobId; - /// Get the query information from the TLS context. - fn current_query_job(&self) -> Option; - fn try_collect_active_jobs(&self) -> Option; /// Load side effects associated to the node in the previous session. @@ -111,14 +107,4 @@ pub trait QueryContext: HasDepContext { dep_node_index: DepNodeIndex, side_effects: QuerySideEffects, ); - - /// Executes a job by changing the `ImplicitCtxt` to point to the - /// new query job while it executes. It returns the diagnostics - /// captured during execution and the actual result. - fn start_query( - &self, - token: QueryJobId, - diagnostics: Option<&Lock>>, - compute: impl FnOnce() -> R, - ) -> R; } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 700290d67a41..d94430e26a82 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -162,7 +162,7 @@ where match lock.entry(key) { Entry::Vacant(entry) => { let id = tcx.next_job_id(); - let job = tcx.current_query_job(); + let job = crate::tls::current_query_job(); let job = QueryJob::new(id, span, job); let key = entry.key().clone(); @@ -182,7 +182,7 @@ where // so we just return the error. return TryGetJob::Cycle(id.find_cycle_in_stack( tcx.try_collect_active_jobs().unwrap(), - &tcx.current_query_job(), + &crate::tls::current_query_job(), span, )); } @@ -200,7 +200,7 @@ where // With parallel queries we might just have to wait on some other // thread. - let result = latch.wait_on(tcx.current_query_job(), span); + let result = latch.wait_on(crate::tls::current_query_job(), span); match result { Ok(()) => TryGetJob::JobCompleted(query_blocked_prof_timer), @@ -330,7 +330,7 @@ fn try_execute_query( cache: &C, span: Span, key: C::Key, - dep_node: Option>, + dep_node: Option, query: &QueryVtable, ) -> (C::Stored, Option) where @@ -367,7 +367,7 @@ where fn execute_job( tcx: CTX, key: K, - mut dep_node_opt: Option>, + mut dep_node_opt: Option, query: &QueryVtable, job_id: QueryJobId, ) -> (V, DepNodeIndex) @@ -381,7 +381,8 @@ where // Fast path for when incr. comp. is off. if !dep_graph.is_fully_enabled() { let prof_timer = tcx.dep_context().profiler().query_provider(); - let result = tcx.start_query(job_id, None, || query.compute(*tcx.dep_context(), key)); + let result = + crate::tls::start_query(job_id, None, || query.compute(*tcx.dep_context(), key)); let dep_node_index = dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); return (result, dep_node_index); @@ -394,7 +395,7 @@ where // The diagnostics for this query will be promoted to the current session during // `try_mark_green()`, so we can ignore them here. - if let Some(ret) = tcx.start_query(job_id, None, || { + if let Some(ret) = crate::tls::start_query(job_id, None, || { try_load_from_disk_and_cache_in_memory(tcx, &key, &dep_node, query) }) { return ret; @@ -404,7 +405,7 @@ where let prof_timer = tcx.dep_context().profiler().query_provider(); let diagnostics = Lock::new(ThinVec::new()); - let (result, dep_node_index) = tcx.start_query(job_id, Some(&diagnostics), || { + let (result, dep_node_index) = crate::tls::start_query(job_id, Some(&diagnostics), || { if query.anon { return dep_graph.with_anon_task(*tcx.dep_context(), query.dep_kind, || { query.compute(*tcx.dep_context(), key) @@ -436,11 +437,11 @@ where fn try_load_from_disk_and_cache_in_memory( tcx: CTX, key: &K, - dep_node: &DepNode, + dep_node: &DepNode, query: &QueryVtable, ) -> Option<(V, DepNodeIndex)> where - K: Clone, + K: Clone + DepNodeParams, CTX: QueryContext, V: Debug, { @@ -527,7 +528,7 @@ where fn incremental_verify_ich( tcx: CTX::DepContext, result: &V, - dep_node: &DepNode, + dep_node: &DepNode, query: &QueryVtable, ) where CTX: QueryContext, @@ -642,7 +643,7 @@ fn ensure_must_run( tcx: CTX, key: &K, query: &QueryVtable, -) -> (bool, Option>) +) -> (bool, Option) where K: crate::dep_graph::DepNodeParams, CTX: QueryContext, @@ -713,7 +714,7 @@ where Some(result) } -pub fn force_query(tcx: CTX, key: Q::Key, dep_node: DepNode) +pub fn force_query(tcx: CTX, key: Q::Key, dep_node: DepNode) where Q: QueryDescription, Q::Key: DepNodeParams, diff --git a/compiler/rustc_query_system/src/tls.rs b/compiler/rustc_query_system/src/tls.rs new file mode 100644 index 000000000000..a4cb605dee0c --- /dev/null +++ b/compiler/rustc_query_system/src/tls.rs @@ -0,0 +1,164 @@ +use crate::dep_graph::TaskDepsRef; +use crate::query::QueryJobId; +use rustc_data_structures::sync::{self, Lock}; +use rustc_data_structures::thin_vec::ThinVec; +use rustc_errors::Diagnostic; + +#[cfg(not(parallel_compiler))] +use std::cell::Cell; + +#[cfg(parallel_compiler)] +use rustc_rayon_core as rayon_core; + +/// This is the implicit state of rustc. It contains the current +/// query. It is updated when +/// executing a new query. +#[derive(Clone, Default)] +struct ImplicitCtxt<'a> { + /// The current query job, if any. This is updated by `JobOwner::start` in + /// `ty::query::plumbing` when executing a query. + query: Option, + + /// Where to store diagnostics for the current query job, if any. + /// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query. + diagnostics: Option<&'a Lock>>, + + /// Used to prevent layout from recursing too deeply. + layout_depth: usize, + + /// The current dep graph task. This is used to add dependencies to queries + /// when executing them. + task_deps: TaskDepsRef<'a>, +} + +/// Sets Rayon's thread-local variable, which is preserved for Rayon jobs +/// to `value` during the call to `f`. It is restored to its previous value after. +/// This is used to set the pointer to the new `ImplicitCtxt`. +#[cfg(parallel_compiler)] +#[inline] +fn set_tlv R, R>(value: usize, f: F) -> R { + rayon_core::tlv::with(value, f) +} + +/// Gets Rayon's thread-local variable, which is preserved for Rayon jobs. +/// This is used to get the pointer to the current `ImplicitCtxt`. +#[cfg(parallel_compiler)] +#[inline] +pub fn get_tlv() -> usize { + rayon_core::tlv::get() +} + +#[cfg(not(parallel_compiler))] +thread_local! { + /// A thread local variable that stores a pointer to the current `ImplicitCtxt`. + static TLV: Cell = const { Cell::new(0) }; +} + +/// Sets TLV to `value` during the call to `f`. +/// It is restored to its previous value after. +/// This is used to set the pointer to the new `ImplicitCtxt`. +#[cfg(not(parallel_compiler))] +#[inline] +fn set_tlv R, R>(value: usize, f: F) -> R { + let old = get_tlv(); + let _reset = rustc_data_structures::OnDrop(move || TLV.with(|tlv| tlv.set(old))); + TLV.with(|tlv| tlv.set(value)); + f() +} + +/// Gets the pointer to the current `ImplicitCtxt`. +#[cfg(not(parallel_compiler))] +#[inline] +fn get_tlv() -> usize { + TLV.with(|tlv| tlv.get()) +} + +/// Sets `context` as the new current `ImplicitCtxt` for the duration of the function `f`. +#[inline] +fn enter_context<'a, F, R>(context: &ImplicitCtxt<'a>, f: F) -> R +where + F: FnOnce() -> R, +{ + set_tlv(context as *const _ as usize, || f()) +} + +/// Allows access to the current `ImplicitCtxt` in a closure if one is available. +#[inline] +fn with_context_opt(f: F) -> R +where + F: for<'a> FnOnce(Option<&ImplicitCtxt<'a>>) -> R, +{ + // We could get a `ImplicitCtxt` pointer from another thread. + // Ensure that `ImplicitCtxt` is `Sync`. + sync::assert_sync::>(); + + let context = get_tlv(); + let context = + if context == 0 { None } else { unsafe { Some(&*(context as *const ImplicitCtxt<'_>)) } }; + f(context) +} + +/// This is a callback from `rustc_ast` as it cannot access the implicit state +/// in `rustc_middle` otherwise. It is used to when diagnostic messages are +/// emitted and stores them in the current query, if there is one. +pub fn track_diagnostic(diagnostic: &Diagnostic) { + with_context_opt(|icx| { + if let Some(icx) = icx { + if let Some(ref diagnostics) = icx.diagnostics { + let mut diagnostics = diagnostics.lock(); + diagnostics.extend(Some(diagnostic.clone())); + } + } + }) +} + +pub fn with_deps(task_deps: TaskDepsRef<'_>, op: impl FnOnce() -> R) -> R { + crate::tls::with_context_opt(|icx| { + let icx = crate::tls::ImplicitCtxt { task_deps, ..icx.cloned().unwrap_or_default() }; + + crate::tls::enter_context(&icx, op) + }) +} + +pub fn read_deps(op: impl FnOnce(TaskDepsRef<'_>)) { + crate::tls::with_context_opt(|icx| if let Some(icx) = icx { op(icx.task_deps) } else { return }) +} + +/// Get the query information from the TLS context. +#[inline(always)] +pub fn current_query_job() -> Option { + with_context_opt(|icx| icx?.query) +} + +/// Executes a job by changing the `ImplicitCtxt` to point to the +/// new query job while it executes. It returns the diagnostics +/// captured during execution and the actual result. +#[inline(always)] +pub fn start_query( + token: QueryJobId, + diagnostics: Option<&Lock>>, + compute: impl FnOnce() -> R, +) -> R { + with_context_opt(move |current_icx| { + let task_deps = current_icx.map_or(TaskDepsRef::Ignore, |icx| icx.task_deps); + let layout_depth = current_icx.map_or(0, |icx| icx.layout_depth); + + // Update the `ImplicitCtxt` to point to our new query job. + let new_icx = ImplicitCtxt { query: Some(token), diagnostics, task_deps, layout_depth }; + + // Use the `ImplicitCtxt` while we execute the query. + enter_context(&new_icx, || rustc_data_structures::stack::ensure_sufficient_stack(compute)) + }) +} + +pub fn with_increased_layout_depth(f: impl FnOnce(usize) -> R) -> R { + with_context_opt(|icx| { + let layout_depth = icx.map_or(0, |icx| icx.layout_depth); + + // Update the ImplicitCtxt to increase the layout_depth + let icx = + ImplicitCtxt { layout_depth: layout_depth + 1, ..icx.cloned().unwrap_or_default() }; + + enter_context(&icx, || f(layout_depth)) + }) +} diff --git a/compiler/rustc_session/src/cstore.rs b/compiler/rustc_session/src/cstore.rs index c1fd3c7c61b9..658e969fcfe9 100644 --- a/compiler/rustc_session/src/cstore.rs +++ b/compiler/rustc_session/src/cstore.rs @@ -215,4 +215,4 @@ pub trait CrateStore: std::fmt::Debug { fn import_source_files(&self, sess: &Session, cnum: CrateNum); } -pub type CrateStoreDyn = dyn CrateStore + sync::Sync; +pub type CrateStoreDyn = dyn CrateStore + sync::Sync + sync::Send; diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index e69bf6eca90e..75ea252a73a7 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -30,7 +30,7 @@ use rustc_errors::{Diagnostic, ErrorGuaranteed}; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_infer::infer::LateBoundRegionConversionTime; -use rustc_middle::dep_graph::{DepKind, DepNodeIndex}; +use rustc_middle::dep_graph::{dep_kind, DepNodeIndex}; use rustc_middle::mir::interpret::ErrorHandled; use rustc_middle::thir::abstract_const::NotConstEvaluatable; use rustc_middle::ty::fast_reject::{self, TreatParams}; @@ -1128,7 +1128,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { OP: FnOnce(&mut Self) -> R, { let (result, dep_node) = - self.tcx().dep_graph.with_anon_task(self.tcx(), DepKind::TraitSelect, || op(self)); + self.tcx().dep_graph.with_anon_task(self.tcx(), dep_kind::TraitSelect, || op(self)); self.tcx().dep_graph.read_index(dep_node); (result, dep_node) }