diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 4c94c993ab405..f39e9ebff8a58 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -202,196 +202,245 @@ impl DepGraph { arg: A, task: fn(C, A) -> R) -> (R, DepNodeIndex) - where C: DepGraphSafe + StableHashingContextProvider<'gcx>, + where C: DepGraphSafe + StableHashingContextProvider<'gcx> + Clone, R: HashStable<StableHashingContext<'gcx>>, { - self.with_task_impl(key, cx, arg, false, task, - |key| OpenTask::Regular(Lock::new(RegularOpenTask { + if let Some(ref data) = self.data { + let open_task = OpenTask::Regular(Lock::new(RegularOpenTask { node: key, reads: SmallVec::new(), read_set: Default::default(), - })), - |data, key, task| data.borrow_mut().complete_task(key, task)) + })); + let result = ty::tls::with_context(|icx| { + let icx = ty::tls::ImplicitCtxt { + task: &open_task, + ..icx.clone() + }; + + ty::tls::enter_context(&icx, |_| { + task(cx.clone(), arg) + }) + }); + let dep_node_index = data.current.borrow_mut().complete_task(key, open_task); + self.finish_task_incr_on(data, key, cx, &result, dep_node_index); + (result, dep_node_index) + } else { + let result = task(cx.clone(), arg); + self.finish_task_incr_off(key, cx, &result); + (result, DepNodeIndex::INVALID) + } } - /// Creates a new dep-graph input with value `input` - pub fn input_task<'gcx, C, R>(&self, - key: DepNode, - cx: C, - input: R) - -> (R, DepNodeIndex) - where C: DepGraphSafe + StableHashingContextProvider<'gcx>, - R: HashStable<StableHashingContext<'gcx>>, + /// Execute something within an "eval-always" task which is a task + // that runs whenever anything changes. + // FIXME: Find a way to make F: DepGraphSafe + pub fn with_eval_always_task<'a, F, R>( + &self, + tcx: TyCtxt<'a, '_, '_>, + key: DepNode, + task: F, + ) -> (R, DepNodeIndex) + where F: FnOnce(&OpenTask) -> R, + R: HashStable<StableHashingContext<'a>>, { - fn identity_fn<C, A>(_: C, arg: A) -> A { - arg + if let Some(ref data) = self.data { + let open_task = OpenTask::EvalAlways { node: key }; + let result = task(&open_task); + let dep_node_index = data.current.borrow_mut() + .complete_eval_always_task(key, open_task); + self.finish_task_incr_on(data, key, tcx, &result, dep_node_index); + (result, dep_node_index) + } else { + debug_assert!(!key.kind.fingerprint_needed_for_crate_hash()); + (task(&OpenTask::Ignore), DepNodeIndex::INVALID) } + } - self.with_task_impl(key, cx, input, true, identity_fn, - |_| OpenTask::Ignore, - |data, key, _| data.borrow_mut().alloc_node(key, SmallVec::new())) + // FIXME: Merge with with_task? + pub fn with_query_task<'a, F, R>( + &self, + tcx: TyCtxt<'a, '_, '_>, + key: DepNode, + task: F, + ) -> (R, DepNodeIndex) + where F: FnOnce(&OpenTask) -> R, + R: HashStable<StableHashingContext<'a>>, + { + if let Some(ref data) = self.data { + let open_task = OpenTask::Regular(Lock::new(RegularOpenTask { + node: key, + reads: SmallVec::new(), + read_set: Default::default(), + })); + let result = task(&open_task); + // FIXME: Look at `complete_task` and the same for other functions + let dep_node_index = data.current.borrow_mut().complete_task(key, open_task); + self.finish_task_incr_on(data, key, tcx, &result, dep_node_index); + (result, dep_node_index) + } else { + debug_assert!(!key.kind.fingerprint_needed_for_crate_hash()); + // with_task runs finish_task_incr_off here + (task(&OpenTask::Ignore), DepNodeIndex::INVALID) + } + } + + /// Creates a new dep-graph input with value `input` + pub fn input_dep_index<'gcx, R>( + &self, + key: DepNode, + cx: &StableHashingContext<'gcx>, + input: &R + ) -> DepNodeIndex + where R: HashStable<StableHashingContext<'gcx>>, + { + // This assumes that we don't have an ImplicitCtxt and thus have + // an implicit OpenTask::Ignore task + debug_assert!(ty::tls::with_opt(|tcx| tcx.is_none())); + + if let Some(ref data) = self.data { + let dep_node_index = data.current.borrow_mut().alloc_node(key, SmallVec::new()); + self.finish_task_incr_on(data, key, cx, input, dep_node_index); + dep_node_index + } else { + self.finish_task_incr_off(key, cx, input) + } } - fn with_task_impl<'gcx, C, A, R>( + fn finish_task_incr_on<'gcx, C, R>( &self, + data: &DepGraphData, key: DepNode, cx: C, - arg: A, - no_tcx: bool, - task: fn(C, A) -> R, - create_task: fn(DepNode) -> OpenTask, - finish_task_and_alloc_depnode: fn(&Lock<CurrentDepGraph>, - DepNode, - OpenTask) -> DepNodeIndex - ) -> (R, DepNodeIndex) + result: &R, + dep_node_index: DepNodeIndex, + ) where C: DepGraphSafe + StableHashingContextProvider<'gcx>, R: HashStable<StableHashingContext<'gcx>>, { - if let Some(ref data) = self.data { - let open_task = create_task(key); + // In incremental mode, hash the result of the task. We don't + // do anything with the hash yet, but we are computing it + // anyway so that + // - we make sure that the infrastructure works and + // - we can get an idea of the runtime cost. + let mut hcx = cx.get_stable_hashing_context(); + + if cfg!(debug_assertions) { + profq_msg(hcx.sess(), ProfileQueriesMsg::TaskBegin(key.clone())) + }; - // In incremental mode, hash the result of the task. We don't - // do anything with the hash yet, but we are computing it - // anyway so that - // - we make sure that the infrastructure works and - // - we can get an idea of the runtime cost. - let mut hcx = cx.get_stable_hashing_context(); + if cfg!(debug_assertions) { + profq_msg(hcx.sess(), ProfileQueriesMsg::TaskEnd) + }; - if cfg!(debug_assertions) { - profq_msg(hcx.sess(), ProfileQueriesMsg::TaskBegin(key.clone())) - }; + let mut stable_hasher = StableHasher::new(); + result.hash_stable(&mut hcx, &mut stable_hasher); - let result = if no_tcx { - task(cx, arg) - } else { - ty::tls::with_context(|icx| { - let icx = ty::tls::ImplicitCtxt { - task: &open_task, - ..icx.clone() - }; - - ty::tls::enter_context(&icx, |_| { - task(cx, arg) - }) - }) - }; + let current_fingerprint = stable_hasher.finish(); - if cfg!(debug_assertions) { - profq_msg(hcx.sess(), ProfileQueriesMsg::TaskEnd) - }; + // Store the current fingerprint + { + let mut fingerprints = self.fingerprints.borrow_mut(); - let dep_node_index = finish_task_and_alloc_depnode(&data.current, key, open_task); + if dep_node_index.index() >= fingerprints.len() { + fingerprints.resize(dep_node_index.index() + 1, Fingerprint::ZERO); + } - let mut stable_hasher = StableHasher::new(); - result.hash_stable(&mut hcx, &mut stable_hasher); + debug_assert!(fingerprints[dep_node_index] == Fingerprint::ZERO, + "DepGraph::with_task() - Duplicate fingerprint \ + insertion for {:?}", key); + fingerprints[dep_node_index] = current_fingerprint; + } - let current_fingerprint = stable_hasher.finish(); + // Determine the color of the new DepNode. + if let Some(prev_index) = data.previous.node_to_index_opt(&key) { + let prev_fingerprint = data.previous.fingerprint_by_index(prev_index); - // Store the current fingerprint - { - let mut fingerprints = self.fingerprints.borrow_mut(); + let color = if current_fingerprint == prev_fingerprint { + DepNodeColor::Green(dep_node_index) + } else { + DepNodeColor::Red + }; - if dep_node_index.index() >= fingerprints.len() { - fingerprints.resize(dep_node_index.index() + 1, Fingerprint::ZERO); - } + let mut colors = data.colors.borrow_mut(); + debug_assert!(colors.get(prev_index).is_none(), + "DepGraph::with_task() - Duplicate DepNodeColor \ + insertion for {:?}", key); - debug_assert!(fingerprints[dep_node_index] == Fingerprint::ZERO, - "DepGraph::with_task() - Duplicate fingerprint \ - insertion for {:?}", key); - fingerprints[dep_node_index] = current_fingerprint; - } + colors.insert(prev_index, color); + } + } - // Determine the color of the new DepNode. - if let Some(prev_index) = data.previous.node_to_index_opt(&key) { - let prev_fingerprint = data.previous.fingerprint_by_index(prev_index); + fn finish_task_incr_off<'gcx, C, R>( + &self, + key: DepNode, + cx: C, + result: &R, + ) -> DepNodeIndex + where + C: DepGraphSafe + StableHashingContextProvider<'gcx>, + R: HashStable<StableHashingContext<'gcx>>, + { + debug_assert!(self.data.is_none()); - let color = if current_fingerprint == prev_fingerprint { - DepNodeColor::Green(dep_node_index) - } else { - DepNodeColor::Red - }; + if key.kind.fingerprint_needed_for_crate_hash() { + let mut hcx = cx.get_stable_hashing_context(); + let mut stable_hasher = StableHasher::new(); + result.hash_stable(&mut hcx, &mut stable_hasher); + let fingerprint = stable_hasher.finish(); - let mut colors = data.colors.borrow_mut(); - debug_assert!(colors.get(prev_index).is_none(), - "DepGraph::with_task() - Duplicate DepNodeColor \ - insertion for {:?}", key); + let mut fingerprints = self.fingerprints.borrow_mut(); + let dep_node_index = DepNodeIndex::new(fingerprints.len()); + fingerprints.push(fingerprint); - colors.insert(prev_index, color); - } + debug_assert!(fingerprints[dep_node_index] == fingerprint, + "DepGraph::with_task() - Assigned fingerprint to \ + unexpected index for {:?}", key); - (result, dep_node_index) + dep_node_index } else { - if key.kind.fingerprint_needed_for_crate_hash() { - let mut hcx = cx.get_stable_hashing_context(); - let result = task(cx, arg); - let mut stable_hasher = StableHasher::new(); - result.hash_stable(&mut hcx, &mut stable_hasher); - let fingerprint = stable_hasher.finish(); - - let mut fingerprints = self.fingerprints.borrow_mut(); - let dep_node_index = DepNodeIndex::new(fingerprints.len()); - fingerprints.push(fingerprint); - - debug_assert!(fingerprints[dep_node_index] == fingerprint, - "DepGraph::with_task() - Assigned fingerprint to \ - unexpected index for {:?}", key); - - (result, dep_node_index) - } else { - (task(cx, arg), DepNodeIndex::INVALID) - } + DepNodeIndex::INVALID } } /// Execute something within an "anonymous" task, that is, a task the /// DepNode of which is determined by the list of inputs it read from. - pub fn with_anon_task<OP,R>(&self, dep_kind: DepKind, op: OP) -> (R, DepNodeIndex) - where OP: FnOnce() -> R + pub fn with_anon_open_task<OP,R>(&self, dep_kind: DepKind, op: OP) -> (R, DepNodeIndex) + where OP: FnOnce(&OpenTask) -> R { if let Some(ref data) = self.data { - let (result, open_task) = ty::tls::with_context(|icx| { - let task = OpenTask::Anon(Lock::new(AnonOpenTask { - reads: SmallVec::new(), - read_set: Default::default(), - })); - - let r = { - let icx = ty::tls::ImplicitCtxt { - task: &task, - ..icx.clone() - }; - - ty::tls::enter_context(&icx, |_| { - op() - }) - }; + let task = OpenTask::Anon(Lock::new(AnonOpenTask { + reads: SmallVec::new(), + read_set: Default::default(), + })); - (r, task) - }); + let result = op(&task); let dep_node_index = data.current .borrow_mut() - .pop_anon_task(dep_kind, open_task); + .pop_anon_task(dep_kind, task); (result, dep_node_index) } else { - (op(), DepNodeIndex::INVALID) + (op(&OpenTask::Ignore), DepNodeIndex::INVALID) } } - /// Execute something within an "eval-always" task which is a task - // that runs whenever anything changes. - pub fn with_eval_always_task<'gcx, C, A, R>(&self, - key: DepNode, - cx: C, - arg: A, - task: fn(C, A) -> R) - -> (R, DepNodeIndex) - where C: DepGraphSafe + StableHashingContextProvider<'gcx>, - R: HashStable<StableHashingContext<'gcx>>, + /// Execute something within an "anonymous" task, that is, a task the + /// DepNode of which is determined by the list of inputs it read from. + pub fn with_anon_task<OP,R>(&self, dep_kind: DepKind, op: OP) -> (R, DepNodeIndex) + where OP: FnOnce() -> R { - self.with_task_impl(key, cx, arg, false, task, - |key| OpenTask::EvalAlways { node: key }, - |data, key, task| data.borrow_mut().complete_eval_always_task(key, task)) + self.with_anon_open_task(dep_kind, |task| { + ty::tls::with_context(|icx| { + let icx = ty::tls::ImplicitCtxt { + task, + ..icx.clone() + }; + + ty::tls::enter_context(&icx, |_| { + op() + }) + }) + }) } #[inline] diff --git a/src/librustc/hir/map/collector.rs b/src/librustc/hir/map/collector.rs index 2917fd7457acf..ea1845530fe84 100644 --- a/src/librustc/hir/map/collector.rs +++ b/src/librustc/hir/map/collector.rs @@ -83,20 +83,20 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { body_ids: _, } = *krate; - root_mod_sig_dep_index = dep_graph.input_task( + root_mod_sig_dep_index = dep_graph.input_dep_index( root_mod_def_path_hash.to_dep_node(DepKind::Hir), &hcx, - HirItemLike { item_like: (module, attrs, span), hash_bodies: false }, - ).1; - root_mod_full_dep_index = dep_graph.input_task( + &HirItemLike { item_like: (module, attrs, span), hash_bodies: false }, + ); + root_mod_full_dep_index = dep_graph.input_dep_index( root_mod_def_path_hash.to_dep_node(DepKind::HirBody), &hcx, - HirItemLike { item_like: (module, attrs, span), hash_bodies: true }, - ).1; + &HirItemLike { item_like: (module, attrs, span), hash_bodies: true }, + ); } { - dep_graph.input_task( + dep_graph.input_dep_index( DepNode::new_no_params(DepKind::AllLocalTraitImpls), &hcx, &krate.trait_impls, @@ -169,11 +169,11 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { source_file_names.sort_unstable(); - let (_, crate_dep_node_index) = self + let crate_dep_node_index = self .dep_graph - .input_task(DepNode::new_no_params(DepKind::Krate), + .input_dep_index(DepNode::new_no_params(DepKind::Krate), &self.hcx, - (((node_hashes, upstream_crates), source_file_names), + &(((node_hashes, upstream_crates), source_file_names), (commandline_args_hash, crate_disambiguator.to_fingerprint()))); @@ -261,17 +261,17 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { let def_path_hash = self.definitions.def_path_hash(dep_node_owner); - self.current_signature_dep_index = self.dep_graph.input_task( + self.current_signature_dep_index = self.dep_graph.input_dep_index( def_path_hash.to_dep_node(DepKind::Hir), &self.hcx, - HirItemLike { item_like, hash_bodies: false }, - ).1; + &HirItemLike { item_like, hash_bodies: false }, + ); - self.current_full_dep_index = self.dep_graph.input_task( + self.current_full_dep_index = self.dep_graph.input_dep_index( def_path_hash.to_dep_node(DepKind::HirBody), &self.hcx, - HirItemLike { item_like, hash_bodies: true }, - ).1; + &HirItemLike { item_like, hash_bodies: true }, + ); self.hir_body_nodes.push((def_path_hash, self.current_full_dep_index)); diff --git a/src/librustc/ty/query/job.rs b/src/librustc/ty/query/job.rs index 1439e41bb31fd..2bd5f133ed3af 100644 --- a/src/librustc/ty/query/job.rs +++ b/src/librustc/ty/query/job.rs @@ -79,6 +79,10 @@ impl<'tcx> QueryJob<'tcx> { } } + pub fn extract_diagnostics(&self) -> Vec<Diagnostic> { + mem::replace(&mut *self.diagnostics.lock(), Vec::new()) + } + /// Awaits for the query job to complete. /// /// For single threaded rustc there's no concurrent jobs running, so if we are waiting for any diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 5f33d466c4a19..44f21dc624210 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -12,10 +12,9 @@ //! that generate the actual methods on tcx which find and execute the //! provider, manage the caches, and so forth. -use dep_graph::{DepNodeIndex, DepNode, DepKind, DepNodeColor}; +use dep_graph::{DepNodeIndex, DepNode, DepKind, DepNodeColor, OpenTask}; use errors::DiagnosticBuilder; use errors::Level; -use errors::Diagnostic; use errors::FatalError; use ty::tls; use ty::{TyCtxt}; @@ -30,6 +29,7 @@ use rustc_data_structures::fx::{FxHashMap}; use rustc_data_structures::sync::{Lrc, Lock}; use std::mem; use std::ptr; +use std::intrinsics::unlikely; use std::collections::hash_map::Entry; use syntax_pos::Span; use syntax::source_map::DUMMY_SP; @@ -183,36 +183,33 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> { /// Executes a job by changing the ImplicitCtxt to point to the /// new query job while it executes. It returns the diagnostics /// captured during execution and the actual result. - pub(super) fn start<'lcx, F, R>( + pub(super) fn with_context<'lcx, F, R>( &self, tcx: TyCtxt<'_, 'tcx, 'lcx>, + task: &OpenTask, compute: F) - -> (R, Vec<Diagnostic>) + -> R where F: for<'b> FnOnce(TyCtxt<'b, 'tcx, 'lcx>) -> R { // The TyCtxt stored in TLS has the same global interner lifetime // as `tcx`, so we use `with_related_context` to relate the 'gcx lifetimes // when accessing the ImplicitCtxt - let r = tls::with_related_context(tcx, move |current_icx| { + tls::with_related_context(tcx, move |current_icx| { // Update the ImplicitCtxt to point to our new query job let new_icx = tls::ImplicitCtxt { tcx, query: Some(self.job.clone()), + // FIXME: Remove `layout_depth` to avoid accessing ImplicitCtxt here layout_depth: current_icx.layout_depth, - task: current_icx.task, + task, }; // Use the ImplicitCtxt while we execute the query tls::enter_context(&new_icx, |_| { compute(tcx) }) - }); - - // Extract the diagnostic from the job - let diagnostics = mem::replace(&mut *self.job.diagnostics.lock(), Vec::new()); - - (r, diagnostics) + }) } } @@ -389,20 +386,18 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { profq_msg!(self, ProfileQueriesMsg::ProviderBegin); self.sess.profiler(|p| p.start_activity(Q::CATEGORY)); - let res = job.start(self, |tcx| { - tcx.dep_graph.with_anon_task(dep_node.kind, || { - Q::compute(tcx.global_tcx(), key) - }) + let res = self.dep_graph.with_anon_open_task(dep_node.kind, |open_task| { + job.with_context(self, open_task, |tcx | Q::compute(tcx.global_tcx(), key)) }); self.sess.profiler(|p| p.end_activity(Q::CATEGORY)); profq_msg!(self, ProfileQueriesMsg::ProviderEnd); - let ((result, dep_node_index), diagnostics) = res; + let (result, dep_node_index) = res; self.dep_graph.read_index(dep_node_index); self.queries.on_disk_cache - .store_diagnostics_for_anon_node(dep_node_index, diagnostics); + .store_diagnostics_for_anon_node(dep_node_index, job.job.extract_diagnostics()); job.complete(&result, dep_node_index); @@ -472,19 +467,14 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { // The diagnostics for this query have already been // promoted to the current session during // try_mark_green(), so we can ignore them here. - let (result, _) = job.start(self, |tcx| { - // The dep-graph for this computation is already in - // place - tcx.dep_graph.with_ignore(|| { - Q::compute(tcx, key) - }) - }); - result + // The dep-graph for this computation is already in + // place so we pass OpenTask::Ignore. + job.with_context(self, &OpenTask::Ignore, |tcx| Q::compute(tcx, key)) }; // If -Zincremental-verify-ich is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. - if self.sess.opts.debugging_opts.incremental_verify_ich { + if unsafe { unlikely(self.sess.opts.debugging_opts.incremental_verify_ich) } { use rustc_data_structures::stable_hasher::{StableHasher, HashStable}; use ich::Fingerprint; @@ -508,7 +498,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { for {:?}", dep_node); } - if self.sess.opts.debugging_opts.query_dep_graph { + if unsafe { unlikely(self.sess.opts.debugging_opts.query_dep_graph) } { self.dep_graph.mark_loaded_from_cache(dep_node_index, true); } @@ -517,6 +507,8 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { Ok(result) } + // FIXME: Inline this so LLVM can tell what kind of DepNode we are using + #[inline(always)] fn force_query_with_job<Q: QueryDescription<'gcx>>( self, key: Q::Key, @@ -540,32 +532,28 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { p.record_query(Q::CATEGORY); }); - let res = job.start(self, |tcx| { - if dep_node.kind.is_eval_always() { - tcx.dep_graph.with_eval_always_task(dep_node, - tcx, - key, - Q::compute) - } else { - tcx.dep_graph.with_task(dep_node, - tcx, - key, - Q::compute) - } - }); + let res = if dep_node.kind.is_eval_always() { + self.dep_graph.with_eval_always_task(self, dep_node, |task| { + job.with_context(self, task, |tcx| Q::compute(tcx, key)) + }) + } else { + self.dep_graph.with_query_task(self, dep_node, |task| { + job.with_context(self, task, |tcx| Q::compute(tcx, key)) + }) + }; self.sess.profiler(|p| p.end_activity(Q::CATEGORY)); profq_msg!(self, ProfileQueriesMsg::ProviderEnd); - let ((result, dep_node_index), diagnostics) = res; + let (result, dep_node_index) = res; - if self.sess.opts.debugging_opts.query_dep_graph { + if unsafe { unlikely(self.sess.opts.debugging_opts.query_dep_graph) } { self.dep_graph.mark_loaded_from_cache(dep_node_index, false); } if dep_node.kind != ::dep_graph::DepKind::Null { self.queries.on_disk_cache - .store_diagnostics(dep_node_index, diagnostics); + .store_diagnostics(dep_node_index, job.job.extract_diagnostics()); } job.complete(&result, dep_node_index);