Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Only keep one dep-graph in memory #82345

Closed
wants to merge 14 commits into from
Prev Previous commit
Next Next commit
Make try_mark_previous_green lock-free.
  • Loading branch information
cjgillot committed Feb 24, 2021
commit 064bbb8db6e7bfab21c10431e27061883e2eb730
11 changes: 10 additions & 1 deletion compiler/rustc_data_structures/src/sync.rs
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ cfg_if! {
}
}

use std::ops::Add;
use std::ops::{Add, BitOr};
use std::panic::{resume_unwind, catch_unwind, AssertUnwindSafe};

/// This is a single threaded variant of AtomicCell provided by crossbeam.
@@ -147,6 +147,15 @@ cfg_if! {
}
}

impl<T: BitOr<Output=T> + Copy> Atomic<T> {
#[inline]
pub fn fetch_or(&self, val: T, _: Ordering) -> T {
let old = self.0.get();
self.0.set(old | val);
old
}
}

pub type AtomicUsize = Atomic<usize>;
pub type AtomicBool = Atomic<bool>;
pub type AtomicU32 = Atomic<u32>;
92 changes: 51 additions & 41 deletions compiler/rustc_query_system/src/dep_graph/graph.rs
Original file line number Diff line number Diff line change
@@ -46,6 +46,10 @@ impl std::convert::From<DepNodeIndex> for QueryInvocationId {
}

struct DepGraphData<K: DepKind> {
/// The dep-graph from the previous compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them.
serialized: SerializedDepGraph<K>,

/// The dep-graph from the compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them.
graph: RwLock<CurrentDepGraph<K>>,
@@ -118,9 +122,11 @@ impl<K: DepKind> DepGraph<K> {
Err(_) => None,
};

let graph = RwLock::new(CurrentDepGraph::new(&prev_graph));
DepGraph {
data: Some(Lrc::new(DepGraphData {
graph: RwLock::new(CurrentDepGraph::new(prev_graph)),
serialized: prev_graph,
graph,
emitting_diagnostics: Default::default(),
previous_work_products: prev_work_products,
dep_node_debug: Default::default(),
@@ -148,7 +154,7 @@ impl<K: DepKind> DepGraph<K> {

pub fn query(&self) -> DepGraphQuery<K> {
let data = self.data.as_ref().unwrap();
data.graph.read().query()
data.graph.read().query(&data.serialized)
}

pub fn assert_ignored(&self) {
@@ -239,6 +245,7 @@ impl<K: DepKind> DepGraph<K> {

// Intern the new `DepNode`.
let dep_node_index = data.graph.write().intern_task_node(
&data.serialized,
key,
&edges[..],
current_fingerprint,
@@ -284,8 +291,11 @@ impl<K: DepKind> DepGraph<K> {
hash: data.anon_id_seed.combine(hasher.finish()).into(),
};

let dep_node_index =
data.graph.write().intern_anon_node(target_dep_node, &task_deps.reads[..]);
let dep_node_index = data.graph.write().intern_anon_node(
&data.serialized,
target_dep_node,
&task_deps.reads[..],
);

(result, dep_node_index)
} else {
@@ -334,7 +344,10 @@ impl<K: DepKind> DepGraph<K> {
#[cfg(debug_assertions)]
if let Some(target) = task_deps.node {
if let Some(ref forbidden_edge) = _data.forbidden_edge {
let src = self.dep_node_of(dep_node_index);
let src = _data
.graph
.read()
.index_to_node(&_data.serialized, dep_node_index);
if forbidden_edge.test(&src, &target) {
panic!("forbidden edge {:?} -> {:?} created", src, target)
}
@@ -352,24 +365,18 @@ impl<K: DepKind> DepGraph<K> {
#[inline]
fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
let data = self.data.as_ref()?;
data.graph.read().dep_node_index_of_opt(dep_node)
data.graph.read().dep_node_index_of_opt(&data.serialized, dep_node)
}

#[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
self.dep_node_index_of_opt(dep_node).is_some()
}

#[inline]
fn dep_node_of(&self, dep_node_index: DepNodeIndex) -> DepNode<K> {
let data = self.data.as_ref().unwrap();
data.graph.read().dep_node_of(dep_node_index)
}

#[inline]
pub(crate) fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
let data = self.data.as_ref().unwrap();
data.graph.read().fingerprint_of(dep_node_index)
data.graph.read().fingerprint_of(&data.serialized, dep_node_index)
}

/// Checks whether a previous work product exists for `v` and, if
@@ -405,8 +412,8 @@ impl<K: DepKind> DepGraph<K> {
pub fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
if let Some(ref data) = self.data {
let graph = data.graph.read();
if let Some(prev_index) = graph.node_to_index_opt(dep_node) {
return graph.color(prev_index);
if let Some(prev_index) = graph.node_to_index_opt(&data.serialized, dep_node) {
return data.serialized.color(prev_index);
} else {
// This is a node that did not exist in the previous compilation
// session, so we consider it to be red.
@@ -444,8 +451,8 @@ impl<K: DepKind> DepGraph<K> {
let data = self.data.as_ref()?;

// Return None if the dep node didn't exist in the previous session
let prev_index = data.graph.read().node_to_index_opt(dep_node)?;
let prev_deps = data.graph.read().color_or_edges(prev_index);
let prev_index = data.graph.read().node_to_index_opt(&data.serialized, dep_node)?;
let prev_deps = data.serialized.color_or_edges(prev_index);
let prev_deps = match prev_deps {
Err(prev_deps) => prev_deps,
Ok(DepNodeColor::Green) => return Some((prev_index, prev_index.rejuvenate())),
@@ -468,7 +475,7 @@ impl<K: DepKind> DepGraph<K> {
parent_dep_node_index: SerializedDepNodeIndex,
dep_node: &DepNode<K>,
) -> Option<()> {
let dep_dep_node_color = data.graph.read().color_or_edges(parent_dep_node_index);
let dep_dep_node_color = data.serialized.color_or_edges(parent_dep_node_index);
let prev_deps = match dep_dep_node_color {
Ok(DepNodeColor::Green) => {
// This dependency has been marked as green before, we are
@@ -477,7 +484,7 @@ impl<K: DepKind> DepGraph<K> {
debug!(
"try_mark_parent_green({:?}) --- found dependency {:?} to be immediately green",
dep_node,
data.graph.read().index_to_node(parent_dep_node_index)
data.serialized.index_to_node(parent_dep_node_index)
);
return Some(());
}
@@ -489,7 +496,7 @@ impl<K: DepKind> DepGraph<K> {
debug!(
"try_mark_parent_green({:?}) - END - dependency {:?} was immediately red",
dep_node,
data.graph.read().index_to_node(parent_dep_node_index)
data.serialized.index_to_node(parent_dep_node_index)
);
return None;
}
@@ -503,12 +510,12 @@ impl<K: DepKind> DepGraph<K> {
is unknown, trying to mark it green",
dep_node,
{
let dep_dep_node = data.graph.read().index_to_node(parent_dep_node_index);
let dep_dep_node = data.serialized.index_to_node(parent_dep_node_index);
(dep_dep_node, dep_dep_node.hash)
}
);

let dep_dep_node = &data.graph.read().index_to_node(parent_dep_node_index);
let dep_dep_node = &data.serialized.index_to_node(parent_dep_node_index);
let node_index =
self.try_mark_previous_green(tcx, data, parent_dep_node_index, prev_deps, dep_dep_node);
if node_index.is_some() {
@@ -533,7 +540,7 @@ impl<K: DepKind> DepGraph<K> {
return None;
}

let dep_dep_node_color = data.graph.read().color(parent_dep_node_index);
let dep_dep_node_color = data.serialized.color(parent_dep_node_index);

match dep_dep_node_color {
Some(DepNodeColor::Green) => {
@@ -592,15 +599,15 @@ impl<K: DepKind> DepGraph<K> {

// We never try to mark eval_always nodes as green
debug_assert!(!dep_node.kind.is_eval_always());
debug_assert_eq!(data.graph.read().index_to_node(prev_dep_node_index), *dep_node);
debug_assert_eq!(data.serialized.index_to_node(prev_dep_node_index), *dep_node);

for &dep_dep_node_index in prev_deps {
self.try_mark_parent_green(tcx, data, dep_dep_node_index, dep_node)?
}

#[cfg(not(parallel_compiler))]
debug_assert_eq!(
data.graph.read().color(prev_dep_node_index),
data.serialized.color(prev_dep_node_index),
None,
"DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \
insertion for {:?}",
@@ -616,7 +623,7 @@ impl<K: DepKind> DepGraph<K> {
let dep_node_index = {
// We allocating an entry for the node in the current dependency graph and
// adding all the appropriate edges imported from the previous graph
data.graph.write().intern_dark_green_node(prev_dep_node_index)
data.serialized.intern_dark_green_node(prev_dep_node_index)
};

// ... and emitting any stored diagnostic.
@@ -672,11 +679,10 @@ impl<K: DepKind> DepGraph<K> {
let _prof_timer = tcx.profiler().generic_activity("incr_comp_query_cache_promotion");

let data = self.data.as_ref().unwrap();
let graph = data.graph.read();
for prev_index in graph.serialized_indices() {
match graph.color(prev_index) {
for prev_index in data.serialized.serialized_indices() {
match data.serialized.color(prev_index) {
Some(DepNodeColor::Green) => {
let dep_node = data.graph.read().index_to_node(prev_index);
let dep_node = data.serialized.index_to_node(prev_index);
debug!("PROMOTE {:?} {:?}", prev_index, dep_node);
qcx.try_load_from_on_disk_cache(&dep_node);
}
@@ -692,11 +698,10 @@ impl<K: DepKind> DepGraph<K> {
// Register reused dep nodes (i.e. nodes we've marked red or green) with the context.
pub fn register_reused_dep_nodes<Ctxt: DepContext<DepKind = K>>(&self, tcx: Ctxt) {
let data = self.data.as_ref().unwrap();
let graph = data.graph.read();
for prev_index in graph.serialized_indices() {
match graph.color(prev_index) {
for prev_index in data.serialized.serialized_indices() {
match data.serialized.color(prev_index) {
Some(_) => {
let dep_node = data.graph.read().index_to_node(prev_index);
let dep_node = data.serialized.index_to_node(prev_index);
tcx.register_reused_dep_node(&dep_node);
}
None => {}
@@ -717,17 +722,17 @@ impl<K: DepKind> DepGraph<K> {

let mut stats: FxHashMap<_, Stat<K>> = FxHashMap::with_hasher(Default::default());

for index in prev.live_indices() {
let kind = prev.dep_node_of(index).kind;
let edge_count = prev.edge_targets_from(index).len();
for index in prev.live_indices(&data.serialized) {
let kind = prev.index_to_node(&data.serialized, index).kind;
let edge_count = prev.edge_targets_from(&data.serialized, index).len();

let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stat.node_counter += 1;
stat.edge_counter += edge_count as u64;
}

let total_node_count = prev.node_count();
let total_edge_count = prev.edge_count();
let total_node_count = prev.node_count(&data.serialized);
let total_edge_count = prev.edge_count(&data.serialized);

// Drop the lock guard.
std::mem::drop(prev);
@@ -792,14 +797,19 @@ impl<K: DepKind> DepGraph<K> {
}

pub fn compression_map(&self) -> IndexVec<DepNodeIndex, Option<SerializedDepNodeIndex>> {
self.data.as_ref().unwrap().graph.read().compression_map()
let data = self.data.as_ref().unwrap();
data.graph.read().compression_map(&data.serialized)
}

pub fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), E::Error>
where
K: Encodable<E>,
{
if let Some(data) = &self.data { data.graph.read().encode(encoder) } else { Ok(()) }
if let Some(data) = &self.data {
data.graph.read().encode(&data.serialized, encoder)
} else {
Ok(())
}
}
}

Loading