Skip to content

Commit

Permalink
Encode dep graph edges directly from the previous graph when promoting
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zoxc
Zoxc committed Mar 6, 2024
1 parent 5663a35 commit de02827
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 37 deletions.
11 changes: 4 additions & 7 deletions compiler/rustc_query_system/src/dep_graph/graph.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ impl<D: Deps> DepGraph<D> {
encoder,
record_graph,
record_stats,
prev_graph.clone(),
);

let colors = DepNodeColorMap::new(prev_graph_node_count);
Expand Down Expand Up @@ -1085,6 +1086,7 @@ impl<D: Deps> CurrentDepGraph<D> {
encoder: FileEncoder,
record_graph: bool,
record_stats: bool,
previous: Arc<SerializedDepGraph>,
) -> Self {
use std::time::{SystemTime, UNIX_EPOCH};

Expand Down Expand Up @@ -1117,6 +1119,7 @@ impl<D: Deps> CurrentDepGraph<D> {
record_graph,
record_stats,
profiler,
previous,
),
new_node_to_index: Sharded::new(|| {
FxHashMap::with_capacity_and_hasher(
Expand Down Expand Up @@ -1237,13 +1240,7 @@ impl<D: Deps> CurrentDepGraph<D> {
match prev_index_to_index[prev_index] {
Some(dep_node_index) => dep_node_index,
None => {
let key = prev_graph.index_to_node(prev_index);
let edges = prev_graph
.edge_targets_from(prev_index)
.map(|i| prev_index_to_index[i].unwrap())
.collect();
let fingerprint = prev_graph.fingerprint_by_index(prev_index);
let dep_node_index = self.encoder.send(key, fingerprint, edges);
let dep_node_index = self.encoder.promote(prev_index, &mut *prev_index_to_index);
prev_index_to_index[prev_index] = Some(dep_node_index);
#[cfg(debug_assertions)]
self.record_edge(dep_node_index, key, fingerprint);
Expand Down
168 changes: 138 additions & 30 deletions compiler/rustc_query_system/src/dep_graph/serialized.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ impl SerializedDepGraph {
pub fn edge_targets_from(
&self,
source: SerializedDepNodeIndex,
) -> impl Iterator<Item = SerializedDepNodeIndex> + '_ {
) -> impl Iterator<Item = SerializedDepNodeIndex> + Clone + '_ {
let header = self.edge_list_indices[source];
let mut raw = &self.edge_list_data[header.start()..];
// Figure out where the edge list for `source` ends by getting the start index of the next
Expand Down Expand Up @@ -393,21 +393,24 @@ impl<D: Deps> SerializedNodeHeader<D> {
const MAX_INLINE_LEN: usize = (u16::MAX as usize >> (Self::TOTAL_BITS - Self::LEN_BITS)) - 1;

#[inline]
fn new(node_info: &NodeInfo) -> Self {
fn new(
node: DepNode,
fingerprint: Fingerprint,
edge_max_index: u32,
edge_count: usize,
) -> Self {
debug_assert_eq!(Self::TOTAL_BITS, Self::LEN_BITS + Self::WIDTH_BITS + Self::KIND_BITS);

let NodeInfo { node, fingerprint, edges } = node_info;

let mut head = node.kind.as_inner();

let free_bytes = edges.max_index().leading_zeros() as usize / 8;
let free_bytes = edge_max_index.leading_zeros() as usize / 8;
let bytes_per_index = (DEP_NODE_SIZE - free_bytes).saturating_sub(1);
head |= (bytes_per_index as u16) << Self::KIND_BITS;

// Encode number of edges + 1 so that we can reserve 0 to indicate that the len doesn't fit
// in this bitfield.
if edges.len() <= Self::MAX_INLINE_LEN {
head |= (edges.len() as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS);
if edge_count <= Self::MAX_INLINE_LEN {
head |= (edge_count as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS);
}

let hash: Fingerprint = node.hash.into();
Expand All @@ -421,10 +424,10 @@ impl<D: Deps> SerializedNodeHeader<D> {
#[cfg(debug_assertions)]
{
let res = Self { bytes, _marker: PhantomData };
assert_eq!(node_info.fingerprint, res.fingerprint());
assert_eq!(node_info.node, res.node());
assert_eq!(fingerprint, res.fingerprint());
assert_eq!(node, res.node());
if let Some(len) = res.len() {
assert_eq!(node_info.edges.len(), len);
assert_eq!(edge_count, len);
}
}
Self { bytes, _marker: PhantomData }
Expand Down Expand Up @@ -487,20 +490,55 @@ struct NodeInfo {

impl NodeInfo {
fn encode<D: Deps>(&self, e: &mut FileEncoder) {
let header = SerializedNodeHeader::<D>::new(self);
let NodeInfo { node, fingerprint, ref edges } = *self;
let header =
SerializedNodeHeader::<D>::new(node, fingerprint, edges.max_index(), edges.len());
e.write_array(header.bytes);

if header.len().is_none() {
e.emit_usize(edges.len());
}

let bytes_per_index = header.bytes_per_index();
for node_index in edges.iter() {
e.write_with(|dest| {
*dest = node_index.as_u32().to_le_bytes();
bytes_per_index
});
}
}

#[inline]
fn encode_promoted<D: Deps>(
e: &mut FileEncoder,
node: DepNode,
fingerprint: Fingerprint,
prev_index: SerializedDepNodeIndex,
prev_index_to_index: &mut IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
previous: &SerializedDepGraph,
) -> usize {
let edges = previous.edge_targets_from(prev_index);
let edge_count = edges.size_hint().0;
let edge_max =
edges.clone().map(|i| prev_index_to_index[i].unwrap().as_u32()).max().unwrap_or(0);

let header = SerializedNodeHeader::<D>::new(node, fingerprint, edge_max, edge_count);
e.write_array(header.bytes);

if header.len().is_none() {
e.emit_usize(self.edges.len());
e.emit_usize(edge_count);
}

let bytes_per_index = header.bytes_per_index();
for node_index in self.edges.iter() {
for node_index in edges {
let node_index = prev_index_to_index[node_index].unwrap();
e.write_with(|dest| {
*dest = node_index.as_u32().to_le_bytes();
bytes_per_index
});
}

edge_count
}
}

Expand All @@ -511,6 +549,7 @@ struct Stat {
}

struct EncoderState<D: Deps> {
previous: Arc<SerializedDepGraph>,
encoder: FileEncoder,
total_node_count: usize,
total_edge_count: usize,
Expand All @@ -522,8 +561,9 @@ struct EncoderState<D: Deps> {
}

impl<D: Deps> EncoderState<D> {
fn new(encoder: FileEncoder, record_stats: bool) -> Self {
fn new(encoder: FileEncoder, record_stats: bool, previous: Arc<SerializedDepGraph>) -> Self {
Self {
previous,
encoder,
total_edge_count: 0,
total_node_count: 0,
Expand All @@ -533,38 +573,90 @@ impl<D: Deps> EncoderState<D> {
}
}

fn encode_node(
#[inline]
fn record(
&mut self,
node: &NodeInfo,
node: DepNode,
edge_count: usize,
edges: impl FnOnce(&mut Self) -> Vec<DepNodeIndex>,
record_graph: &Option<Lock<DepGraphQuery>>,
) -> DepNodeIndex {
let index = DepNodeIndex::new(self.total_node_count);
self.total_node_count += 1;
self.kind_stats[node.node.kind.as_usize()] += 1;

let edge_count = node.edges.len();
self.total_node_count += 1;
self.kind_stats[node.kind.as_usize()] += 1;
self.total_edge_count += edge_count;

if let Some(record_graph) = &record_graph {
// Do not ICE when a query is called from within `with_query`.
if let Some(record_graph) = &mut record_graph.try_lock() {
record_graph.push(index, node.node, &node.edges);
}
let edges = edges(self);
outline(move || {
// Do not ICE when a query is called from within `with_query`.
if let Some(record_graph) = &mut record_graph.try_lock() {
record_graph.push(index, node, &edges);
}
});
}

if let Some(stats) = &mut self.stats {
let kind = node.node.kind;
let kind = node.kind;

let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stat.node_counter += 1;
stat.edge_counter += edge_count as u64;
outline(move || {
let stat =
stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stat.node_counter += 1;
stat.edge_counter += edge_count as u64;
});
}

let encoder = &mut self.encoder;
node.encode::<D>(encoder);
index
}

fn encode_node(
&mut self,
node: &NodeInfo,
record_graph: &Option<Lock<DepGraphQuery>>,
) -> DepNodeIndex {
node.encode::<D>(&mut self.encoder);
self.record(
node.node,
node.edges.len(),
|_| node.edges[..].iter().copied().collect(),
record_graph,
)
}

#[inline]
fn promote_node(
&mut self,
prev_index: SerializedDepNodeIndex,
record_graph: &Option<Lock<DepGraphQuery>>,
prev_index_to_index: &mut IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
) -> DepNodeIndex {
let node = self.previous.index_to_node(prev_index);

let fingerprint = self.previous.fingerprint_by_index(prev_index);
let edge_count = NodeInfo::encode_promoted::<D>(
&mut self.encoder,
node,
fingerprint,
prev_index,
prev_index_to_index,
&self.previous,
);

self.record(
node,
edge_count,
|this| {
this.previous
.edge_targets_from(prev_index)
.map(|i| prev_index_to_index[i].unwrap())
.collect()
},
record_graph,
)
}

fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
let Self {
mut encoder,
Expand All @@ -573,6 +665,7 @@ impl<D: Deps> EncoderState<D> {
stats: _,
kind_stats,
marker: _,
previous: _,
} = self;

let node_count = total_node_count.try_into().unwrap();
Expand Down Expand Up @@ -612,9 +705,10 @@ impl<D: Deps> GraphEncoder<D> {
record_graph: bool,
record_stats: bool,
profiler: &SelfProfilerRef,
previous: Arc<SerializedDepGraph>,
) -> Self {
let record_graph = record_graph.then(|| Lock::new(DepGraphQuery::new(prev_node_count)));
let status = Lock::new(Some(EncoderState::new(encoder, record_stats)));
let status = Lock::new(Some(EncoderState::new(encoder, record_stats, previous)));
GraphEncoder { status, record_graph, profiler: profiler.clone() }
}

Expand Down Expand Up @@ -688,6 +782,20 @@ impl<D: Deps> GraphEncoder<D> {
self.status.lock().as_mut().unwrap().encode_node(&node, &self.record_graph)
}

#[inline]
pub(crate) fn promote(
&self,
prev_index: SerializedDepNodeIndex,
prev_index_to_index: &mut IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
) -> DepNodeIndex {
let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph");
self.status.lock().as_mut().unwrap().promote_node(
prev_index,
&self.record_graph,
prev_index_to_index,
)
}

pub fn finish(&self) -> FileEncodeResult {
let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph_finish");

Expand Down

0 comments on commit de02827

Please sign in to comment.