Skip to content

Commit

Permalink
Auto merge of rust-lang#115693 - Zoxc:early-dep-graph-load, r=<try>
Browse files Browse the repository at this point in the history 
Load the dep graph before parsing

This changes the dep graph to start loading before parsing. We don't have access to the crate name defined in the source code at this point, so if it's not specified on the command line, incremental compilation will fallback to the file path as a crate identifier.

r? `@oli-obk`
  • Loading branch information
@bors
bors committed Sep 9, 2023
2 parents 6cc1898 + 360ee87 commit bcdfb0b
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 43 deletions.
44 changes: 35 additions & 9 deletions compiler/rustc_incremental/src/persist/fs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,15 +105,18 @@
use crate::errors;
use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
use rustc_data_structures::stable_hasher::{Hash64, StableHasher};
use rustc_data_structures::svh::Svh;
use rustc_data_structures::unord::{UnordMap, UnordSet};
use rustc_data_structures::{base_n, flock};
use rustc_errors::ErrorGuaranteed;
use rustc_fs_util::{link_or_copy, try_canonicalize, LinkOrCopy};
use rustc_session::config::{CrateType, Input};
use rustc_session::{Session, StableCrateId};
use rustc_span::Symbol;
use rustc_span::symbol;

use std::fs as std_fs;
use std::hash::Hash;
use std::io::{self, ErrorKind};
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
Expand Down Expand Up @@ -202,11 +205,7 @@ pub fn in_incr_comp_dir(incr_comp_session_dir: &Path, file_name: &str) -> PathBu
/// The garbage collection will take care of it.
///
/// [`rustc_interface::queries::dep_graph`]: ../../rustc_interface/struct.Queries.html#structfield.dep_graph
pub fn prepare_session_directory(
sess: &Session,
crate_name: Symbol,
stable_crate_id: StableCrateId,
) -> Result<(), ErrorGuaranteed> {
pub fn prepare_session_directory(sess: &Session) -> Result<(), ErrorGuaranteed> {
if sess.opts.incremental.is_none() {
return Ok(());
}
Expand All @@ -216,7 +215,7 @@ pub fn prepare_session_directory(
debug!("prepare_session_directory");

// {incr-comp-dir}/{crate-name-and-disambiguator}
let crate_dir = crate_path(sess, crate_name, stable_crate_id);
let crate_dir = crate_path(sess);
debug!("crate-dir: {}", crate_dir.display());
create_dir(sess, &crate_dir, "crate")?;

Expand Down Expand Up @@ -601,12 +600,39 @@ fn string_to_timestamp(s: &str) -> Result<SystemTime, &'static str> {
Ok(UNIX_EPOCH + duration)
}

fn crate_path(sess: &Session, crate_name: Symbol, stable_crate_id: StableCrateId) -> PathBuf {
fn crate_path(sess: &Session) -> PathBuf {
// Use the crate name from the command line if available and valid
let crate_name = sess.opts.crate_name.as_ref().and_then(|name| {
name.as_str().chars().all(|c| c.is_alphanumeric() || c == '_').then_some(&name[..])
});

// Create a hash for metadata and the rust version
let metadata_and_version = StableCrateId::new(
symbol::kw::Empty,
// JUSTIFICATION: before wrapper fn is available
#[allow(rustc::bad_opt_access)]
sess.opts.crate_types.contains(&CrateType::Executable),
sess.opts.cg.metadata.clone(),
sess.cfg_version,
);

// Find a file path to differentiate crates if crate name is missing
let canonical_file_path = crate_name.is_none().then(|| match &sess.io.input {
Input::File(file) => try_canonicalize(file).ok(),
Input::Str { .. } => None,
});

let mut hasher = StableHasher::new();
crate_name.hash(&mut hasher);
metadata_and_version.hash(&mut hasher);
canonical_file_path.hash(&mut hasher);
let stable_crate_id: Hash64 = hasher.finish();

let incr_dir = sess.opts.incremental.as_ref().unwrap().clone();

let stable_crate_id = base_n::encode(stable_crate_id.as_u64() as u128, INT_ENCODE_BASE);

let crate_name = format!("{crate_name}-{stable_crate_id}");
let crate_name = format!("{}-{stable_crate_id}", crate_name.unwrap_or("unknown.crate"));
incr_dir.join(crate_name)
}

Expand Down
70 changes: 36 additions & 34 deletions compiler/rustc_interface/src/queries.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ use rustc_session::config::{self, CrateType, OutputFilenames, OutputType};
use rustc_session::cstore::Untracked;
use rustc_session::{output::find_crate_name, Session};
use rustc_span::symbol::sym;
use rustc_span::Symbol;
use std::any::Any;
use std::cell::{RefCell, RefMut};
use std::sync::Arc;
Expand Down Expand Up @@ -85,6 +84,7 @@ pub struct Queries<'tcx> {
arena: WorkerLocal<Arena<'tcx>>,
hir_arena: WorkerLocal<rustc_hir::Arena<'tcx>>,

dep_graph_future: Query<Option<DepGraphFuture>>,
parse: Query<ast::Crate>,
pre_configure: Query<(ast::Crate, ast::AttrVec)>,
// This just points to what's in `gcx_cell`.
Expand All @@ -98,6 +98,7 @@ impl<'tcx> Queries<'tcx> {
gcx_cell: OnceLock::new(),
arena: WorkerLocal::new(|_| Arena::default()),
hir_arena: WorkerLocal::new(|_| rustc_hir::Arena::default()),
dep_graph_future: Default::default(),
parse: Default::default(),
pre_configure: Default::default(),
gcx: Default::default(),
Expand All @@ -112,8 +113,13 @@ impl<'tcx> Queries<'tcx> {
}

pub fn parse(&self) -> Result<QueryResult<'_, ast::Crate>> {
self.parse
.compute(|| passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit()))
self.parse.compute(|| {
// Compute the dependency graph (in the background). We want to do this as early as
// possible, to give the DepGraph maximum time to load before `dep_graph` is called.
self.dep_graph_future()?;

passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit())
})
}

pub fn pre_configure(&self) -> Result<QueryResult<'_, (ast::Crate, ast::AttrVec)>> {
Expand All @@ -133,41 +139,41 @@ impl<'tcx> Queries<'tcx> {
})
}

fn dep_graph_future(
&self,
crate_name: Symbol,
stable_crate_id: StableCrateId,
) -> Result<Option<DepGraphFuture>> {
let sess = self.session();

// `load_dep_graph` can only be called after `prepare_session_directory`.
rustc_incremental::prepare_session_directory(sess, crate_name, stable_crate_id)?;
let res = sess.opts.build_dep_graph().then(|| rustc_incremental::load_dep_graph(sess));

if sess.opts.incremental.is_some() {
sess.time("incr_comp_garbage_collect_session_directories", || {
if let Err(e) = rustc_incremental::garbage_collect_session_directories(sess) {
warn!(
"Error while trying to garbage collect incremental \
compilation cache directory: {}",
e
);
}
});
}
fn dep_graph_future(&self) -> Result<QueryResult<'_, Option<DepGraphFuture>>> {
self.dep_graph_future.compute(|| {
let sess = self.session();

// `load_dep_graph` can only be called after `prepare_session_directory`.
rustc_incremental::prepare_session_directory(sess)?;
let res = sess.opts.build_dep_graph().then(|| rustc_incremental::load_dep_graph(sess));

if sess.opts.incremental.is_some() {
sess.time("incr_comp_garbage_collect_session_directories", || {
if let Err(e) = rustc_incremental::garbage_collect_session_directories(sess) {
warn!(
"Error while trying to garbage collect incremental \
compilation cache directory: {}",
e
);
}
});
}

Ok(res)
Ok(res)
})
}

fn dep_graph(&self, dep_graph_future: Option<DepGraphFuture>) -> DepGraph {
dep_graph_future
fn dep_graph(&self) -> Result<DepGraph> {
Ok(self
.dep_graph_future()?
.steal()
.and_then(|future| {
let sess = self.session();
let (prev_graph, prev_work_products) =
sess.time("blocked_on_dep_graph_loading", || future.open().open(sess));
rustc_incremental::build_dep_graph(sess, prev_graph, prev_work_products)
})
.unwrap_or_else(DepGraph::new_disabled)
.unwrap_or_else(DepGraph::new_disabled))
}

pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
Expand All @@ -185,10 +191,6 @@ impl<'tcx> Queries<'tcx> {
sess.cfg_version,
);

// Compute the dependency graph (in the background). We want to do this as early as
// possible, to give the DepGraph maximum time to load before `dep_graph` is called.
let dep_graph_future = self.dep_graph_future(crate_name, stable_crate_id)?;

let lint_store = Lrc::new(passes::create_lint_store(
sess,
&*self.codegen_backend().metadata_loader(),
Expand All @@ -210,7 +212,7 @@ impl<'tcx> Queries<'tcx> {
crate_types,
stable_crate_id,
lint_store,
self.dep_graph(dep_graph_future),
self.dep_graph()?,
untracked,
&self.gcx_cell,
&self.arena,
Expand Down

0 comments on commit bcdfb0b

Please sign in to comment.