Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Load the dep graph before parsing #115693

Closed
wants to merge 1 commit into from
Closed

Load the dep graph before parsing #115693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 35 additions & 9 deletions compiler/rustc_incremental/src/persist/fs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,15 +105,18 @@

use crate::errors;
use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
use rustc_data_structures::stable_hasher::{Hash64, StableHasher};
use rustc_data_structures::svh::Svh;
use rustc_data_structures::unord::{UnordMap, UnordSet};
use rustc_data_structures::{base_n, flock};
use rustc_errors::ErrorGuaranteed;
use rustc_fs_util::{link_or_copy, try_canonicalize, LinkOrCopy};
use rustc_session::config::{CrateType, Input};
use rustc_session::{Session, StableCrateId};
use rustc_span::Symbol;
use rustc_span::symbol;

use std::fs as std_fs;
use std::hash::Hash;
use std::io::{self, ErrorKind};
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
Expand Down Expand Up @@ -202,11 +205,7 @@ pub fn in_incr_comp_dir(incr_comp_session_dir: &Path, file_name: &str) -> PathBu
/// The garbage collection will take care of it.
///
/// [`rustc_interface::queries::dep_graph`]: ../../rustc_interface/struct.Queries.html#structfield.dep_graph
pub fn prepare_session_directory(
sess: &Session,
crate_name: Symbol,
stable_crate_id: StableCrateId,
) -> Result<(), ErrorGuaranteed> {
pub fn prepare_session_directory(sess: &Session) -> Result<(), ErrorGuaranteed> {
if sess.opts.incremental.is_none() {
return Ok(());
}
Expand All @@ -216,7 +215,7 @@ pub fn prepare_session_directory(
debug!("prepare_session_directory");

// {incr-comp-dir}/{crate-name-and-disambiguator}
let crate_dir = crate_path(sess, crate_name, stable_crate_id);
let crate_dir = crate_path(sess);
debug!("crate-dir: {}", crate_dir.display());
create_dir(sess, &crate_dir, "crate")?;

Expand Down Expand Up @@ -601,12 +600,39 @@ fn string_to_timestamp(s: &str) -> Result<SystemTime, &'static str> {
Ok(UNIX_EPOCH + duration)
}

fn crate_path(sess: &Session, crate_name: Symbol, stable_crate_id: StableCrateId) -> PathBuf {
fn crate_path(sess: &Session) -> PathBuf {
// Use the crate name from the command line if available and valid
let crate_name = sess.opts.crate_name.as_ref().and_then(|name| {
name.as_str().chars().all(|c| c.is_alphanumeric() || c == '_').then_some(&name[..])
});
Comment on lines +604 to +607
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this duplicate some logic from somewhere?

We should probably ensure they stay in sync (ideally by deduplicating, maybe via a Sess method?)


// Create a hash for metadata and the rust version
let metadata_and_version = StableCrateId::new(
symbol::kw::Empty,
// JUSTIFICATION: before wrapper fn is available
#[allow(rustc::bad_opt_access)]
sess.opts.crate_types.contains(&CrateType::Executable),
sess.opts.cg.metadata.clone(),
sess.cfg_version,
);

// Find a file path to differentiate crates if crate name is missing
let canonical_file_path = crate_name.is_none().then(|| match &sess.io.input {
Input::File(file) => try_canonicalize(file).ok(),
Input::Str { .. } => None,
});

let mut hasher = StableHasher::new();
crate_name.hash(&mut hasher);
metadata_and_version.hash(&mut hasher);
canonical_file_path.hash(&mut hasher);
let stable_crate_id: Hash64 = hasher.finish();

let incr_dir = sess.opts.incremental.as_ref().unwrap().clone();

let stable_crate_id = base_n::encode(stable_crate_id.as_u64() as u128, INT_ENCODE_BASE);

let crate_name = format!("{crate_name}-{stable_crate_id}");
let crate_name = format!("{}-{stable_crate_id}", crate_name.unwrap_or("unknown.crate"));
incr_dir.join(crate_name)
}

Expand Down
70 changes: 36 additions & 34 deletions compiler/rustc_interface/src/queries.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ use rustc_session::config::{self, CrateType, OutputFilenames, OutputType};
use rustc_session::cstore::Untracked;
use rustc_session::{output::find_crate_name, Session};
use rustc_span::symbol::sym;
use rustc_span::Symbol;
use std::any::Any;
use std::cell::{RefCell, RefMut};
use std::sync::Arc;
Expand Down Expand Up @@ -85,6 +84,7 @@ pub struct Queries<'tcx> {
arena: WorkerLocal<Arena<'tcx>>,
hir_arena: WorkerLocal<rustc_hir::Arena<'tcx>>,

dep_graph_future: Query<Option<DepGraphFuture>>,
parse: Query<ast::Crate>,
pre_configure: Query<(ast::Crate, ast::AttrVec)>,
// This just points to what's in `gcx_cell`.
Expand All @@ -98,6 +98,7 @@ impl<'tcx> Queries<'tcx> {
gcx_cell: OnceLock::new(),
arena: WorkerLocal::new(|_| Arena::default()),
hir_arena: WorkerLocal::new(|_| rustc_hir::Arena::default()),
dep_graph_future: Default::default(),
parse: Default::default(),
pre_configure: Default::default(),
gcx: Default::default(),
Expand All @@ -112,8 +113,13 @@ impl<'tcx> Queries<'tcx> {
}

pub fn parse(&self) -> Result<QueryResult<'_, ast::Crate>> {
self.parse
.compute(|| passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit()))
self.parse.compute(|| {
// Compute the dependency graph (in the background). We want to do this as early as
// possible, to give the DepGraph maximum time to load before `dep_graph` is called.
self.dep_graph_future()?;

passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit())
})
}

pub fn pre_configure(&self) -> Result<QueryResult<'_, (ast::Crate, ast::AttrVec)>> {
Expand All @@ -133,41 +139,41 @@ impl<'tcx> Queries<'tcx> {
})
}

fn dep_graph_future(
&self,
crate_name: Symbol,
stable_crate_id: StableCrateId,
) -> Result<Option<DepGraphFuture>> {
let sess = self.session();

// `load_dep_graph` can only be called after `prepare_session_directory`.
rustc_incremental::prepare_session_directory(sess, crate_name, stable_crate_id)?;
let res = sess.opts.build_dep_graph().then(|| rustc_incremental::load_dep_graph(sess));

if sess.opts.incremental.is_some() {
sess.time("incr_comp_garbage_collect_session_directories", || {
if let Err(e) = rustc_incremental::garbage_collect_session_directories(sess) {
warn!(
"Error while trying to garbage collect incremental \
compilation cache directory: {}",
e
);
}
});
}
fn dep_graph_future(&self) -> Result<QueryResult<'_, Option<DepGraphFuture>>> {
self.dep_graph_future.compute(|| {
let sess = self.session();

// `load_dep_graph` can only be called after `prepare_session_directory`.
rustc_incremental::prepare_session_directory(sess)?;
let res = sess.opts.build_dep_graph().then(|| rustc_incremental::load_dep_graph(sess));

if sess.opts.incremental.is_some() {
sess.time("incr_comp_garbage_collect_session_directories", || {
if let Err(e) = rustc_incremental::garbage_collect_session_directories(sess) {
warn!(
"Error while trying to garbage collect incremental \
compilation cache directory: {}",
e
);
}
});
}

Ok(res)
Ok(res)
})
}

fn dep_graph(&self, dep_graph_future: Option<DepGraphFuture>) -> DepGraph {
dep_graph_future
fn dep_graph(&self) -> Result<DepGraph> {
Ok(self
.dep_graph_future()?
.steal()
.and_then(|future| {
let sess = self.session();
let (prev_graph, prev_work_products) =
sess.time("blocked_on_dep_graph_loading", || future.open().open(sess));
rustc_incremental::build_dep_graph(sess, prev_graph, prev_work_products)
})
.unwrap_or_else(DepGraph::new_disabled)
.unwrap_or_else(DepGraph::new_disabled))
}

pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
Expand All @@ -185,10 +191,6 @@ impl<'tcx> Queries<'tcx> {
sess.cfg_version,
);

// Compute the dependency graph (in the background). We want to do this as early as
// possible, to give the DepGraph maximum time to load before `dep_graph` is called.
let dep_graph_future = self.dep_graph_future(crate_name, stable_crate_id)?;

let lint_store = Lrc::new(passes::create_lint_store(
sess,
&*self.codegen_backend().metadata_loader(),
Expand All @@ -210,7 +212,7 @@ impl<'tcx> Queries<'tcx> {
crate_types,
stable_crate_id,
lint_store,
self.dep_graph(dep_graph_future),
self.dep_graph()?,
untracked,
&self.gcx_cell,
&self.arena,
Expand Down
Loading