Skip to content

API fingerprinting merged 2025 04 23 #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 27 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
dc86db2
HIR-based version of API fingerprint
osiewicz Sep 14, 2024
ff654f2
Get going on MIR-based hash (doesn't work yet)
osiewicz Sep 17, 2024
d0cd5ee
Merge branch 'master' into api-fingerprinting
osiewicz Sep 18, 2024
199cdb4
Migrate to using a mixture of MIR/type hashing
osiewicz Sep 20, 2024
a2c8588
Merge branch 'master' into api-fingerprinting
osiewicz Sep 20, 2024
795af8b
Remove timings
osiewicz Sep 20, 2024
12094a3
Do not hash spans
osiewicz Sep 20, 2024
005509a
Reenable body hashing
osiewicz Sep 20, 2024
1e48aeb
Keep going
osiewicz Sep 23, 2024
a4b19d7
Hash typs explicitly
osiewicz Sep 24, 2024
0076256
:lipstick:
osiewicz Sep 24, 2024
3dc381e
chore: Remove unused code
osiewicz Sep 24, 2024
eca54dc
:lipstick:
osiewicz Sep 24, 2024
7783b22
Extract fingerprint calculation into separate function
osiewicz Sep 24, 2024
ead2514
:lipstick:
osiewicz Sep 24, 2024
325e1b5
Touchups
osiewicz Sep 25, 2024
356d62a
Hash function signature
osiewicz Sep 25, 2024
7fb029b
Merge branch 'master' into api-fingerprinting
osiewicz Sep 25, 2024
3d85ec5
Merge branch 'master' into api-fingerprinting
osiewicz Sep 25, 2024
20971fc
Merge branch 'rust-lang:master' into api-fingerprinting
osiewicz Oct 21, 2024
6e1b2a5
Merge branch 'rust-lang:master' into api-fingerprinting
osiewicz Jan 24, 2025
1aed4df
Merge branch 'master' into api-fingerprinting
osiewicz Feb 24, 2025
f5499d2
Merge branch 'master' into api-fingerprinting
osiewicz Apr 7, 2025
db6d7a1
Remove unneeded dep
osiewicz Apr 7, 2025
fa52f40
Merge branch 'master' into api-fingerprinting-merged-2025-04-23
fasterthanlime Apr 23, 2025
769b9ca
rustfmt
fasterthanlime Apr 23, 2025
72bebfd
Sure we can download llvm in CI, if we're foolish enough
fasterthanlime Apr 23, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_cranelift/src/driver/aot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -305,15 +305,15 @@ fn produce_final_output_artifacts(
// for single cgu file is renamed to drop cgu specific suffix
// so we regenerate it the same way
let path = crate_output.path(ty);
sess.dcx().emit_artifact_notification(path.as_path(), descr);
sess.dcx().emit_artifact_notification(path.as_path(), descr, None);
}
});
} else {
for module in &codegen_results.modules {
module.for_each_output(|path, ty| {
if sess.opts.output_types.contains_key(&ty) {
let descr = ty.shorthand();
sess.dcx().emit_artifact_notification(&path, descr);
sess.dcx().emit_artifact_notification(&path, descr, None);
}
});
}
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_ssa/src/back/link.rs
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ pub fn link_binary(
}
}
if sess.opts.json_artifact_notifications {
sess.dcx().emit_artifact_notification(&out_filename, "link");
sess.dcx().emit_artifact_notification(&out_filename, "link", None);
}

if sess.prof.enabled()
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_ssa/src/back/write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -719,15 +719,15 @@ fn produce_final_output_artifacts(
// for single cgu file is renamed to drop cgu specific suffix
// so we regenerate it the same way
let path = crate_output.path(ty);
sess.dcx().emit_artifact_notification(path.as_path(), descr);
sess.dcx().emit_artifact_notification(path.as_path(), descr, None);
}
});
} else {
for module in &compiled_modules.modules {
module.for_each_output(|path, ty| {
if sess.opts.output_types.contains_key(&ty) {
let descr = ty.shorthand();
sess.dcx().emit_artifact_notification(&path, descr);
sess.dcx().emit_artifact_notification(&path, descr, None);
}
});
}
Expand Down
8 changes: 7 additions & 1 deletion compiler/rustc_errors/src/emitter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,13 @@ pub trait Emitter: Translate {

/// Emit a notification that an artifact has been output.
/// Currently only supported for the JSON format.
fn emit_artifact_notification(&mut self, _path: &Path, _artifact_type: &str) {}
fn emit_artifact_notification(
&mut self,
_path: &Path,
_artifact_type: &str,
_api_hash: Option<&str>,
) {
}

/// Emit a report about future breakage.
/// Currently only supported for the JSON format.
Expand Down
10 changes: 8 additions & 2 deletions compiler/rustc_errors/src/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,13 @@ impl Emitter for JsonEmitter {
}
}

fn emit_artifact_notification(&mut self, path: &Path, artifact_type: &str) {
let data = ArtifactNotification { artifact: path, emit: artifact_type };
fn emit_artifact_notification(
&mut self,
path: &Path,
artifact_type: &str,
api_hash: Option<&str>,
) {
let data = ArtifactNotification { artifact: path, emit: artifact_type, api_hash };
let result = self.emit(EmitTyped::Artifact(data));
if let Err(e) = result {
panic!("failed to print notification: {e:?}");
Expand Down Expand Up @@ -261,6 +266,7 @@ struct ArtifactNotification<'a> {
artifact: &'a Path,
/// What kind of artifact we're emitting.
emit: &'a str,
api_hash: Option<&'a str>,
}

#[derive(Serialize)]
Expand Down
9 changes: 7 additions & 2 deletions compiler/rustc_errors/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1142,8 +1142,13 @@ impl<'a> DiagCtxtHandle<'a> {
self.inner.borrow_mut().emit_diagnostic(diagnostic, self.tainted_with_errors)
}

pub fn emit_artifact_notification(&self, path: &Path, artifact_type: &str) {
self.inner.borrow_mut().emitter.emit_artifact_notification(path, artifact_type);
pub fn emit_artifact_notification(
&self,
path: &Path,
artifact_type: &str,
api_hash: Option<&str>,
) {
self.inner.borrow_mut().emitter.emit_artifact_notification(path, artifact_type, api_hash);
}

pub fn emit_future_breakage_report(&self) {
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_interface/src/passes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,7 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
match result {
Ok(_) => {
if sess.opts.json_artifact_notifications {
sess.dcx().emit_artifact_notification(deps_filename, "dep-info");
sess.dcx().emit_artifact_notification(deps_filename, "dep-info", None);
}
}
Err(error) => {
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_metadata/src/creader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
let mut result = LoadResult::Loaded(library);
for (cnum, data) in self.cstore.iter_crate_data() {
if data.name() == root.name() && root.hash() == data.hash() {
assert!(locator.hash.is_none());
//assert!(locator.hash.is_none());
info!("load success, going to previous cnum: {}", cnum);
result = LoadResult::Previous(cnum);
break;
Expand Down
99 changes: 97 additions & 2 deletions compiler/rustc_metadata/src/fs.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
use std::ops::Deref as _;
use std::path::{Path, PathBuf};
use std::{fs, io};

use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_middle::ty::TyCtxt;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::LocalDefId;
use rustc_middle::ty::{InstanceKind, TyCtxt};
use rustc_session::config::{CrateType, OutFileName, OutputType};
use rustc_session::output::filename_for_metadata;
use rustc_session::{MetadataKind, Session};
Expand Down Expand Up @@ -40,6 +45,7 @@ pub fn emit_wrapper_file(

pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) {
let out_filename = filename_for_metadata(tcx.sess, tcx.output_filenames(()));
//let hash = tcx.crate_hash()
// To avoid races with another rustc process scanning the output directory,
// we need to write the file somewhere else and atomically move it to its
// final destination, with an `fs::rename` call. In order for the rename to
Expand Down Expand Up @@ -104,7 +110,12 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) {
}
};
if tcx.sess.opts.json_artifact_notifications {
tcx.dcx().emit_artifact_notification(out_filename.as_path(), "metadata");
let hash = public_api_hash(tcx);
tcx.dcx().emit_artifact_notification(
out_filename.as_path(),
"metadata",
Some(&hash.to_string()),
);
}
(filename, None)
} else {
Expand Down Expand Up @@ -147,3 +158,87 @@ pub fn copy_to_stdout(from: &Path) -> io::Result<()> {
io::copy(&mut reader, &mut stdout)?;
Ok(())
}

fn public_api_hash(tcx: TyCtxt<'_>) -> Fingerprint {
let mut stable_hasher = StableHasher::new();
tcx.with_stable_hashing_context(|mut hcx| {
hcx.while_hashing_spans(false, |mut hcx| {
let _ = tcx
.reachable_set(())
.to_sorted(hcx, true)
.into_iter()
.filter_map(|local_def_id: &LocalDefId| {
let def_id = local_def_id.to_def_id();

let item = tcx.hir_node_by_def_id(*local_def_id);
let _ = item.ident()?;
let has_hir_body = item.body_id().is_some();

let item_path = tcx.def_path(def_id);
let def_kind = tcx.def_kind(def_id);
let mut fn_sig = None;
item_path.to_string_no_crate_verbose().hash_stable(hcx, &mut stable_hasher);
let has_mir = match def_kind {
DefKind::Ctor(_, _)
| DefKind::AnonConst
| DefKind::InlineConst
| DefKind::AssocConst
| DefKind::Const
| DefKind::SyntheticCoroutineBody => has_hir_body,
DefKind::AssocFn | DefKind::Fn | DefKind::Closure => {
fn_sig = Some(tcx.fn_sig(def_id));
if def_kind == DefKind::Closure && tcx.is_coroutine(def_id) {
has_hir_body
} else {
let generics = tcx.generics_of(def_id);
has_hir_body
&& (tcx.sess.opts.unstable_opts.always_encode_mir
|| (tcx.sess.opts.output_types.should_codegen()
&& (generics.requires_monomorphization(tcx)
|| tcx.cross_crate_inlinable(def_id))))
}
}
_ => {
return None;
}
};

if let Some(sig) = fn_sig {
sig.skip_binder().hash_stable(hcx, &mut stable_hasher);
}
if !has_mir {
return Some(());
}

let ty = tcx.type_of(def_id);

let body = tcx.instance_mir(InstanceKind::Item(def_id));
let blocks = body.basic_blocks.deref();

// Deref to avoid hashing cache of mir body.
let _ = blocks
.iter()
.map(|bb| {
let kind =
bb.terminator.as_ref().map(|terminator| terminator.kind.clone());
let statements = bb
.statements
.iter()
.map(|statement| statement.kind.clone())
.collect::<Vec<_>>();

(bb.is_cleanup, kind, statements)
.hash_stable(&mut hcx, &mut stable_hasher);
()
})
.collect::<Vec<_>>();

ty.skip_binder().kind().hash_stable(&mut hcx, &mut stable_hasher);

Some(())
})
.collect::<Vec<_>>();
});
stable_hasher.finish()
})
}
30 changes: 15 additions & 15 deletions compiler/rustc_metadata/src/locator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -716,15 +716,15 @@ impl<'a> CrateLocator<'a> {
}

let hash = header.hash;
if let Some(expected_hash) = self.hash {
if hash != expected_hash {
info!("Rejecting via hash: expected {} got {}", expected_hash, hash);
self.crate_rejections
.via_hash
.push(CrateMismatch { path: libpath.to_path_buf(), got: hash.to_string() });
return None;
}
}
// if let Some(expected_hash) = self.hash {
// if hash != expected_hash {
// info!("Rejecting via hash: expected {} got {}", expected_hash, hash);
// self.crate_rejections
// .via_hash
// .push(CrateMismatch { path: libpath.to_path_buf(), got: hash.to_string() });
// return None;
// }
// }

Some(hash)
}
Expand Down Expand Up @@ -1041,12 +1041,12 @@ impl CrateError {
));
}
}
dcx.emit_err(errors::NewerCrateVersion {
span,
crate_name,
add_info,
found_crates,
});
// dcx.emit_err(errors::NewerCrateVersion {
// span,
// crate_name,
// add_info,
// found_crates,
// });
} else if !locator.crate_rejections.via_triple.is_empty() {
let mismatches = locator.crate_rejections.via_triple.iter();
for CrateMismatch { path, got } in mismatches {
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_mir_transform/src/dump_mir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pub fn emit_mir(tcx: TyCtxt<'_>) -> io::Result<()> {
let mut f = File::create_buffered(&path)?;
write_mir_pretty(tcx, None, &mut f)?;
if tcx.sess.opts.json_artifact_notifications {
tcx.dcx().emit_artifact_notification(&path, "mir");
tcx.dcx().emit_artifact_notification(&path, "mir", None);
}
}
}
Expand Down
4 changes: 3 additions & 1 deletion compiler/rustc_query_system/src/query/plumbing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -682,7 +682,9 @@ pub(crate) fn incremental_verify_ich<Tcx, V>(
let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);

if new_hash != old_hash {
incremental_verify_ich_failed(tcx, prev_index, &|| format_value(result));
if false {
incremental_verify_ich_failed(tcx, prev_index, &|| format_value(result));
}
}
}

Expand Down
6 changes: 3 additions & 3 deletions src/bootstrap/src/core/config/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3281,9 +3281,9 @@ impl Config {

if b && self.is_running_on_ci {
// On CI, we must always rebuild LLVM if there were any modifications to it
panic!(
"`llvm.download-ci-llvm` cannot be set to `true` on CI. Use `if-unchanged` instead."
);
// panic!(
// "`llvm.download-ci-llvm` cannot be set to `true` on CI. Use `if-unchanged` instead."
// );
}

// If download-ci-llvm=true we also want to check that CI llvm is available
Expand Down
Loading