Print reason for failure during assess run (rust-lang#2182)

Add a sorted list of "Failed to assess packages" to the output of scan for the scan regression script. The list looks like:

```
Running assess scan test:
... (redacted)
Assessed 2 successfully, with 2 failures.
Failed to assess packages:
  - `compile_error`: Failed to execute cargo (exit status: 101). Found 3 compilation errors.
  - `manifest_error`: Failed to get cargo metadata.
... (redacted)
```

kani-driver/src/assess/scan.rs

@@ -8,10 +8,12 @@ use std::process::Command;
 use std::time::Instant;
 
 use anyhow::Result;
+use cargo_metadata::Package;
 
 use crate::session::KaniSession;
 
 use super::args::ScanArgs;
+use super::metadata::AssessMetadata;
 use super::metadata::{aggregate_metadata, read_metadata};
 
 /// `cargo kani assess scan` is not a normal invocation of `cargo kani`: we don't directly build anything.
@@ -106,17 +108,21 @@ pub(crate) fn assess_scan_main(session: KaniSession, args: &ScanArgs) -> Result<
  invoke_assess(&session, name, manifest, &outfile, &logfile)
  };
 
- if result.is_err() {
- println!("Failed: {name}");
- failed_packages.push(package);
- } else {
- let meta = read_metadata(&outfile);
- if let Ok(meta) = meta {
- success_metas.push(meta);
- } else {
+ let meta = read_metadata(&outfile);
+ if let Ok(meta) = meta {
+ if meta.error.is_some() {
  println!("Failed: {name}");
- failed_packages.push(package);
+ // Some execution error that we have collected.
+ failed_packages.push((package, Some(meta)))
+ } else {
+ success_metas.push(meta);
  }
+ } else {
+ println!("Failed: {name}");
+ failed_packages.push((
+ package,
+ result.err().map(|err| AssessMetadata::from_error(err.as_ref())),
+ ));
  }
  //TODO: cargo clean?
  println!(
@@ -134,7 +140,9 @@ pub(crate) fn assess_scan_main(session: KaniSession, args: &ScanArgs) -> Result<
  failed_packages.len()
  );
  let results = aggregate_metadata(success_metas);
+ print_failures(failed_packages);
  println!("{}", results.unsupported_features.render());
+
  if !session.args.only_codegen {
  println!("{}", results.failure_reasons.render());
  println!("{}", results.promising_tests.render());
@@ -203,3 +211,22 @@ fn scan_cargo_projects(path: PathBuf, accumulator: &mut Vec<PathBuf>) {
  }
  }
 }
+
+/// Print failures if any happened.
+fn print_failures(mut failures: Vec<(&Package, Option<AssessMetadata>)>) {
+ if !failures.is_empty() {
+ println!("Failed to assess packages:");
+ let unknown = "Unknown".to_string();
+ failures.sort_by_key(|(pkg, _)| &pkg.name);
+ for (pkg, meta) in failures {
+ println!(
+ " - `{}`: {}",
+ pkg.name,
+ meta.as_ref().map_or(&unknown, |md| {
+ md.error.as_ref().map_or(&unknown, |error| &error.msg)
+ }),
+ );
+ }
+ println!();
+ }
+}

scripts/assess-scan-regression.sh

@@ -15,20 +15,30 @@ cargo kani --enable-unstable assess scan
 # Clean up
 (cd foo && cargo clean)
 (cd bar && cargo clean)
+(cd compile_error && cargo clean)
+(cd manifest_error && cargo clean)
 
 # Check for expected files (and clean up)
 EXPECTED_FILES=(
- bar/bar.kani-assess-metadata.json
- foo/foo.kani-assess-metadata.json
  bar/bar.kani-assess.log
+ bar/bar.kani-assess-metadata.json
+ compile_error/compile_error.kani-assess.log
+ compile_error/compile_error.kani-assess-metadata.json
+ manifest_error/manifest_error.kani-assess.log
+ manifest_error/manifest_error.kani-assess-metadata.json
  foo/foo.kani-assess.log
+ foo/foo.kani-assess-metadata.json
 )
+
+errors=0
 for file in ${EXPECTED_FILES[@]}; do
  if [ -f $KANI_DIR/tests/assess-scan-test-scaffold/$file ]; then
  rm $KANI_DIR/tests/assess-scan-test-scaffold/$file
  else
- echo "Failed to find $file" && exit 1
+ errors=1
+ echo "Failed to find $file"
  fi
 done
 
 echo "Done with assess scan test"
+exit $errors

tests/assess-scan-test-scaffold/compile_error/Cargo.toml

@@ -0,0 +1,11 @@
+# Copyright Kani Contributors
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+
+[package]
+name = "compile_error"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

tests/assess-scan-test-scaffold/compile_error/src/lib.rs

@@ -0,0 +1,7 @@
+// Copyright Kani Contributors
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+/// Function with a compilation error
+pub fn with_error(left: usize, right: u32) -> usize {
+ left + right
+}

tests/assess-scan-test-scaffold/manifest_error/Cargo.toml

@@ -0,0 +1,12 @@
+# Copyright Kani Contributors
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+
+[package]
+name = "manifest_error"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+unknown = { path="./does/not/exist" }

tests/assess-scan-test-scaffold/manifest_error/src/lib.rs

@@ -0,0 +1,5 @@
+// Copyright Kani Contributors
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+/// Nothing here
+pub fn void() {}