From 68b85d6054805cd450eb43171f9b580b21d62108 Mon Sep 17 00:00:00 2001 From: "Celina G. Val" Date: Mon, 6 Feb 2023 15:40:03 -0800 Subject: [PATCH] Print reason for failure during assess run (#2182) Add a sorted list of "Failed to assess packages" to the output of scan for the scan regression script. The list looks like: ``` Running assess scan test: ... (redacted) Assessed 2 successfully, with 2 failures. Failed to assess packages: - `compile_error`: Failed to execute cargo (exit status: 101). Found 3 compilation errors. - `manifest_error`: Failed to get cargo metadata. ... (redacted) ``` --- kani-driver/src/assess/scan.rs | 45 +++++++++++++++---- scripts/assess-scan-regression.sh | 16 +++++-- .../compile_error/Cargo.toml | 11 +++++ .../compile_error/src/lib.rs | 7 +++ .../manifest_error/Cargo.toml | 12 +++++ .../manifest_error/src/lib.rs | 5 +++ 6 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 tests/assess-scan-test-scaffold/compile_error/Cargo.toml create mode 100644 tests/assess-scan-test-scaffold/compile_error/src/lib.rs create mode 100644 tests/assess-scan-test-scaffold/manifest_error/Cargo.toml create mode 100644 tests/assess-scan-test-scaffold/manifest_error/src/lib.rs diff --git a/kani-driver/src/assess/scan.rs b/kani-driver/src/assess/scan.rs index dbe43f211f027..ddf18076a68ef 100644 --- a/kani-driver/src/assess/scan.rs +++ b/kani-driver/src/assess/scan.rs @@ -8,10 +8,12 @@ use std::process::Command; use std::time::Instant; use anyhow::Result; +use cargo_metadata::Package; use crate::session::KaniSession; use super::args::ScanArgs; +use super::metadata::AssessMetadata; use super::metadata::{aggregate_metadata, read_metadata}; /// `cargo kani assess scan` is not a normal invocation of `cargo kani`: we don't directly build anything. @@ -106,17 +108,21 @@ pub(crate) fn assess_scan_main(session: KaniSession, args: &ScanArgs) -> Result< invoke_assess(&session, name, manifest, &outfile, &logfile) }; - if result.is_err() { - println!("Failed: {name}"); - failed_packages.push(package); - } else { - let meta = read_metadata(&outfile); - if let Ok(meta) = meta { - success_metas.push(meta); - } else { + let meta = read_metadata(&outfile); + if let Ok(meta) = meta { + if meta.error.is_some() { println!("Failed: {name}"); - failed_packages.push(package); + // Some execution error that we have collected. + failed_packages.push((package, Some(meta))) + } else { + success_metas.push(meta); } + } else { + println!("Failed: {name}"); + failed_packages.push(( + package, + result.err().map(|err| AssessMetadata::from_error(err.as_ref())), + )); } //TODO: cargo clean? println!( @@ -134,7 +140,9 @@ pub(crate) fn assess_scan_main(session: KaniSession, args: &ScanArgs) -> Result< failed_packages.len() ); let results = aggregate_metadata(success_metas); + print_failures(failed_packages); println!("{}", results.unsupported_features.render()); + if !session.args.only_codegen { println!("{}", results.failure_reasons.render()); println!("{}", results.promising_tests.render()); @@ -203,3 +211,22 @@ fn scan_cargo_projects(path: PathBuf, accumulator: &mut Vec) { } } } + +/// Print failures if any happened. +fn print_failures(mut failures: Vec<(&Package, Option)>) { + if !failures.is_empty() { + println!("Failed to assess packages:"); + let unknown = "Unknown".to_string(); + failures.sort_by_key(|(pkg, _)| &pkg.name); + for (pkg, meta) in failures { + println!( + " - `{}`: {}", + pkg.name, + meta.as_ref().map_or(&unknown, |md| { + md.error.as_ref().map_or(&unknown, |error| &error.msg) + }), + ); + } + println!(); + } +} diff --git a/scripts/assess-scan-regression.sh b/scripts/assess-scan-regression.sh index 14bf57d34c567..f349219d53892 100755 --- a/scripts/assess-scan-regression.sh +++ b/scripts/assess-scan-regression.sh @@ -15,20 +15,30 @@ cargo kani --enable-unstable assess scan # Clean up (cd foo && cargo clean) (cd bar && cargo clean) +(cd compile_error && cargo clean) +(cd manifest_error && cargo clean) # Check for expected files (and clean up) EXPECTED_FILES=( - bar/bar.kani-assess-metadata.json - foo/foo.kani-assess-metadata.json bar/bar.kani-assess.log + bar/bar.kani-assess-metadata.json + compile_error/compile_error.kani-assess.log + compile_error/compile_error.kani-assess-metadata.json + manifest_error/manifest_error.kani-assess.log + manifest_error/manifest_error.kani-assess-metadata.json foo/foo.kani-assess.log + foo/foo.kani-assess-metadata.json ) + +errors=0 for file in ${EXPECTED_FILES[@]}; do if [ -f $KANI_DIR/tests/assess-scan-test-scaffold/$file ]; then rm $KANI_DIR/tests/assess-scan-test-scaffold/$file else - echo "Failed to find $file" && exit 1 + errors=1 + echo "Failed to find $file" fi done echo "Done with assess scan test" +exit $errors diff --git a/tests/assess-scan-test-scaffold/compile_error/Cargo.toml b/tests/assess-scan-test-scaffold/compile_error/Cargo.toml new file mode 100644 index 0000000000000..1189e08d0956e --- /dev/null +++ b/tests/assess-scan-test-scaffold/compile_error/Cargo.toml @@ -0,0 +1,11 @@ +# Copyright Kani Contributors +# SPDX-License-Identifier: Apache-2.0 OR MIT + +[package] +name = "compile_error" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/tests/assess-scan-test-scaffold/compile_error/src/lib.rs b/tests/assess-scan-test-scaffold/compile_error/src/lib.rs new file mode 100644 index 0000000000000..2bad1be422f98 --- /dev/null +++ b/tests/assess-scan-test-scaffold/compile_error/src/lib.rs @@ -0,0 +1,7 @@ +// Copyright Kani Contributors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Function with a compilation error +pub fn with_error(left: usize, right: u32) -> usize { + left + right +} diff --git a/tests/assess-scan-test-scaffold/manifest_error/Cargo.toml b/tests/assess-scan-test-scaffold/manifest_error/Cargo.toml new file mode 100644 index 0000000000000..09b611dc645cf --- /dev/null +++ b/tests/assess-scan-test-scaffold/manifest_error/Cargo.toml @@ -0,0 +1,12 @@ +# Copyright Kani Contributors +# SPDX-License-Identifier: Apache-2.0 OR MIT + +[package] +name = "manifest_error" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +unknown = { path="./does/not/exist" } diff --git a/tests/assess-scan-test-scaffold/manifest_error/src/lib.rs b/tests/assess-scan-test-scaffold/manifest_error/src/lib.rs new file mode 100644 index 0000000000000..0058f8ca1b587 --- /dev/null +++ b/tests/assess-scan-test-scaffold/manifest_error/src/lib.rs @@ -0,0 +1,5 @@ +// Copyright Kani Contributors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Nothing here +pub fn void() {}