diff --git a/src/custom_flags/run.rs b/src/custom_flags/run.rs index 1f53a310..68baf40e 100644 --- a/src/custom_flags/run.rs +++ b/src/custom_flags/run.rs @@ -1,7 +1,11 @@ //! Types used for running tests after they pass compilation use bstr::ByteSlice; -use std::process::{Command, Output}; +use spanned::{Span, Spanned}; +use std::{ + path::PathBuf, + process::{Command, Output}, +}; use crate::{build_manager::BuildManager, per_test_config::TestConfig, Error, Errored}; @@ -67,6 +71,12 @@ impl Flag for Run { mode: format!("run({exit_code})"), status, expected: exit_code, + reason: match (exit_code, status.code()) { + (_, Some(101)) => get_panic_span(&output.stderr), + (0, _) => Spanned::dummy("the test was expected to run successfully".into()), + (101, _) => Spanned::dummy("the test was expected to panic".into()), + _ => Spanned::dummy(String::new()), + }, }) } if errors.is_empty() { @@ -81,3 +91,41 @@ impl Flag for Run { } } } + +fn get_panic_span(stderr: &[u8]) -> Spanned { + let mut lines = stderr.lines(); + while let Some(line) = lines.next() { + if let Some((_, location)) = line.split_once_str(b"panicked at ") { + let mut parts = location.split(|&c| c == b':'); + let Some(filename) = parts.next() else { + continue; + }; + let Some(line) = parts.next() else { continue }; + let Some(col) = parts.next() else { continue }; + let message = lines + .next() + .and_then(|msg| msg.to_str().ok()) + .unwrap_or("the test panicked during execution"); + let Ok(line) = line.to_str() else { continue }; + let Ok(col) = col.to_str() else { continue }; + let Ok(filename) = filename.to_str() else { + continue; + }; + let Ok(line) = line.parse() else { + continue; + }; + let Ok(col) = col.parse() else { + continue; + }; + let span = Span { + file: PathBuf::from(filename), + line_start: line, + line_end: line, + col_start: col, + col_end: col, + }; + return Spanned::new(message.into(), span); + } + } + Spanned::dummy("".into()) +} diff --git a/src/custom_flags/rustfix.rs b/src/custom_flags/rustfix.rs index 05e29c9e..16d152aa 100644 --- a/src/custom_flags/rustfix.rs +++ b/src/custom_flags/rustfix.rs @@ -5,7 +5,7 @@ use std::{ process::{Command, Output}, }; -use spanned::Span; +use spanned::{Span, Spanned}; use crate::{ build_manager::BuildManager, @@ -178,14 +178,25 @@ impl Flag for RustfixMode { if output.status.success() { Ok(None) } else { + let diagnostics = config.process(&output.stderr); Err(Errored { command: cmd, errors: vec![Error::ExitStatus { mode: "rustfix".into(), expected: 0, status: output.status, + reason: Spanned::new( + "after rustfix is applied, all errors should be gone, but weren't".into(), + diagnostics + .messages + .iter() + .flatten() + .chain(diagnostics.messages_from_unknown_file_or_line.iter()) + .find_map(|message| message.line_col.clone()) + .unwrap_or_default(), + ), }], - stderr: config.process(&output.stderr).rendered, + stderr: diagnostics.rendered, stdout: output.stdout, }) } diff --git a/src/error.rs b/src/error.rs index 0e2896a4..c142a48b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -16,6 +16,8 @@ pub enum Error { status: ExitStatus, /// The expected exit status as set in the file or derived from the mode. expected: i32, + /// A reason for why the expected exit status was expected + reason: Spanned, }, /// A pattern was declared but had no matching error. PatternNotFound { diff --git a/src/mode.rs b/src/mode.rs index 6468633d..c75cc58d 100644 --- a/src/mode.rs +++ b/src/mode.rs @@ -1,3 +1,5 @@ +use spanned::Spanned; + use super::Error; use std::fmt::Display; use std::process::ExitStatus; @@ -34,6 +36,15 @@ impl Mode { mode: self.to_string(), status, expected, + reason: Spanned::dummy( + match (expected, status.code()) { + (_, Some(101)) => "the compiler panicked", + (0, Some(1)) => "compilation failed, but was expected to succeed", + (1, Some(0)) => "compilation succeeded, but was expected to fail", + _ => "", + } + .into(), + ), }) } } diff --git a/src/status_emitter.rs b/src/status_emitter.rs index e1917d05..8a9e4596 100644 --- a/src/status_emitter.rs +++ b/src/status_emitter.rs @@ -486,11 +486,17 @@ fn print_error(error: &Error, path: &Path) { mode, status, expected, + reason, } => { // `status` prints as `exit status: N`. - print_error_header(format_args!( - "{mode} test got {status}, but expected {expected}" - )) + create_error( + format!("{mode} test got {status}, but expected {expected}"), + &[( + &[(reason, Some(reason.span.clone()))], + reason.span.line_start, + )], + path, + ) } Error::Command { kind, status } => { // `status` prints as `exit status: N`. @@ -683,10 +689,10 @@ fn create_error( }), slices: lines .iter() - .map(|(label, line)| { - let source = source[line.get() - 1]; + .filter_map(|(label, line)| { + let source = source.get(line.get() - 1)?; let len = source.chars().count(); - Slice { + Some(Slice { source, line_start: line.get(), origin: Some(&file), @@ -714,10 +720,23 @@ fn create_error( }) .collect(), fold: false, + }) + }) + .collect(), + footer: lines + .iter() + .filter_map(|(label, line)| { + if source.get(line.get() - 1).is_some() { + return None; } + Some(label.iter().map(|(label, _)| Annotation { + id: None, + annotation_type: AnnotationType::Note, + label: Some(label), + })) }) + .flatten() .collect(), - footer: vec![], }; let renderer = if colored::control::SHOULD_COLORIZE.should_colorize() { Renderer::styled() @@ -733,11 +752,13 @@ fn gha_error(error: &Error, test_path: &str, revision: &str) { mode, status, expected, + reason, } => { - github_actions::error( + let mut err = github_actions::error( test_path, format!("{mode} test{revision} got {status}, but expected {expected}"), ); + err.write_str(reason).unwrap(); } Error::Command { kind, status } => { github_actions::error(test_path, format!("{kind}{revision} failed with {status}")); diff --git a/tests/integrations/basic-fail/Cargo.stdout b/tests/integrations/basic-fail/Cargo.stdout index 7e872508..d3b1daa8 100644 --- a/tests/integrations/basic-fail/Cargo.stdout +++ b/tests/integrations/basic-fail/Cargo.stdout @@ -96,6 +96,7 @@ FAILED TEST: tests/actual_tests/executable_compile_err.rs command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests/executable_compile_err.rs" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" error: pass test got exit status: 1, but expected 0 + = note: compilation failed, but was expected to succeed error: actual output differed from expected Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable_compile_err.stderr` to the actual output @@ -140,6 +141,7 @@ FAILED TEST: tests/actual_tests/exit_code_fail.rs command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests/exit_code_fail.rs" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" error: fail test got exit status: 0, but expected 1 + = note: compilation succeeded, but was expected to fail error: no error patterns found in fail test @@ -349,6 +351,7 @@ FAILED TEST: tests/actual_tests/rustc_ice.rs command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests/rustc_ice.rs" "-Ztreat-err-as-bug" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" error: fail test got exit status: 101, but expected 1 + = note: the compiler panicked error: `mismatched types` not found in diagnostics on line 8 --> tests/actual_tests/rustc_ice.rs:9:17 @@ -607,6 +610,11 @@ FAILED TEST: tests/actual_tests_bless/failing_executable.rs command: "$CMD" error: run(0) test got exit status: 101, but expected 0 + --> tests/actual_tests_bless/failing_executable.rs:4:5 + | +4 | assert_eq!(5, 6); + | ^ assertion `left == right` failed + | full stderr: @@ -618,6 +626,7 @@ FAILED TEST: tests/actual_tests_bless/no_main.rs command: "rustc" "--error-format=json" "--crate-type=lib" "--out-dir" "$TMP "tests/actual_tests_bless/no_main.rs" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" error: fail test got exit status: 0, but expected 1 + = note: compilation succeeded, but was expected to fail error: no error patterns found in fail test @@ -663,6 +672,7 @@ FAILED TEST: tests/actual_tests_bless/no_test.rs command: "rustc" "--error-format=json" "--test" "--out-dir" "$TMP "tests/actual_tests_bless/no_test.rs" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" error: fail test got exit status: 0, but expected 1 + = note: compilation succeeded, but was expected to fail error: no error patterns found in fail test @@ -747,6 +757,7 @@ FAILED TEST: tests/actual_tests_bless/revisioned_executable.rs (revision `panic` command: "$CMD" error: run(101) test got exit status: 0, but expected 101 + = note: the test was expected to panic full stderr: @@ -758,6 +769,11 @@ FAILED TEST: tests/actual_tests_bless/revisioned_executable_panic.rs (revision ` command: "$CMD" error: run(0) test got exit status: 101, but expected 0 + --> tests/actual_tests_bless/revisioned_executable_panic.rs + | +6 | panic!() + | ^ explicit panic + | full stderr: @@ -801,6 +817,11 @@ FAILED TEST: tests/actual_tests_bless/rustfix-fail-revisions.rs (revision `a`) command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests_bless/rustfix-fail-revisions.a.fixed" "--cfg=a" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" "--crate-name" "rustfix_fail_revisions" error: rustfix test got exit status: 1, but expected 0 + --> tests/actual_tests_bless/rustfix-fail-revisions.rs:11:6 + | +11 | x; + | ^ after rustfix is applied, all errors should be gone, but weren't + | full stderr: error[E0382]: use of moved value: `x` @@ -831,6 +852,11 @@ FAILED TEST: tests/actual_tests_bless/rustfix-fail-revisions.rs (revision `b`) command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests_bless/rustfix-fail-revisions.b.fixed" "--cfg=b" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" "--crate-name" "rustfix_fail_revisions" error: rustfix test got exit status: 1, but expected 0 + --> tests/actual_tests_bless/rustfix-fail-revisions.rs:11:6 + | +11 | x; + | ^ after rustfix is applied, all errors should be gone, but weren't + | full stderr: error[E0382]: use of moved value: `x` @@ -861,6 +887,11 @@ FAILED TEST: tests/actual_tests_bless/rustfix-fail.rs command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests_bless/rustfix-fail.fixed" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" "--crate-name" "rustfix_fail" error: rustfix test got exit status: 1, but expected 0 + --> tests/actual_tests_bless/rustfix-fail.rs:10:6 + | +10 | x; + | ^ after rustfix is applied, all errors should be gone, but weren't + | full stderr: error[E0382]: use of moved value: `x`