Auto merge of #143002 - Enselic:tests-ui-run-fail-exit-vs-signal, r=<try>

tests: Require `run-fail` ui tests to have an exit code (`SIGABRT` not ok)

Normally a `run-fail` ui test shall not be terminated by a signal like `SIGABRT`. So begin having that as a hard requirement.

Some of our current tests do terminate by a signal however. Introduce and use `run-fail-without-exit-code` for those tests.

This adds further (cc #142304, #142886) protection against the regression in #123733 since that bug also manifested as `SIGABRT` in `tests/ui/panics/panic-main.rs` (shown as `Aborted (core dumped)` in the logs attached to that issue, and I have also been able to reproduce this locally).

### TODO
- [ ] **Q:** what about on Windows?
    **A:** we'll treat any exit code outside of 1 - 127 as "crashed", and we'll do the same on unix.
- [x] test all permutations of actual vs expected
    **Done:** See #143002 (comment).

### Zulip discussion

See https://rust-lang.zulipchat.com/#narrow/channel/122651-general/topic/compiletest.3A.20terminate.20by.20signal.20vs.20exit.20with.20error/with/525611235

try-job: x86_64-msvc-1
try-job: x86_64-msvc-2

Author: rust-bors[bot]

src/doc/rustc-dev-guide/src/tests/directives.md

@@ -75,8 +75,9 @@ expectations](ui.md#controlling-passfail-expectations).
 | `check-fail`                | Building (no codegen) should fail           | `ui`, `crashes`                           | N/A             |
 | `build-pass`                | Building should pass                        | `ui`, `crashes`, `codegen`, `incremental` | N/A             |
 | `build-fail`                | Building should fail                        | `ui`, `crashes`                           | N/A             |
-| `run-pass`                  | Running the test binary should pass         | `ui`, `crashes`, `incremental`            | N/A             |
-| `run-fail`                  | Running the test binary should fail         | `ui`, `crashes`                           | N/A             |
+| `run-pass`                  | Program must exit with code `0`             | `ui`, `crashes`, `incremental`            | N/A             |
+| `run-fail`                  | Program must exit with code `1..=127`       | `ui`, `crashes`                           | N/A             |
+| `run-crash`                 | Program must crash                          | `ui`,                                     | N/A             |
 | `ignore-pass`               | Ignore `--pass` flag                        | `ui`, `crashes`, `codegen`, `incremental` | N/A             |
 | `dont-check-failure-status` | Don't check exact failure status (i.e. `1`) | `ui`, `incremental`                       | N/A             |
 | `failure-status`            | Check                                       | `ui`, `crashes`                           | Any `u16`       |

src/doc/rustc-dev-guide/src/tests/ui.md

@@ -443,7 +443,7 @@ even run the resulting program. Just add one of the following
   - `//@ build-pass` — compilation and linking should succeed but do
     not run the resulting binary.
   - `//@ run-pass` — compilation should succeed and running the resulting
-    binary should also succeed.
+    binary should make it exit with code 0 which indicates success.
 - Fail directives:
   - `//@ check-fail` — compilation should fail (the codegen phase is skipped).
     This is the default for UI tests.
@@ -452,7 +452,10 @@ even run the resulting program. Just add one of the following
     without the codegen phase, then a second time the full compile should
     fail.
   - `//@ run-fail` — compilation should succeed, but running the resulting
-    binary should fail.
+    binary should make it exit with a code in the `1..=127` which indicates
+    regular failure.
+  - `//@ run-crash` — compilation should succeed, but running the resulting
+    binary should fail with a crash (e.g. `SIGABRT` or `SIGSEGV`).
 
 For `run-pass` and `run-fail` tests, by default the output of the program itself
 is not checked.

src/tools/compiletest/src/common.rs

@@ -110,11 +110,23 @@ string_enum! {
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
+pub enum RunFailMode {
+    /// Running the program must make it exit with a regular failure exit code
+    /// in the range `1..=127`. If the program is terminated by e.g. a signal
+    /// the test will fail.
+    ExitWithFailure,
+    /// Running the program must result in a crash, e.g. by `SIGABRT` or
+    /// `SIGSEGV` on Unix or on Windows by having an appropriate NTSTATUS high
+    /// bit in the exit code.
+    Crash,
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
 pub enum FailMode {
     Check,
     Build,
-    Run,
+    Run(RunFailMode),
 }
 
 string_enum! {

src/tools/compiletest/src/directive-list.rs

@@ -241,6 +241,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "regex-error-pattern",
     "remap-src-base",
     "revisions",
+    "run-crash",
     "run-fail",
     "run-flags",
     "run-pass",

src/tools/compiletest/src/header.rs

@@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf};
 use semver::Version;
 use tracing::*;
 
-use crate::common::{Config, Debugger, FailMode, Mode, PassMode};
+use crate::common::{Config, Debugger, FailMode, Mode, PassMode, RunFailMode};
 use crate::debuggers::{extract_cdb_version, extract_gdb_version};
 use crate::errors::ErrorKind;
 use crate::executor::{CollectedTestDesc, ShouldPanic};
@@ -655,7 +655,10 @@ impl TestProps {
             Some(FailMode::Build)
         } else if config.parse_name_directive(ln, "run-fail") {
             check_ui("run");
-            Some(FailMode::Run)
+            Some(FailMode::Run(RunFailMode::ExitWithFailure))
+        } else if config.parse_name_directive(ln, "run-crash") {
+            check_ui("run");
+            Some(FailMode::Run(RunFailMode::Crash))
         } else {
             None
         };

src/tools/compiletest/src/runtest.rs

@@ -17,10 +17,10 @@ use tracing::*;
 
 use crate::common::{
     Assembly, Codegen, CodegenUnits, CompareMode, Config, CoverageMap, CoverageRun, Crashes,
-    DebugInfo, Debugger, FailMode, Incremental, MirOpt, PassMode, Pretty, RunMake, Rustdoc,
-    RustdocJs, RustdocJson, TestPaths, UI_EXTENSIONS, UI_FIXED, UI_RUN_STDERR, UI_RUN_STDOUT,
-    UI_STDERR, UI_STDOUT, UI_SVG, UI_WINDOWS_SVG, Ui, expected_output_path, incremental_dir,
-    output_base_dir, output_base_name, output_testname_unique,
+    DebugInfo, Debugger, FailMode, Incremental, MirOpt, PassMode, Pretty, RunFailMode, RunMake,
+    Rustdoc, RustdocJs, RustdocJson, TestPaths, UI_EXTENSIONS, UI_FIXED, UI_RUN_STDERR,
+    UI_RUN_STDOUT, UI_STDERR, UI_STDOUT, UI_SVG, UI_WINDOWS_SVG, Ui, expected_output_path,
+    incremental_dir, output_base_dir, output_base_name, output_testname_unique,
 };
 use crate::compute_diff::{DiffLine, make_diff, write_diff, write_filtered_diff};
 use crate::errors::{Error, ErrorKind, load_errors};
@@ -277,7 +277,11 @@ impl<'test> TestCx<'test> {
 
     fn should_run(&self, pm: Option<PassMode>) -> WillExecute {
         let test_should_run = match self.config.mode {
-            Ui if pm == Some(PassMode::Run) || self.props.fail_mode == Some(FailMode::Run) => true,
+            Ui if pm == Some(PassMode::Run)
+                || matches!(self.props.fail_mode, Some(FailMode::Run(_))) =>
+            {
+                true
+            }
             MirOpt if pm == Some(PassMode::Run) => true,
             Ui | MirOpt => false,
             mode => panic!("unimplemented for mode {:?}", mode),
@@ -296,6 +300,13 @@ impl<'test> TestCx<'test> {
         }
     }
 
+    fn should_exit_with_failure(&self) -> bool {
+        match self.config.mode {
+            Ui => self.props.fail_mode == Some(FailMode::Run(RunFailMode::ExitWithFailure)),
+            mode => panic!("unimplemented for mode {:?}", mode),
+        }
+    }
+
     fn should_compile_successfully(&self, pm: Option<PassMode>) -> bool {
         match self.config.mode {
             RustdocJs => true,

src/tools/compiletest/src/runtest/ui.rs

@@ -140,14 +140,32 @@ impl TestCx<'_> {
                     &proc_res,
                 );
             }
+            let code = proc_res.status.code();
+            let exit_with_success = proc_res.status.success();
+            // A normal failure exit code is between 1 and 127. If we don't have
+            // an exit code or if it is outside that range (and not 0) we
+            // consider it a crash.
+            let exit_with_failure = code.is_some_and(|c| c >= 1 && c <= 127);
             if self.should_run_successfully(pm) {
-                if !proc_res.status.success() {
-                    self.fatal_proc_rec("test run failed!", &proc_res);
+                if !exit_with_success {
+                    self.fatal_proc_rec(&format!("test run failed! code={code:?}"), &proc_res);
+                }
+            } else if self.should_exit_with_failure() {
+                if !exit_with_failure {
+                    self.fatal_proc_rec(
+                        &format!("test did not exit with failure! code={code:?}"),
+                        &proc_res,
+                    );
+                }
+            } else {
+                // If we get here it means we should not run successfully and we
+                // should not exit with failure. So we should crash. And if we
+                // did exit with success or did exit with failure it means we
+                // did NOT crash.
+                if exit_with_success || exit_with_failure {
+                    self.fatal_proc_rec(&format!("test did not crash! code={code:?}"), &proc_res);
                 }
-            } else if proc_res.status.success() {
-                self.fatal_proc_rec("test run succeeded!", &proc_res);
             }
-
             self.get_output(&proc_res)
         } else {
             self.get_output(&proc_res)

tests/ui/contracts/contract-attributes-generics.rs

@@ -5,9 +5,9 @@
 //@ [unchk_pass] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
-//@ [chk_const_fail] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
+//@ [chk_const_fail] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //

tests/ui/contracts/contract-attributes-nest.rs

@@ -5,8 +5,8 @@
 //@ [unchk_fail_post] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [unchk_fail_pre] compile-flags: -Zcontract-checks=no

tests/ui/contracts/contract-attributes-tail.rs

@@ -5,8 +5,8 @@
 //@ [unchk_fail_post] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [unchk_fail_pre] compile-flags: -Zcontract-checks=no