diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 72b57d91c234e..1a4101e4de80b 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -374,11 +374,12 @@ impl<'test> TestCx<'test> {
 
         // if a test does not crash, consider it an error
         if proc_res.status.success() || matches!(proc_res.status.code(), Some(1 | 0)) {
-            self.fatal(
-                "test no longer crashes/triggers ICE! Please give it a mearningful name, \
+            self.fatal(&format!(
+                "crashtest no longer crashes/triggers ICE, horray! Please give it a meaningful name, \
             add a doc-comment to the start of the test explaining why it exists and \
-            move it to tests/ui or wherever you see fit.",
-            );
+            move it to tests/ui or wherever you see fit. Adding 'Fixes #<issueNr>' to your PR description \
+            ensures that the corresponding ticket is auto-closed upon merge."
+            ));
         }
     }
 
diff --git a/triagebot.toml b/triagebot.toml
index 3abff0f1c784e..70fb14c332848 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -811,6 +811,9 @@ If appropriate, please update `CONFIG_CHANGE_HISTORY` in `src/bootstrap/src/util
 [mentions."src/bootstrap/src/core/build_steps/llvm.rs"]
 message = "This PR changes how LLVM is built. Consider updating src/bootstrap/download-ci-llvm-stamp."
 
+[mentions."test/crashes"]
+message = "This PR changes a file inside `tests/crashes`. If a crash was fixed, please move into the correspondig `ui` subdir and add 'Fixes #<issueNr>' to the pr description to autoclose the issue upon merge."
+
 [mentions."tests/ui/deriving/deriving-all-codegen.stdout"]
 message = "Changes to the code generated for builtin derived traits."
 cc = ["@nnethercote"]