diff --git a/src/tools/run-make-support/src/lib.rs b/src/tools/run-make-support/src/lib.rs
index d96c8b891278b..35d7b65bec085 100644
--- a/src/tools/run-make-support/src/lib.rs
+++ b/src/tools/run-make-support/src/lib.rs
@@ -268,6 +268,17 @@ pub fn recursive_diff(dir1: impl AsRef<Path>, dir2: impl AsRef<Path>) {
     }
 }
 
+/// Check that `haystack` does not contain `needle`. Panic otherwise.
+pub fn assert_not_contains(haystack: &str, needle: &str) {
+    if haystack.contains(needle) {
+        eprintln!("=== HAYSTACK ===");
+        eprintln!("{}", haystack);
+        eprintln!("=== NEEDLE ===");
+        eprintln!("{}", needle);
+        panic!("needle was unexpectedly found in haystack");
+    }
+}
+
 /// Implement common helpers for command wrappers. This assumes that the command wrapper is a struct
 /// containing a `cmd: Command` field and a `output` function. The provided helpers are:
 ///
diff --git a/src/tools/tidy/src/allowed_run_make_makefiles.txt b/src/tools/tidy/src/allowed_run_make_makefiles.txt
index 2329b8b44dea0..9ca27834cf720 100644
--- a/src/tools/tidy/src/allowed_run_make_makefiles.txt
+++ b/src/tools/tidy/src/allowed_run_make_makefiles.txt
@@ -1,6 +1,5 @@
 run-make/allocator-shim-circular-deps/Makefile
 run-make/allow-non-lint-warnings-cmdline/Makefile
-run-make/allow-warnings-cmdline-stability/Makefile
 run-make/archive-duplicate-names/Makefile
 run-make/atomic-lock-free/Makefile
 run-make/bare-outfile/Makefile
diff --git a/tests/run-make/allow-warnings-cmdline-stability/Makefile b/tests/run-make/allow-warnings-cmdline-stability/Makefile
deleted file mode 100644
index 368a39af6bfe2..0000000000000
--- a/tests/run-make/allow-warnings-cmdline-stability/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-# ignore-cross-compile
-include ../tools.mk
-
-# Test that -A warnings makes the 'empty trait list for derive' warning go away
-DEP=$(shell $(RUSTC) bar.rs)
-OUT=$(shell $(RUSTC) foo.rs -A warnings 2>&1 | grep "warning" )
-
-all: foo bar
-	test -z '$(OUT)'
-
-# These are just to ensure that the above commands actually work
-bar:
-	$(RUSTC) bar.rs
-
-foo: bar
-	$(RUSTC) foo.rs -A warnings
diff --git a/tests/run-make/allow-warnings-cmdline-stability/rmake.rs b/tests/run-make/allow-warnings-cmdline-stability/rmake.rs
new file mode 100644
index 0000000000000..8f6fe6bd0b66a
--- /dev/null
+++ b/tests/run-make/allow-warnings-cmdline-stability/rmake.rs
@@ -0,0 +1,11 @@
+// Test that `-Awarnings` suppresses warnings for unstable APIs.
+
+use run_make_support::{assert_not_contains, rustc};
+
+fn main() {
+    rustc().input("bar.rs").run();
+    let output = rustc().input("foo.rs").arg("-Awarnings").run();
+
+    assert_not_contains(&String::from_utf8(output.stdout).unwrap(), "warning");
+    assert_not_contains(&String::from_utf8(output.stderr).unwrap(), "warning");
+}