gh-62432: unittest runner: Exit code 5 if no tests were run

Doc/library/unittest.rst

@@ -2281,7 +2281,8 @@ Loading and running tests
 
  The *testRunner* argument can either be a test runner class or an already
  created instance of it. By default ``main`` calls :func:`sys.exit` with
- an exit code indicating success or failure of the tests run.
+ an exit code indicating success (0) or failure (1) of the tests run.
+ An exit code of 5 indicates that no tests were run.
 
  The *testLoader* argument has to be a :class:`TestLoader` instance,
  and defaults to :data:`defaultTestLoader`.

Lib/test/test_unittest/test_program.py

@@ -71,15 +71,22 @@ def testExpectedFailure(self):
  def testUnexpectedSuccess(self):
  pass
 
- class FooBarLoader(unittest.TestLoader):
- """Test loader that returns a suite containing FooBar."""
+ class Empty(unittest.TestCase):
+ pass
+
+ class TestLoader(unittest.TestLoader):
+ """Test loader that returns a suite containing the supplied testcase."""
+
+ def __init__(self, testcase):
+ self.testcase = testcase
+
  def loadTestsFromModule(self, module):
  return self.suiteClass(
- [self.loadTestsFromTestCase(Test_TestProgram.FooBar)])
+ [self.loadTestsFromTestCase(self.testcase)])
 
  def loadTestsFromNames(self, names, module):
  return self.suiteClass(
- [self.loadTestsFromTestCase(Test_TestProgram.FooBar)])
+ [self.loadTestsFromTestCase(self.testcase)])
 
  def test_defaultTest_with_string(self):
  class FakeRunner(object):
@@ -92,7 +99,7 @@ def run(self, test):
  runner = FakeRunner()
  program = unittest.TestProgram(testRunner=runner, exit=False,
  defaultTest='test.test_unittest',
- testLoader=self.FooBarLoader())
+ testLoader=self.TestLoader(self.FooBar))
  sys.argv = old_argv
  self.assertEqual(('test.test_unittest',), program.testNames)
 
@@ -108,7 +115,7 @@ def run(self, test):
  program = unittest.TestProgram(
  testRunner=runner, exit=False,
  defaultTest=['test.test_unittest', 'test.test_unittest2'],
- testLoader=self.FooBarLoader())
+ testLoader=self.TestLoader(self.FooBar))
  sys.argv = old_argv
  self.assertEqual(['test.test_unittest', 'test.test_unittest2'],
  program.testNames)
@@ -118,7 +125,7 @@ def test_NonExit(self):
  program = unittest.main(exit=False,
  argv=["foobar"],
  testRunner=unittest.TextTestRunner(stream=stream),
- testLoader=self.FooBarLoader())
+ testLoader=self.TestLoader(self.FooBar))
  self.assertTrue(hasattr(program, 'result'))
  out = stream.getvalue()
  self.assertIn('\nFAIL: testFail ', out)
@@ -130,13 +137,13 @@ def test_NonExit(self):
 
  def test_Exit(self):
  stream = BufferedWriter()
- self.assertRaises(
- SystemExit,
- unittest.main,
- argv=["foobar"],
- testRunner=unittest.TextTestRunner(stream=stream),
- exit=True,
-  testLoader=self.FooBarLoader())
+ with self.assertRaises(SystemExit) as cm:
+ unittest.main(
+  argv=["foobar"],
+  testRunner=unittest.TextTestRunner(stream=stream),
+  exit=True,
+  testLoader=self.TestLoader(self.FooBar))
+ self.assertEqual(cm.exception.code, 1)
  out = stream.getvalue()
  self.assertIn('\nFAIL: testFail ', out)
  self.assertIn('\nERROR: testError ', out)
@@ -147,12 +154,11 @@ def test_Exit(self):
 
  def test_ExitAsDefault(self):
  stream = BufferedWriter()
- self.assertRaises(
- SystemExit,
- unittest.main,
- argv=["foobar"],
- testRunner=unittest.TextTestRunner(stream=stream),
- testLoader=self.FooBarLoader())
+ with self.assertRaises(SystemExit):
+ unittest.main(
+ argv=["foobar"],
+ testRunner=unittest.TextTestRunner(stream=stream),
+ testLoader=self.TestLoader(self.FooBar))
  out = stream.getvalue()
  self.assertIn('\nFAIL: testFail ', out)
  self.assertIn('\nERROR: testError ', out)
@@ -161,6 +167,17 @@ def test_ExitAsDefault(self):
  'expected failures=1, unexpected successes=1)\n')
  self.assertTrue(out.endswith(expected))
 
+ def test_ExitEmptySuite(self):
+ stream = BufferedWriter()
+ with self.assertRaises(SystemExit) as cm:
+ unittest.main(
+ argv=["empty"],
+ testRunner=unittest.TextTestRunner(stream=stream),
+ testLoader=self.TestLoader(self.Empty))
+ self.assertEqual(cm.exception.code, 5)
+ out = stream.getvalue()
+ self.assertIn('\nNO TESTS RAN\n', out)
+
 
 class InitialisableProgram(unittest.TestProgram):
  exit = False

Lib/test/test_unittest/test_result.py

@@ -451,6 +451,7 @@ def testFailFastSetByRunner(self):
  stream = BufferedWriter()
  runner = unittest.TextTestRunner(stream=stream, failfast=True)
  def test(result):
+ result.testsRun += 1
  self.assertTrue(result.failfast)
  result = runner.run(test)
  stream.flush()

Lib/test/test_unittest/test_runner.py

@@ -577,6 +577,16 @@ def test(self):
  'inner setup', 'inner test', 'inner cleanup',
  'end outer test', 'outer cleanup'])
 
+ def test_run_empty_suite_error_message(self):
+ class EmptyTest(unittest.TestCase):
+ pass
+
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(EmptyTest)
+ runner = getRunner()
+ runner.run(suite)
+
+ self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue())
+
 
 class TestModuleCleanUp(unittest.TestCase):
  def test_add_and_do_ModuleCleanup(self):

Lib/unittest/main.py

@@ -9,6 +9,7 @@
 from .signals import installHandler
 
 __unittest = True
+_NO_TESTS_EXITCODE = 5
 
 MAIN_EXAMPLES = """\
 Examples:
@@ -279,6 +280,12 @@ def runTests(self):
  testRunner = self.testRunner
  self.result = testRunner.run(self.test)
  if self.exit:
- sys.exit(not self.result.wasSuccessful())
+ if self.result.testsRun == 0:
+ sys.exit(_NO_TESTS_EXITCODE)
+ elif self.result.wasSuccessful():
+ sys.exit(0)
+ else:
+ sys.exit(1)
+
 
 main = TestProgram

Lib/unittest/runner.py

@@ -274,6 +274,8 @@ def run(self, test):
  infos.append("failures=%d" % failed)
  if errored:
  infos.append("errors=%d" % errored)
+ elif run == 0:
+ self.stream.write("NO TESTS RAN")
  else:
  self.stream.write("OK")
  if skipped:

Misc/ACKS

@@ -1513,6 +1513,7 @@ Vlad Riscutia
 Wes Rishel
 Daniel Riti
 Juan M. Bello Rivas
+Stefano Rivera
 Llandy Riveron Del Risco
 Mohd Sanad Zaki Rizvi
 Davide Rizzo

Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst

@@ -0,0 +1,3 @@
+The :mod:`unittest` runner will now exit with status code 5 if no tests
+were run. It is common for test runner misconfiguration to fail to find
+any tests, this should be an error.