diff --git a/README.md b/README.md index 8ae64c1..987c911 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,20 @@ unittest-parallel -t . -s tests --coverage-branch - [Source code on GitHub](https://github.com/craigahobbs/unittest-parallel) +## How it works + +unittest-parallel uses Python's built-in unit test discovery to find all test cases in your project. +It then runs all test cases in a Python multi-processing pool of the requested size. + +### Class and Module Fixtures + +Python's unittest framework supports +[class and module fixtures.](https://docs.python.org/3/library/unittest.html#class-and-module-fixtures) +Use the "--class-fixtures" option to execute class fixtures correctly. Use the "--module-fixtures" +option to execute module fixtures correctly. Note that these options reduce the amount of +parallelism. + + ## Example output ``` @@ -112,12 +126,6 @@ coverage options: ``` -## How it works - -unittest-parallel uses Python's built-in unit test discovery to find all of the TestCase classes in -your project. It then runs all tests in a Python multi-processing pool of the requested size. - - ## Development This project is developed using [Python Build](https://github.com/craigahobbs/python-build#readme). diff --git a/src/tests/test_main.py b/src/tests/test_main.py index 2a4b00a..fa2a839 100644 --- a/src/tests/test_main.py +++ b/src/tests/test_main.py @@ -38,6 +38,16 @@ def mock_3(self): self.assertIsNotNone(self) +class SuccessTestCase2(SuccessTestCase): + def mock_1(self): + self.assertIsNotNone(self) + + +class SuccessTestCase3(SuccessTestCase): + def mock_1(self): + self.assertIsNotNone(self) + + class SuccessWithOutputTestCase(unittest.TestCase): def mock_1(self): self.assertIsNotNone(self) @@ -99,16 +109,6 @@ def mock_3(self): self.assertIsNotNone(self) -def _create_test_suite(test_case_class): - return unittest.TestSuite(tests=[ - unittest.TestSuite(tests=[ - test_case_class('mock_1'), - test_case_class('mock_2'), - test_case_class('mock_3') - ]) - ]) - - class TestMain(unittest.TestCase): def assert_output(self, actual, expected): @@ -149,6 +149,7 @@ def test_jobs(self): cpu_count_mock.assert_not_called() self.assertEqual(stdout.getvalue(), '') self.assertEqual(stderr.getvalue(), '''\ +Running 0 test suites (0 total tests) across 1 processes ---------------------------------------------------------------------- Ran 0 test in 0.000s @@ -156,8 +157,9 @@ def test_jobs(self): OK ''') - def test_pool_no_tests(self): + def test_no_tests(self): with patch('multiprocessing.cpu_count', Mock(return_value=1)), \ + patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ patch('unittest.TestLoader.discover', Mock(return_value=unittest.TestSuite())): @@ -165,6 +167,7 @@ def test_pool_no_tests(self): self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 0 test suites (0 total tests) across 1 processes ---------------------------------------------------------------------- Ran 0 test in s @@ -172,31 +175,148 @@ def test_pool_no_tests(self): OK ''') - def test_pool_success(self): + def test_success(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]), + unittest.TestSuite(tests=[SuccessTestCase2('mock_1')]) + ]), + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase3('mock_1')]) + ]) + ]) with patch('multiprocessing.cpu_count', Mock(return_value=1)), \ + patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): - main([]) + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): + main(['-v']) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 5 test suites (5 total tests) across 1 processes + +mock_1 (tests.test_main.SuccessTestCase) ... ok +mock_2 (tests.test_main.SuccessTestCase) ... ok +mock_3 (tests.test_main.SuccessTestCase) ... ok +mock_1 (tests.test_main.SuccessTestCase2) ... ok +mock_1 (tests.test_main.SuccessTestCase3) ... ok +---------------------------------------------------------------------- +Ran 5 tests in s + +OK +''') + + def test_success_max_suites(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]), + ]) + ]) + with patch('multiprocessing.cpu_count', Mock(return_value=2)), \ + patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ + patch('sys.stdout', StringIO()) as stdout, \ + patch('sys.stderr', StringIO()) as stderr, \ + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): + main([]) + + self.assertEqual(stdout.getvalue(), '') + self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 2 processes +... ---------------------------------------------------------------------- Ran 3 tests in s OK ''') - def test_success(self): - with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ + def test_success_class_fixtures(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]), + unittest.TestSuite(tests=[SuccessTestCase2('mock_1')]) + ]), + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase3('mock_1')]) + ]) + ]) + with patch('multiprocessing.cpu_count', Mock(return_value=1)), \ + patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): - main(['-v']) + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): + main(['-v', '--class-fixtures']) + + self.assertEqual(stdout.getvalue(), '') + self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (5 total tests) across 1 processes + +mock_1 (tests.test_main.SuccessTestCase) ... ok +mock_2 (tests.test_main.SuccessTestCase) ... ok +mock_3 (tests.test_main.SuccessTestCase) ... ok +mock_1 (tests.test_main.SuccessTestCase2) ... ok +mock_1 (tests.test_main.SuccessTestCase3) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in s + +OK +''') + + def test_success_module_fixtures(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]), + unittest.TestSuite(tests=[SuccessTestCase2('mock_1')]) + ]), + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase3('mock_1')]) + ]) + ]) + with patch('multiprocessing.cpu_count', Mock(return_value=1)), \ + patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ + patch('sys.stdout', StringIO()) as stdout, \ + patch('sys.stderr', StringIO()) as stderr, \ + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): + main(['-v', '--module-fixtures']) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 2 test suites (5 total tests) across 1 processes + +mock_1 (tests.test_main.SuccessTestCase) ... ok +mock_2 (tests.test_main.SuccessTestCase) ... ok +mock_3 (tests.test_main.SuccessTestCase) ... ok +mock_1 (tests.test_main.SuccessTestCase2) ... ok +mock_1 (tests.test_main.SuccessTestCase3) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in s + +OK +''') + + def test_success_module_fixtures_empty(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]), + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[]) + ]) + ]) + with patch('multiprocessing.cpu_count', Mock(return_value=1)), \ + patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ + patch('sys.stdout', StringIO()) as stdout, \ + patch('sys.stderr', StringIO()) as stderr, \ + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): + main(['-v', '--module-fixtures']) + + self.assertEqual(stdout.getvalue(), '') + self.assert_output(stderr.getvalue(), '''\ +Running 1 test suites (3 total tests) across 1 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -208,14 +328,20 @@ def test_success(self): ''') def test_success_dots(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]), + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): main([]) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 3 processes ... ---------------------------------------------------------------------- Ran 3 tests in s @@ -224,14 +350,20 @@ def test_success_dots(self): ''') def test_success_quiet(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): main(['-q']) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 3 processes ---------------------------------------------------------------------- Ran 3 tests in s @@ -239,14 +371,21 @@ def test_success_quiet(self): ''') def test_success_verbose(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): main(['-q', '-v']) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -258,14 +397,25 @@ def test_success_verbose(self): ''') def test_success_buffer(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + SuccessWithOutputTestCase('mock_1'), + SuccessWithOutputTestCase('mock_2'), + SuccessWithOutputTestCase('mock_3') + ]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessWithOutputTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): main(['-v', '-b']) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessWithOutputTestCase) ... ok mock_2 (tests.test_main.SuccessWithOutputTestCase) ... ok mock_3 (tests.test_main.SuccessWithOutputTestCase) ... ok @@ -277,16 +427,27 @@ def test_success_buffer(self): ''') def test_success_buffer_off(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + SuccessWithOutputTestCase('mock_1'), + SuccessWithOutputTestCase('mock_2'), + SuccessWithOutputTestCase('mock_3') + ]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessWithOutputTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): main(['-v']) self.assertEqual(stdout.getvalue(), '''\ Hello stdout! ''') self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessWithOutputTestCase) ... ok mock_2 (tests.test_main.SuccessWithOutputTestCase) ... ok Hello stderr! @@ -299,16 +460,23 @@ def test_success_buffer_off(self): ''') def test_failure(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[FailureTestCase('mock_1'), FailureTestCase('mock_2'), FailureTestCase('mock_3')]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(FailureTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): with self.assertRaises(SystemExit) as cm_exc: main(['-v']) self.assertEqual(cm_exc.exception.code, 1) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.FailureTestCase) ... ok mock_2 (tests.test_main.FailureTestCase) ... FAIL mock_3 (tests.test_main.FailureTestCase) ... ok @@ -328,16 +496,23 @@ def test_failure(self): ''') def test_error(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ErrorTestCase('mock_1'), ErrorTestCase('mock_2'), ErrorTestCase('mock_3')]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(ErrorTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): with self.assertRaises(SystemExit) as cm_exc: main(['-v']) self.assertEqual(cm_exc.exception.code, 1) self.assertEqual(stdout.getvalue(), '') self.assert_output(re.sub(r'File ".*?", line \d+', 'File "", line ', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.ErrorTestCase) ... ok mock_2 (tests.test_main.ErrorTestCase) ... ERROR mock_3 (tests.test_main.ErrorTestCase) ... ok @@ -357,14 +532,21 @@ def test_error(self): ''') def test_skipped(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SkipTestCase('mock_1'), SkipTestCase('mock_2'), SkipTestCase('mock_3')]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SkipTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): main(['-v']) self.assertEqual(stdout.getvalue(), '') self.assert_output(stderr.getvalue(), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SkipTestCase) ... ok mock_2 (tests.test_main.SkipTestCase) ... skipped 'skip reason' mock_3 (tests.test_main.SkipTestCase) ... ok @@ -376,16 +558,27 @@ def test_skipped(self): ''') def test_expected_failure(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + ExpectedFailureTestCase('mock_1'), + ExpectedFailureTestCase('mock_2'), + ExpectedFailureTestCase('mock_3') + ]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(ExpectedFailureTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): with self.assertRaises(SystemExit) as cm_exc: main(['-v']) self.assertEqual(cm_exc.exception.code, 1) self.assertEqual(stdout.getvalue(), '') self.assert_output(re.sub(r'File ".*?", line \d+', 'File "", line ', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.ExpectedFailureTestCase) ... ok mock_2 (tests.test_main.ExpectedFailureTestCase) ... expected failure mock_3 (tests.test_main.ExpectedFailureTestCase) ... unexpected success @@ -397,14 +590,21 @@ def test_expected_failure(self): ''') def test_run_tests_coverage(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): main(['-v']) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -416,11 +616,16 @@ def test_run_tests_coverage(self): ''') def test_coverage(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('coverage.Coverage') as coverage_mock, \ patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): coverage_instance = coverage_mock.return_value coverage_instance.report.return_value = 100. main(['-v', '--coverage']) @@ -451,6 +656,8 @@ def test_coverage(self): ) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -465,11 +672,16 @@ def test_coverage(self): ''') def test_coverage_branch(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('coverage.Coverage') as coverage_mock, \ patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): coverage_instance = coverage_mock.return_value coverage_instance.report.return_value = 100. main(['-v', '--coverage-branch']) @@ -500,6 +712,8 @@ def test_coverage_branch(self): ) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -537,6 +751,7 @@ def test_coverage_no_tests(self): ) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 0 test suites (0 total tests) across 1 processes ---------------------------------------------------------------------- Ran 0 test in s @@ -548,11 +763,16 @@ def test_coverage_no_tests(self): ''') def test_coverage_html(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('coverage.Coverage') as coverage_mock, \ patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): coverage_instance = coverage_mock.return_value coverage_instance.report.return_value = 100. main(['-v', '--coverage-branch', '--coverage-html', 'html_dir']) @@ -584,6 +804,8 @@ def test_coverage_html(self): ) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -598,11 +820,16 @@ def test_coverage_html(self): ''') def test_coverage_xml(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('coverage.Coverage') as coverage_mock, \ patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): coverage_instance = coverage_mock.return_value coverage_instance.report.return_value = 100. main(['-v', '--coverage-branch', '--coverage-xml', 'xml_dir']) @@ -634,6 +861,8 @@ def test_coverage_xml(self): ) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -648,11 +877,16 @@ def test_coverage_xml(self): ''') def test_coverage_fail_under(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('coverage.Coverage') as coverage_mock, \ patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): coverage_instance = coverage_mock.return_value coverage_instance.report.return_value = 99. with self.assertRaises(SystemExit) as cm_exc: @@ -685,6 +919,8 @@ def test_coverage_fail_under(self): ) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok @@ -699,11 +935,16 @@ def test_coverage_fail_under(self): ''') def test_coverage_other(self): + discover_suite = unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[ + unittest.TestSuite(tests=[SuccessTestCase('mock_1'), SuccessTestCase('mock_2'), SuccessTestCase('mock_3')]) + ]) + ]) with patch('coverage.Coverage') as coverage_mock, \ patch('multiprocessing.Pool', new=MockMultiprocessingPool), \ patch('sys.stdout', StringIO()) as stdout, \ patch('sys.stderr', StringIO()) as stderr, \ - patch('unittest.TestLoader.discover', Mock(return_value=_create_test_suite(SuccessTestCase))): + patch('unittest.TestLoader.discover', Mock(return_value=discover_suite)): coverage_instance = coverage_mock.return_value coverage_instance.report.return_value = 100. main([ @@ -745,6 +986,8 @@ def test_coverage_other(self): ) self.assertEqual(stdout.getvalue(), '') self.assertEqual(re.sub(r'\d+\.\d{3}s', 's', stderr.getvalue()), '''\ +Running 3 test suites (3 total tests) across 3 processes + mock_1 (tests.test_main.SuccessTestCase) ... ok mock_2 (tests.test_main.SuccessTestCase) ... ok mock_3 (tests.test_main.SuccessTestCase) ... ok diff --git a/src/unittest_parallel/main.py b/src/unittest_parallel/main.py index 5499328..5019ee8 100644 --- a/src/unittest_parallel/main.py +++ b/src/unittest_parallel/main.py @@ -28,6 +28,10 @@ def main(argv=None): help='Buffer stdout and stderr during tests') parser.add_argument('-j', '--jobs', metavar='COUNT', type=int, default=0, help='The number of test processes (default is 0, all cores)') + parser.add_argument('--class-fixtures', action='store_true', default=False, + help='One or more TestCase class has a setUpClass or tearDownClass method') + parser.add_argument('--module-fixtures', action='store_true', default=False, + help='One or more test module has a setUpModule or tearDownModule method') parser.add_argument('--version', action='store_true', help='show version number and quit') group_unittest = parser.add_argument_group('unittest options') @@ -72,15 +76,31 @@ def main(argv=None): # Discover tests with _coverage(args, temp_dir): test_loader = unittest.TestLoader() - test_suites = test_loader.discover(args.start_directory, pattern=args.pattern, top_level_dir=args.top_level_directory) + discover_suite = test_loader.discover(args.start_directory, pattern=args.pattern, top_level_dir=args.top_level_directory) + + # Get the parallelizable test suites + if args.module_fixtures: + test_suites = list(_iter_module_suites(discover_suite)) + elif args.class_fixtures: + test_suites = list(_iter_class_suites(discover_suite)) + else: + test_suites = list(_iter_test_cases(discover_suite)) + + # Don't use more processes than test suites + process_count = max(1, min(len(test_suites), process_count)) + + # Report test suites and processes + print( + f'Running {len(test_suites)} test suites ({discover_suite.countTestCases()} total tests) across {process_count} processes', + file=sys.stderr + ) + if args.verbose > 1: + print(file=sys.stderr) # Run the tests in parallel start_time = time.perf_counter() with multiprocessing.Pool(process_count) as pool: - results = pool.map( - _run_tests, - ((test_case, args, temp_dir) for test_case in _iter_test_cases(test_suites)) - ) + results = pool.map(_run_tests, ((suite, args, temp_dir) for suite in test_suites)) stop_time = time.perf_counter() test_duration = stop_time - start_time @@ -173,7 +193,7 @@ def _coverage(args, temp_dir): data_file=coverage_file.name, branch=args.coverage_branch, include=args.coverage_include, - omit=list((args.coverage_omit if args.coverage_omit else []) + [__file__]), + omit=(args.coverage_omit if args.coverage_omit else []) + [__file__], source=args.coverage_source ) try: @@ -193,12 +213,30 @@ def _coverage(args, temp_dir): yield None +# A "module suite" is a top-level test suite returned from TestLoader.discover +def _iter_module_suites(test_suite): + for module_suite in test_suite: + if module_suite.countTestCases(): + yield module_suite + + +# A "class suite" is a test suite that contains test cases +def _iter_class_suites(test_suite): + has_cases = any(isinstance(suite, unittest.TestCase) for suite in test_suite) + if has_cases: + yield test_suite + else: + for suite in test_suite: + yield from _iter_class_suites(suite) + + def _iter_test_cases(test_suite): if isinstance(test_suite, unittest.TestCase): yield test_suite else: - for sub_test_suite in test_suite: - yield from _iter_test_cases(sub_test_suite) + for suite in test_suite: + yield from _iter_test_cases(suite) + class ParallelTextTestResult(unittest.TextTestResult): @@ -253,10 +291,10 @@ def printErrors(self): def _run_tests(pool_args): - test_case, args, temp_dir = pool_args + test_suite, args, temp_dir = pool_args with _coverage(args, temp_dir): runner = unittest.TextTestRunner(stream=StringIO(), resultclass=ParallelTextTestResult, verbosity=args.verbose, buffer=args.buffer) - result = runner.run(test_case) + result = runner.run(test_suite) return ( result.testsRun, [_format_error(result, error) for error in result.errors],