From 8ea7554b1c2509f7c8b8a7d5c0e9c323b19657fc Mon Sep 17 00:00:00 2001 From: XonqNopp Date: Mon, 19 Nov 2018 16:04:23 +0100 Subject: [PATCH 1/4] [result] trying to fix subtest failures --- xmlrunner/result.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/xmlrunner/result.py b/xmlrunner/result.py index a49184f..27ddb58 100644 --- a/xmlrunner/result.py +++ b/xmlrunner/result.py @@ -337,14 +337,31 @@ def addSubTest(self, testcase, test, err): Called when a subTest method raises an error. """ if err is not None: + + errorText = None + errorValue = None + errorList = None + if issubclass(err[0], test.failureException): + # FAIL + errorText = 'FAIL' + errorValue = self.infoclass.FAILURE + errorList = self.failures + + else: + # ERROR + errorText = 'ERROR' + errorValue = self.infoclass.ERROR + errorList = self.errors + self._save_output_data() + testinfo = self.infoclass( - self, testcase, self.infoclass.ERROR, err, subTest=test) - self.errors.append(( + self, testcase, errorValue, err, subTest=test) + errorList.append(( testinfo, self._exc_info_to_string(err, testcase) )) - self._prepare_callback(testinfo, [], 'ERROR', 'E') + self._prepare_callback(testinfo, [], errorText, errorText[0]) def addSkip(self, test, reason): """ From 13da6702c8926a0baa691992c72e90e7af6e001b Mon Sep 17 00:00:00 2001 From: XonqNopp Date: Mon, 19 Nov 2018 16:04:26 +0100 Subject: [PATCH 2/4] [result] adding missing methods to add expected failure and unexpected success --- xmlrunner/result.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/xmlrunner/result.py b/xmlrunner/result.py index 27ddb58..8336aeb 100644 --- a/xmlrunner/result.py +++ b/xmlrunner/result.py @@ -373,6 +373,39 @@ def addSkip(self, test, reason): self.skipped.append((testinfo, reason)) self._prepare_callback(testinfo, [], 'SKIP', 'S') + def addExpectedFailure(self, test, err): + """ + Missing in xmlrunner, copy-pasted from xmlrunner addError. + """ + self._save_output_data() + + outcome = self.infoclass.ERROR # this (easy) way the XML reports as error, which is not success neither fail + + testinfo = self.infoclass(self, test, outcome, err) + testinfo.test_exception_name = 'ExpectedFailure' + testinfo.test_exception_message = 'EXPECTED FAILURE: {}'.format(testinfo.test_exception_message) + + self.expectedFailures.append((testinfo, self._exc_info_to_string(err, test))) + self._prepare_callback(testinfo, [], 'EXPECTED FAILURE', 'EF') + + @failfast + def addUnexpectedSuccess(self, test): + """ + Missing in xmlrunner, copy-pasted from xmlrunner addSuccess. + """ + self._save_output_data() + + outcome = self.infoclass.ERROR # this (easy) way the XML reports as error, which is not success neither fail + + testinfo = self.infoclass(self, test) # do not set outcome here because it will need exception + testinfo.outcome = outcome + # But since we want to have error outcome, we need to provide additional fields: + testinfo.test_exception_name = 'UnexpectedSuccess' + testinfo.test_exception_message = 'This test was marked as expected failure but passed, please review it' + + self.unexpectedSuccesses.append(testinfo) + self._prepare_callback(testinfo, [], 'UNEXPECTED SUCCESS', 'US') + def printErrorList(self, flavour, errors): """ Writes information about the FAIL or ERROR to the stream. From 47a787802a777b3f1c8ab545fd6f8fa0dea110f1 Mon Sep 17 00:00:00 2001 From: XonqNopp Date: Mon, 19 Nov 2018 16:04:27 +0100 Subject: [PATCH 3/4] [result] trying to add missing subdescription --- xmlrunner/result.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/xmlrunner/result.py b/xmlrunner/result.py index 8336aeb..e955e2b 100644 --- a/xmlrunner/result.py +++ b/xmlrunner/result.py @@ -156,8 +156,10 @@ def __init__(self, test_result, test_method, outcome=SUCCESS, err=None, subTest= self.test_name = testcase_name(test_method) self.test_id = test_method.id() + self.subDescription = None if subTest: self.test_id = subTest.id() + self.subDescription = subTest._subDescription() def id(self): return self.test_id @@ -174,7 +176,12 @@ def get_description(self): """ Return a text representation of the test method. """ - return self.test_description + description = self.test_description + + if self.subDescription is not None: + description += ' ' + self.subDescription + + return description def get_error_info(self): """ @@ -342,13 +349,11 @@ def addSubTest(self, testcase, test, err): errorValue = None errorList = None if issubclass(err[0], test.failureException): - # FAIL errorText = 'FAIL' errorValue = self.infoclass.FAILURE errorList = self.failures else: - # ERROR errorText = 'ERROR' errorValue = self.infoclass.ERROR errorList = self.errors @@ -379,14 +384,12 @@ def addExpectedFailure(self, test, err): """ self._save_output_data() - outcome = self.infoclass.ERROR # this (easy) way the XML reports as error, which is not success neither fail - - testinfo = self.infoclass(self, test, outcome, err) + testinfo = self.infoclass(self, test, self.infoclass.ERROR, err) testinfo.test_exception_name = 'ExpectedFailure' testinfo.test_exception_message = 'EXPECTED FAILURE: {}'.format(testinfo.test_exception_message) self.expectedFailures.append((testinfo, self._exc_info_to_string(err, test))) - self._prepare_callback(testinfo, [], 'EXPECTED FAILURE', 'EF') + self._prepare_callback(testinfo, [], 'EXPECTED FAILURE', 'X') @failfast def addUnexpectedSuccess(self, test): @@ -395,16 +398,15 @@ def addUnexpectedSuccess(self, test): """ self._save_output_data() - outcome = self.infoclass.ERROR # this (easy) way the XML reports as error, which is not success neither fail - testinfo = self.infoclass(self, test) # do not set outcome here because it will need exception - testinfo.outcome = outcome + testinfo.outcome = self.infoclass.ERROR # But since we want to have error outcome, we need to provide additional fields: testinfo.test_exception_name = 'UnexpectedSuccess' - testinfo.test_exception_message = 'This test was marked as expected failure but passed, please review it' + testinfo.test_exception_message = ('UNEXPECTED SUCCESS: This test was marked as expected failure but passed, ' + 'please review it') self.unexpectedSuccesses.append(testinfo) - self._prepare_callback(testinfo, [], 'UNEXPECTED SUCCESS', 'US') + self._prepare_callback(testinfo, [], 'UNEXPECTED SUCCESS', 'U') def printErrorList(self, flavour, errors): """ From a1ed8c47e835e6e1f7b8a06dcef61b3c3ef5269b Mon Sep 17 00:00:00 2001 From: XonqNopp Date: Mon, 19 Nov 2018 17:11:03 +0100 Subject: [PATCH 4/4] [tests] adding missing test for subtest error --- tests/testsuite.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/testsuite.py b/tests/testsuite.py index 1cacf05..774def2 100755 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -142,6 +142,11 @@ def test_subTest_fail(self): with self.subTest(i=i): self.fail('this is a subtest.') + def test_subTest_error(self): + for i in range(2): + with self.subTest(i=i): + raise Exception('this is a subtest') + class DummyErrorInCallTest(unittest.TestCase): def __call__(self, result): @@ -411,6 +416,30 @@ def test_unittest_subTest_fail(self): br'(XMLTestRunnerTestCase\.)?DummySubTest" ' br'name="test_subTest_fail \(i=1\)"') + @unittest.skipIf(not hasattr(unittest.TestCase, 'subTest'), + 'unittest.TestCase.subTest not present.') + def test_unittest_subTest_error(self): + # test for issue #155 + outdir = BytesIO() + runner = xmlrunner.XMLTestRunner( + stream=self.stream, output=outdir, verbosity=self.verbosity, + **self.runner_kwargs) + suite = unittest.TestSuite() + suite.addTest(self.DummySubTest('test_subTest_error')) + runner.run(suite) + outdir.seek(0) + output = outdir.read() + self.assertRegexpMatches( + output, + br'