Expand Testing for Unittest Run in PythonFiles (#21028)

closes #21031

pythonFiles/tests/unittestadapter/.data/test_fail_simple.py

@@ -0,0 +1,21 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+# Test class for the test_fail_simple test.
+# The test_failed_tests function should return a dictionary with a "success" status
+# and the two tests with their outcome as "failed".
+
+class RunFailSimple(unittest.TestCase):
+ """Test class for the test_fail_simple test.
+
+ The test_failed_tests function should return a dictionary with a "success" status
+ and the two tests with their outcome as "failed".
+ """
+
+ def test_one_fail(self) -> None:
+ self.assertGreater(2, 3)
+
+ def test_two_fail(self) -> None:
+ self.assertNotEqual(1, 1)

pythonFiles/tests/unittestadapter/.data/test_two_classes.py

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+# Test class which runs for the test_multiple_ids_run test with the two class parameters.
+# Both test functions will be returned in a dictionary with a "success" status,
+# and the two tests with their outcome as "success".
+
+
+class ClassOne(unittest.TestCase):
+
+ def test_one(self) -> None:
+ self.assertGreater(2, 1)
+
+class ClassTwo(unittest.TestCase):
+
+ def test_two(self) -> None:
+ self.assertGreater(2, 1)
+

pythonFiles/tests/unittestadapter/.data/two_patterns/pattern_a_test.py

@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import unittest
+
+# Test class for the two file pattern test. It is pattern *test.py.
+# The test_ids_multiple_runs function should return a dictionary with a "success" status,
+# and the two tests with their outcome as "success".
+
+
+
+class DiscoveryA(unittest.TestCase):
+ """Test class for the two file pattern test. It is pattern *test.py
+
+ The test_ids_multiple_runs function should return a dictionary with a "success" status,
+ and the two tests with their outcome as "success".
+ """
+
+ def test_one_a(self) -> None:
+ self.assertGreater(2, 1)
+
+ def test_two_a(self) -> None:
+ self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/two_patterns/test_pattern_b.py

@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import unittest
+
+# Test class for the two file pattern test. This file is pattern test*.py.
+# The test_ids_multiple_runs function should return a dictionary with a "success" status,
+# and the two tests with their outcome as "success".
+
+class DiscoveryB(unittest.TestCase):
+
+ def test_one_b(self) -> None:
+ self.assertGreater(2, 1)
+
+ def test_two_b(self) -> None:
+ self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/unittest_folder/test_add.py

@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import unittest
+
+# Test class which runs for the test_multiple_ids_run test with the two test
+# files in the same folder. The cwd is set to the parent folder. This should return
+# a dictionary with a "success" status and the two tests with their outcome as "success".
+
+def add(a, b):
+ return a + b
+
+
+class TestAddFunction(unittest.TestCase):
+
+ def test_add_positive_numbers(self): 
+ result = add(2, 3)
+ self.assertEqual(result, 5)
+
+
+ def test_add_negative_numbers(self): 
+ result = add(-2, -3)
+ self.assertEqual(result, -5)

pythonFiles/tests/unittestadapter/.data/unittest_folder/test_subtract.py

@@ -0,0 +1,21 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import unittest
+
+# Test class which runs for the test_multiple_ids_run test with the two test
+# files in the same folder. The cwd is set to the parent folder. This should return
+# a dictionary with a "success" status and the two tests with their outcome as "success".
+
+def subtract(a, b):
+ return a - b
+
+
+class TestSubtractFunction(unittest.TestCase):
+ def test_subtract_positive_numbers(self):
+ result = subtract(5, 3)
+ self.assertEqual(result, 2)
+
+
+ def test_subtract_negative_numbers(self):
+ result = subtract(-2, -3)
+ self.assertEqual(result, 1)

pythonFiles/tests/unittestadapter/test_execution.py

@@ -92,3 +92,163 @@ def test_single_ids_run() -> None:
  assert id_result is not None
  assert "outcome" in id_result
  assert id_result["outcome"] == "success"
+
+
+@pytest.mark.parametrize(
+ "test_ids, pattern, cwd, expected_outcome",
+ [
+ (
+ [
+ "test_add.TestAddFunction.test_add_negative_numbers",
+ "test_add.TestAddFunction.test_add_positive_numbers",
+ ],
+ "test_add.py",
+ os.fspath(TEST_DATA_PATH / "unittest_folder"),
+ "success",
+ ),
+ (
+ [
+ "test_add.TestAddFunction.test_add_negative_numbers",
+ "test_add.TestAddFunction.test_add_positive_numbers",
+ "test_subtract.TestSubtractFunction.test_subtract_negative_numbers",
+ "test_subtract.TestSubtractFunction.test_subtract_positive_numbers",
+ ],
+ "test*",
+ os.fspath(TEST_DATA_PATH / "unittest_folder"),
+ "success",
+ ),
+ (
+ [
+ "pattern_a_test.DiscoveryA.test_one_a",
+ "pattern_a_test.DiscoveryA.test_two_a",
+ ],
+ "*test",
+ os.fspath(TEST_DATA_PATH / "two_patterns"),
+ "success",
+ ),
+ (
+ [
+ "test_pattern_b.DiscoveryB.test_one_b",
+ "test_pattern_b.DiscoveryB.test_two_b",
+ ],
+ "test_*",
+ os.fspath(TEST_DATA_PATH / "two_patterns"),
+ "success",
+ ),
+ (
+ [
+ "file_one.CaseTwoFileOne.test_one",
+ "file_one.CaseTwoFileOne.test_two",
+ "folder.file_two.CaseTwoFileTwo.test_one",
+ "folder.file_two.CaseTwoFileTwo.test_two",
+ ],
+ "*",
+ os.fspath(TEST_DATA_PATH / "utils_nested_cases"),
+ "success",
+ ),
+ (
+ [
+ "test_two_classes.ClassOne.test_one",
+ "test_two_classes.ClassTwo.test_two",
+ ],
+ "test_two_classes.py",
+ os.fspath(TEST_DATA_PATH),
+ "success",
+ ),
+ ],
+)
+def test_multiple_ids_run(test_ids, pattern, cwd, expected_outcome) -> None:
+ """
+ The following are all successful tests of different formats.
+
+ # 1. Two tests with the `pattern` specified as a file
+ # 2. Two test files in the same folder called `unittest_folder`
+ # 3. A folder with two different test file patterns, this test gathers pattern `*test`
+ # 4. A folder with two different test file patterns, this test gathers pattern `test_*`
+ # 5. A nested structure where a test file is on the same level as a folder containing a test file
+ # 6. Test file with two test classes
+
+ All tests should have the outcome of `success`.
+ """
+ actual = run_tests(cwd, test_ids, pattern, None, "fake-uuid")
+ assert actual
+ assert all(item in actual for item in ("cwd", "status"))
+ assert actual["status"] == "success"
+ assert actual["cwd"] == cwd
+ assert "result" in actual
+ result = actual["result"]
+ assert len(result) == len(test_ids)
+ for test_id in test_ids:
+ assert test_id in result
+ id_result = result[test_id]
+ assert id_result is not None
+ assert "outcome" in id_result
+ assert id_result["outcome"] == expected_outcome
+ assert True
+
+
+def test_failed_tests():
+ """This test runs on a single file `test_fail` with two tests that fail."""
+ test_ids = [
+ "test_fail_simple.RunFailSimple.test_one_fail",
+ "test_fail_simple.RunFailSimple.test_two_fail",
+ ]
+ actual = run_tests(
+ os.fspath(TEST_DATA_PATH), test_ids, "test_fail_simple*", None, "fake-uuid"
+ )
+ assert actual
+ assert all(item in actual for item in ("cwd", "status"))
+ assert actual["status"] == "success"
+ assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
+ assert "result" in actual
+ result = actual["result"]
+ assert len(result) == len(test_ids)
+ for test_id in test_ids:
+ assert test_id in result
+ id_result = result[test_id]
+ assert id_result is not None
+ assert "outcome" in id_result
+ assert id_result["outcome"] == "failure"
+ assert "message" and "traceback" in id_result
+ assert True
+
+
+def test_unknown_id():
+ """This test runs on a unknown test_id, therefore it should return
+ an error as the outcome as it attempts to find the given test.
+ """
+ test_ids = ["unknown_id"]
+ actual = run_tests(
+ os.fspath(TEST_DATA_PATH), test_ids, "test_fail_simple*", None, "fake-uuid"
+ )
+ assert actual
+ assert all(item in actual for item in ("cwd", "status"))
+ assert actual["status"] == "success"
+ assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
+ assert "result" in actual
+ result = actual["result"]
+ assert len(result) == len(test_ids)
+ assert "unittest.loader._FailedTest.unknown_id" in result
+ id_result = result["unittest.loader._FailedTest.unknown_id"]
+ assert id_result is not None
+ assert "outcome" in id_result
+ assert id_result["outcome"] == "error"
+ assert "message" and "traceback" in id_result
+
+
+def test_incorrect_path():
+ """This test runs on a non existent path, therefore it should return
+ an error as the outcome as it attempts to find the given folder.
+ """
+ test_ids = ["unknown_id"]
+ actual = run_tests(
+ os.fspath(TEST_DATA_PATH / "unknown_folder"),
+ test_ids,
+ "test_fail_simple*",
+ None,
+ "fake-uuid",
+ )
+ assert actual
+ assert all(item in actual for item in ("cwd", "status", "error"))
+ assert actual["status"] == "error"
+ assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "unknown_folder")

pythonFiles/unittestadapter/execution.py

@@ -62,7 +62,9 @@ class TestOutcomeEnum(str, enum.Enum):
 
 
 class UnittestTestResult(unittest.TextTestResult):
- formatted: Dict[str, Dict[str, Union[str, None]]] = dict()
+ def __init__(self, *args, **kwargs):
+ self.formatted: Dict[str, Dict[str, Union[str, None]]] = dict()
+ super(UnittestTestResult, self).__init__(*args, **kwargs)
 
  def startTest(self, test: unittest.TestCase):
  super(UnittestTestResult, self).startTest(test)