bpo-37957: Allow regrtest to receive a file with test (and subtests) to ignore (pythonGH-16989)

When building Python in some uncommon platforms there are some known tests that will fail. Right now, the test suite has the ability to ignore entire tests using the -x option and to receive a filter file using the --matchfile filter. The problem with the --matchfile option is that it receives a file with patterns to accept and when you want to ignore a couple of tests and subtests, is too cumbersome to lists ALL tests that are not the ones that you want to accept and he problem with -x is that is not easy to ignore just a subtests that fail and the whole test needs to be ignored.

For these reasons, add a new option to allow to ignore a list of test and subtests for these situations.

Author: pablogsal

Lib/test/libregrtest/cmdline.py

@@ -207,10 +207,17 @@ def _create_parser():
     group.add_argument('-m', '--match', metavar='PAT',
                        dest='match_tests', action='append',
                        help='match test cases and methods with glob pattern PAT')
+    group.add_argument('-i', '--ignore', metavar='PAT',
+                       dest='ignore_tests', action='append',
+                       help='ignore test cases and methods with glob pattern PAT')
     group.add_argument('--matchfile', metavar='FILENAME',
                        dest='match_filename',
                        help='similar to --match but get patterns from a '
                             'text file, one pattern per line')
+    group.add_argument('--ignorefile', metavar='FILENAME',
+                       dest='ignore_filename',
+                       help='similar to --matchfile but it receives patterns '
+                            'from text file to ignore')
     group.add_argument('-G', '--failfast', action='store_true',
                        help='fail as soon as a test fails (only with -v or -W)')
     group.add_argument('-u', '--use', metavar='RES1,RES2,...',
@@ -317,7 +324,8 @@ def _parse_args(args, **kwargs):
          findleaks=1, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
          random_seed=None, use_mp=None, verbose3=False, forever=False,
-         header=False, failfast=False, match_tests=None, pgo=False)
+         header=False, failfast=False, match_tests=None, ignore_tests=None,
+         pgo=False)
     for k, v in kwargs.items():
         if not hasattr(ns, k):
             raise TypeError('%r is an invalid keyword argument '
@@ -395,6 +403,12 @@ def _parse_args(args, **kwargs):
         with open(ns.match_filename) as fp:
             for line in fp:
                 ns.match_tests.append(line.strip())
+    if ns.ignore_filename:
+        if ns.ignore_tests is None:
+            ns.ignore_tests = []
+        with open(ns.ignore_filename) as fp:
+            for line in fp:
+                ns.ignore_tests.append(line.strip())
     if ns.forever:
         # --forever implies --failfast
         ns.failfast = True

Lib/test/libregrtest/main.py

@@ -287,7 +287,7 @@ def _list_cases(self, suite):
 
     def list_cases(self):
         support.verbose = False
-        support.set_match_tests(self.ns.match_tests)
+        support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests)
 
         for test_name in self.selected:
             abstest = get_abs_module(self.ns, test_name)

Lib/test/libregrtest/runtest.py

@@ -123,7 +123,7 @@ def _runtest(ns, test_name):
 
     start_time = time.perf_counter()
     try:
-        support.set_match_tests(ns.match_tests)
+        support.set_match_tests(ns.match_tests, ns.ignore_tests)
         support.junit_xml_list = xml_list = [] if ns.xmlpath else None
         if ns.failfast:
             support.failfast = True

Lib/test/support/__init__.py

@@ -2096,7 +2096,9 @@ def _run_suite(suite):
 
 # By default, don't filter tests
 _match_test_func = None
-_match_test_patterns = None
+
+_accept_test_patterns = None
+_ignore_test_patterns = None
 
 
 def match_test(test):
@@ -2112,18 +2114,45 @@ def _is_full_match_test(pattern):
     # as a full test identifier.
     # Example: 'test.test_os.FileTests.test_access'.
     #
-    # Reject patterns which contain fnmatch patterns: '*', '?', '[...]'
-    # or '[!...]'. For example, reject 'test_access*'.
+    # ignore patterns which contain fnmatch patterns: '*', '?', '[...]'
+    # or '[!...]'. For example, ignore 'test_access*'.
     return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern))
 
 
-def set_match_tests(patterns):
-    global _match_test_func, _match_test_patterns
+def set_match_tests(accept_patterns=None, ignore_patterns=None):
+    global _match_test_func, _accept_test_patterns, _ignore_test_patterns
 
-    if patterns == _match_test_patterns:
-        # No change: no need to recompile patterns.
-        return
 
+    if accept_patterns is None:
+        accept_patterns = ()
+    if ignore_patterns is None:
+        ignore_patterns = ()
+
+    accept_func = ignore_func = None
+
+    if accept_patterns != _accept_test_patterns:
+        accept_patterns, accept_func = _compile_match_function(accept_patterns)
+    if ignore_patterns != _ignore_test_patterns:
+        ignore_patterns, ignore_func = _compile_match_function(ignore_patterns)
+
+    # Create a copy since patterns can be mutable and so modified later
+    _accept_test_patterns = tuple(accept_patterns)
+    _ignore_test_patterns = tuple(ignore_patterns)
+
+    if accept_func is not None or ignore_func is not None:
+        def match_function(test_id):
+            accept = True
+            ignore = False
+            if accept_func:
+                accept = accept_func(test_id)
+            if ignore_func:
+                ignore = ignore_func(test_id)
+            return accept and not ignore
+
+        _match_test_func = match_function
+
+
+def _compile_match_function(patterns):
     if not patterns:
         func = None
         # set_match_tests(None) behaves as set_match_tests(())
@@ -2151,10 +2180,7 @@ def match_test_regex(test_id):
 
         func = match_test_regex
 
-    # Create a copy since patterns can be mutable and so modified later
-    _match_test_patterns = tuple(patterns)
-    _match_test_func = func
-
+    return patterns, func
 
 
 def run_unittest(*classes):

Lib/test/test_regrtest.py

@@ -155,6 +155,24 @@ def test_single(self):
                 self.assertTrue(ns.single)
                 self.checkError([opt, '-f', 'foo'], "don't go together")
 
+    def test_ignore(self):
+        for opt in '-i', '--ignore':
+            with self.subTest(opt=opt):
+                ns = libregrtest._parse_args([opt, 'pattern'])
+                self.assertEqual(ns.ignore_tests, ['pattern'])
+                self.checkError([opt], 'expected one argument')
+
+        self.addCleanup(support.unlink, support.TESTFN)
+        with open(support.TESTFN, "w") as fp:
+            print('matchfile1', file=fp)
+            print('matchfile2', file=fp)
+
+        filename = os.path.abspath(support.TESTFN)
+        ns = libregrtest._parse_args(['-m', 'match',
+                                      '--ignorefile', filename])
+        self.assertEqual(ns.ignore_tests,
+                         ['matchfile1', 'matchfile2'])
+
     def test_match(self):
         for opt in '-m', '--match':
             with self.subTest(opt=opt):
@@ -961,6 +979,42 @@ def parse_methods(self, output):
         regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
         return [match.group(1) for match in regex.finditer(output)]
 
+    def test_ignorefile(self):
+        code = textwrap.dedent("""
+            import unittest
+
+            class Tests(unittest.TestCase):
+                def test_method1(self):
+                    pass
+                def test_method2(self):
+                    pass
+                def test_method3(self):
+                    pass
+                def test_method4(self):
+                    pass
+        """)
+        all_methods = ['test_method1', 'test_method2',
+                       'test_method3', 'test_method4']
+        testname = self.create_test(code=code)
+
+        # only run a subset
+        filename = support.TESTFN
+        self.addCleanup(support.unlink, filename)
+
+        subset = [
+            # only ignore the method name
+            'test_method1',
+            # ignore the full identifier
+            '%s.Tests.test_method3' % testname]
+        with open(filename, "w") as fp:
+            for name in subset:
+                print(name, file=fp)
+
+        output = self.run_tests("-v", "--ignorefile", filename, testname)
+        methods = self.parse_methods(output)
+        subset = ['test_method2', 'test_method4']
+        self.assertEqual(methods, subset)
+
     def test_matchfile(self):
         code = textwrap.dedent("""
             import unittest

Lib/test/test_support.py

@@ -527,52 +527,100 @@ def id(self):
         test_access = Test('test.test_os.FileTests.test_access')
         test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir')
 
+        # Test acceptance
         with support.swap_attr(support, '_match_test_func', None):
             # match all
             support.set_match_tests([])
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # match all using None
-            support.set_match_tests(None)
+            support.set_match_tests(None, None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # match the full test identifier
-            support.set_match_tests([test_access.id()])
+            support.set_match_tests([test_access.id()], None)
             self.assertTrue(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
 
             # match the module name
-            support.set_match_tests(['test_os'])
+            support.set_match_tests(['test_os'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # Test '*' pattern
-            support.set_match_tests(['test_*'])
+            support.set_match_tests(['test_*'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # Test case sensitivity
-            support.set_match_tests(['filetests'])
+            support.set_match_tests(['filetests'], None)
             self.assertFalse(support.match_test(test_access))
-            support.set_match_tests(['FileTests'])
+            support.set_match_tests(['FileTests'], None)
             self.assertTrue(support.match_test(test_access))
 
             # Test pattern containing '.' and a '*' metacharacter
-            support.set_match_tests(['*test_os.*.test_*'])
+            support.set_match_tests(['*test_os.*.test_*'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # Multiple patterns
-            support.set_match_tests([test_access.id(), test_chdir.id()])
+            support.set_match_tests([test_access.id(), test_chdir.id()], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
-            support.set_match_tests(['test_access', 'DONTMATCH'])
+            support.set_match_tests(['test_access', 'DONTMATCH'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
 
+        # Test rejection
+        with support.swap_attr(support, '_match_test_func', None):
+            # match all
+            support.set_match_tests(ignore_patterns=[])
+            self.assertTrue(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
+            # match all using None
+            support.set_match_tests(None, None)
+            self.assertTrue(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
+            # match the full test identifier
+            support.set_match_tests(None, [test_access.id()])
+            self.assertFalse(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
+            # match the module name
+            support.set_match_tests(None, ['test_os'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            # Test '*' pattern
+            support.set_match_tests(None, ['test_*'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            # Test case sensitivity
+            support.set_match_tests(None, ['filetests'])
+            self.assertTrue(support.match_test(test_access))
+            support.set_match_tests(None, ['FileTests'])
+            self.assertFalse(support.match_test(test_access))
+
+            # Test pattern containing '.' and a '*' metacharacter
+            support.set_match_tests(None, ['*test_os.*.test_*'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            # Multiple patterns
+            support.set_match_tests(None, [test_access.id(), test_chdir.id()])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            support.set_match_tests(None, ['test_access', 'DONTMATCH'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
     def test_fd_count(self):
         # We cannot test the absolute value of fd_count(): on old Linux
         # kernel or glibc versions, os.urandom() keeps a FD open on

Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst

@@ -0,0 +1,3 @@
+test.regrtest now can receive a list of test patterns to ignore (using the
+-i/--ignore argument) or a file with a list of patterns to ignore (using the
+--ignore-file argument). Patch by Pablo Galindo.