Add an xfailed test for issue #9852 (#10932)

As part of doing this I upgraded the pep561 tests to support
incremental tests, and removed at least a little of the duplication
between different incremental test suites.

mypy/test/data.py

@@ -335,7 +335,6 @@ def module_from_path(path: str) -> str:
     path = re.sub(r'\.pyi?$', '', path)
     # We can have a mix of Unix-style and Windows-style separators.
     parts = re.split(r'[/\\]', path)
-    assert parts[0] == test_temp_dir
     del parts[0]
     module = '.'.join(parts)
     module = re.sub(r'\.__init__$', '', module)

mypy/test/helpers.py

@@ -5,7 +5,7 @@
 import shutil
 import contextlib
 
-from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator
+from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator, Union
 
 from mypy import defaults
 import mypy.api as api
@@ -18,7 +18,9 @@
 
 from mypy.main import process_options
 from mypy.options import Options
-from mypy.test.data import DataDrivenTestCase, fix_cobertura_filename
+from mypy.test.data import (
+    DataDrivenTestCase, fix_cobertura_filename, UpdateFile, DeleteFile
+)
 from mypy.test.config import test_temp_dir
 import mypy.version
 
@@ -423,6 +425,24 @@ def copy_and_fudge_mtime(source_path: str, target_path: str) -> None:
         os.utime(target_path, times=(new_time, new_time))
 
 
+def perform_file_operations(
+        operations: List[Union[UpdateFile, DeleteFile]]) -> None:
+    for op in operations:
+        if isinstance(op, UpdateFile):
+            # Modify/create file
+            copy_and_fudge_mtime(op.source_path, op.target_path)
+        else:
+            # Delete file/directory
+            if os.path.isdir(op.path):
+                # Sanity check to avoid unexpected deletions
+                assert op.path.startswith('tmp')
+                shutil.rmtree(op.path)
+            else:
+                # Use retries to work around potential flakiness on Windows (AppVeyor).
+                path = op.path
+                retry_on_error(lambda: os.remove(path))
+
+
 def check_test_output_files(testcase: DataDrivenTestCase,
                             step: int,
                             strip_prefix: str = '') -> None:

mypy/test/testcheck.py

@@ -2,7 +2,6 @@
 
 import os
 import re
-import shutil
 import sys
 
 from typing import Dict, List, Set, Tuple
@@ -12,12 +11,12 @@
 from mypy.modulefinder import BuildSource, SearchPaths, FindModuleCache
 from mypy.test.config import test_temp_dir, test_data_prefix
 from mypy.test.data import (
-    DataDrivenTestCase, DataSuite, FileOperation, UpdateFile, module_from_path
+    DataDrivenTestCase, DataSuite, FileOperation, module_from_path
 )
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages, assert_module_equivalence,
-    retry_on_error, update_testcase_output, parse_options,
-    copy_and_fudge_mtime, assert_target_equivalence, check_test_output_files
+    update_testcase_output, parse_options,
+    assert_target_equivalence, check_test_output_files, perform_file_operations,
 )
 from mypy.errors import CompileError
 from mypy.semanal_main import core_modules
@@ -156,19 +155,7 @@ def run_case_once(self, testcase: DataDrivenTestCase,
                     break
         elif incremental_step > 1:
             # In runs 2+, copy *.[num] files to * files.
-            for op in operations:
-                if isinstance(op, UpdateFile):
-                    # Modify/create file
-                    copy_and_fudge_mtime(op.source_path, op.target_path)
-                else:
-                    # Delete file/directory
-                    path = op.path
-                    if os.path.isdir(path):
-                        # Sanity check to avoid unexpected deletions
-                        assert path.startswith('tmp')
-                        shutil.rmtree(path)
-                    # Use retries to work around potential flakiness on Windows (AppVeyor).
-                    retry_on_error(lambda: os.remove(path))
+            perform_file_operations(operations)
 
         # Parse options after moving files (in case mypy.ini is being moved).
         options = parse_options(original_program_text, testcase, incremental_step)

mypy/test/testfinegrained.py

@@ -14,7 +14,6 @@
 
 import os
 import re
-import shutil
 
 from typing import List, Dict, Any, Tuple, Union, cast
 
@@ -27,8 +26,8 @@
     DataDrivenTestCase, DataSuite, UpdateFile, DeleteFile
 )
 from mypy.test.helpers import (
-    assert_string_arrays_equal, parse_options, copy_and_fudge_mtime, assert_module_equivalence,
-    assert_target_equivalence
+    assert_string_arrays_equal, parse_options, assert_module_equivalence,
+    assert_target_equivalence, perform_file_operations,
 )
 from mypy.server.mergecheck import check_consistency
 from mypy.dmypy_util import DEFAULT_STATUS_FILE
@@ -211,18 +210,7 @@ def perform_step(self,
 
         Return (mypy output, triggered targets).
         """
-        for op in operations:
-            if isinstance(op, UpdateFile):
-                # Modify/create file
-                copy_and_fudge_mtime(op.source_path, op.target_path)
-            else:
-                # Delete file/directory
-                if os.path.isdir(op.path):
-                    # Sanity check to avoid unexpected deletions
-                    assert op.path.startswith('tmp')
-                    shutil.rmtree(op.path)
-                else:
-                    os.remove(op.path)
+        perform_file_operations(operations)
         sources = self.parse_sources(main_src, step, options)
 
         if step <= num_regular_incremental_steps:

mypy/test/testpep561.py

@@ -13,7 +13,7 @@
 from mypy.util import try_find_python2_interpreter
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.config import test_temp_dir
-from mypy.test.helpers import assert_string_arrays_equal
+from mypy.test.helpers import assert_string_arrays_equal, perform_file_operations
 
 
 # NOTE: options.use_builtins_fixtures should not be set in these
@@ -38,6 +38,7 @@ class PEP561Suite(DataSuite):
     files = [
         'pep561.test',
     ]
+    base_path = '.'
 
     def run_case(self, test_case: DataDrivenTestCase) -> None:
         test_pep561(test_case)
@@ -102,6 +103,7 @@ def test_pep561(testcase: DataDrivenTestCase) -> None:
             pytest.skip()
     else:
         python = sys.executable
+
     assert python is not None, "Should be impossible"
     pkgs, pip_args = parse_pkgs(testcase.input[0])
     mypy_args = parse_mypy_args(testcase.input[1])
@@ -118,34 +120,30 @@ def test_pep561(testcase: DataDrivenTestCase) -> None:
         for pkg in pkgs:
             install_package(pkg, python_executable, use_pip, editable)
 
-        if venv_dir is not None:
-            old_dir = os.getcwd()
-            os.chdir(venv_dir)
-        try:
-            cmd_line = list(mypy_args)
-            has_program = not ('-p' in cmd_line or '--package' in cmd_line)
-            if has_program:
-                program = testcase.name + '.py'
-                with open(program, 'w', encoding='utf-8') as f:
-                    for s in testcase.input:
-                        f.write('{}\n'.format(s))
-                cmd_line.append(program)
-            cmd_line.extend(['--no-incremental', '--no-error-summary'])
-            if python_executable != sys.executable:
-                cmd_line.append('--python-executable={}'.format(python_executable))
-            if testcase.files != []:
-                for name, content in testcase.files:
-                    if 'mypy.ini' in name:
-                        with open('mypy.ini', 'w') as m:
-                            m.write(content)
-                    elif 'pyproject.toml' in name:
-                        with open('pyproject.toml', 'w') as m:
-                            m.write(content)
+        cmd_line = list(mypy_args)
+        has_program = not ('-p' in cmd_line or '--package' in cmd_line)
+        if has_program:
+            program = testcase.name + '.py'
+            with open(program, 'w', encoding='utf-8') as f:
+                for s in testcase.input:
+                    f.write('{}\n'.format(s))
+            cmd_line.append(program)
+
+        cmd_line.extend(['--no-error-summary'])
+        if python_executable != sys.executable:
+            cmd_line.append('--python-executable={}'.format(python_executable))
+
+        steps = testcase.find_steps()
+        if steps != [[]]:
+            steps = [[]] + steps  # type: ignore[operator,assignment]
+
+        for i, operations in enumerate(steps):
+            perform_file_operations(operations)
+
             output = []
             # Type check the module
             out, err, returncode = mypy.api.run(cmd_line)
-            if has_program:
-                os.remove(program)
+
             # split lines, remove newlines, and remove directory of test case
             for line in (out + err).splitlines():
                 if line.startswith(test_temp_dir + os.sep):
@@ -154,12 +152,15 @@ def test_pep561(testcase: DataDrivenTestCase) -> None:
                     # Normalize paths so that the output is the same on Windows and Linux/macOS.
                     line = line.replace(test_temp_dir + os.sep, test_temp_dir + '/')
                     output.append(line.rstrip("\r\n"))
-            assert_string_arrays_equal([line for line in testcase.output], output,
-                               'Invalid output ({}, line {})'.format(
-                                   testcase.file, testcase.line))
-        finally:
-            if venv_dir is not None:
-                os.chdir(old_dir)
+            iter_count = '' if i == 0 else ' on iteration {}'.format(i + 1)
+            expected = testcase.output if i == 0 else testcase.output2.get(i + 1, [])
+
+            assert_string_arrays_equal(expected, output,
+                               'Invalid output ({}, line {}){}'.format(
+                                   testcase.file, testcase.line, iter_count))
+
+        if has_program:
+            os.remove(program)
 
 
 def parse_pkgs(comment: str) -> Tuple[List[str], List[str]]:

mypyc/test-data/run-multimodule.test

@@ -780,6 +780,7 @@ assert other_a.foo() == 10
 [file other_a.py]
 def foo() -> int: return 10
 
+[file build/__native_other_a.c]
 
 [delete build/__native_other_a.c.2]
 

mypyc/test/test_run.py

@@ -12,11 +12,11 @@
 from typing import Any, Iterator, List, cast
 
 from mypy import build
-from mypy.test.data import DataDrivenTestCase, UpdateFile
+from mypy.test.data import DataDrivenTestCase
 from mypy.test.config import test_temp_dir
 from mypy.errors import CompileError
 from mypy.options import Options
-from mypy.test.helpers import copy_and_fudge_mtime, assert_module_equivalence
+from mypy.test.helpers import assert_module_equivalence, perform_file_operations
 
 from mypyc.codegen import emitmodule
 from mypyc.options import CompilerOptions
@@ -135,7 +135,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             self.run_case_inner(testcase)
 
     def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
-        os.mkdir(WORKDIR)
+        if not os.path.isdir(WORKDIR):  # (one test puts something in build...)
+            os.mkdir(WORKDIR)
 
         text = '\n'.join(testcase.input)
 
@@ -161,16 +162,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
 
             step += 1
             with chdir_manager('..'):
-                for op in operations:
-                    if isinstance(op, UpdateFile):
-                        # Modify/create file
-                        copy_and_fudge_mtime(op.source_path, op.target_path)
-                    else:
-                        # Delete file
-                        try:
-                            os.remove(op.path)
-                        except FileNotFoundError:
-                            pass
+                perform_file_operations(operations)
             self.run_case_step(testcase, step)
 
     def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> None:

test-data/unit/pep561.test

@@ -175,3 +175,23 @@ testTypedPkgNamespaceRegImport.py:4: error: Cannot find implementation or librar
 testTypedPkgNamespaceRegImport.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 testTypedPkgNamespaceRegImport.py:10: error: Argument 1 has incompatible type "bool"; expected "str"
 testTypedPkgNamespaceRegImport.py:11: error: Argument 1 has incompatible type "int"; expected "bool"
+
+-- This is really testing the test framework to make sure incremental works
+[case testPep561TestIncremental]
+# pkgs: typedpkg
+import a
+[file a.py]
+[file a.py.2]
+1 + 'no'
+[out]
+[out2]
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+
+-- Test for issue #9852, among others
+[case testTypedPkgNamespaceRegFromImportTwice-xfail]
+# pkgs: typedpkg, typedpkg_ns
+from typedpkg_ns import ns
+-- dummy should trigger a second iteration
+[file dummy.py.2]
+[out]
+[out2]