From f2eca248de4b6de09ee635b7c11a9b9d76467bb8 Mon Sep 17 00:00:00 2001 From: Lukasz Langa Date: Tue, 3 Jan 2017 12:44:40 -0800 Subject: [PATCH] Externalize flake8 linting, introduce additional checks This change is a step towards removing `runtests.py` (see #1673). The exclusion list in the flake8 configuration in `setup.cfg` has been updated to enable running the linter from the root of the project by simply invoking `flake8`. This enables it to leverage its own file discovery and its own multiprocessing queue without excessive subprocessing for linting every file. This gives a minor speed up in local test runs. Before: total time in lint: 130.914682 After: total time in lint: 20.379915 There's an additional speedup on Travis because linting is now only performed on Python 3.6. More importantly, this means flake8 is now running over all files unless explicitly excluded in `setup.cfg`. This will help avoiding unintentional omissions in the future (see comments on #2637). Note: running `flake8` as a single lazy subprocess in `runtests.py` doesn't sacrifice any parallelism because the linter has its own process pool. Minimal whitespace changes were required to `mypy_extensions.py` but in return flake8 will check it now exactly like it checks the rest of the `mypy/*` codebase. Those are also done on #2637 but that hasn't landed yet. Finally, flake8-bugbear and flake8-pyi were added to test requirements to make the linter configuration consistent with typeshed. I hope the additional checks will speed up future pull requests by automating bigger parts of the code review. The pyi plugin enables forward reference support when linting .pyi files. That means it's now possible to run `flake8` inside the typeshed submodule or on arbitrary .pyi files during development (which your editor could do for you), for example on fixtures. See discussion on #2629 on checks that are disabled and why. --- .travis.yml | 6 ++++-- extensions/mypy_extensions.py | 3 +++ runtests.py | 12 ++++-------- setup.cfg | 27 +++++++++++++++++++++++---- test-data/.flake8 | 1 + test-requirements.txt | 2 ++ 6 files changed, 37 insertions(+), 14 deletions(-) create mode 120000 test-data/.flake8 diff --git a/.travis.yml b/.travis.yml index 801216b0798e..4c25d0e4bdd6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,8 @@ python: - "3.4" # Specifically request 3.5.1 because we need to be compatible with that. - "3.5.1" - - "3.6-dev" + - "3.6" + - "3.7-dev" # Pypy build is disabled because it doubles the travis build time, and it rarely fails # unless one one of the other builds fails. # - "pypy3" @@ -15,4 +16,5 @@ install: - python setup.py install script: - - python runtests.py + - python runtests.py -x lint + - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then flake8; fi diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index efa3c35e8e03..8305902778cb 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -22,9 +22,11 @@ def _check_fails(cls, other): pass return False + def _dict_new(cls, *args, **kwargs): return dict(*args, **kwargs) + def _typeddict_new(cls, _typename, _fields=None, **kwargs): if _fields is None: _fields = kwargs @@ -33,6 +35,7 @@ def _typeddict_new(cls, _typename, _fields=None, **kwargs): " but not both") return _TypedDictMeta(_typename, (), {'__annotations__': dict(_fields)}) + class _TypedDictMeta(type): def __new__(cls, name, bases, ns): # Create new typed dict class object. diff --git a/runtests.py b/runtests.py index 0df7cfaeeffd..42c3fce4f4bb 100755 --- a/runtests.py +++ b/runtests.py @@ -151,11 +151,11 @@ def add_python2(self, name: str, *args: str, cwd: Optional[str] = None) -> None: env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) - def add_flake8(self, name: str, file: str, cwd: Optional[str] = None) -> None: - name = 'lint %s' % name + def add_flake8(self, cwd: Optional[str] = None) -> None: + name = 'lint' if not self.allow(name): return - largs = ['flake8', file] + largs = ['flake8', '-j{}'.format(self.waiter.limit)] env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) @@ -167,13 +167,9 @@ def list_tasks(self) -> None: def add_basic(driver: Driver) -> None: if False: driver.add_mypy('file setup.py', 'setup.py') - driver.add_flake8('file setup.py', 'setup.py') driver.add_mypy('file runtests.py', 'runtests.py') - driver.add_flake8('file runtests.py', 'runtests.py') driver.add_mypy('legacy entry script', 'scripts/mypy') - driver.add_flake8('legacy entry script', 'scripts/mypy') driver.add_mypy('legacy myunit script', 'scripts/myunit') - driver.add_flake8('legacy myunit script', 'scripts/myunit') # needs typed_ast installed: driver.add_mypy('fast-parse', '--fast-parse', 'test-data/samples/hello.py') @@ -206,7 +202,6 @@ def add_imports(driver: Driver) -> None: mod = file_to_module(f) if not mod.endswith('.__main__'): driver.add_python_string('import %s' % mod, 'import %s' % mod) - driver.add_flake8('module %s' % mod, f) PYTEST_FILES = ['mypy/test/{}.py'.format(name) for name in [ @@ -411,6 +406,7 @@ def main() -> None: add_stubs(driver) add_stdlibsamples(driver) add_samples(driver) + driver.add_flake8() if list_only: driver.list_tasks() diff --git a/setup.cfg b/setup.cfg index 084dab95b5e1..43ab1b3b9a03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,24 @@ [flake8] max-line-length = 99 -exclude = mypy/codec/* -# Thing to ignore: +# typeshed and unit test fixtures have .pyi-specific flake8 configuration +exclude = + # Sphinx configuration is irrelevant + docs/source/conf.py, + # external library with incompatible style + lib-typing/*, + # conflicting styles + misc/*, + # external library with incompatible style + pinfer/*, + # conflicting styles + scripts/*, + # tests have more relaxed styling requirements + # fixtures have their own .pyi-specific configuration + test-data/*, + # typeshed has its own .pyi-specific configuration + typeshed/* + +# Things to ignore: # E251: spaces around default arg value (against our style) # E128: continuation line under-indented (too noisy) # F401: unused identifiers (useless, as it doesn't see inside # type: comments) @@ -10,8 +27,10 @@ exclude = mypy/codec/* # W503: line break before binary operator # E704: multiple statements on one line (def) # E402: module level import not at top of file -# B???: flake8-bugbear errors -ignore = E251,E128,F401,W601,E701,W503,E704,E402,B +# B3??: Python 3 compatibility warnings +# B006: use of mutable defaults in function signatures +# B007: Loop control variable not used within the loop body. +ignore = E251,E128,F401,W601,E701,W503,E704,E402,B3,B006,B007 [coverage:run] branch = true diff --git a/test-data/.flake8 b/test-data/.flake8 new file mode 120000 index 000000000000..3ec8bd526b2f --- /dev/null +++ b/test-data/.flake8 @@ -0,0 +1 @@ +../typeshed/.flake8 \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index a2219e46f1e5..99834a7e50e1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,6 @@ flake8 +flake8-bugbear; python_version >= '3.5' +flake8-pyi; python_version >= '3.5' lxml typed-ast>=0.6.1; sys_platform != 'win32' pytest>=2.8