diff --git a/ChangeLog b/ChangeLog index 5059142991..c10a70c3b8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,12 @@ Release date: TBA .. Put new features here and also in 'doc/whatsnew/2.13.rst' +* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. Running pylint with + ``--recursive=y`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided + as parameter to pylint. + + Closes #352 + * Add ``modified-iterating-list``, ``modified-iterating-dict`` and ``modified-iterating-set``, emitted when items are added to or removed from respectively a list, dictionary or set being iterated through. diff --git a/doc/faq.rst b/doc/faq.rst index 47dc9b501c..f5ca1a9997 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -116,6 +116,27 @@ For example:: Much probably. Read :ref:`ide-integration` +3.5 I need to run pylint over all modules and packages in my project directory. +------------------------------------------------------------------------------- + +By default the ``pylint`` command only accepts a list of python modules and packages. Using a +directory which is not a package results in an error:: + + pylint mydir + ************* Module mydir + mydir/__init__.py:1:0: F0010: error while code parsing: Unable to load file mydir/__init__.py: + [Errno 2] No such file or directory: 'mydir/__init__.py' (parse-error) + +To execute pylint over all modules and packages under the directory, the ``--recursive=y`` option must +be provided. This option makes ``pylint`` attempt to discover all modules (files ending with ``.py`` extension) +and all packages (all directories containing a ``__init__.py`` file). +Those modules and packages are then analyzed:: + + pylint --recursive=y mydir + +When ``--recursive=y`` option is used, modules and packages are also accepted as parameters:: + + pylint --recursive=y mydir mymodule mypackage 4. Message Control ================== diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index ace2009866..5501e23065 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -33,6 +33,10 @@ will work if ``directory`` is a python package (i.e. has an __init__.py file or it is an implicit namespace package) or if "directory" is in the python path. +By default, pylint will exit with an error when one of the arguments is a directory which is not +a python package. In order to run pylint over all modules and packages within the provided +subtree of a directory, the ``--recursive=y`` option must be provided. + For more details on this see the :ref:`faq`. It is also possible to call Pylint from another Python program, diff --git a/doc/whatsnew/2.13.rst b/doc/whatsnew/2.13.rst index 1486b135cc..9d5330c437 100644 --- a/doc/whatsnew/2.13.rst +++ b/doc/whatsnew/2.13.rst @@ -92,6 +92,12 @@ Extensions Other Changes ============= +* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. Running pylint with + ``--recursive=y`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided + as parameter to pylint. + + Closes #352 + * Fix false-negative for ``assignment-from-none`` checker with list.sort() method. Closes #5722 diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 92733a39cb..be5cd6fd58 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -515,6 +515,15 @@ def make_options() -> Tuple[Tuple[str, OptionDict], ...]: ), }, ), + ( + "recursive", + { + "type": "yn", + "metavar": "", + "default": False, + "help": "Discover python modules and packages in the file system subtree.", + }, + ), ( "py-version", { @@ -1005,6 +1014,33 @@ def initialize(self): if not msg.may_be_emitted(): self._msgs_state[msg.msgid] = False + @staticmethod + def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]: + """Discover python modules and packages in subdirectory. + + Returns iterator of paths to discovered modules and packages. + """ + for something in files_or_modules: + if os.path.isdir(something) and not os.path.isfile( + os.path.join(something, "__init__.py") + ): + skip_subtrees: List[str] = [] + for root, _, files in os.walk(something): + if any(root.startswith(s) for s in skip_subtrees): + # Skip subtree of already discovered package. + continue + if "__init__.py" in files: + skip_subtrees.append(root) + yield root + else: + yield from ( + os.path.join(root, file) + for file in files + if file.endswith(".py") + ) + else: + yield something + def check(self, files_or_modules: Union[Sequence[str], str]) -> None: """main checking entry: check a list of files or modules from their name. @@ -1019,6 +1055,8 @@ def check(self, files_or_modules: Union[Sequence[str], str]) -> None: DeprecationWarning, ) files_or_modules = (files_or_modules,) # type: ignore[assignment] + if self.config.recursive: + files_or_modules = tuple(self._discover_files(files_or_modules)) if self.config.from_stdin: if len(files_or_modules) != 1: raise exceptions.InvalidArgsError( diff --git a/tests/regrtest_data/directory/package/__init__.py b/tests/regrtest_data/directory/package/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/package/module.py b/tests/regrtest_data/directory/package/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/package/subpackage/__init__.py b/tests/regrtest_data/directory/package/subpackage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/package/subpackage/module.py b/tests/regrtest_data/directory/package/subpackage/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/subdirectory/module.py b/tests/regrtest_data/directory/subdirectory/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/subdirectory/subsubdirectory/module.py b/tests/regrtest_data/directory/subdirectory/subsubdirectory/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_self.py b/tests/test_self.py index 5703cacfe3..8b41654d20 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -100,6 +100,24 @@ def _configure_lc_ctype(lc_ctype: str) -> Iterator: os.environ[lc_ctype_env] = original_lctype +@contextlib.contextmanager +def _test_sys_path() -> Generator[None, None, None]: + original_path = sys.path + try: + yield + finally: + sys.path = original_path + + +@contextlib.contextmanager +def _test_cwd() -> Generator[None, None, None]: + original_dir = os.getcwd() + try: + yield + finally: + os.chdir(original_dir) + + class MultiReporter(BaseReporter): def __init__(self, reporters: List[BaseReporter]) -> None: # pylint: disable=super-init-not-called @@ -810,14 +828,6 @@ def test_fail_on_edge_case(self, opts, out): @staticmethod def test_modify_sys_path() -> None: - @contextlib.contextmanager - def test_sys_path() -> Generator[None, None, None]: - original_path = sys.path - try: - yield - finally: - sys.path = original_path - @contextlib.contextmanager def test_environ_pythonpath( new_pythonpath: Optional[str], @@ -837,7 +847,7 @@ def test_environ_pythonpath( # Only delete PYTHONPATH if new_pythonpath wasn't None del os.environ["PYTHONPATH"] - with test_sys_path(), patch("os.getcwd") as mock_getcwd: + with _test_sys_path(), patch("os.getcwd") as mock_getcwd: cwd = "/tmp/pytest-of-root/pytest-0/test_do_not_import_files_from_0" mock_getcwd.return_value = cwd default_paths = [ @@ -1284,3 +1294,47 @@ def test_regex_paths_csv_validator() -> None: with pytest.raises(SystemExit) as ex: Run(["--ignore-paths", "test", join(HERE, "regrtest_data", "empty.py")]) assert ex.value.code == 0 + + def test_regression_recursive(self): + self._test_output( + [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=n"], + expected_output="No such file or directory", + ) + + def test_recursive(self): + self._runtest( + [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"], + code=0, + ) + + def test_recursive_current_dir(self): + with _test_sys_path(): + # pytest is including directory HERE/regrtest_data to sys.path which causes + # astroid to believe that directory is a package. + sys.path = [ + path + for path in sys.path + if not os.path.basename(path) == "regrtest_data" + ] + with _test_cwd(): + os.chdir(join(HERE, "regrtest_data", "directory")) + self._runtest( + [".", "--recursive=y"], + code=0, + ) + + def test_regression_recursive_current_dir(self): + with _test_sys_path(): + # pytest is including directory HERE/regrtest_data to sys.path which causes + # astroid to believe that directory is a package. + sys.path = [ + path + for path in sys.path + if not os.path.basename(path) == "regrtest_data" + ] + with _test_cwd(): + os.chdir(join(HERE, "regrtest_data", "directory")) + self._test_output( + ["."], + expected_output="No such file or directory", + )