From e44977db0417e87e5f1d5654d4833e112c4f73d5 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 14 Jan 2022 23:34:41 +0100 Subject: [PATCH 01/29] Added --recursive option --- pylint/lint/pylinter.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 8b1993b4ea..770368a8e1 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -515,6 +515,13 @@ def make_options() -> Tuple[Tuple[str, OptionDict], ...]: ), }, ), + ( + "recursive", + { + "action": "store_true", + "help": ("Discover python files in file system subtree."), + }, + ), ( "py-version", { @@ -1005,6 +1012,22 @@ def initialize(self): if not msg.may_be_emitted(): self._msgs_state[msg.msgid] = False + def _discover_files(self, files_or_modules): + for something in files_or_modules: + if os.path.isdir(something) and not os.path.isfile(os.path.join(something, '__init__.py')): + skip_subtrees = [] + for root, dirs, files in os.walk(something): + if any(root.startswith(s) for s in skip_subtrees): + # Skip subtree of already discovered package + continue + elif '__init__.py' in files: + skip_subtrees.append(root) + yield root + else: + yield from (os.path.join(root, file) for file in files if file.endswith('.py')) + else: + yield something + def check(self, files_or_modules: Union[Sequence[str], str]) -> None: """main checking entry: check a list of files or modules from their name. @@ -1019,6 +1042,8 @@ def check(self, files_or_modules: Union[Sequence[str], str]) -> None: DeprecationWarning, ) files_or_modules = (files_or_modules,) # type: ignore[assignment] + if self.config.recursive: + files_or_modules = tuple(self._discover_files(files_or_modules)) if self.config.from_stdin: if len(files_or_modules) != 1: raise exceptions.InvalidArgsError( From 9b23124fe93195cd2d19adfa3c2beb0ccf7ec6fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 22:54:36 +0000 Subject: [PATCH 02/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pylint/lint/pylinter.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 770368a8e1..a75a55dc6d 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -1014,17 +1014,23 @@ def initialize(self): def _discover_files(self, files_or_modules): for something in files_or_modules: - if os.path.isdir(something) and not os.path.isfile(os.path.join(something, '__init__.py')): + if os.path.isdir(something) and not os.path.isfile( + os.path.join(something, "__init__.py") + ): skip_subtrees = [] for root, dirs, files in os.walk(something): if any(root.startswith(s) for s in skip_subtrees): # Skip subtree of already discovered package continue - elif '__init__.py' in files: + elif "__init__.py" in files: skip_subtrees.append(root) yield root else: - yield from (os.path.join(root, file) for file in files if file.endswith('.py')) + yield from ( + os.path.join(root, file) + for file in files + if file.endswith(".py") + ) else: yield something From 20ad40891678210e46e821ab5b92cf32b8d89bab Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sat, 15 Jan 2022 22:05:31 +0100 Subject: [PATCH 03/29] Added tests for --recursive option --- tests/regrtest_data/directory/package/__init__.py | 0 tests/regrtest_data/directory/package/module.py | 0 .../directory/package/subpackage/__init__.py | 0 .../directory/package/subpackage/module.py | 0 tests/regrtest_data/directory/subdirectory/module.py | 0 .../directory/subdirectory/subsubdirectory/module.py | 0 tests/test_self.py | 12 ++++++++++++ 7 files changed, 12 insertions(+) create mode 100644 tests/regrtest_data/directory/package/__init__.py create mode 100644 tests/regrtest_data/directory/package/module.py create mode 100644 tests/regrtest_data/directory/package/subpackage/__init__.py create mode 100644 tests/regrtest_data/directory/package/subpackage/module.py create mode 100644 tests/regrtest_data/directory/subdirectory/module.py create mode 100644 tests/regrtest_data/directory/subdirectory/subsubdirectory/module.py diff --git a/tests/regrtest_data/directory/package/__init__.py b/tests/regrtest_data/directory/package/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/package/module.py b/tests/regrtest_data/directory/package/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/package/subpackage/__init__.py b/tests/regrtest_data/directory/package/subpackage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/package/subpackage/module.py b/tests/regrtest_data/directory/package/subpackage/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/subdirectory/module.py b/tests/regrtest_data/directory/subdirectory/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regrtest_data/directory/subdirectory/subsubdirectory/module.py b/tests/regrtest_data/directory/subdirectory/subsubdirectory/module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_self.py b/tests/test_self.py index 444a5076b9..d6fdbc41fa 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1290,3 +1290,15 @@ def test_regex_paths_csv_validator() -> None: with pytest.raises(SystemExit) as ex: Run(["--ignore-paths", "test", join(HERE, "regrtest_data", "empty.py")]) assert ex.value.code == 0 + + def test_regression_recursive(self): + self._test_output( + [join(HERE, "regrtest_data", "directory", "subdirectory")], + expected_output="No such file or directory", + ) + + def test_recursive(self): + self._runtest( + [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive"], + code=0, + ) From 2aceafbb9f87ecd9e3148910887fe79d68e2e891 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sat, 15 Jan 2022 22:22:50 +0100 Subject: [PATCH 04/29] Added comments, types and make linters happy --- pylint/lint/pylinter.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index a75a55dc6d..65c7bc75b3 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -1012,17 +1012,22 @@ def initialize(self): if not msg.may_be_emitted(): self._msgs_state[msg.msgid] = False - def _discover_files(self, files_or_modules): + @staticmethod + def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]: + """Discover python modules and packages in subdirectory. + + Returns iterator of paths to discovered modules and packages. + """ for something in files_or_modules: if os.path.isdir(something) and not os.path.isfile( os.path.join(something, "__init__.py") ): - skip_subtrees = [] - for root, dirs, files in os.walk(something): + skip_subtrees: List[str] = [] + for root, _, files in os.walk(something): if any(root.startswith(s) for s in skip_subtrees): - # Skip subtree of already discovered package + # Skip subtree of already discovered package. continue - elif "__init__.py" in files: + if "__init__.py" in files: skip_subtrees.append(root) yield root else: From 404bb9b2f7da9e794046c953f993fe202dfbe040 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sat, 15 Jan 2022 22:44:35 +0100 Subject: [PATCH 05/29] Updated Changelog and whatsnew --- ChangeLog | 4 ++++ doc/whatsnew/2.13.rst | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/ChangeLog b/ChangeLog index ef6aa6bdcc..0e62956372 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,10 @@ Release date: TBA .. Put new features here and also in 'doc/whatsnew/2.13.rst' +* Add ``--recursive`` option to allow linting recursively all modules and packages in subtree. + + Closes #352 + * Added several checkers to deal with unicode security issues (see `Trojan Sources `_ and `PEP 672 `_ for details) that also diff --git a/doc/whatsnew/2.13.rst b/doc/whatsnew/2.13.rst index 3d6a1b263e..435ecdd0a4 100644 --- a/doc/whatsnew/2.13.rst +++ b/doc/whatsnew/2.13.rst @@ -77,6 +77,12 @@ Extensions Other Changes ============= +* Add ``--recursive`` option to allow linting recursively all modules and packages in subtree. Running pylint with + ``--recursive`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided + as parameter to pylint. + + Closes #352 + * Fixed false positive ``consider-using-dict-comprehension`` when creating a dict using a list of tuples where key AND value vary depending on the same condition. From 44f1919c42c86766388481a42559416c1df0521c Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sat, 15 Jan 2022 22:47:15 +0100 Subject: [PATCH 06/29] Improved help message --- pylint/lint/pylinter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 65c7bc75b3..2730efb07b 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -519,7 +519,7 @@ def make_options() -> Tuple[Tuple[str, OptionDict], ...]: "recursive", { "action": "store_true", - "help": ("Discover python files in file system subtree."), + "help": "Discover python modules and packages in file system subtree.", }, ), ( From 3ebb1d7797e80585b9158a15e65e936f1e8700eb Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sun, 16 Jan 2022 22:53:05 +0100 Subject: [PATCH 07/29] Mention --recursive option in faq and user guide --- doc/faq.rst | 21 +++++++++++++++++++++ doc/user_guide/run.rst | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/doc/faq.rst b/doc/faq.rst index 47dc9b501c..a3a84d7cf0 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -116,6 +116,27 @@ For example:: Much probably. Read :ref:`ide-integration` +3.5 I need to run pylint over all modules and packages in my project directory. +------------------------------------------------------------------------------- + +``pylint`` command by default accepts only list of python modules and packages. Putting +directory which is not a package results in an error:: + + pylint mydir + ************* Module mydir + mydir/__init__.py:1:0: F0010: error while code parsing: Unable to load file mydir/__init__.py: + [Errno 2] No such file or directory: 'mydir/__init__.py' (parse-error) + +To execute pylint over all modules and packages under the directory, ``--recursive`` option must +be provided. This option causes that pylint tries to discover all modules (files ending with ``.py`` extention) +and all packages (all directories containing ``__init__.py`` file). +Those modules and packages are then subject for analysis:: + + pylint --recursive mydir + +When ``--recursive`` option is used, also module and packages are accepted as parameters:: + + pylint --recursive mydir mymodule mypackage 4. Message Control ================== diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index de28a44136..2e3787cb9d 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -33,6 +33,10 @@ will work if ``directory`` is a python package (i.e. has an __init__.py file or it is an implicit namespace package) or if "directory" is in the python path. +By default, pylint will result with an error when one of the arguments is directory which is not +a python package. In order to run pylint over all modules and packages within file system +subtree of direcotry, ``--recursive`` option must be provided. + For more details on this see the :ref:`faq`. It is also possible to call Pylint from another Python program, From 8741104368bfd68cb77b633b238e6036ee0ec9b8 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 17 Jan 2022 09:38:19 +0100 Subject: [PATCH 08/29] Fix typos --- doc/faq.rst | 2 +- doc/user_guide/run.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index a3a84d7cf0..39fdef206d 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -128,7 +128,7 @@ directory which is not a package results in an error:: [Errno 2] No such file or directory: 'mydir/__init__.py' (parse-error) To execute pylint over all modules and packages under the directory, ``--recursive`` option must -be provided. This option causes that pylint tries to discover all modules (files ending with ``.py`` extention) +be provided. This option causes that pylint tries to discover all modules (files ending with ``.py`` extension) and all packages (all directories containing ``__init__.py`` file). Those modules and packages are then subject for analysis:: diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 2e3787cb9d..8f74452bc1 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -35,7 +35,7 @@ python path. By default, pylint will result with an error when one of the arguments is directory which is not a python package. In order to run pylint over all modules and packages within file system -subtree of direcotry, ``--recursive`` option must be provided. +subtree of directory, ``--recursive`` option must be provided. For more details on this see the :ref:`faq`. From 91fe1063754dfbfff70726549dc954804be2aac3 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Tue, 18 Jan 2022 19:15:40 +0100 Subject: [PATCH 09/29] Use --recursive=y/n instead of just --recursive --- doc/faq.rst | 8 ++++---- doc/user_guide/run.rst | 2 +- pylint/lint/pylinter.py | 4 +++- tests/test_self.py | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 39fdef206d..a32a04d775 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -127,16 +127,16 @@ directory which is not a package results in an error:: mydir/__init__.py:1:0: F0010: error while code parsing: Unable to load file mydir/__init__.py: [Errno 2] No such file or directory: 'mydir/__init__.py' (parse-error) -To execute pylint over all modules and packages under the directory, ``--recursive`` option must +To execute pylint over all modules and packages under the directory, ``--recursive=y`` option must be provided. This option causes that pylint tries to discover all modules (files ending with ``.py`` extension) and all packages (all directories containing ``__init__.py`` file). Those modules and packages are then subject for analysis:: - pylint --recursive mydir + pylint --recursive=y mydir -When ``--recursive`` option is used, also module and packages are accepted as parameters:: +When ``--recursive=y`` option is used, also module and packages are accepted as parameters:: - pylint --recursive mydir mymodule mypackage + pylint --recursive=y mydir mymodule mypackage 4. Message Control ================== diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 8f74452bc1..2fa35f46db 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -35,7 +35,7 @@ python path. By default, pylint will result with an error when one of the arguments is directory which is not a python package. In order to run pylint over all modules and packages within file system -subtree of directory, ``--recursive`` option must be provided. +subtree of directory, ``--recursive=y`` option must be provided. For more details on this see the :ref:`faq`. diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 2730efb07b..087c016e47 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -518,7 +518,9 @@ def make_options() -> Tuple[Tuple[str, OptionDict], ...]: ( "recursive", { - "action": "store_true", + "type": "yn", + "metavar": "", + "default": False, "help": "Discover python modules and packages in file system subtree.", }, ), diff --git a/tests/test_self.py b/tests/test_self.py index d6fdbc41fa..8045d8f0ea 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1293,12 +1293,12 @@ def test_regex_paths_csv_validator() -> None: def test_regression_recursive(self): self._test_output( - [join(HERE, "regrtest_data", "directory", "subdirectory")], + [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=n"], expected_output="No such file or directory", ) def test_recursive(self): self._runtest( - [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive"], + [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"], code=0, ) From 6623eb1cb4ef0756f6bf68460fc2976f41b5053c Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 19 Jan 2022 20:31:50 +0100 Subject: [PATCH 10/29] Enable recursive mode when pylint is executed over current directory --- doc/faq.rst | 4 ++++ doc/user_guide/run.rst | 3 ++- pylint/lint/pylinter.py | 5 ++++- tests/test_self.py | 10 ++++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index a32a04d775..14924f7ad9 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -138,6 +138,10 @@ When ``--recursive=y`` option is used, also module and packages are accepted as pylint --recursive=y mydir mymodule mypackage +Recursive mode is enabled by default only when pylint scans current directory:: + + pylint . + 4. Message Control ================== diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 2fa35f46db..92ff2d7465 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -35,7 +35,8 @@ python path. By default, pylint will result with an error when one of the arguments is directory which is not a python package. In order to run pylint over all modules and packages within file system -subtree of directory, ``--recursive=y`` option must be provided. +subtree of directory, ``--recursive=y`` option must be provided. Running pylint over current directory +(`pylint .`) enables recursive mode by default. For more details on this see the :ref:`faq`. diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 087c016e47..442c7ab939 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -1055,7 +1055,10 @@ def check(self, files_or_modules: Union[Sequence[str], str]) -> None: DeprecationWarning, ) files_or_modules = (files_or_modules,) # type: ignore[assignment] - if self.config.recursive: + # We enable recursive mode when --recursive=y is set or when running command: pylint . + if self.config.recursive or ( + len(files_or_modules) == 1 and files_or_modules[0] == "." + ): files_or_modules = tuple(self._discover_files(files_or_modules)) if self.config.from_stdin: if len(files_or_modules) != 1: diff --git a/tests/test_self.py b/tests/test_self.py index 8045d8f0ea..6d4627d050 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1297,6 +1297,16 @@ def test_regression_recursive(self): expected_output="No such file or directory", ) + def test_recursive_current_dir(self): + os.chdir(join(HERE, "regrtest_data", "directory")) + try: + self._runtest( + ["."], + code=0, + ) + finally: + os.chdir(HERE) + def test_recursive(self): self._runtest( [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"], From bc4573638244eb3a8bf272b59cb7b0a3004b635a Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 21 Jan 2022 17:43:46 +0100 Subject: [PATCH 11/29] Revert "Enable recursive mode when pylint is executed over current directory" This reverts commit 6623eb1cb4ef0756f6bf68460fc2976f41b5053c. --- doc/faq.rst | 4 ---- doc/user_guide/run.rst | 3 +-- pylint/lint/pylinter.py | 5 +---- tests/test_self.py | 10 ---------- 4 files changed, 2 insertions(+), 20 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 14924f7ad9..a32a04d775 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -138,10 +138,6 @@ When ``--recursive=y`` option is used, also module and packages are accepted as pylint --recursive=y mydir mymodule mypackage -Recursive mode is enabled by default only when pylint scans current directory:: - - pylint . - 4. Message Control ================== diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 92ff2d7465..2fa35f46db 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -35,8 +35,7 @@ python path. By default, pylint will result with an error when one of the arguments is directory which is not a python package. In order to run pylint over all modules and packages within file system -subtree of directory, ``--recursive=y`` option must be provided. Running pylint over current directory -(`pylint .`) enables recursive mode by default. +subtree of directory, ``--recursive=y`` option must be provided. For more details on this see the :ref:`faq`. diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 442c7ab939..087c016e47 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -1055,10 +1055,7 @@ def check(self, files_or_modules: Union[Sequence[str], str]) -> None: DeprecationWarning, ) files_or_modules = (files_or_modules,) # type: ignore[assignment] - # We enable recursive mode when --recursive=y is set or when running command: pylint . - if self.config.recursive or ( - len(files_or_modules) == 1 and files_or_modules[0] == "." - ): + if self.config.recursive: files_or_modules = tuple(self._discover_files(files_or_modules)) if self.config.from_stdin: if len(files_or_modules) != 1: diff --git a/tests/test_self.py b/tests/test_self.py index 6d4627d050..8045d8f0ea 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1297,16 +1297,6 @@ def test_regression_recursive(self): expected_output="No such file or directory", ) - def test_recursive_current_dir(self): - os.chdir(join(HERE, "regrtest_data", "directory")) - try: - self._runtest( - ["."], - code=0, - ) - finally: - os.chdir(HERE) - def test_recursive(self): self._runtest( [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"], From 8ae1438631736bcac1ba1848831f60f208f117e3 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:50:42 +0100 Subject: [PATCH 12/29] Update doc/faq.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index a32a04d775..52a4df4206 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -119,7 +119,7 @@ Much probably. Read :ref:`ide-integration` 3.5 I need to run pylint over all modules and packages in my project directory. ------------------------------------------------------------------------------- -``pylint`` command by default accepts only list of python modules and packages. Putting +By default the ``pylint`` command only accepts a list of python modules and packages. Using a directory which is not a package results in an error:: pylint mydir From 756acc562393534363051cbd40f18d3ffe12c35f Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:51:02 +0100 Subject: [PATCH 13/29] Update pylint/lint/pylinter.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- pylint/lint/pylinter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 087c016e47..5cde581193 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -521,7 +521,7 @@ def make_options() -> Tuple[Tuple[str, OptionDict], ...]: "type": "yn", "metavar": "", "default": False, - "help": "Discover python modules and packages in file system subtree.", + "help": "Discover python modules and packages in the file system subtree.", }, ), ( From e83f9fc2f1a4dbf77a3f12a1036144aad258b9d8 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:51:17 +0100 Subject: [PATCH 14/29] Update doc/user_guide/run.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/user_guide/run.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 2fa35f46db..2ee7240e84 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -35,7 +35,7 @@ python path. By default, pylint will result with an error when one of the arguments is directory which is not a python package. In order to run pylint over all modules and packages within file system -subtree of directory, ``--recursive=y`` option must be provided. +subtree of a directory, the ``--recursive=y`` option must be provided. For more details on this see the :ref:`faq`. From ca13575982fc4e4dc7f2f4c9a6b8fa8687e68ef1 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:51:24 +0100 Subject: [PATCH 15/29] Update doc/user_guide/run.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/user_guide/run.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 2ee7240e84..1ac5f952bd 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -34,7 +34,7 @@ file or it is an implicit namespace package) or if "directory" is in the python path. By default, pylint will result with an error when one of the arguments is directory which is not -a python package. In order to run pylint over all modules and packages within file system +a python package. In order to run pylint over all modules and packages within the provided subtree of a directory, the ``--recursive=y`` option must be provided. For more details on this see the :ref:`faq`. From c9e571e7f2107e9a9970ab6bc9dd40396bf3811c Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:51:40 +0100 Subject: [PATCH 16/29] Update doc/faq.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index 52a4df4206..ccdd025356 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -127,7 +127,7 @@ directory which is not a package results in an error:: mydir/__init__.py:1:0: F0010: error while code parsing: Unable to load file mydir/__init__.py: [Errno 2] No such file or directory: 'mydir/__init__.py' (parse-error) -To execute pylint over all modules and packages under the directory, ``--recursive=y`` option must +To execute pylint over all modules and packages under the directory, the ``--recursive=y`` option must be provided. This option causes that pylint tries to discover all modules (files ending with ``.py`` extension) and all packages (all directories containing ``__init__.py`` file). Those modules and packages are then subject for analysis:: From 68529b5bd38df597c76bc2eba0df4e6f1ed1fa33 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:51:50 +0100 Subject: [PATCH 17/29] Update doc/faq.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index ccdd025356..ed7ec7ab4d 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -128,7 +128,7 @@ directory which is not a package results in an error:: [Errno 2] No such file or directory: 'mydir/__init__.py' (parse-error) To execute pylint over all modules and packages under the directory, the ``--recursive=y`` option must -be provided. This option causes that pylint tries to discover all modules (files ending with ``.py`` extension) +be provided. This option makes ``pylint`` attempt to discover all modules (files ending with ``.py`` extension) and all packages (all directories containing ``__init__.py`` file). Those modules and packages are then subject for analysis:: From b5e5bb698849d36c93acf6e328427073045f7331 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:51:57 +0100 Subject: [PATCH 18/29] Update doc/faq.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index ed7ec7ab4d..d6e9404e89 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -129,7 +129,7 @@ directory which is not a package results in an error:: To execute pylint over all modules and packages under the directory, the ``--recursive=y`` option must be provided. This option makes ``pylint`` attempt to discover all modules (files ending with ``.py`` extension) -and all packages (all directories containing ``__init__.py`` file). +and all packages (all directories containing a ``__init__.py`` file). Those modules and packages are then subject for analysis:: pylint --recursive=y mydir From 6199d4e98bdffebeaccff5030a40347a63d2d934 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:52:03 +0100 Subject: [PATCH 19/29] Update doc/faq.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index d6e9404e89..5fdc3c317e 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -130,7 +130,7 @@ directory which is not a package results in an error:: To execute pylint over all modules and packages under the directory, the ``--recursive=y`` option must be provided. This option makes ``pylint`` attempt to discover all modules (files ending with ``.py`` extension) and all packages (all directories containing a ``__init__.py`` file). -Those modules and packages are then subject for analysis:: +Those modules and packages are then analyzed:: pylint --recursive=y mydir From 59480dcd290216d580f0b82669e0b284703baba7 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 07:52:09 +0100 Subject: [PATCH 20/29] Update doc/user_guide/run.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/user_guide/run.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 1ac5f952bd..b12c903062 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -33,7 +33,7 @@ will work if ``directory`` is a python package (i.e. has an __init__.py file or it is an implicit namespace package) or if "directory" is in the python path. -By default, pylint will result with an error when one of the arguments is directory which is not +By default, pylint will exits an error when one of the arguments is a directory which is not a python package. In order to run pylint over all modules and packages within the provided subtree of a directory, the ``--recursive=y`` option must be provided. From f47c9b808b773c92f84a2270f0bf2d0f14346310 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 17:10:16 +0100 Subject: [PATCH 21/29] Added unittests for current directory --- tests/test_self.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_self.py b/tests/test_self.py index 8045d8f0ea..010c2802c4 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1302,3 +1302,23 @@ def test_recursive(self): [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"], code=0, ) + + def test_recursive_current_dir(self): + os.chdir(join(HERE, "regrtest_data", "directory")) + try: + self._runtest( + [".", "--recursive=y"], + code=0, + ) + finally: + os.chdir(HERE) + + def test_regression_recursive_current_dir(self): + os.chdir(join(HERE, "regrtest_data", "directory")) + try: + self._test_output( + ["."], + expected_output="No such file or directory", + ) + finally: + os.chdir(HERE) From 1de19990e0f4b676d58382c54e952f71267ffb99 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 19:00:02 +0100 Subject: [PATCH 22/29] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- ChangeLog | 2 +- doc/faq.rst | 2 +- doc/whatsnew/2.13.rst | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6ba58c9a94..240948c3a7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,7 +9,7 @@ Release date: TBA .. Put new features here and also in 'doc/whatsnew/2.13.rst' -* Add ``--recursive`` option to allow linting recursively all modules and packages in subtree. +* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. Closes #352 diff --git a/doc/faq.rst b/doc/faq.rst index 5fdc3c317e..f5ca1a9997 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -134,7 +134,7 @@ Those modules and packages are then analyzed:: pylint --recursive=y mydir -When ``--recursive=y`` option is used, also module and packages are accepted as parameters:: +When ``--recursive=y`` option is used, modules and packages are also accepted as parameters:: pylint --recursive=y mydir mymodule mypackage diff --git a/doc/whatsnew/2.13.rst b/doc/whatsnew/2.13.rst index 0155ea2320..243efe3f4d 100644 --- a/doc/whatsnew/2.13.rst +++ b/doc/whatsnew/2.13.rst @@ -77,8 +77,8 @@ Extensions Other Changes ============= -* Add ``--recursive`` option to allow linting recursively all modules and packages in subtree. Running pylint with - ``--recursive`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided +* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. Running pylint with + ``--recursive=y`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided as parameter to pylint. Closes #352 From a86cf1451548ea61758efbbc5476eaeec0d00516 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Wed, 26 Jan 2022 19:02:37 +0100 Subject: [PATCH 23/29] Make changelog entry same as entry in doc/whatsnew/2.13.rst --- ChangeLog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 240948c3a7..1f15162a4a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,7 +9,9 @@ Release date: TBA .. Put new features here and also in 'doc/whatsnew/2.13.rst' -* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. +* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. Running pylint with + ``--recursive=y`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided + as parameter to pylint. Closes #352 From 3d965989d1a893483937e8fe6c51e492881a8fa3 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Thu, 27 Jan 2022 08:08:31 +0100 Subject: [PATCH 24/29] correctly revert chdir in unittests --- tests/test_self.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_self.py b/tests/test_self.py index 010c2802c4..23a5e03e66 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1304,6 +1304,7 @@ def test_recursive(self): ) def test_recursive_current_dir(self): + _dir = os.getcwd() os.chdir(join(HERE, "regrtest_data", "directory")) try: self._runtest( @@ -1311,9 +1312,10 @@ def test_recursive_current_dir(self): code=0, ) finally: - os.chdir(HERE) + os.chdir(_dir) def test_regression_recursive_current_dir(self): + _dir = os.getcwd() os.chdir(join(HERE, "regrtest_data", "directory")) try: self._test_output( @@ -1321,4 +1323,4 @@ def test_regression_recursive_current_dir(self): expected_output="No such file or directory", ) finally: - os.chdir(HERE) + os.chdir(_dir) From 20dcce41d02903ad6f5755e35e476ad2d4525653 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sat, 29 Jan 2022 19:40:01 +0100 Subject: [PATCH 25/29] Fix failing test_regression_recursive_current_dir --- tests/test_self.py | 66 +++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/tests/test_self.py b/tests/test_self.py index 23a5e03e66..d799d7c5f7 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -100,6 +100,24 @@ def _configure_lc_ctype(lc_ctype: str) -> Iterator: os.environ[lc_ctype_env] = original_lctype +@contextlib.contextmanager +def _test_sys_path() -> Generator[None, None, None]: + original_path = sys.path + try: + yield + finally: + sys.path = original_path + + +@contextlib.contextmanager +def _test_cwd() -> Generator[None, None, None]: + original_dir = os.getcwd() + try: + yield + finally: + os.chdir(original_dir) + + class MultiReporter(BaseReporter): def __init__(self, reporters: List[BaseReporter]) -> None: # pylint: disable=super-init-not-called @@ -810,14 +828,6 @@ def test_fail_on_edge_case(self, opts, out): @staticmethod def test_modify_sys_path() -> None: - @contextlib.contextmanager - def test_sys_path() -> Generator[None, None, None]: - original_path = sys.path - try: - yield - finally: - sys.path = original_path - @contextlib.contextmanager def test_environ_pythonpath( new_pythonpath: Optional[str], @@ -837,7 +847,7 @@ def test_environ_pythonpath( # Only delete PYTHONPATH if new_pythonpath wasn't None del os.environ["PYTHONPATH"] - with test_sys_path(), patch("os.getcwd") as mock_getcwd: + with _test_sys_path(), patch("os.getcwd") as mock_getcwd: cwd = "/tmp/pytest-of-root/pytest-0/test_do_not_import_files_from_0" mock_getcwd.return_value = cwd default_paths = [ @@ -1304,23 +1314,25 @@ def test_recursive(self): ) def test_recursive_current_dir(self): - _dir = os.getcwd() - os.chdir(join(HERE, "regrtest_data", "directory")) - try: - self._runtest( - [".", "--recursive=y"], - code=0, - ) - finally: - os.chdir(_dir) + with _test_sys_path(): + sys.path = [path for path in sys.path if not os.path.basename(path) == 'regrtest_data'] + # pytest is including directory HERE/regrtest_data to sys.path which causes + # astroid believe that directory is a package. + with _test_cwd(): + os.chdir(join(HERE, "regrtest_data", "directory")) + self._runtest( + [".", "--recursive=y"], + code=0, + ) def test_regression_recursive_current_dir(self): - _dir = os.getcwd() - os.chdir(join(HERE, "regrtest_data", "directory")) - try: - self._test_output( - ["."], - expected_output="No such file or directory", - ) - finally: - os.chdir(_dir) + with _test_sys_path(): + # pytest is including directory HERE/regrtest_data to sys.path which causes + # astroid believe that directory is a package. + sys.path = [path for path in sys.path if not os.path.basename(path) == 'regrtest_data'] + with _test_cwd(): + os.chdir(join(HERE, "regrtest_data", "directory")) + self._test_output( + ["."], + expected_output="No such file or directory", + ) \ No newline at end of file From 1aa4fdc81040a4986138c6827c58b014d7d59867 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 29 Jan 2022 18:41:05 +0000 Subject: [PATCH 26/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_self.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_self.py b/tests/test_self.py index d799d7c5f7..dc67112648 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1315,7 +1315,11 @@ def test_recursive(self): def test_recursive_current_dir(self): with _test_sys_path(): - sys.path = [path for path in sys.path if not os.path.basename(path) == 'regrtest_data'] + sys.path = [ + path + for path in sys.path + if not os.path.basename(path) == "regrtest_data" + ] # pytest is including directory HERE/regrtest_data to sys.path which causes # astroid believe that directory is a package. with _test_cwd(): @@ -1329,10 +1333,14 @@ def test_regression_recursive_current_dir(self): with _test_sys_path(): # pytest is including directory HERE/regrtest_data to sys.path which causes # astroid believe that directory is a package. - sys.path = [path for path in sys.path if not os.path.basename(path) == 'regrtest_data'] + sys.path = [ + path + for path in sys.path + if not os.path.basename(path) == "regrtest_data" + ] with _test_cwd(): os.chdir(join(HERE, "regrtest_data", "directory")) self._test_output( ["."], expected_output="No such file or directory", - ) \ No newline at end of file + ) From 154f7d916d557574a0c5a1ea3be37b3e87402141 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sat, 29 Jan 2022 20:40:26 +0100 Subject: [PATCH 27/29] Cosmetics --- tests/test_self.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_self.py b/tests/test_self.py index dc67112648..5b818f9b24 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1315,13 +1315,13 @@ def test_recursive(self): def test_recursive_current_dir(self): with _test_sys_path(): + # pytest is including directory HERE/regrtest_data to sys.path which causes + # astroid believe that directory is a package. sys.path = [ path for path in sys.path if not os.path.basename(path) == "regrtest_data" ] - # pytest is including directory HERE/regrtest_data to sys.path which causes - # astroid believe that directory is a package. with _test_cwd(): os.chdir(join(HERE, "regrtest_data", "directory")) self._runtest( From 573561859933decdb7c75a7bd840aea6beb37c50 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 4 Feb 2022 20:07:33 +0100 Subject: [PATCH 28/29] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/user_guide/run.rst | 2 +- tests/test_self.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/user_guide/run.rst b/doc/user_guide/run.rst index 3863d47b9d..5501e23065 100644 --- a/doc/user_guide/run.rst +++ b/doc/user_guide/run.rst @@ -33,7 +33,7 @@ will work if ``directory`` is a python package (i.e. has an __init__.py file or it is an implicit namespace package) or if "directory" is in the python path. -By default, pylint will exits an error when one of the arguments is a directory which is not +By default, pylint will exit with an error when one of the arguments is a directory which is not a python package. In order to run pylint over all modules and packages within the provided subtree of a directory, the ``--recursive=y`` option must be provided. diff --git a/tests/test_self.py b/tests/test_self.py index 0764ba09fb..e7097ef671 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1310,7 +1310,7 @@ def test_recursive(self): def test_recursive_current_dir(self): with _test_sys_path(): # pytest is including directory HERE/regrtest_data to sys.path which causes - # astroid believe that directory is a package. + # astroid believe to that directory is a package. sys.path = [ path for path in sys.path @@ -1326,7 +1326,7 @@ def test_recursive_current_dir(self): def test_regression_recursive_current_dir(self): with _test_sys_path(): # pytest is including directory HERE/regrtest_data to sys.path which causes - # astroid believe that directory is a package. + # astroid believe to that directory is a package. sys.path = [ path for path in sys.path From ca79daeedc2099bbe9980929f87baf5c167847e5 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sat, 5 Feb 2022 17:28:05 +0100 Subject: [PATCH 29/29] Apply suggestions from code review Co-authored-by: Jacob Walls --- tests/test_self.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_self.py b/tests/test_self.py index e7097ef671..8b41654d20 100644 --- a/tests/test_self.py +++ b/tests/test_self.py @@ -1310,7 +1310,7 @@ def test_recursive(self): def test_recursive_current_dir(self): with _test_sys_path(): # pytest is including directory HERE/regrtest_data to sys.path which causes - # astroid believe to that directory is a package. + # astroid to believe that directory is a package. sys.path = [ path for path in sys.path @@ -1326,7 +1326,7 @@ def test_recursive_current_dir(self): def test_regression_recursive_current_dir(self): with _test_sys_path(): # pytest is including directory HERE/regrtest_data to sys.path which causes - # astroid believe to that directory is a package. + # astroid to believe that directory is a package. sys.path = [ path for path in sys.path