Lint all files in a directory

pylint/lint/expand_modules.py

@@ -6,6 +6,8 @@
 
 from pylint.typing import ErrorDescriptionDict, ModuleDescriptionDict
 
+PATH = sys.path.copy()
+
 
 def _modpath_from_file(filename, is_namespace, path=None):
     def _is_package_cb(inner_path, parts):
@@ -37,6 +39,11 @@ def _is_in_ignore_list_re(element: str, ignore_list_re: List[Pattern]) -> bool:
     return any(file_pattern.match(element) for file_pattern in ignore_list_re)
 
 
+def _get_additional_search_path(something):
+    module_path = get_python_path(something)
+    return [".", module_path] + PATH
+
+
 def expand_modules(
     files_or_modules: List[str],
     ignore_list: List[str],
@@ -48,18 +55,12 @@ def expand_modules(
     """
     result: List[ModuleDescriptionDict] = []
     errors: List[ErrorDescriptionDict] = []
-    path = sys.path.copy()
 
     for something in files_or_modules:
         basename = os.path.basename(something)
-        if (
-            basename in ignore_list
-            or _is_in_ignore_list_re(os.path.basename(something), ignore_list_re)
-            or _is_in_ignore_list_re(something, ignore_list_paths_re)
-        ):
+        if _skip_file(something, ignore_list, ignore_list_re, ignore_list_paths_re):
             continue
-        module_path = get_python_path(something)
-        additional_search_path = [".", module_path] + path
+        additional_search_path = _get_additional_search_path(something)
         if os.path.exists(something):
             # this is a file or a directory
             try:
@@ -138,3 +139,76 @@ def expand_modules(
                     }
                 )
     return result, errors
+
+
+def _skip_file(filepath, ignore_list, ignore_list_re, ignore_list_paths_re):
+    return (
+        os.path.basename(filepath) in ignore_list
+        or _is_in_ignore_list_re(os.path.basename(filepath), ignore_list_re)
+        or _is_in_ignore_list_re(filepath, ignore_list_paths_re)
+    )
+
+
+def _discover_module(filename):
+    result = None
+    error = None
+    modname = ".".join(
+        modutils.modpath_from_file(filename, path=_get_additional_search_path(filename))
+    )
+    try:
+        filepath = modutils.file_from_modpath(
+            modname.split("."), path=_get_additional_search_path(filename)
+        )
+    except (ImportError, SyntaxError) as ex:
+        # The SyntaxError is a Python bug and should be
+        # removed once we move away from imp.find_module: https://bugs.python.org/issue10588
+        error = {"key": "fatal", "mod": modname, "ex": ex}
+    else:
+        result = {
+            "path": os.path.normpath(filepath),
+            "name": modname,
+            "isarg": False,
+            "basepath": os.path.normpath(filepath),
+            "basename": modname,
+        }
+    return result, error
+
+
+def discover_modules(
+    files_or_modules: List[str],
+    ignore_list: List[str],
+    ignore_list_re: List[Pattern],
+    ignore_list_paths_re: List[Pattern[str]],
+) -> Tuple[List[ModuleDescriptionDict], List[ErrorDescriptionDict]]:
+    result: List[ModuleDescriptionDict] = []
+    errors: List[ErrorDescriptionDict] = []
+    for something in files_or_modules:
+        if _skip_file(something, ignore_list, ignore_list_re, ignore_list_paths_re):
+            continue
+        if os.path.isfile(something):
+            moddesc, error = _discover_module(something)
+            if error:
+                errors.append(error)
+                continue
+            if moddesc:
+                result.append(moddesc)
+        else:
+            for root, _, files in os.walk(top=something, topdown=True):
+                for f in files:
+                    filepath = os.path.join(root, f)
+                    if _skip_file(
+                        filepath,
+                        ignore_list,
+                        ignore_list_re,
+                        ignore_list_paths_re,
+                    ):
+                        continue
+                    if not f.endswith(".py"):
+                        continue
+                    moddesc, error = _discover_module(filepath)
+                    if error:
+                        errors.append(error)
+                        continue
+                    if moddesc:
+                        result.append(moddesc)
+    return result, errors

pylint/lint/pylinter.py

@@ -39,7 +39,7 @@
     MSG_TYPES_LONG,
     MSG_TYPES_STATUS,
 )
-from pylint.lint.expand_modules import expand_modules
+from pylint.lint.expand_modules import discover_modules, expand_modules
 from pylint.lint.parallel import check_parallel
 from pylint.lint.report_functions import (
     report_messages_by_module_stats,
@@ -515,6 +515,13 @@ def make_options() -> Tuple[Tuple[str, OptionDict], ...]:
                     ),
                 },
             ),
+            (
+                "recursive",
+                {
+                    "action": "store_true",
+                    "help": ("Discover python files in file system subtree."),
+                },
+            ),
             (
                 "py-version",
                 {
@@ -1143,12 +1150,20 @@ def _iterate_file_descrs(self, files_or_modules) -> Iterator[FileItem]:
 
     def _expand_files(self, modules) -> List[ModuleDescriptionDict]:
         """get modules and errors from a list of modules and handle errors"""
-        result, errors = expand_modules(
-            modules,
-            self.config.black_list,
-            self.config.black_list_re,
-            self._ignore_paths,
-        )
+        if self.config.recursive:
+            result, errors = discover_modules(
+                modules,
+                self.config.black_list,
+                self.config.black_list_re,
+                self._ignore_paths,
+            )
+        else:
+            result, errors = expand_modules(
+                modules,
+                self.config.black_list,
+                self.config.black_list_re,
+                self._ignore_paths,
+            )
         for error in errors:
             message = modname = error["mod"]
             key = error["key"]