[Python APIView] Add error diagnostic if pylint parsing fails rather …

…than fail APIView (and thus CI) (#3884)

* Make pylint parsing failure create an APIView error instead of failing the tool (and thus CI).

* Closes #3842.

packages/python-packages/api-stub-generator/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Release History
 
+## Version 0.3.4 (Unreleased)
+Fixed issue so that APIView is still generated even if pylint parsing fails.
+
 ## Version 0.3.3 (2022-08-03)
 Fixed issue in module order to get consistent order
 

packages/python-packages/api-stub-generator/apistub/_apiview.py

@@ -54,6 +54,7 @@ def __init__(self, *, pkg_name="", namespace = "", metadata_map=None, source_url
         self.metadata_map = metadata_map or MetadataMap("")
         self.add_token(Token("", TokenKind.SkipDiffRangeStart))
         self.add_literal(HEADER_TEXT)
+        self.add_line_marker("GLOBAL")
         if source_url:
             self.set_blank_lines(1)
             self.add_literal(f"# Source URL: {source_url}")

packages/python-packages/api-stub-generator/apistub/_stub_generator.py

@@ -195,6 +195,7 @@ def _generate_tokens(self, pkg_root_path, package_name, namespace, *, source_url
         # Import ModuleNode.
         # Importing it globally can cause circular dependency since it needs NodeIndex that is defined in this file
         from apistub.nodes._module_node import ModuleNode
+        from apistub.nodes import PylintParser
 
         self.module_dict = {}
         mapping = MetadataMap(pkg_root_path, mapping_path=self.mapping_path)
@@ -222,6 +223,11 @@ def _generate_tokens(self, pkg_root_path, package_name, namespace, *, source_url
         navigation.tags = NavigationTag(Kind.type_package)
         apiview.add_navigation(navigation)
 
+        # Generate any global diagnostics
+        global_errors = PylintParser.get_items("GLOBAL")
+        for g in global_errors or []:
+            g.generate_tokens(apiview, "GLOBAL")
+
         # Generate tokens
         modules = self.module_dict.keys()
         for m in modules:

packages/python-packages/api-stub-generator/apistub/_version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-VERSION = "0.3.3"
+VERSION = "0.3.4"

packages/python-packages/api-stub-generator/apistub/nodes/_pylint_parser.py

@@ -25,9 +25,9 @@ def __init__(self, pkg_name, **kwargs):
         self.message = kwargs.pop('message', None)
         self.message_id = kwargs.pop('message-id', None)
         self.help_link = None
-        if self.path.startswith(pkg_name):
+        if self.path and self.path.startswith(pkg_name):
             self.path = self.path[(len(f"{pkg_name}\\\\") - 1):]
-        code = self.symbol[0]
+        code = self.symbol[0] if self.symbol else ""
         self.level = DiagnosticLevel.ERROR if code in "EF" else DiagnosticLevel.WARNING
         self.owner = None
         self._parse_help_link()
@@ -66,32 +66,40 @@ def parse(cls, path):
             plugin_failed = any([x["symbol"] == "bad-plugin-value" for x in json_items])
             if plugin_failed:
                 logging.error(f"Unable to load pylint_guidelines_checker. Check that it is installed.")
-        except json.JSONDecodeError as err:
+            cls.items = [PylintError(pkg_name, **x) for x in json_items if x["message-id"][1:3] == PylintParser.AZURE_CHECKER_CODE]
+        except Exception as err:
+            from apistub import DiagnosticLevel
             logging.error(f"Error decoding pylint output:\n{stderr_str}")
             logging.error(f"Error content:\n{err}")
-            logging.error(f"==STDOUT==\n{stdout_lines}")
-            raise err
-        cls.items = [PylintError(pkg_name, **x) for x in json_items if x["message-id"][1:3] == PylintParser.AZURE_CHECKER_CODE]
+            logging.error(f"==STDOUT==\n{stdout_lines}\n==END STDOUT==")
+            # instead of raising an error, we will log a pylint error
+            error = PylintError(pkg_name)
+            error.level = DiagnosticLevel.ERROR
+            error.owner = "GLOBAL"
+            error.symbol = "apiview-pylint-parse-error"
+            error.message = "Failure parsing pylint output. Please post an issue in the `Azure/azure-sdk-tools` repository."
+            cls.items = [error]
 
     @classmethod
     def match_items(cls, obj) -> None:
         try:
             source_file = inspect.getsourcefile(obj)
             (source_lines, start_line) = inspect.getsourcelines(obj)
             end_line = start_line + len(source_lines) - 1
-        except Exception as err:
+        except Exception:
             return
         for item in cls.items:
             item_path = item.path
-            if source_file.endswith(item_path):
+            if item_path and source_file.endswith(item_path):
                 # nested items will overwrite the ownership of their
                 # containing parent.
                 if item.line >= start_line and item.line <= end_line:
                     item.owner = str(obj)
 
     @classmethod
     def get_items(cls, obj) -> List[PylintError]:
-        return [x for x in cls.items if x.owner == str(obj)]
+        items = [x for x in cls.items if x.owner == str(obj)]
+        return items
 
     @classmethod
     def get_unclaimed(cls) -> List[PylintError]:

packages/python-packages/api-stub-generator/apistubgen.py

@@ -1,5 +1,5 @@
-import logging
 import sys
+import traceback
 
 from apistub import console_entry_point
 
@@ -8,5 +8,6 @@
         console_entry_point()
         sys.exit(0)
     except Exception as err:
-        logging.error(err)
+        exc_type, exc_val, exc_tb = sys.exc_info()
+        traceback.print_exception(exc_type, exc_val, exc_tb, file=sys.stderr)
         sys.exit(1)

tools/pylint-extensions/pylint-guidelines-checker/pylint_guidelines_checker.py

@@ -1983,7 +1983,7 @@ def register(linter):
 
 
     # disabled by default, use pylint --enable=check-docstrings if you want to use it
-    # linter.register_checker(CheckDocstringParameters(linter))
+    linter.register_checker(CheckDocstringParameters(linter))
 
     # Rules are disabled until false positive rate improved
     # linter.register_checker(CheckForPolicyUse(linter))