Stricter more organized normalized functional tests

pylint/testutils/functional/functional_test_normalizer.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+from typing import List, Union
+
+from pylint.message import MessageDefinition, MessageDefinitionStore
+
+
+class FunctionalTestNormalizer:
+
+    """Help normalize the functional tests based on the content of the MessageStore."""
+
+    def __init__(
+        self,
+        base_functional_directory: Union[Path, str],
+        msg_store: MessageDefinitionStore,
+    ):
+        self.base_functional_directory = Path(base_functional_directory)
+        self.message_store = msg_store
+
+    def __iter__(self):
+        yield from self.message_store.messages
+
+    def expected_directories(self, message: MessageDefinition) -> List[Path]:
+        """The normalized directory that should contain the functional test for a message"""
+        result = []
+        intermediate_dirs = (message.checker_name, message.symbol[0])
+        leaf_dirs = (message.symbol, message.symbol.replace("-", "_"))
+        for intermediate_dir in intermediate_dirs:
+            for leaf_dir in leaf_dirs:
+                result.append(
+                    self.base_functional_directory / intermediate_dir / leaf_dir
+                )
+        return result

tests/functional/a/arguments.py

@@ -58,13 +58,13 @@ def function_default_arg(one=1, two=2):
 function_1_arg(bob=4)  # [unexpected-keyword-arg,no-value-for-parameter]
 function_default_arg(1, 4, coin="hello")  # [unexpected-keyword-arg]
 
-function_default_arg(1, one=5)  # [redundant-keyword-arg]
 
 # Remaining tests are for coverage of correct names in messages.
 LAMBDA = lambda arg: 1
 
 LAMBDA()  # [no-value-for-parameter]
 
+
 def method_tests():
     """Method invocations."""
     demo = DemoClass()
@@ -135,13 +135,6 @@ def test(self):
 
 Test().lam() # [no-value-for-parameter]
 
-# Don't emit a redundant-keyword-arg for this example,
-# it's perfectly valid
-
-class Issue642(object):
-    attr = 0
-    def __str__(self):
-        return "{self.attr}".format(self=self)
 
 # These should not emit anything regarding the number of arguments,
 # since they have something invalid.

tests/functional/a/arguments.txt

@@ -8,8 +8,7 @@ too-many-function-args:53:0:53:41::Too many positional arguments for function ca
 no-value-for-parameter:58:0:58:21::No value for argument 'first_argument' in function call:UNDEFINED
 unexpected-keyword-arg:58:0:58:21::Unexpected keyword argument 'bob' in function call:UNDEFINED
 unexpected-keyword-arg:59:0:59:40::Unexpected keyword argument 'coin' in function call:UNDEFINED
-redundant-keyword-arg:61:0:61:30::Argument 'one' passed by position and keyword in function call:UNDEFINED
-no-value-for-parameter:66:0:66:8::No value for argument 'arg' in lambda call:UNDEFINED
+no-value-for-parameter:65:0:65:8::No value for argument 'arg' in lambda call:UNDEFINED
 no-value-for-parameter:71:4:71:24:method_tests:No value for argument 'arg' in staticmethod call:UNDEFINED
 no-value-for-parameter:72:4:72:29:method_tests:No value for argument 'arg' in staticmethod call:UNDEFINED
 no-value-for-parameter:74:4:74:23:method_tests:No value for argument 'arg' in classmethod call:UNDEFINED
@@ -28,12 +27,12 @@ unexpected-keyword-arg:122:8:122:29:TypeCheckConstructor.test:Unexpected keyword
 no-value-for-parameter:133:8:133:18:Test.test:No value for argument 'icon' in method call:UNDEFINED
 too-many-function-args:134:8:134:25:Test.test:Too many positional arguments for method call:UNDEFINED
 no-value-for-parameter:136:0:136:12::No value for argument 'icon' in method call:UNDEFINED
-no-value-for-parameter:163:4:163:29:no_context_but_redefined:No value for argument 'three' in function call:UNDEFINED
-no-value-for-parameter:163:4:163:29:no_context_but_redefined:No value for argument 'two' in function call:UNDEFINED
-no-value-for-parameter:166:4:166:22:no_context_one_elem:No value for argument 'three' in function call:UNDEFINED
-no-value-for-parameter:166:4:166:22:no_context_one_elem:No value for argument 'two' in function call:UNDEFINED
-unexpected-keyword-arg:202:23:202:56:namedtuple_replace_issue_1036:Unexpected keyword argument 'd' in method call:UNDEFINED
-unexpected-keyword-arg:202:23:202:56:namedtuple_replace_issue_1036:Unexpected keyword argument 'e' in method call:UNDEFINED
-no-value-for-parameter:215:0:215:24::No value for argument 'third' in function call:UNDEFINED
-no-value-for-parameter:216:0:216:30::No value for argument 'second' in function call:UNDEFINED
-unexpected-keyword-arg:217:0:217:43::Unexpected keyword argument 'fourth' in function call:UNDEFINED
+no-value-for-parameter:156:4:156:29:no_context_but_redefined:No value for argument 'three' in function call:UNDEFINED
+no-value-for-parameter:156:4:156:29:no_context_but_redefined:No value for argument 'two' in function call:UNDEFINED
+no-value-for-parameter:159:4:159:22:no_context_one_elem:No value for argument 'three' in function call:UNDEFINED
+no-value-for-parameter:159:4:159:22:no_context_one_elem:No value for argument 'two' in function call:UNDEFINED
+unexpected-keyword-arg:195:23:195:56:namedtuple_replace_issue_1036:Unexpected keyword argument 'd' in method call:UNDEFINED
+unexpected-keyword-arg:195:23:195:56:namedtuple_replace_issue_1036:Unexpected keyword argument 'e' in method call:UNDEFINED
+no-value-for-parameter:208:0:208:24::No value for argument 'third' in function call:UNDEFINED
+no-value-for-parameter:209:0:209:30::No value for argument 'second' in function call:UNDEFINED
+unexpected-keyword-arg:210:0:210:43::Unexpected keyword argument 'fourth' in function call:UNDEFINED

tests/functional/i/import_error.py

@@ -30,7 +30,7 @@
     pass
 
 
-from functional.s.syntax_error import toto  # [no-name-in-module,syntax-error]
+from functional.s.syntax_error.syntax_error import toto  # [no-name-in-module,syntax-error]
 
 
 # Don't emit `import-error` or `no-name-in-module`

tests/functional/i/import_error.txt

@@ -1,5 +1,5 @@
 import-error:3:0:3:22::Unable to import 'totally_missing':UNDEFINED
 import-error:21:4:21:26::Unable to import 'maybe_missing_2':UNDEFINED
-no-name-in-module:33:0:33:42::No name 'syntax_error' in module 'functional.s':UNDEFINED
-syntax-error:33:0:None:None::Cannot import 'functional.s.syntax_error' due to syntax error 'invalid syntax (<unknown>, line 1)':UNDEFINED
+no-name-in-module:33:0:33:55::No name 'syntax_error' in module 'functional.s.syntax_error':UNDEFINED
+syntax-error:33:0:None:None::Cannot import 'functional.s.syntax_error.syntax_error' due to syntax error 'invalid syntax (<unknown>, line 1)':UNDEFINED
 multiple-imports:78:0:78:15::Multiple imports on one line (foo, bar):UNDEFINED

tests/functional/i/invalid-envvar-default/invalid_envvar_default.py

@@ -0,0 +1,54 @@
+# pylint: disable=useless-return,missing-docstring
+from os import getenv
+
+
+def function_returning_list():
+    return []
+
+
+def function_returning_none():
+    return None
+
+
+def function_returning_string():
+    return "Result"
+
+
+def function_returning_bytes():
+    return b"Result"
+
+
+def deep_function_returning_string():
+    return function_returning_string()
+
+
+def deep_function_returning_bytes():
+    return function_returning_bytes()
+
+
+getenv("TEST", "value")
+getenv("TEST", [])  # [invalid-envvar-default]
+getenv("TEST", None)
+getenv("TEST", b"123")  # [invalid-envvar-default]
+getenv("TEST", function_returning_list())  # [invalid-envvar-default]
+getenv("TEST", function_returning_none())
+getenv("TEST", function_returning_string())
+getenv("TEST", function_returning_bytes())  # [invalid-envvar-default]
+
+getenv("TEST", default="value")
+getenv("TEST", default=[])  # [invalid-envvar-default]
+getenv("TEST", default=None)
+getenv("TEST", default=b"123")  # [invalid-envvar-default]
+getenv("TEST", default=function_returning_list())  # [invalid-envvar-default]
+getenv("TEST", default=function_returning_none())
+getenv("TEST", default=function_returning_string())
+getenv("TEST", default=function_returning_bytes())  # [invalid-envvar-default]
+
+getenv(key="TEST")
+getenv(key="TEST", default="value")
+getenv(key="TEST", default=b"value")  # [invalid-envvar-default]
+getenv(key="TEST", default=["Crap"])  # [invalid-envvar-default]
+getenv(key="TEST", default=function_returning_list())  # [invalid-envvar-default]
+getenv(key="TEST", default=function_returning_none())
+getenv(key="TEST", default=function_returning_string())
+getenv(key="TEST", default=function_returning_bytes())  # [invalid-envvar-default]

tests/functional/i/invalid-envvar-default/invalid_envvar_default.txt

@@ -0,0 +1,12 @@
+invalid-envvar-default:30:0:30:18::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
+invalid-envvar-default:32:0:32:22::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
+invalid-envvar-default:33:0:33:41::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
+invalid-envvar-default:36:0:36:42::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
+invalid-envvar-default:39:0:39:26::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
+invalid-envvar-default:41:0:41:30::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
+invalid-envvar-default:42:0:42:49::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
+invalid-envvar-default:45:0:45:50::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
+invalid-envvar-default:49:0:49:36::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
+invalid-envvar-default:50:0:50:36::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
+invalid-envvar-default:51:0:51:53::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
+invalid-envvar-default:54:0:54:54::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED

tests/functional/i/invalid/e/invalid_envvar_value.py → tests/functional/i/invalid-envvar-value/invalid_envvar_value.py

@@ -64,21 +64,3 @@ def deep_function_returning_bytes():
 getenv('TEST', function_returning_none())
 getenv('TEST', function_returning_string())
 getenv('TEST', function_returning_bytes())   # [invalid-envvar-default]
-
-getenv('TEST', default="value")
-getenv('TEST', default=[])  # [invalid-envvar-default]
-getenv('TEST', default=None)
-getenv('TEST', default=b"123")  # [invalid-envvar-default]
-getenv('TEST', default=function_returning_list())  # [invalid-envvar-default]
-getenv('TEST', default=function_returning_none())
-getenv('TEST', default=function_returning_string())
-getenv('TEST', default=function_returning_bytes())  # [invalid-envvar-default]
-
-getenv(key='TEST')
-getenv(key='TEST', default="value")
-getenv(key='TEST', default=b"value")  # [invalid-envvar-default]
-getenv(key='TEST', default=["Crap"])  # [invalid-envvar-default]
-getenv(key='TEST', default=function_returning_list())  # [invalid-envvar-default]
-getenv(key='TEST', default=function_returning_none())
-getenv(key='TEST', default=function_returning_string())
-getenv(key='TEST', default=function_returning_bytes())  # [invalid-envvar-default]

tests/functional/i/invalid/e/invalid_envvar_value.txt → tests/functional/i/invalid-envvar-value/invalid_envvar_value.txt

@@ -21,11 +21,3 @@ invalid-envvar-default:60:0:60:18::os.getenv default type is builtins.list. Expe
 invalid-envvar-default:62:0:62:22::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
 invalid-envvar-default:63:0:63:41::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
 invalid-envvar-default:66:0:66:42::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
-invalid-envvar-default:69:0:69:26::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
-invalid-envvar-default:71:0:71:30::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
-invalid-envvar-default:72:0:72:49::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
-invalid-envvar-default:75:0:75:50::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
-invalid-envvar-default:79:0:79:36::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED
-invalid-envvar-default:80:0:80:36::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
-invalid-envvar-default:81:0:81:53::os.getenv default type is builtins.list. Expected str or None.:UNDEFINED
-invalid-envvar-default:84:0:84:54::os.getenv default type is builtins.bytes. Expected str or None.:UNDEFINED

tests/functional/i/iterable_context.py

@@ -1,8 +1,8 @@
-"""
-Checks that primitive values are not used in an
-iterating/mapping context.
-"""
-# pylint: disable=missing-docstring,invalid-name,too-few-public-methods,no-init,no-self-use,import-error,unused-argument,bad-mcs-method-argument,wrong-import-position,no-else-return, useless-object-inheritance, unnecessary-comprehension,redundant-u-string-prefix
+"""Checks that primitive values are not used in an iterating/mapping context."""
+# pylint: disable=missing-docstring,invalid-name,too-few-public-methods,no-init,no-self-use,import-error,
+# pylint: disable=unused-argument,bad-mcs-method-argument,wrong-import-position,no-else-return
+# pylint: disable=useless-object-inheritance, unnecessary-comprehension,redundant-u-string-prefix
+
 from __future__ import print_function
 
 # primitives

tests/functional/r/redundant-keyword-arg/redundant_keyword_arg.py

@@ -0,0 +1,19 @@
+# pylint: disable=missing-docstring, too-few-public-methods,useless-object-inheritance, consider-using-f-string
+
+
+def function_default_arg(one=1, two=2):
+    """function with default value"""
+    return two, one
+
+
+function_default_arg(1, one=5)  # [redundant-keyword-arg]
+
+# Don't emit a redundant-keyword-arg for this example,
+# it's perfectly valid
+
+
+class Issue642(object):
+    attr = 0
+
+    def __str__(self):
+        return "{self.attr}".format(self=self)

tests/functional/r/redundant-keyword-arg/redundant_keyword_arg.txt

@@ -0,0 +1 @@
+redundant-keyword-arg:9:0:9:30::Argument 'one' passed by position and keyword in function call:UNDEFINED

tests/functional/t/too-many-ancestors/__init__.py

@@ -0,0 +1 @@
+# Required for ignored-parents option in too_many_ancestors_ignored_parents.py

tests/functional/t/too-many-ancestors/too_many_ancestors_ignored_parents.rc

@@ -0,0 +1,3 @@
+[DESIGN]
+max-parents=2
+ignored-parents=functional.t.too-many-ancestors.too_many_ancestors_ignored_parents.E

tests/test_functional.py

@@ -31,18 +31,18 @@
 from _pytest.recwarn import WarningsRecorder
 
 from pylint import testutils
+from pylint.lint import PyLinter
 from pylint.testutils import UPDATE_FILE, UPDATE_OPTION
 from pylint.testutils.functional import (
     FunctionalTestFile,
-    LintModuleOutputUpdate,
     get_functional_test_files_from_directory,
 )
+from pylint.testutils.functional.functional_test_normalizer import (
+    FunctionalTestNormalizer,
+)
+from pylint.testutils.functional.lint_module_output_update import LintModuleOutputUpdate
 from pylint.utils import HAS_ISORT_5
 
-# TODOs
-#  - implement exhaustivity tests
-
-
 FUNCTIONAL_DIR = Path(__file__).parent.resolve() / "functional"
 
 
@@ -57,6 +57,61 @@
     "future_unicode_literals",
     "anomalous_unicode_escape_py3",
 ]
+NO_TESTS_LIST = {
+    "bad-configuration-section",
+    "bad-plugin-value",
+    "fatal",
+    "astroid-error",
+    "parse-error",
+    "config-parse-error",
+    "raw-checker-failed",
+    "file-ignored",
+}
+NORMALIZE_LATER_MAYBE = {
+    "too-many-format-args",
+    "too-few-format-args",
+    "logging-unsupported-format",
+    "logging-format-truncated",
+    "suppressed-message",
+    "locally-disabled",
+    "no-value-for-parameter",
+    "too-many-function-args",
+    "bad-string-format-type",
+    "unexpected-keyword-arg",
+    "assignment-from-none",
+    "unsubscriptable-object",
+    "bad-str-strip-call",
+    "c-extension-no-member",
+    "bad-format-string-key",
+    "unused-format-string-key",
+    "bad-format-string",
+    "missing-format-argument-key",
+    "unused-format-string-argument",
+    "format-combined-specification",
+    "missing-format-attribute",
+    "invalid-format-index",
+    "truncated-format-string",
+    "format-needs-mapping",
+    "not-an-iterable",
+    "boolean-datetime",
+    "deprecated-argument",  # move unittest
+    "duplicate-string-formatting-argument",
+    "format-string-without-interpolation",
+}
+
+
+def test_functional_check_existing(linter: PyLinter):
+    normalizer = FunctionalTestNormalizer(FUNCTIONAL_DIR, linter.msgs_store)
+    SKIP = NO_TESTS_LIST
+    SKIP |= NORMALIZE_LATER_MAYBE
+    for message in normalizer:
+        if message.symbol in SKIP:
+            continue
+        msg = f"One of '{{}}' should exists to test '{message.symbol}'."
+        expected_directories = normalizer.expected_directories(message)
+        assert any(d.exists() for d in expected_directories), msg.format(
+            expected_directories
+        )
 
 
 @pytest.mark.parametrize("test_file", TESTS, ids=TESTS_NAMES)