Fix all type errors (#579)

* bump pyre version
* make sure CI-pyre uses working copy
* remove unused pyre suppressions
* suppress invalid decorations
* fix undefined attributes
* fix missing return annotations
* fix tuple concatenation issues
* add native stubs
* fix invalid typing of **kwargs in test_apply_type_annotations
* only install pyre on non-windows
* update test fixture to reflect changes in recent pyre versions
* suppress errors related to mismatched positions

.github/workflows/.pyre_configuration.template

@@ -8,5 +8,6 @@
     "search_path": [
         "stubs", "{{ .python_site_packages }}"
     ],
+    "workers": 3,
     "strict": true
 }

.github/workflows/build.yml

@@ -113,8 +113,10 @@ jobs:
             python_site_packages: ${{ steps.python-info.outputs.SITE_PACKAGES }}
     - name: Write Pyre Config
       run: echo '${{ steps.pyre_template.outputs.result }}' > .pyre_configuration
+    - name: Make sure Pyre uses the working copy
+      run: pip install -e .
     - run: pyre --version
-    - run: pyre check
+    - run: pyre -n check
     - run: python libcst/tests/test_pyre_integration.py
     - run: git diff --exit-code
 

.pyre_configuration.example

@@ -6,7 +6,8 @@
         "stubs"
     ],
     "exclude": [
-        ".*/\\.tox/.*"
+        ".*/\\.tox/.*",
+        ".*/native/.*"
     ],
     "strict": true
 }

libcst/_add_slots.py

@@ -34,9 +34,13 @@ def add_slots(cls: Type[_T]) -> Type[_T]:
         # by the removal of GenericMeta. We should just be able to use cls.__bases__ in the
         # future.
         bases = getattr(cls, "__orig_bases__", cls.__bases__)
+        # pyre-fixme[9]: cls has type `Type[Variable[_T]]`; used as `_T`.
+        # pyre-fixme[19]: Expected 0 positional arguments.
         cls = type(cls)(cls.__name__, bases, cls_dict)
     except TypeError:
         # We're in py3.7 and should use cls.__bases__
+        # pyre-fixme[9]: cls has type `Type[Variable[_T]]`; used as `_T`.
+        # pyre-fixme[19]: Expected 0 positional arguments.
         cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
     if qualname is not None:
         cls.__qualname__ = qualname

libcst/_nodes/base.py

@@ -423,7 +423,7 @@ def with_deep_changes(
             raise Exception("Logic error, cannot get a *Sentinel here!")
         return new_tree
 
-    def __eq__(self: _CSTNodeSelfT, other: _CSTNodeSelfT) -> bool:
+    def __eq__(self: _CSTNodeSelfT, other: object) -> bool:
         """
         CSTNodes are only treated as equal by identity. This matches the behavior of
         CPython's AST nodes.

libcst/_nodes/expression.py

@@ -1844,7 +1844,6 @@ def _validate_stars_sequence(self, vals: Sequence[Param], *, section: str) -> No
         if len(vals) == 0:
             return
         for val in vals:
-            # pyre-ignore Pyre seems to think val.star.__eq__ is not callable
             if isinstance(val.star, str) and val.star != "":
                 raise CSTValidationError(
                     f"Expecting a star prefix of '' for {section} Param."
@@ -1864,6 +1863,8 @@ def _validate_kwonly_star(self) -> None:
 
     def _validate_defaults(self) -> None:
         seen_default = False
+        # pyre-fixme[60]: Concatenation not yet support for multiple variadic
+        #  tuples: `*self.posonly_params, *self.params`.
         for param in (*self.posonly_params, *self.params):
             if param.default:
                 # Mark that we've moved onto defaults
@@ -1891,7 +1892,6 @@ def _validate_stars(self) -> None:
         if (
             isinstance(star_arg, Param)
             and isinstance(star_arg.star, str)
-            # pyre-ignore Pyre seems to think star_kwarg.star.__eq__ is not callable
             and star_arg.star != "*"
         ):
             raise CSTValidationError(
@@ -1903,7 +1903,6 @@ def _validate_stars(self) -> None:
         if (
             star_kwarg is not None
             and isinstance(star_kwarg.star, str)
-            # pyre-ignore Pyre seems to think star_kwarg.star.__eq__ is not callable
             and star_kwarg.star != "**"
         ):
             raise CSTValidationError(
@@ -2194,9 +2193,7 @@ class _BaseExpressionWithArgs(BaseExpression, ABC):
     #: Sequence of arguments that will be passed to the function call.
     args: Sequence[Arg] = ()
 
-    def _check_kwargs_or_keywords(
-        self, arg: Arg
-    ) -> Optional[Callable[[Arg], Callable]]:
+    def _check_kwargs_or_keywords(self, arg: Arg) -> None:
         """
         Validates that we only have a mix of "keyword=arg" and "**arg" expansion.
         """
@@ -2220,7 +2217,7 @@ def _check_kwargs_or_keywords(
 
     def _check_starred_or_keywords(
         self, arg: Arg
-    ) -> Optional[Callable[[Arg], Callable]]:
+    ) -> Optional[Callable[[Arg], Callable[[Arg], None]]]:
         """
         Validates that we only have a mix of "*arg" expansion and "keyword=arg".
         """
@@ -2243,7 +2240,9 @@ def _check_starred_or_keywords(
                 "Cannot have positional argument after keyword argument."
             )
 
-    def _check_positional(self, arg: Arg) -> Optional[Callable[[Arg], Callable]]:
+    def _check_positional(
+        self, arg: Arg
+    ) -> Optional[Callable[[Arg], Callable[[Arg], Callable[[Arg], None]]]]:
         """
         Validates that we only have a mix of positional args and "*arg" expansion.
         """
@@ -2267,6 +2266,8 @@ def _check_positional(self, arg: Arg) -> Optional[Callable[[Arg], Callable]]:
             # Valid, allowed to have positional arguments here
             return None
 
+    # pyre-fixme[30]: Pyre gave up inferring some types - function `_validate` was
+    #  too complex.
     def _validate(self) -> None:
         # Validate any super-class stuff, whatever it may be.
         super()._validate()

libcst/_nodes/statement.py

@@ -2308,7 +2308,6 @@ def _validate(self) -> None:
                 not isinstance(whitespace_before_from, MaybeSentinel)
                 and whitespace_before_from.empty
             )
-            # pyre-ignore Pyre thinks exc is Optional
             if has_no_gap and not exc._safe_to_use_with_word_operator(
                 ExpressionPosition.LEFT
             ):
@@ -2590,6 +2589,7 @@ class MatchPattern(_BaseParenthesizedNode, ABC):
 
 @add_slots
 @dataclass(frozen=True)
+# pyre-fixme[13]: Attribute `body` is never initialized.
 class Match(BaseCompoundStatement):
     """
     A ``match`` statement.
@@ -2731,9 +2731,13 @@ def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "CSTNode":
                 self, "whitespace_after_case", self.whitespace_after_case, visitor
             ),
             pattern=visit_required(self, "pattern", self.pattern, visitor),
+            # pyre-fixme[6]: Expected `SimpleWhitespace` for 4th param but got
+            #  `Optional[SimpleWhitespace]`.
             whitespace_before_if=visit_optional(
                 self, "whitespace_before_if", self.whitespace_before_if, visitor
             ),
+            # pyre-fixme[6]: Expected `SimpleWhitespace` for 5th param but got
+            #  `Optional[SimpleWhitespace]`.
             whitespace_after_if=visit_optional(
                 self, "whitespace_after_if", self.whitespace_after_if, visitor
             ),
@@ -2817,6 +2821,7 @@ def lpar(self) -> Sequence[LeftParen]:
 
     @lpar.setter
     def lpar(self, value: Sequence[LeftParen]) -> None:
+        # pyre-fixme[41]: Cannot reassign final attribute `lpar`.
         self.value.lpar = value
 
 

libcst/_nodes/tests/test_assign.py

@@ -422,7 +422,7 @@ def test_valid(self, **kwargs: Any) -> None:
                             operator=cst.Add(),
                             right=cst.Integer("1"),
                         ),
-                        operator=cst.Add(),
+                        operator=cst.AddAssign(),
                         value=cst.Name("y"),
                     )
                 ),

libcst/_nodes/tests/test_match.py

@@ -3,19 +3,20 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any
+from typing import Any, Callable, Optional
 
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
 from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
-parser = parse_statement if is_native() else None
+parser: Optional[Callable[[str], cst.CSTNode]] = (
+    parse_statement if is_native() else None
+)
 
 
 class MatchTest(CSTNodeTest):
-    # pyre-fixme[56]: Invalid decoration - Pyre was not able to infer the type
     @data_provider(
         (
             # Values and singletons

libcst/_nodes/tests/test_try.py

@@ -3,7 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any
+from typing import Any, Callable, Optional
 
 import libcst as cst
 from libcst import parse_statement
@@ -12,7 +12,9 @@
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
-native_parse_statement = parse_statement if is_native() else None
+native_parse_statement: Optional[Callable[[str], cst.CSTNode]] = (
+    parse_statement if is_native() else None
+)
 
 
 class TryTest(CSTNodeTest):

libcst/_parser/base_parser.py

@@ -204,7 +204,6 @@ def _add_token(self, token: _TokenT) -> None:
                 )
 
         # Logically, `plan` is always defined, but pyre can't reasonably determine that.
-        # pyre-fixme[18]: Global name `plan` is undefined.
         stack[-1].dfa = plan.next_dfa
 
         for push in plan.dfa_pushes:

libcst/_parser/conversions/expression.py

@@ -616,6 +616,8 @@ def convert_atom_expr_trailer(
                     config, trailer.whitespace_before
                 ),
                 lbracket=trailer.lbracket,
+                # pyre-fixme[6]: Expected `Sequence[SubscriptElement]` for 4th param
+                #  but got `Union[typing.Sequence[SubscriptElement], Index, Slice]`.
                 slice=trailer.slice,
                 rbracket=trailer.rbracket,
             )
@@ -643,6 +645,8 @@ def convert_atom_expr_trailer(
                     config, trailer.lpar.whitespace_before
                 ),
                 whitespace_before_args=trailer.lpar.value.whitespace_after,
+                # pyre-fixme[6]: Expected `Sequence[Arg]` for 4th param but got
+                #  `Tuple[object, ...]`.
                 args=tuple(args),
             )
         else:
@@ -778,13 +782,10 @@ def convert_subscript(
             first_colon=Colon(
                 whitespace_before=parse_parenthesizable_whitespace(
                     config,
-                    # pyre-fixme[16]: Optional type has no attribute
-                    #  `whitespace_before`.
                     first_colon.whitespace_before,
                 ),
                 whitespace_after=parse_parenthesizable_whitespace(
                     config,
-                    # pyre-fixme[16]: Optional type has no attribute `whitespace_after`.
                     first_colon.whitespace_after,
                 ),
             ),
@@ -948,7 +949,10 @@ def convert_atom_parens(
         inner_atom = atoms[0].value
         return WithLeadingWhitespace(
             inner_atom.with_changes(
-                lpar=(lpar, *inner_atom.lpar), rpar=(*inner_atom.rpar, rpar)
+                # pyre-fixme[60]: Expected to unpack an iterable, but got `unknown`.
+                lpar=(lpar, *inner_atom.lpar),
+                # pyre-fixme[60]: Expected to unpack an iterable, but got `unknown`.
+                rpar=(*inner_atom.rpar, rpar),
             ),
             lpar_tok.whitespace_before,
         )
@@ -1237,7 +1241,6 @@ def _convert_sequencelike(
 
     # lpar/rpar are the responsibility of our parent
     return WithLeadingWhitespace(
-        # pyre-ignore[29]: `Union[Type[List], Type[Set], Type[Tuple]]` is not a function.
         sequence_type(elements, lpar=(), rpar=()),
         children[0].whitespace_before,
     )

libcst/_parser/conversions/params.py

@@ -140,7 +140,6 @@ def add_param(
                     "Cannot have multiple slash ('/') markers in a single argument "
                     + "list."
                 )
-        # pyre-ignore Pyre seems to think param.star.__eq__ is not callable
         elif isinstance(param.star, str) and param.star == "" and param.default is None:
             # Can only add this if we're in the params or kwonly_params section
             if current_param is params and not seen_default:
@@ -156,7 +155,6 @@ def add_param(
                 )
         elif (
             isinstance(param.star, str)
-            # pyre-ignore Pyre seems to think param.star.__eq__ is not callable
             and param.star == ""
             and param.default is not None
         ):
@@ -172,10 +170,7 @@ def add_param(
                 # This should be unreachable, the grammar already disallows it.
                 raise Exception("Cannot have any arguments after a kwargs expansion.")
         elif (
-            isinstance(param.star, str)
-            # pyre-ignore Pyre seems to think param.star.__eq__ is not callable
-            and param.star == "*"
-            and param.default is None
+            isinstance(param.star, str) and param.star == "*" and param.default is None
         ):
             # Can only add this if we're in params, since we only allow one of
             # "*" or "*param".
@@ -191,10 +186,7 @@ def add_param(
                     + "argument expansion."
                 )
         elif (
-            isinstance(param.star, str)
-            # pyre-ignore Pyre seems to think param.star.__eq__ is not callable
-            and param.star == "**"
-            and param.default is None
+            isinstance(param.star, str) and param.star == "**" and param.default is None
         ):
             # Can add this in all cases where we don't have a star_kwarg
             # yet.

libcst/_parser/conversions/statement.py

@@ -1300,6 +1300,10 @@ def convert_decorated(config: ParserConfig, children: Sequence[Any]) -> Any:
     # Now, modify the original function or class to add the decorators.
     return class_or_func.with_changes(
         leading_lines=leading_lines,
+        # pyre-fixme[60]: Concatenation not yet support for multiple variadic
+        #  tuples: `*class_or_func.leading_lines,
+        #  *class_or_func.lines_after_decorators`.
+        # pyre-fixme[60]: Expected to unpack an iterable, but got `unknown`.
         lines_after_decorators=(
             *class_or_func.leading_lines,
             *class_or_func.lines_after_decorators,

libcst/_parser/parso/python/tokenize.py

@@ -580,13 +580,15 @@ def dedent_if_necessary(start):
             if not pseudomatch:  # scan for tokens
                 match = token_collection.whitespace.match(line, pos)
                 if pos == 0:
+                    # pyre-fixme[16]: `Optional` has no attribute `end`.
                     yield from dedent_if_necessary(match.end())
                 pos = match.end()
                 new_line = False
                 yield PythonToken(
                     ERRORTOKEN,
                     line[pos],
                     (lnum, pos),
+                    # pyre-fixme[16]: `Optional` has no attribute `group`.
                     additional_prefix + match.group(0),
                 )
                 additional_prefix = ""
@@ -935,6 +937,7 @@ def dedent_if_necessary(start):
             if not pseudomatch:  # scan for tokens
                 match = token_collection.whitespace.match(line, pos)
                 if pos == 0:
+                    # pyre-fixme[16]: `Optional` has no attribute `end`.
                     for t in dedent_if_necessary(match.end()):
                         yield t
                 pos = match.end()
@@ -943,6 +946,7 @@ def dedent_if_necessary(start):
                     ERRORTOKEN,
                     line[pos],
                     (lnum, pos),
+                    # pyre-fixme[16]: `Optional` has no attribute `group`.
                     additional_prefix + match.group(0),
                 )
                 additional_prefix = ""

libcst/_parser/production_decorator.py

@@ -21,6 +21,10 @@ def with_production(
     *,
     version: Optional[str] = None,
     future: Optional[str] = None,
+    # pyre-fixme[34]: `Variable[_NonterminalConversionT (bound to
+    #  typing.Callable[[libcst_native.parser_config.ParserConfig,
+    #  typing.Sequence[typing.Any]], typing.Any])]` isn't present in the function's
+    #  parameters.
 ) -> Callable[[_NonterminalConversionT], _NonterminalConversionT]:
     """
     Attaches a bit of grammar to a conversion function. The parser extracts all of these

libcst/_parser/types/config.py

@@ -23,7 +23,6 @@
 except ImportError:
     from libcst._parser.types import py_config as config_mod
 
-    # pyre-fixme[9]: This is a small implementation difference between native and python
     MockWhitespaceParserConfig = config_mod.MockWhitespaceParserConfig
 
 BaseWhitespaceParserConfig = config_mod.BaseWhitespaceParserConfig

libcst/_parser/whitespace_parser.py

@@ -23,11 +23,7 @@
 except ImportError:
     from libcst._parser import py_whitespace_parser as mod
 
-# pyre-fixme[5]: There's no sane way to type these re-exports
 parse_simple_whitespace = mod.parse_simple_whitespace
-# pyre-fixme[5]: There's no sane way to type these re-exports
 parse_empty_lines = mod.parse_empty_lines
-# pyre-fixme[5]: There's no sane way to type these re-exports
 parse_trailing_whitespace = mod.parse_trailing_whitespace
-# pyre-fixme[5]: There's no sane way to type these re-exports
 parse_parenthesizable_whitespace = mod.parse_parenthesizable_whitespace