Add type annotations to the python codebase (#283)

* Add mypy config

* Add some type annotations

* Add types to python parser

* Fix types

* Add more type declarations

* Add missing type annotations in `stream/*.py`

* WIP start typing ast_builder and ast_node

* Finish typing ast_builder and ast_node

* Add type annotations for `dialect.py`

* Add type annotations for `errors.py`

* Add type annotations for `gherkin_line.py`

* Remove useless imports

* Remove entry added by mistake

* Fix annotations for toke_formatter_builder

* Fix annotations for token_matcher

* Add type annotations for token_scanner

* Add missing types to token_matcher_markdown

* Add `py.typed` file marker

* Sync parser.py

* Fully type-annotate the `parser.py`

* Location.column is not always present

* Fix type error

* Fix python3.8 compatibility

* Add changelog entry

* Remove debugging statement committed by mistake

* Fix another python3 compatibility issue

* Undo change slipped in by mistake

* `Container` -> `Envelope`

* Remove unnecessary cast

.gitignore

@@ -2,4 +2,4 @@
 .idea/
 .valid-events
 
-*.iml
+*.iml

CHANGELOG.md

@@ -9,6 +9,7 @@ This document is formatted according to the principles of [Keep A CHANGELOG](htt
 ## [Unreleased]
 ### Added
 - [PHP, Java, Ruby, JavaScript] update dependency messages up to v26
+- [Python] Added type annotations ([#283](https://github.com/cucumber/gherkin/pull/283))
 
 ### Changed
 - [.NET] Drop unsupported frameworks. Now supported target frameworks are .NET 8, .NET Standard 2.0 ([#265](https://github.com/cucumber/gherkin/pull/265))

python/gherkin-python.razor

@@ -26,13 +26,23 @@
 @helper MatchToken(TokenType tokenType)
 {<text>match_@(tokenType)(context, token)</text>}
 # This file is generated. Do not edit! Edit gherkin-python.razor instead.
-import sys
+from __future__ import annotations
+
 from collections import deque
+from collections.abc import Callable
+from typing import cast, TypeVar
+
 from .ast_builder import AstBuilder
+from .token import Token
 from .token_matcher import TokenMatcher
 from .token_scanner import TokenScanner
+from .parser_types import GherkinDocument
 from .errors import UnexpectedEOFException, UnexpectedTokenException, ParserException, CompositeParserException
 
+_T = TypeVar('_T')
+_U = TypeVar('_U')
+_V = TypeVar('_V')
+
 RULE_TYPE = [
     'None',
     @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule))
@@ -42,19 +52,29 @@ RULE_TYPE = [
 
 
 class ParserContext:
-    def __init__(self, token_scanner, token_matcher, token_queue, errors):
+    def __init__(
+        self,
+        token_scanner: TokenScanner,
+        token_matcher: TokenMatcher,
+        token_queue: deque[Token],
+        errors: list[ParserException],
+    ) -> None:
         self.token_scanner = token_scanner
         self.token_matcher = token_matcher
         self.token_queue = token_queue
         self.errors = errors
 
 
 class @(Model.ParserClassName):
-    def __init__(self, ast_builder=None):
+    def __init__(self, ast_builder: AstBuilder | None = None) -> None:
         self.ast_builder = ast_builder if ast_builder is not None else AstBuilder()
         self.stop_at_first_error = False
 
-    def parse(self, token_scanner_or_str, token_matcher=None):
+    def parse(
+        self,
+        token_scanner_or_str: TokenScanner | str,
+        token_matcher: TokenMatcher | None = None,
+    ) -> GherkinDocument:
         token_scanner = TokenScanner(token_scanner_or_str) if isinstance(token_scanner_or_str, str) else token_scanner_or_str
         self.ast_builder.reset()
         if token_matcher is None:
@@ -80,34 +100,34 @@ class @(Model.ParserClassName):
         if context.errors:
             raise CompositeParserException(context.errors)
 
-        return self.get_result()
+        return cast(GherkinDocument, self.get_result())
 
-    def build(self, context, token):
+    def build(self, context: ParserContext, token: Token) -> None:
         self.handle_ast_error(context, token, self.ast_builder.build)
 
-    def add_error(self, context, error):
+    def add_error(self, context: ParserContext, error: ParserException) -> None:
         if str(error) not in (str(e) for e in context.errors):
             context.errors.append(error)
             if len(context.errors) > 10:
                 raise CompositeParserException(context.errors)
 
-    def start_rule(self, context, rule_type):
+    def start_rule(self, context: ParserContext, rule_type: str) -> None:
         self.handle_ast_error(context, rule_type, self.ast_builder.start_rule)
 
-    def end_rule(self, context, rule_type):
+    def end_rule(self, context: ParserContext, rule_type: str) -> None:
         self.handle_ast_error(context, rule_type, self.ast_builder.end_rule)
 
-    def get_result(self):
+    def get_result(self) -> object:
         return self.ast_builder.get_result()
 
-    def read_token(self, context):
+    def read_token(self, context: ParserContext) -> Token:
         if context.token_queue:
             return context.token_queue.popleft()
         else:
             return context.token_scanner.read()
 @foreach(var rule in Model.RuleSet.TokenRules)
 {<text>
-    def match_@(rule.Name.Replace("#", ""))(self, context, token):
+    def match_@(rule.Name.Replace("#", ""))(self, context: ParserContext, token: Token) -> bool:
         @if (rule.Name != "#EOF")
         {
         @:if token.eof():
@@ -116,8 +136,8 @@ class @(Model.ParserClassName):
         return self.handle_external_error(context, False, token, context.token_matcher.match_@(rule.Name.Replace("#", "")))</text>
 }
 
-    def match_token(self, state, token, context):
-        state_map = {
+    def match_token(self, state: int, token: Token, context: ParserContext) -> int:
+        state_map: dict[int, Callable[[Token, ParserContext], int]]= {
         @foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
         {
         @:    @state.Id: self.match_token_at_@(state.Id),
@@ -130,7 +150,7 @@ class @(Model.ParserClassName):
 @foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
 {<text>
     # @Raw(state.Comment)
-    def match_token_at_@(state.Id)(self, token, context):
+    def match_token_at_@(state.Id)(self, token: Token, context: ParserContext) -> int:
         @foreach(var transition in state.Transitions)
         {
         @:if self.@MatchToken(transition.TokenType):
@@ -150,7 +170,7 @@ class @(Model.ParserClassName):
 @foreach(var lookAheadHint in Model.RuleSet.LookAheadHints)
 {
 <text>
-    def lookahead_@(lookAheadHint.Id)(self, context, currentToken):
+    def lookahead_@(lookAheadHint.Id)(self, context: ParserContext, currentToken: Token) -> bool:
         currentToken.detach
         token = None
         queue = []
@@ -174,10 +194,21 @@ class @(Model.ParserClassName):
 
     # private
 
-    def handle_ast_error(self, context, argument, action):
+    def handle_ast_error(
+        self,
+        context: ParserContext,
+        argument: _T,
+        action: Callable[[_T], object],
+    ) -> None:
         self.handle_external_error(context, True, argument, action)
 
-    def handle_external_error(self, context, default_value, argument, action):
+    def handle_external_error(
+        self,
+        context: ParserContext,
+        default_value: _U,
+        argument: _T,
+        action: Callable[[_T], _V],
+    ) -> _V | _U:
         if self.stop_at_first_error:
             return action(argument)
 

python/gherkin/__main__.py

@@ -15,7 +15,13 @@
 (options, args) = parser.parse_args()
 
 source_events = SourceEvents(args)
-gherkin_events = GherkinEvents(options)
+gherkin_events = GherkinEvents(
+    GherkinEvents.Options(
+        print_source=options.print_source,
+        print_ast=options.print_ast,
+        print_pickles=options.print_pickles,
+    )
+)
 
 for source_event in source_events.enum():
     for event in gherkin_events.enum(source_event):