fix: Token now pattern matches correctly

lark/lexer.py

@@ -5,9 +5,10 @@
 from contextlib import suppress
 from typing import (
     TypeVar, Type, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
-    Pattern as REPattern, ClassVar, TYPE_CHECKING
+    Pattern as REPattern, ClassVar, TYPE_CHECKING, overload
 )
 from types import ModuleType
+import warnings
 if TYPE_CHECKING:
     from .common import LexerConf
 
@@ -147,37 +148,106 @@ class Token(str):
     """
     __slots__ = ('type', 'start_pos', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos')
 
+    __match_args__ = ('type', 'value')
+
     type: str
-    start_pos: int
+    start_pos: Optional[int]
     value: Any
-    line: int
-    column: int
-    end_line: int
-    end_column: int
-    end_pos: int
-
-    def __new__(cls, type_, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None):
+    line: Optional[int]
+    column: Optional[int]
+    end_line: Optional[int]
+    end_column: Optional[int]
+    end_pos: Optional[int]
+
+
+    @overload
+    def __new__(
+        cls,
+        type: str,
+        value: Any,
+        start_pos: Optional[int]=None,
+        line: Optional[int]=None,
+        column: Optional[int]=None,
+        end_line: Optional[int]=None,
+        end_column: Optional[int]=None,
+        end_pos: Optional[int]=None
+    ) -> 'Token':
+        ...
+
+    @overload
+    def __new__(
+        cls,
+        type_: str,
+        value: Any,
+        start_pos: Optional[int]=None,
+        line: Optional[int]=None,
+        column: Optional[int]=None,
+        end_line: Optional[int]=None,
+        end_column: Optional[int]=None,
+        end_pos: Optional[int]=None
+    ) -> 'Token':        ...
+
+    def __new__(cls, *args, **kwargs):
+        if "type_" in kwargs:
+            warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning)
+
+            if "type" in kwargs:
+                raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.")
+            kwargs["type"] = kwargs.pop("type_")
+
+        return cls._future_new(*args, **kwargs)
+
+
+    @classmethod
+    def _future_new(cls, type, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None):
         inst = super(Token, cls).__new__(cls, value)
-        inst.type = type_
+
+        inst.type = type
         inst.start_pos = start_pos
         inst.value = value
         inst.line = line
         inst.column = column
         inst.end_line = end_line
         inst.end_column = end_column
         inst.end_pos = end_pos
-        return inst
+        return inst 
 
+    @overload
+    def update(self, type: Optional[str]=None, value: Optional[Any]=None) -> 'Token':
+        ...
+
+    @overload
     def update(self, type_: Optional[str]=None, value: Optional[Any]=None) -> 'Token':
+        ...
+
+    def update(self, *args, **kwargs):
+        if "type_" in kwargs:
+            warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning)
+
+            if "type" in kwargs:
+                raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.")
+            kwargs["type"] = kwargs.pop("type_")
+
+        return self._future_update(*args, **kwargs)
+
+    def _future_update(self, type: Optional[str]=None, value: Optional[Any]=None) -> 'Token':
         return Token.new_borrow_pos(
-            type_ if type_ is not None else self.type,
+            type if type is not None else self.type,
             value if value is not None else self.value,
             self
         )
 
     @classmethod
     def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') -> _T:
         return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos)
+
+    @property
+    def type_(self) -> str:
+        return self.type
+
+    @type_.setter
+    def type_(self, value: str) -> None:
+        self.type = value
 
     def __reduce__(self):
         return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column))

lark/tools/standalone.py

@@ -29,7 +29,7 @@
 from types import ModuleType
 from typing import (
     TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
-    Union, Iterable, IO, TYPE_CHECKING,
+    Union, Iterable, IO, TYPE_CHECKING, overload,
     Pattern as REPattern, ClassVar, Set, Mapping
 )
 ###}

tests/__main__.py

@@ -2,6 +2,7 @@
 
 import unittest
 import logging
+import sys
 from lark import logger
 
 from .test_trees import TestTrees
@@ -26,6 +27,9 @@
 
 from .test_parser import *  # We define __all__ to list which TestSuites to run
 
+if sys.version_info >= (3, 10):
+    from .test_pattern_matching import TestPatternMatching
+
 logger.setLevel(logging.INFO)
 
 if __name__ == '__main__':

tests/test_pattern_matching.py

@@ -0,0 +1,52 @@
+from unittest import TestCase, main
+
+from lark import Token
+
+
+class TestPatternMatching(TestCase):
+    token = Token('A', 'a')
+
+    def setUp(self):
+        pass
+
+    def test_matches_with_string(self):
+        match self.token:
+            case 'a':
+                pass
+            case _:
+                assert False
+
+    def test_matches_with_str_positional_arg(self):
+        match self.token:
+            case str('a'):
+                pass
+            case _:
+                assert False
+
+    def test_matches_with_token_positional_arg(self):
+        match self.token:
+            case Token('a'):
+                assert False
+            case Token('A'):
+                pass
+            case _:
+                assert False
+
+    def test_matches_with_token_kwarg_type(self):
+        match self.token:
+            case Token(type='A'):
+                pass
+            case _:
+                assert False
+
+    def test_matches_with_token_kwarg_type_(self):
+        match self.token:
+            case Token(type_='A'):
+                pass
+            case _:
+                assert False
+
+
+
+if __name__ == '__main__':
+    main()