Merge pull request #1252 from MegaIng/fix-1154

Fix 1154

lark/lexer.py

@@ -63,7 +63,7 @@ def _get_flags(self, value):
 
 
 class PatternStr(Pattern):
- __serialize_fields__ = 'value', 'flags'
+ __serialize_fields__ = 'value', 'flags', 'raw'
 
  type: ClassVar[str] = "str"
 
@@ -80,7 +80,7 @@ def max_width(self) -> int:
 
 
 class PatternRE(Pattern):
- __serialize_fields__ = 'value', 'flags', '_width'
+ __serialize_fields__ = 'value', 'flags', 'raw', '_width'
 
  type: ClassVar[str] = "re"
 

tests/test_cache.py

@@ -3,7 +3,7 @@
 import logging
 from unittest import TestCase, main, skipIf
 
-from lark import Lark, Tree, Transformer
+from lark import Lark, Tree, Transformer, UnexpectedInput
 from lark.lexer import Lexer, Token
 import lark.lark as lark_module
 
@@ -168,7 +168,26 @@ def test_recursive_pattern(self):
  self.assertCountEqual(cm.output, ["ERROR:lark:dummy message"])
 
 
+ def test_error_message(self):
+ # Checks that error message generation works
+ # This is especially important since sometimes the `str` method fails with
+ # the mysterious "<unprintable UnexpectedCharacters object>" or similar
+ g = r"""
+ start: add+
+ add: /\d+/ "+" /\d+/
+ %ignore " "
+ """
+ texts = ("1+", "+1", "", "1 1+1")
 
+ parser1 = Lark(g, parser='lalr', cache=True)
+ parser2 = Lark(g, parser='lalr', cache=True)
+ assert len(self.mock_fs.files) == 1
+ for text in texts:
+ with self.assertRaises((UnexpectedInput)) as cm1:
+ parser1.parse(text)
+ with self.assertRaises((UnexpectedInput)) as cm2:
+ parser2.parse(text)
+ self.assertEqual(str(cm1.exception), str(cm2.exception))
 
 if __name__ == '__main__':
  main()