From 61ea19b07145846403e7d83f1dc68800541b93d2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 24 May 2022 17:57:17 -0400 Subject: [PATCH 1/3] chore: add pre-commit basic checks Signed-off-by: Henry Schreiner --- .github/workflows/mypy.yml | 10 +++++++++- .pre-commit-config.yaml | 32 ++++++++++++++++++++++++++++++++ docs/requirements.txt | 4 ++-- tox.ini | 11 ++++++++++- 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 21450d6f..9624cd9c 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -1,7 +1,7 @@ name: Python type check on: [push, pull_request] jobs: - build: + type: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -9,3 +9,11 @@ jobs: submodules: recursive - name: Lint with mypy run: pipx run tox -e type + + pre-commit: + name: Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v2.0.3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..2852d8ed --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://github.com/pre-commit/pre-commit + +repos: +# Standard hooks +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.2.0" + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + exclude: '(^tests/.*\.lark|\.svg)$' + - id: mixed-line-ending + - id: requirements-txt-fixer + - id: trailing-whitespace + exclude: '(^tests/.*\.lark|\.svg)$' diff --git a/docs/requirements.txt b/docs/requirements.txt index 5509c25c..92ccc7f2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ # https://docs.readthedocs.io/en/stable/guides/specifying-dependencies.html#specifying-a-requirements-file +pillow +recommonmark sphinx-gallery sphinx_markdown_tables -recommonmark sphinx_rtd_theme -pillow \ No newline at end of file diff --git a/tox.ini b/tox.ini index ef895658..1372f10e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, py310, pypy3, type +envlist = lint, type, py36, py37, py38, py39, py310, pypy3 skip_missing_interpreters = true [testenv] @@ -28,3 +28,12 @@ deps = rich commands = mypy + +[testenv:lint] +description = run linters on code base +skip_install = true +recreate = false +deps = + pre-commit +commands = + pre-commit run --all-files --show-diff-on-failure From 5236ff95a37cf464ba7049b27622d87b662c1918 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 25 May 2022 10:05:13 -0400 Subject: [PATCH 2/3] style: run pre-commit Signed-off-by: Henry Schreiner --- .github/ISSUE_TEMPLATE/other.md | 2 -- .gitignore | 2 +- CHANGELOG.md | 2 +- LICENSE | 1 - README.md | 3 +-- docs/Makefile | 2 +- docs/_static/sppf/sppf.html | 2 +- docs/classes.rst | 4 ++-- docs/conf.py | 2 +- docs/features.md | 1 - docs/how_to_develop.md | 2 +- docs/ide/app/app.py | 1 - docs/ide/app/examples.py | 2 +- docs/ide/app/files.json | 2 +- docs/ide/app/html5.py | 2 -- docs/philosophy.md | 1 - docs/recipes.md | 2 +- docs/tools.md | 1 - docs/visitors.rst | 2 +- examples/advanced/error_handling.py | 1 - examples/advanced/error_reporting_earley.py | 2 -- examples/advanced/error_reporting_lalr.py | 2 -- examples/advanced/py3to2.py | 8 ++++---- examples/advanced/python2.lark | 1 - examples/advanced/python_parser.py | 2 +- examples/advanced/reconstruct_python.py | 2 +- examples/advanced/templates.py | 2 +- examples/composition/README.md | 2 +- examples/composition/main.py | 2 +- examples/composition/storage.lark | 1 - examples/grammars/README.md | 2 +- examples/grammars/verilog.lark | 6 +++--- examples/indented_tree.py | 1 - examples/standalone/README.md | 1 - examples/standalone/json_parser_main.py | 1 - examples/tests/no_newline_at_end.lark | 2 +- lark/__pyinstaller/__init__.py | 2 +- lark/ast_utils.py | 2 +- lark/exceptions.py | 6 +++--- lark/grammars/python.lark | 14 +++++++------- lark/lark.py | 8 ++++---- lark/parser_frontends.py | 4 ++-- lark/parsers/earley.py | 2 +- lark/parsers/earley_common.py | 2 +- lark/parsers/lalr_interactive_parser.py | 9 ++++----- lark/parsers/lalr_parser.py | 1 - lark/tools/__init__.py | 2 +- lark/tools/standalone.py | 2 +- lark/visitors.py | 4 ++-- setup.py | 1 - test-requirements.txt | 2 +- tests/test_cache.py | 14 +++++++------- tests/test_grammar.py | 3 --- tests/test_lexer.py | 2 +- tests/test_parser.py | 16 ++++++++-------- tests/test_python_grammar.py | 2 +- 56 files changed, 73 insertions(+), 99 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/other.md b/.github/ISSUE_TEMPLATE/other.md index 1c8fcda2..a8f765fc 100644 --- a/.github/ISSUE_TEMPLATE/other.md +++ b/.github/ISSUE_TEMPLATE/other.md @@ -6,5 +6,3 @@ labels: '' assignees: '' --- - - diff --git a/.gitignore b/.gitignore index b30399ed..26275e4b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ tags /dist /build docs/_build -docs/examples \ No newline at end of file +docs/examples diff --git a/CHANGELOG.md b/CHANGELOG.md index 22f4b8c1..7e94cf99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,4 +10,4 @@ v1.0 - `use_accepts` in `UnexpectedInput.match_examples()` is now True by default -- `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)` \ No newline at end of file +- `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)` diff --git a/LICENSE b/LICENSE index efcb9665..aaf210b1 100644 --- a/LICENSE +++ b/LICENSE @@ -16,4 +16,3 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/README.md b/README.md index 23184f7a..4468243b 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Lark is great at handling ambiguity. Here is the result of parsing the phrase "f - Grammar composition - Import terminals and rules from other grammars - Standard library of terminals (strings, numbers, names, etc.) - Import grammars from Nearley.js ([read more](/docs/tools.md#importing-grammars-from-nearleyjs)) - - Extensive test suite [![codecov](https://codecov.io/gh/lark-parser/lark/branch/master/graph/badge.svg?token=lPxgVhCVPK)](https://codecov.io/gh/lark-parser/lark) + - Extensive test suite [![codecov](https://codecov.io/gh/lark-parser/lark/branch/master/graph/badge.svg?token=lPxgVhCVPK)](https://codecov.io/gh/lark-parser/lark) - Type annotations (MyPy support) - And much more! @@ -193,4 +193,3 @@ Questions about code are best asked on [gitter](https://gitter.im/lark-parser/Lo For anything else, I can be reached by email at erezshin at gmail com. -- [Erez](https://github.com/erezsh) - diff --git a/docs/Makefile b/docs/Makefile index 58127b42..ff51a731 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/sppf/sppf.html b/docs/_static/sppf/sppf.html index c9c3d218..c2fd532a 100644 --- a/docs/_static/sppf/sppf.html +++ b/docs/_static/sppf/sppf.html @@ -209,4 +209,4 @@

Transformation to an abstract - \ No newline at end of file + diff --git a/docs/classes.rst b/docs/classes.rst index 6e88fae4..965cbccd 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -81,7 +81,7 @@ InteractiveParser ast_utils --------- -For an example of using ``ast_utils``, see `/examples/advanced/create_ast.py`_ +For an example of using ``ast_utils``, see `/examples/advanced/create_ast.py`_ .. autoclass:: lark.ast_utils.Ast @@ -89,4 +89,4 @@ For an example of using ``ast_utils``, see `/examples/advanced/create_ast.py`_ .. autofunction:: lark.ast_utils.create_transformer -.. _/examples/advanced/create_ast.py: examples/advanced/create_ast.html \ No newline at end of file +.. _/examples/advanced/create_ast.py: examples/advanced/create_ast.html diff --git a/docs/conf.py b/docs/conf.py index c3d89fe4..0cf22a53 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -182,4 +182,4 @@ sphinx_gallery_conf = { 'examples_dirs': ['../examples'], 'gallery_dirs': ['examples'], -} \ No newline at end of file +} diff --git a/docs/features.md b/docs/features.md index 5baa16eb..121089e6 100644 --- a/docs/features.md +++ b/docs/features.md @@ -29,4 +29,3 @@ - Visualize your parse trees as dot or png files ([see_example](https://github.com/lark-parser/lark/blob/master/examples/fruitflies.py)) - Automatic reconstruction of input from parse-tree (see examples) - Use Lark grammars in Julia and Javascript. - diff --git a/docs/how_to_develop.md b/docs/how_to_develop.md index 311ad313..cf95fed0 100644 --- a/docs/how_to_develop.md +++ b/docs/how_to_develop.md @@ -63,5 +63,5 @@ pytest tests Another way to run the tests is using setup.py: ```bash -python setup.py test +python setup.py test ``` diff --git a/docs/ide/app/app.py b/docs/ide/app/app.py index 146aee98..69745462 100644 --- a/docs/ide/app/app.py +++ b/docs/ide/app/app.py @@ -80,4 +80,3 @@ def start(): html5.Body().appendChild( App() ) - diff --git a/docs/ide/app/examples.py b/docs/ide/app/examples.py index af9c38c4..afe003a7 100644 --- a/docs/ide/app/examples.py +++ b/docs/ide/app/examples.py @@ -147,4 +147,4 @@ "favoriteFruit": "strawberry" } ]""") -} \ No newline at end of file +} diff --git a/docs/ide/app/files.json b/docs/ide/app/files.json index b2308999..98dfdb44 100644 --- a/docs/ide/app/files.json +++ b/docs/ide/app/files.json @@ -6,4 +6,4 @@ "ext.py", "ignite.py", "utils.py" -] \ No newline at end of file +] diff --git a/docs/ide/app/html5.py b/docs/ide/app/html5.py index b62a821b..64d201f2 100644 --- a/docs/ide/app/html5.py +++ b/docs/ide/app/html5.py @@ -2,5 +2,3 @@ from .core import * from . import ext, utils, ignite - - diff --git a/docs/philosophy.md b/docs/philosophy.md index ebdbd4fc..d2baa39a 100644 --- a/docs/philosophy.md +++ b/docs/philosophy.md @@ -62,4 +62,3 @@ In short, "Premature optimization is the root of all evil." - Automatically resolve terminal collisions whenever possible - Automatically keep track of line & column numbers - diff --git a/docs/recipes.md b/docs/recipes.md index 1aadd047..336be23e 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -171,4 +171,4 @@ try: print( t.transform(tree)) except VisitError as e: raise e.orig_exc -``` \ No newline at end of file +``` diff --git a/docs/tools.md b/docs/tools.md index ee9d2cfc..bf445d52 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -68,4 +68,3 @@ python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main near - Lark currently cannot export grammars to Nearley These might get added in the future, if enough users ask for them. - diff --git a/docs/visitors.rst b/docs/visitors.rst index 11d1b643..323c1837 100644 --- a/docs/visitors.rst +++ b/docs/visitors.rst @@ -119,4 +119,4 @@ Discard VisitError ---------- -.. autoclass:: lark.exceptions.VisitError \ No newline at end of file +.. autoclass:: lark.exceptions.VisitError diff --git a/examples/advanced/error_handling.py b/examples/advanced/error_handling.py index 1fb9be3f..aeaa21b4 100644 --- a/examples/advanced/error_handling.py +++ b/examples/advanced/error_handling.py @@ -34,4 +34,3 @@ def main(): print(res) # prints [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] main() - diff --git a/examples/advanced/error_reporting_earley.py b/examples/advanced/error_reporting_earley.py index f0bcc20d..3d47a53e 100644 --- a/examples/advanced/error_reporting_earley.py +++ b/examples/advanced/error_reporting_earley.py @@ -75,5 +75,3 @@ def test(): if __name__ == '__main__': test() - - diff --git a/examples/advanced/error_reporting_lalr.py b/examples/advanced/error_reporting_lalr.py index c2cb2393..722a71fd 100644 --- a/examples/advanced/error_reporting_lalr.py +++ b/examples/advanced/error_reporting_lalr.py @@ -75,5 +75,3 @@ def test(): if __name__ == '__main__': test() - - diff --git a/examples/advanced/py3to2.py b/examples/advanced/py3to2.py index e08967c8..3c474c28 100644 --- a/examples/advanced/py3to2.py +++ b/examples/advanced/py3to2.py @@ -45,12 +45,12 @@ def parse_code(s): # # 2. Define translations using templates (each template code is parsed to a template tree) -# +# pytemplate = TemplateConf(parse=parse_template) translations_3to2 = { - 'yield from $a': + 'yield from $a': 'for _tmp in $a: yield _tmp', 'raise $e from $x': @@ -63,7 +63,7 @@ def parse_code(s): # # 3. Translate and reconstruct Python 3 code into valid Python 2 code -# +# python_reconstruct = PythonReconstructor(parser) @@ -91,4 +91,4 @@ def test(): print(translate_py3to2(_TEST_CODE)) if __name__ == '__main__': - test() \ No newline at end of file + test() diff --git a/examples/advanced/python2.lark b/examples/advanced/python2.lark index 6fbae459..fd49e4dd 100644 --- a/examples/advanced/python2.lark +++ b/examples/advanced/python2.lark @@ -165,4 +165,3 @@ IMAG_NUMBER: (_INT | FLOAT) ("j"|"J") %ignore /\\[\t \f]*\r?\n/ // LINE_CONT %ignore COMMENT %declare _INDENT _DEDENT - diff --git a/examples/advanced/python_parser.py b/examples/advanced/python_parser.py index 776f6e12..78b91ca0 100644 --- a/examples/advanced/python_parser.py +++ b/examples/advanced/python_parser.py @@ -76,4 +76,4 @@ def test_earley_equals_lalr(): if __name__ == '__main__': test_python_lib() # test_earley_equals_lalr() - # python_parser3.parse(_read(sys.argv[1]) + '\n') \ No newline at end of file + # python_parser3.parse(_read(sys.argv[1]) + '\n') diff --git a/examples/advanced/reconstruct_python.py b/examples/advanced/reconstruct_python.py index a318485f..95fb4798 100644 --- a/examples/advanced/reconstruct_python.py +++ b/examples/advanced/reconstruct_python.py @@ -83,4 +83,4 @@ def test(): if __name__ == '__main__': - test() \ No newline at end of file + test() diff --git a/examples/advanced/templates.py b/examples/advanced/templates.py index ac59b7a9..6d4b6056 100644 --- a/examples/advanced/templates.py +++ b/examples/advanced/templates.py @@ -26,4 +26,4 @@ parser = Lark(grammar) print(parser.parse('[1, "a", 2]')) -print(parser.parse('{"a": 2, "b": 6}')) \ No newline at end of file +print(parser.parse('{"a": 2, "b": 6}')) diff --git a/examples/composition/README.md b/examples/composition/README.md index 259a66a5..1c32e59a 100644 --- a/examples/composition/README.md +++ b/examples/composition/README.md @@ -7,4 +7,4 @@ file format that allows both CSV and JSON to co-exist. We show how, by using namespaces, Lark grammars and their transformers can be fully reused - they don't need to care if their grammar is used directly, or being imported, or who is doing the importing. -See [``main.py``](main.py) for more details. \ No newline at end of file +See [``main.py``](main.py) for more details. diff --git a/examples/composition/main.py b/examples/composition/main.py index c6f150ff..4c803f5a 100644 --- a/examples/composition/main.py +++ b/examples/composition/main.py @@ -48,4 +48,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/composition/storage.lark b/examples/composition/storage.lark index 09bb0ae7..fc730277 100644 --- a/examples/composition/storage.lark +++ b/examples/composition/storage.lark @@ -6,4 +6,3 @@ start: (csv__start | json__start _NL?)+ %import .json.start -> json__start %import .csv._NL -> _NL - diff --git a/examples/grammars/README.md b/examples/grammars/README.md index cdd3b75b..71b3232d 100644 --- a/examples/grammars/README.md +++ b/examples/grammars/README.md @@ -2,4 +2,4 @@ This directory is a collection of lark grammars, taken from real world projects. -- [Verilog](verilog.lark) - Taken from https://github.com/circuitgraph/circuitgraph/blob/master/circuitgraph/parsing/verilog.lark \ No newline at end of file +- [Verilog](verilog.lark) - Taken from https://github.com/circuitgraph/circuitgraph/blob/master/circuitgraph/parsing/verilog.lark diff --git a/examples/grammars/verilog.lark b/examples/grammars/verilog.lark index 0120fe71..0b0a4c35 100644 --- a/examples/grammars/verilog.lark +++ b/examples/grammars/verilog.lark @@ -3,9 +3,9 @@ // 1. Source Text start: description* - + ?description: module - + module: "module" name_of_module list_of_ports? ";" module_item* "endmodule" ?name_of_module: IDENTIFIER @@ -13,7 +13,7 @@ module: "module" name_of_module list_of_ports? ";" module_item* "endmodule" list_of_ports: "(" port ("," port)* ")" ?port: IDENTIFIER - + ?module_item: input_declaration | output_declaration | net_declaration diff --git a/examples/indented_tree.py b/examples/indented_tree.py index 6cdaf374..5ac928c7 100644 --- a/examples/indented_tree.py +++ b/examples/indented_tree.py @@ -52,4 +52,3 @@ def test(): if __name__ == '__main__': test() - diff --git a/examples/standalone/README.md b/examples/standalone/README.md index 6ec80355..2f4541d8 100644 --- a/examples/standalone/README.md +++ b/examples/standalone/README.md @@ -17,4 +17,3 @@ Then run using: ```bash python json_parser_main.py ``` - diff --git a/examples/standalone/json_parser_main.py b/examples/standalone/json_parser_main.py index 3d9b5a6d..0afabde6 100644 --- a/examples/standalone/json_parser_main.py +++ b/examples/standalone/json_parser_main.py @@ -34,4 +34,3 @@ def string(self, s): if __name__ == '__main__': with open(sys.argv[1]) as f: print(parser.parse(f.read())) - diff --git a/examples/tests/no_newline_at_end.lark b/examples/tests/no_newline_at_end.lark index 8ab64d7a..9beaff7f 100644 --- a/examples/tests/no_newline_at_end.lark +++ b/examples/tests/no_newline_at_end.lark @@ -1 +1 @@ -start: "a" \ No newline at end of file +start: "a" diff --git a/lark/__pyinstaller/__init__.py b/lark/__pyinstaller/__init__.py index fa02fc92..9da62a33 100644 --- a/lark/__pyinstaller/__init__.py +++ b/lark/__pyinstaller/__init__.py @@ -3,4 +3,4 @@ import os def get_hook_dirs(): - return [os.path.dirname(__file__)] \ No newline at end of file + return [os.path.dirname(__file__)] diff --git a/lark/ast_utils.py b/lark/ast_utils.py index faa17d0f..0ceee98f 100644 --- a/lark/ast_utils.py +++ b/lark/ast_utils.py @@ -44,7 +44,7 @@ def create_transformer(ast_module: types.ModuleType, Parameters: ast_module: A Python module containing all the subclasses of ``ast_utils.Ast`` transformer (Optional[Transformer]): An initial transformer. Its attributes may be overwritten. - decorator_factory (Callable): An optional callable accepting two booleans, inline, and meta, + decorator_factory (Callable): An optional callable accepting two booleans, inline, and meta, and returning a decorator for the methods of ``transformer``. (default: ``v_args``). """ t = transformer or Transformer() diff --git a/lark/exceptions.py b/lark/exceptions.py index da982e33..35b986af 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -72,7 +72,7 @@ def get_context(self, text: str, span: int=40) -> str: after = text[pos:end].split(b'\n', 1)[0] return (before + after + b'\n' + b' ' * len(before.expandtabs()) + b'^\n').decode("ascii", "backslashreplace") - def match_examples(self, parse_fn: 'Callable[[str], Tree]', + def match_examples(self, parse_fn: 'Callable[[str], Tree]', examples: Union[Mapping[T, Iterable[str]], Iterable[Tuple[T, Iterable[str]]]], token_type_match_fallback: bool=False, use_accepts: bool=True @@ -168,7 +168,7 @@ def __str__(self): class UnexpectedCharacters(LexError, UnexpectedInput): - """An exception that is raised by the lexer, when it cannot match the next + """An exception that is raised by the lexer, when it cannot match the next string of characters to any of its terminals. """ @@ -229,7 +229,7 @@ class UnexpectedToken(ParseError, UnexpectedInput): def __init__(self, token, expected, considered_rules=None, state=None, interactive_parser=None, terminals_by_name=None, token_history=None): super(UnexpectedToken, self).__init__() - + # TODO considered_rules and expected can be figured out using state self.line = getattr(token, 'line', '?') self.column = getattr(token, 'column', '?') diff --git a/lark/grammars/python.lark b/lark/grammars/python.lark index 8286c856..ab4a26a4 100644 --- a/lark/grammars/python.lark +++ b/lark/grammars/python.lark @@ -50,14 +50,14 @@ lambda_kwparams: "**" name ","? ?stmt: simple_stmt | compound_stmt ?simple_stmt: small_stmt (";" small_stmt)* [";"] _NEWLINE ?small_stmt: (expr_stmt | assign_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) -expr_stmt: testlist_star_expr +expr_stmt: testlist_star_expr assign_stmt: annassign | augassign | assign annassign: testlist_star_expr ":" test ["=" test] assign: testlist_star_expr ("=" (yield_expr|testlist_star_expr))+ augassign: testlist_star_expr augassign_op (yield_expr|testlist) !augassign_op: "+=" | "-=" | "*=" | "@=" | "/=" | "%=" | "&=" | "|=" | "^=" | "<<=" | ">>=" | "**=" | "//=" -?testlist_star_expr: test_or_star_expr +?testlist_star_expr: test_or_star_expr | test_or_star_expr ("," test_or_star_expr)+ ","? -> tuple | test_or_star_expr "," -> tuple @@ -95,13 +95,13 @@ for_stmt: "for" exprlist "in" testlist ":" suite ["else" ":" suite] try_stmt: "try" ":" suite except_clauses ["else" ":" suite] [finally] | "try" ":" suite finally -> try_finally finally: "finally" ":" suite -except_clauses: except_clause+ +except_clauses: except_clause+ except_clause: "except" [test ["as" name]] ":" suite // NB compile.c makes sure that the default except clause is last with_stmt: "with" with_items ":" suite -with_items: with_item ("," with_item)* +with_items: with_item ("," with_item)* with_item: test ["as" name] match_stmt: "match" test ":" _NEWLINE _INDENT case+ _DEDENT @@ -204,7 +204,7 @@ AWAIT: "await" | "{" _set_exprlist "}" -> set | "{" comprehension{test} "}" -> set_comprehension | name -> var - | number + | number | string_concat | "(" test ")" | "..." -> ellipsis @@ -217,7 +217,7 @@ AWAIT: "await" _testlist_comp: test | _tuple_inner _tuple_inner: test_or_star_expr (("," test_or_star_expr)+ [","] | ",") - + ?test_or_star_expr: test | star_expr @@ -253,7 +253,7 @@ kwargs: "**" test ("," argvalue)* comprehension{comp_result}: comp_result comp_fors [comp_if] -comp_fors: comp_for+ +comp_fors: comp_for+ comp_for: [ASYNC] "for" exprlist "in" or_test ASYNC: "async" ?comp_if: "if" test_nocond diff --git a/lark/lark.py b/lark/lark.py index 40b74be4..7a4c730b 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -14,7 +14,7 @@ from typing import Literal else: from typing_extensions import Literal - + from .exceptions import ConfigurationError, assert_config, UnexpectedInput from .utils import Serialize, SerializeMemoizer, FS, isascii, logger from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource @@ -307,7 +307,7 @@ def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None: else: if self.options.cache is not True: raise ConfigurationError("cache argument must be bool or str") - + cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % (cache_md5, *sys.version_info[:2]) if FS.exists(cache_fn): @@ -326,7 +326,7 @@ def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None: return except Exception: # We should probably narrow done which errors we catch here. logger.exception("Failed to load Lark from cache: %r. We will try to carry on." % cache_fn) - + # In theory, the Lark instance might have been messed up by the call to `_load`. # In practice the only relevant thing that might have been overriden should be `options` self.options = old_options @@ -592,7 +592,7 @@ def lex(self, text: str, dont_ignore: bool=False) -> Iterator[Token]: def get_terminal(self, name: str) -> TerminalDef: """Get information about a terminal""" return self._terminals_dict[name] - + def parse_interactive(self, text: Optional[str]=None, start: Optional[str]=None) -> 'InteractiveParser': """Start an interactive parsing session. diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index e162edfa..4e28e361 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -74,7 +74,7 @@ def __init__(self, lexer_conf, parser_conf, options, parser=None): if lexer_conf.postlex: self.lexer = PostLexConnector(self.lexer, lexer_conf.postlex) - + def _verify_start(self, start=None): if start is None: start_decls = self.parser_conf.start @@ -94,7 +94,7 @@ def parse(self, text, start=None, on_error=None): kw = {} if on_error is None else {'on_error': on_error} stream = self._make_lexer_thread(text) return self.parser.parse(stream, chosen_start, **kw) - + def parse_interactive(self, text=None, start=None): chosen_start = self._verify_start(start) if self.parser_conf.parser_type != 'lalr': diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 044f7b05..2a047b03 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -170,7 +170,7 @@ def is_quasi_complete(item): return True # def create_leo_transitives(origin, start): - # ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 + # ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 def scan(i, token, to_scan): """The core Earley Scanner. diff --git a/lark/parsers/earley_common.py b/lark/parsers/earley_common.py index 844ff108..46e242b4 100644 --- a/lark/parsers/earley_common.py +++ b/lark/parsers/earley_common.py @@ -39,4 +39,4 @@ def __repr__(self): # class TransitiveItem(Item): -# ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 \ No newline at end of file +# ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 diff --git a/lark/parsers/lalr_interactive_parser.py b/lark/parsers/lalr_interactive_parser.py index c9658daf..0013ddf3 100644 --- a/lark/parsers/lalr_interactive_parser.py +++ b/lark/parsers/lalr_interactive_parser.py @@ -37,20 +37,20 @@ def iter_parse(self) -> Iterator[Token]: Returns an iterator of the tokens it encounters. - When the parse is over, the resulting tree can be found in ``InteractiveParser.result``. + When the parse is over, the resulting tree can be found in ``InteractiveParser.result``. """ for token in self.lexer_thread.lex(self.parser_state): yield token self.result = self.feed_token(token) - + def exhaust_lexer(self) -> List[Token]: """Try to feed the rest of the lexer state into the interactive parser. - + Note that this modifies the instance in place and does not feed an '$END' Token """ return list(self.iter_parse()) - + def feed_eof(self, last_token=None): """Feed a '$END' Token. Borrows from 'last_token' if given.""" eof = Token.new_borrow_pos('$END', '', last_token) if last_token is not None else self.lexer_thread._Token('$END', '', 0, 1, 1) @@ -146,4 +146,3 @@ def as_mutable(self): """Convert to an ``InteractiveParser``.""" p = copy(self) return InteractiveParser(p.parser, p.parser_state, p.lexer_thread) - diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py index 292d5c24..c9d21333 100644 --- a/lark/parsers/lalr_parser.py +++ b/lark/parsers/lalr_parser.py @@ -196,4 +196,3 @@ def parse_from_state(self, state): raise ###} - diff --git a/lark/tools/__init__.py b/lark/tools/__init__.py index 6b0bd6ab..391f991f 100644 --- a/lark/tools/__init__.py +++ b/lark/tools/__init__.py @@ -53,7 +53,7 @@ def showwarning_as_comment(message, category, filename, lineno, file=None, line= if file is None: file = sys.stderr if file is None: - return + return try: file.write(text) except OSError: diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py index 9989f872..fb01fb6e 100644 --- a/lark/tools/standalone.py +++ b/lark/tools/standalone.py @@ -187,4 +187,4 @@ def main(): gen_standalone(lark_inst, out=out, compress=ns.compress) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/lark/visitors.py b/lark/visitors.py index 44e4caa6..932fbee1 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -72,7 +72,7 @@ def __class_getitem__(cls, _): class Transformer(_Decoratable, ABC, Generic[_Leaf_T, _Return_T]): - """Transformers work bottom-up (or depth-first), starting with visiting the leaves and working + """Transformers work bottom-up (or depth-first), starting with visiting the leaves and working their way up until ending at the root of the tree. For each node visited, the transformer will call the appropriate method (callbacks), according to the @@ -83,7 +83,7 @@ class Transformer(_Decoratable, ABC, Generic[_Leaf_T, _Return_T]): If the transformer cannot find a method with the right name, it will instead call ``__default__``, which by default creates a copy of the node. - + To discard a node, return Discard (``lark.visitors.Discard``). ``Transformer`` can do anything ``Visitor`` can do, but because it reconstructs the tree, diff --git a/setup.py b/setup.py index a6b5bef0..6c4a3676 100644 --- a/setup.py +++ b/setup.py @@ -69,4 +69,3 @@ ] }, ) - diff --git a/test-requirements.txt b/test-requirements.txt index d304ee8f..43014967 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,2 @@ Js2Py==0.68 -regex \ No newline at end of file +regex diff --git a/tests/test_cache.py b/tests/test_cache.py index 479f1a1c..a4ec4502 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -68,12 +68,12 @@ def append_zero(t): class TestCache(TestCase): g = '''start: "a"''' - + def setUp(self): self.fs = lark_module.FS self.mock_fs = MockFS() lark_module.FS = self.mock_fs - + def tearDown(self): self.mock_fs.files = {} lark_module.FS = self.fs @@ -85,7 +85,7 @@ def test_simple(self): assert fn in self.mock_fs.files parser = Lark(self.g, parser='lalr', cache=fn) assert parser.parse('a') == Tree('start', []) - + def test_automatic_naming(self): assert len(self.mock_fs.files) == 0 Lark(self.g, parser='lalr', cache=True) @@ -99,7 +99,7 @@ def test_automatic_naming(self): parser = Lark(self.g, parser='lalr', cache=True) assert parser.parse('a') == Tree('start', []) - + def test_custom_lexer(self): parser = Lark(self.g, parser='lalr', lexer=CustomLexer, cache=True) @@ -112,7 +112,7 @@ def test_options(self): Lark(self.g, parser="lalr", debug=True, cache=True) parser = Lark(self.g, parser="lalr", debug=True, cache=True) assert parser.options.options['debug'] - + def test_inline(self): # Test inline transformer (tree-less) & lexer_callbacks # Note: the Transformer should not be saved to the file, @@ -133,7 +133,7 @@ def test_inline(self): res1 = parser.parse(text) res2 = InlineTestT().transform(Lark(g, parser="lalr", cache=True, lexer_callbacks={'NUM': append_zero}).parse(text)) assert res0 == res1 == res2 == expected - + def test_imports(self): g = """ %import .grammars.ab (startab, expr) @@ -164,7 +164,7 @@ def test_recursive_pattern(self): # should only have the dummy log self.assertCountEqual(cm.output, ["ERROR:lark:dummy message"]) - + if __name__ == '__main__': diff --git a/tests/test_grammar.py b/tests/test_grammar.py index 78161899..1fadb12c 100644 --- a/tests/test_grammar.py +++ b/tests/test_grammar.py @@ -294,6 +294,3 @@ def test_line_breaks(self): if __name__ == '__main__': main() - - - diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 411ef942..0996c897 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -20,4 +20,4 @@ def test_basic(self): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/tests/test_parser.py b/tests/test_parser.py index 94427ca6..b48ea039 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -193,11 +193,11 @@ def test_visit_tokens2(self): class T(base): def add(self, children): return sum(children if isinstance(children, list) else children.children) - + def NUM(self, token): return int(token) - - + + parser = Lark(g, parser='lalr', transformer=T()) result = parser.parse(text) self.assertEqual(result, expected) @@ -385,7 +385,7 @@ def test_lexer_token_limit(self): tokens = {'A%d'%i:'"%d"'%i for i in range(300)} g = """start: %s %s""" % (' '.join(tokens), '\n'.join("%s: %s"%x for x in tokens.items())) - + p = Lark(g, parser='lalr') @@ -2478,7 +2478,7 @@ def test_unicode_word(self): NAME: /[\w]+/ """, regex=True) self.assertEqual(g.parse('வணக்கம்'), 'வணக்கம்') - + @unittest.skipIf(not regex, "regex not installed") def test_regex_width_fallback(self): g = r""" @@ -2488,7 +2488,7 @@ def test_regex_width_fallback(self): self.assertRaises((GrammarError, LexError, re.error), _Lark, g) p = _Lark(g, regex=True) self.assertEqual(p.parse("123abc"), Tree('start', ['123', 'abc'])) - + g = r""" start: NAME NAME? NAME: /(?(?=\d)\d+|\w*)/ @@ -2503,7 +2503,7 @@ def test_parser_interactive_parser(self): A: "a" B: "b" ''') - + ip = g.parse_interactive() self.assertRaises(UnexpectedToken, ip.feed_eof) @@ -2526,7 +2526,7 @@ def test_parser_interactive_parser(self): res = ip.feed_eof(ip.lexer_thread.state.last_token) self.assertEqual(res, Tree('start', ['a', 'b'])) self.assertRaises(UnexpectedToken ,ip.feed_eof) - + self.assertRaises(UnexpectedToken, ip_copy.feed_token, Token('A', 'a')) ip_copy.feed_token(Token('B', 'b')) res = ip_copy.feed_eof() diff --git a/tests/test_python_grammar.py b/tests/test_python_grammar.py index 837369ea..345452b7 100644 --- a/tests/test_python_grammar.py +++ b/tests/test_python_grammar.py @@ -190,7 +190,7 @@ case str() | bytes(): print("Something string-like") case _: - print("Something else") + print("Something else") """), # guards From c1f063d812e3c5a0f848662edbb7f2b57bbed479 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Thu, 27 Oct 2022 15:34:14 -0300 Subject: [PATCH 3/3] Small changes; run pre-commit again --- lark/lark.py | 4 ++-- lark/lexer.py | 10 +++++----- tests/test_pattern_matching.py | 10 +++++----- tox.ini | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lark/lark.py b/lark/lark.py index 9f80ef06..c93e9e19 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod import getpass -import sys, os, pickle, hashlib +import sys, os, pickle import tempfile import types import re @@ -17,7 +17,7 @@ else: from typing_extensions import Literal from .parser_frontends import ParsingFrontend - + from .exceptions import ConfigurationError, assert_config, UnexpectedInput from .utils import Serialize, SerializeMemoizer, FS, isascii, logger from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, md5_digest diff --git a/lark/lexer.py b/lark/lexer.py index 8f912c1c..5e6d6d40 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -173,7 +173,7 @@ def __new__( end_pos: Optional[int]=None ) -> 'Token': ... - + @overload def __new__( cls, @@ -197,7 +197,7 @@ def __new__(cls, *args, **kwargs): return cls._future_new(*args, **kwargs) - + @classmethod def _future_new(cls, type, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): inst = super(Token, cls).__new__(cls, value) @@ -210,12 +210,12 @@ def _future_new(cls, type, value, start_pos=None, line=None, column=None, end_li inst.end_line = end_line inst.end_column = end_column inst.end_pos = end_pos - return inst + return inst @overload def update(self, type: Optional[str]=None, value: Optional[Any]=None) -> 'Token': ... - + @overload def update(self, type_: Optional[str]=None, value: Optional[Any]=None) -> 'Token': ... @@ -240,7 +240,7 @@ def _future_update(self, type: Optional[str]=None, value: Optional[Any]=None) -> @classmethod def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') -> _T: return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) - + def __reduce__(self): return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column)) diff --git a/tests/test_pattern_matching.py b/tests/test_pattern_matching.py index 63fe67e5..b86bd543 100644 --- a/tests/test_pattern_matching.py +++ b/tests/test_pattern_matching.py @@ -22,7 +22,7 @@ def test_matches_with_str_positional_arg(self): pass case _: assert False - + def test_matches_with_token_positional_arg(self): match self.token: case Token('a'): @@ -31,22 +31,22 @@ def test_matches_with_token_positional_arg(self): pass case _: assert False - + def test_matches_with_token_kwarg_type(self): match self.token: case Token(type='A'): pass case _: assert False - + def test_matches_with_bad_token_type(self): match self.token: case Token(type='B'): assert False case _: pass - + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/tox.ini b/tox.ini index 1372f10e..0b1426dc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = lint, type, py36, py37, py38, py39, py310, pypy3 +envlist = lint, type, py36, py37, py38, py39, py310, py311, pypy3 skip_missing_interpreters = true [testenv]