From 8ae70814e2ba3d481551d6791cec4c99676649cd Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 8 Jun 2021 13:44:48 -0400 Subject: [PATCH 1/2] Don't expand unused vars --- src/tinuous/util.py | 76 ++++++++++++++++++++++++++++++++++++++++----- test/test_util.py | 36 ++++++++++++++++++++- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/src/tinuous/util.py b/src/tinuous/util.py index 0ded703..9aa1883 100644 --- a/src/tinuous/util.py +++ b/src/tinuous/util.py @@ -4,8 +4,9 @@ import os from pathlib import Path import re +from string import Formatter import subprocess -from typing import Dict, Iterator, cast +from typing import Any, Dict, Iterator, Mapping, Optional, Sequence, Union import requests @@ -46,17 +47,78 @@ def iterfiles(dirpath: Path) -> Iterator[Path]: yield p +class LazySlicingFormatter(Formatter): + def __init__(self, var_defs: Dict[str, str]): + self.var_defs: Dict[str, str] = var_defs + self.expanded_vars: Dict[str, str] = {} + super().__init__() + + def get_value( + self, key: Union[int, str], args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> Any: + if isinstance(key, int): + return args[key] + elif key in kwargs: + return kwargs[key] + elif key in self.expanded_vars: + return self.expanded_vars[key] + elif key in self.var_defs: + self.expanded_vars[key] = self.format(self.var_defs[key], **kwargs) + return self.expanded_vars[key] + else: + raise KeyError(key) + + def get_field( + self, field_name: str, args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> Any: + m = re.match(r"\w+", field_name) + assert m, f"format field name {field_name!r} does not start with arg_name" + key = m.group() + obj = self.get_value(key, args, kwargs) + s = field_name[m.end() :] + while s: + m = re.match(r"\.(?P\w+)|\[(?P[^]]+)\]", s) + assert m, f"format field name {field_name!r} has invalid attr/index" + s = s[m.end() :] + attr, index = m.group("attr", "index") + if attr is not None: + obj = getattr(obj, attr) + else: + assert index is not None # type: ignore[unreachable] + try: + sl = parse_slice(index) + except ValueError: + if index.isdigit(): + obj = obj[int(index)] + else: + obj = obj[index] + else: + obj = obj[sl] + return obj, key + + def expand_template( template_str: str, fields: Dict[str, str], vars: Dict[str, str] ) -> str: - expanded_vars: Dict[str, str] = {} - for name, tmplt in vars.items(): - expanded_vars[name] = fstring(tmplt, **fields, **expanded_vars) - return fstring(template_str, **fields, **expanded_vars) + return LazySlicingFormatter(vars).format(template_str, **fields) + + +SLICE_RGX = re.compile(r"(?P-?\d+)?:(?P-?\d+)?(?::(?P-?\d+)?)?") -def fstring(s: str, **kwargs: str) -> str: - return cast(str, eval(f"f{s!r}", {}, kwargs)) +def parse_slice(s: str) -> slice: + if m := SLICE_RGX.fullmatch(s): + s_start, s_stop, s_step = m.group("start", "stop", "step") + start: Optional[int] = None if s_start is None else int(s_start) + stop: Optional[int] = None if s_stop is None else int(s_stop) + step: Optional[int] + if s_step is None or s_step == "": + step = None + else: + step = int(s_step) + return slice(start, stop, step) + else: + raise ValueError(s) def get_github_token() -> str: diff --git a/test/test_util.py b/test/test_util.py index fe2e273..351142a 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,4 +1,5 @@ -from tinuous.util import expand_template +import pytest +from tinuous.util import expand_template, parse_slice def test_expand_template() -> None: @@ -24,3 +25,36 @@ def test_expand_template_sliced() -> None: ) == "1234567/A test" ) + + +def test_expand_template_unused_bad() -> None: + assert ( + expand_template( + "{cleesh}", + {"description": "A test commit"}, + {"bad": "{undefined}", "cleesh": "{description[:6]}"}, + ) + == "A test" + ) + + +@pytest.mark.parametrize( + "s,sl", + [ + (":", slice(None)), + ("::", slice(None)), + ("23:", slice(23, None)), + ("23::", slice(23, None)), + (":42", slice(42)), + (":42:", slice(42)), + ("23:42", slice(23, 42)), + ("23:42:", slice(23, 42)), + ("::5", slice(None, None, 5)), + ("23::5", slice(23, None, 5)), + (":42:5", slice(None, 42, 5)), + ("23:42:5", slice(23, 42, 5)), + ("-23:-42:-5", slice(-23, -42, -5)), + ], +) +def test_parse_slice(s: str, sl: slice) -> None: + assert parse_slice(s) == sl From cd1fe7191c30c288d708eb657d96a598e8f779fd Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 8 Jun 2021 15:58:23 -0400 Subject: [PATCH 2/2] Add more tests of custom formatter --- src/tinuous/util.py | 12 +++++++++++- test/test_util.py | 37 ++++++++++++++++++++++++++++++++++++- tox.ini | 1 + 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/tinuous/util.py b/src/tinuous/util.py index 9aa1883..fd62238 100644 --- a/src/tinuous/util.py +++ b/src/tinuous/util.py @@ -48,6 +48,14 @@ def iterfiles(dirpath: Path) -> Iterator[Path]: class LazySlicingFormatter(Formatter): + """ + A `string.Formatter` subclass that: + + - accepts a second set of format kwargs that can refer to the main kwargs + or each other and are only templated as needed + - supports indexing strings & other sequences with slices + """ + def __init__(self, var_defs: Dict[str, str]): self.var_defs: Dict[str, str] = var_defs self.expanded_vars: Dict[str, str] = {} @@ -73,7 +81,9 @@ def get_field( ) -> Any: m = re.match(r"\w+", field_name) assert m, f"format field name {field_name!r} does not start with arg_name" - key = m.group() + key: Union[int, str] = m.group() + if key.isdigit(): + key = int(key) obj = self.get_value(key, args, kwargs) s = field_name[m.end() :] while s: diff --git a/test/test_util.py b/test/test_util.py index 351142a..4ccc510 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,5 +1,8 @@ +from types import SimpleNamespace +from typing import Any, Dict import pytest -from tinuous.util import expand_template, parse_slice +from pytest_mock import MockerFixture +from tinuous.util import LazySlicingFormatter, expand_template, parse_slice def test_expand_template() -> None: @@ -58,3 +61,35 @@ def test_expand_template_unused_bad() -> None: ) def test_parse_slice(s: str, sl: slice) -> None: assert parse_slice(s) == sl + + +@pytest.mark.parametrize( + "fmt,args,kwargs,result", + [ + ("{0}", ["foo"], {}, "foo"), + ( + "{foo.bar.baz}", + [], + {"foo": SimpleNamespace(bar=SimpleNamespace(baz="quux"))}, + "quux", + ), + ("{foo[1][2]}", [], {"foo": ["abc", "def", "ghi"]}, "f"), + ("{foo[bar][baz]}", [], {"foo": {"bar": {"baz": "quux"}}}, "quux"), + ], +) +def test_lazy_slicing_formatter_basics( + fmt: str, args: list, kwargs: Dict[str, Any], result: str +) -> None: + assert LazySlicingFormatter({}).format(fmt, *args, **kwargs) == result + + +def test_lazy_slicing_formatter_undef_key() -> None: + with pytest.raises(KeyError): + LazySlicingFormatter({}).format("{foo}", bar=42) + + +def test_lazy_slicing_formatter_var_reuse(mocker: MockerFixture) -> None: + fmter = LazySlicingFormatter({"foo": "bar"}) + spy = mocker.spy(fmter, "format") + assert fmter.format("-{foo}-{foo}-") == "-bar-bar-" + assert spy.call_args_list == [mocker.call("-{foo}-{foo}-"), mocker.call("bar")] diff --git a/tox.ini b/tox.ini index 03fdb72..f6a9bad 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ minversion = 3.3.0 deps = pytest~=6.0 pytest-cov~=2.0 + pytest-mock~=3.0 commands = # Basic smoketest: tinuous --help