From e4f920f0ab1ffaebea2b7b5ad64e33f54f034a33 Mon Sep 17 00:00:00 2001 From: Robin van der Noord Date: Wed, 14 Jun 2023 12:26:09 +0200 Subject: [PATCH] feat: added JSON and YAML file loading. --- README.md | 4 ++-- examples/example_from_docs.json | 13 +++++++++++ examples/example_from_docs.py | 4 ++-- examples/example_from_docs.yaml | 8 +++++++ pyproject.toml | 2 ++ pytest_examples/example.json | 13 +++++++++++ pytest_examples/example.toml | 7 ++++++ pytest_examples/example.yaml | 8 +++++++ src/configuraptor/core.py | 4 ++-- src/configuraptor/loaders/__init__.py | 26 ++++++++++++++++++++- src/configuraptor/loaders/_types.py | 11 +++++++++ src/configuraptor/loaders/loaders_310.py | 10 ++++---- src/configuraptor/loaders/loaders_311.py | 5 ++-- src/configuraptor/loaders/loaders_shared.py | 26 +++++++++++++++++++++ tests/test_core.py | 9 +++++-- tests/test_json_yaml.py | 21 +++++++++++++++++ 16 files changed, 155 insertions(+), 16 deletions(-) create mode 100644 examples/example_from_docs.json create mode 100644 examples/example_from_docs.yaml create mode 100644 pytest_examples/example.json create mode 100644 pytest_examples/example.toml create mode 100644 pytest_examples/example.yaml create mode 100644 src/configuraptor/loaders/_types.py create mode 100644 src/configuraptor/loaders/loaders_shared.py create mode 100644 tests/test_json_yaml.py diff --git a/README.md b/README.md index 0d3bd9e..1e475c7 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ class Config: if __name__ == '__main__': - my_config = load_into(Config, "example_from_docs.toml") + my_config = load_into(Config, "example_from_docs.toml") # or .json, .yaml print(my_config.name) # Hello World! @@ -100,7 +100,7 @@ class OtherConfig(TypedConfig): if __name__ == '__main__': - my_config = OtherConfig.load("example_from_docs.toml") + my_config = OtherConfig.load("example_from_docs.toml") # or .json, .yaml print(my_config.name) # Hello World! diff --git a/examples/example_from_docs.json b/examples/example_from_docs.json new file mode 100644 index 0000000..2352a41 --- /dev/null +++ b/examples/example_from_docs.json @@ -0,0 +1,13 @@ +{ + "config": { + "name": "Hello World!", + "reference": { + "number": 42, + "numbers": [ + 41, + 43 + ], + "string": "42" + } + } +} \ No newline at end of file diff --git a/examples/example_from_docs.py b/examples/example_from_docs.py index 3be34ca..a6d7c24 100644 --- a/examples/example_from_docs.py +++ b/examples/example_from_docs.py @@ -16,7 +16,7 @@ class Config: if __name__ == '__main__': - my_config = load_into(Config, "example_from_docs.toml") + my_config = load_into(Config, "example_from_docs.json") print(my_config.name) # Hello World! @@ -40,7 +40,7 @@ class OtherConfig(TypedConfig): if __name__ == '__main__': - my_config = OtherConfig.load("example_from_docs.toml") + my_config = OtherConfig.load("example_from_docs.json") print(my_config.name) # Hello World! diff --git a/examples/example_from_docs.yaml b/examples/example_from_docs.yaml new file mode 100644 index 0000000..de1b1c6 --- /dev/null +++ b/examples/example_from_docs.yaml @@ -0,0 +1,8 @@ +config: + name: "Hello World!" + reference: + number: 42 + numbers: + - 41 + - 43 + string: "42" diff --git a/pyproject.toml b/pyproject.toml index 8d1e4cb..28334a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ dependencies = [ "typeguard", "tomlkit; python_version < '3.11'", + "PyYAML", ] [template.plugins.default] @@ -34,6 +35,7 @@ src-layout = true dev = [ "su6[all]", "hatch", + "types-PyYAML", ] [project.urls] diff --git a/pytest_examples/example.json b/pytest_examples/example.json new file mode 100644 index 0000000..2352a41 --- /dev/null +++ b/pytest_examples/example.json @@ -0,0 +1,13 @@ +{ + "config": { + "name": "Hello World!", + "reference": { + "number": 42, + "numbers": [ + 41, + 43 + ], + "string": "42" + } + } +} \ No newline at end of file diff --git a/pytest_examples/example.toml b/pytest_examples/example.toml new file mode 100644 index 0000000..7594d58 --- /dev/null +++ b/pytest_examples/example.toml @@ -0,0 +1,7 @@ +[config] +name = "Hello World!" + +[config.reference] +number = 42 +numbers = [41, 43] +string = "42" diff --git a/pytest_examples/example.yaml b/pytest_examples/example.yaml new file mode 100644 index 0000000..de1b1c6 --- /dev/null +++ b/pytest_examples/example.yaml @@ -0,0 +1,8 @@ +config: + name: "Hello World!" + reference: + number: 42 + numbers: + - 41 + - 43 + string: "42" diff --git a/src/configuraptor/core.py b/src/configuraptor/core.py index 344ebfc..cd74ace 100644 --- a/src/configuraptor/core.py +++ b/src/configuraptor/core.py @@ -62,9 +62,9 @@ def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str if isinstance(data, str): data = Path(data) if isinstance(data, Path): - # todo: more than toml with data.open("rb") as f: - data = loaders.toml(f) + loader = loaders.get(data.suffix) + data = loader(f) if not data: return {} diff --git a/src/configuraptor/loaders/__init__.py b/src/configuraptor/loaders/__init__.py index 3674ff1..33f5613 100644 --- a/src/configuraptor/loaders/__init__.py +++ b/src/configuraptor/loaders/__init__.py @@ -3,10 +3,34 @@ """ import sys +import typing + +from ._types import T_config +from .loaders_shared import json, yaml if sys.version_info > (3, 11): from .loaders_311 import toml else: # pragma: no cover from .loaders_310 import toml -__all__ = ["toml"] +__all__ = ["get", "toml", "json", "yaml"] + +T_loader = typing.Callable[[typing.BinaryIO], T_config] + +LOADERS: dict[str, T_loader] = { + "toml": toml, + "json": json, + "yml": yaml, + "yaml": yaml, +} + + +def get(extension: str) -> T_loader: + """ + Get the right loader for a specific extension. + """ + extension = extension.removeprefix(".") + if loader := LOADERS.get(extension): + return loader + else: + raise ValueError(f"Invalid extension {extension}") diff --git a/src/configuraptor/loaders/_types.py b/src/configuraptor/loaders/_types.py new file mode 100644 index 0000000..d455fd4 --- /dev/null +++ b/src/configuraptor/loaders/_types.py @@ -0,0 +1,11 @@ +import typing + +T_config = dict[str, typing.Any] + + +def as_tconfig(data: typing.Any) -> T_config: + """ + Does not actually do anything, but tells mypy the 'data' of type Any (json, pyyaml, tomlkit) \ + is actually a dict of string keys and Any values. + """ + return typing.cast(T_config, data) diff --git a/src/configuraptor/loaders/loaders_310.py b/src/configuraptor/loaders/loaders_310.py index 0ef37ff..3538858 100644 --- a/src/configuraptor/loaders/loaders_310.py +++ b/src/configuraptor/loaders/loaders_310.py @@ -3,18 +3,18 @@ """ import sys -import typing from typing import BinaryIO +from ._types import T_config, as_tconfig + if sys.version_info > (3, 11): raise EnvironmentError("Wrong Python version!") else: # pragma: no cover import tomlkit - T_toml = dict[str, typing.Any] - - def toml(f: BinaryIO) -> T_toml: + def toml(f: BinaryIO) -> T_config: """ Load a toml file. """ - return typing.cast(T_toml, tomlkit.load(f)) + data = tomlkit.load(f) + return as_tconfig(data) diff --git a/src/configuraptor/loaders/loaders_311.py b/src/configuraptor/loaders/loaders_311.py index 08c1348..71f0a5f 100644 --- a/src/configuraptor/loaders/loaders_311.py +++ b/src/configuraptor/loaders/loaders_311.py @@ -2,15 +2,16 @@ Loaders for Python 3.11+. """ import sys -import typing from typing import BinaryIO +from ._types import T_config + if sys.version_info < (3, 11): # pragma: no cover raise EnvironmentError("Wrong Python version!") else: import tomllib - def toml(f: BinaryIO) -> dict[str, typing.Any]: + def toml(f: BinaryIO) -> T_config: """ Load a toml file. """ diff --git a/src/configuraptor/loaders/loaders_shared.py b/src/configuraptor/loaders/loaders_shared.py new file mode 100644 index 0000000..52e957a --- /dev/null +++ b/src/configuraptor/loaders/loaders_shared.py @@ -0,0 +1,26 @@ +""" +File loaders that work regardless of Python version. +""" + +import json as json_lib +from typing import BinaryIO + +import yaml as yaml_lib + +from ._types import T_config, as_tconfig + + +def json(f: BinaryIO) -> T_config: + """ + Load a JSON file. + """ + data = json_lib.load(f) + return as_tconfig(data) + + +def yaml(f: BinaryIO) -> T_config: + """ + Load a YAML file. + """ + data = yaml_lib.load(f, yaml_lib.SafeLoader) + return as_tconfig(data) diff --git a/tests/test_core.py b/tests/test_core.py index 0285997..87fe8f8 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,7 +4,6 @@ import pytest if sys.version_info > (3, 11): - def test_loader_310_fails(): with pytest.raises(EnvironmentError): from src.configuraptor.loaders.loaders_310 import toml @@ -12,9 +11,15 @@ def test_loader_310_fails(): toml() else: - def test_loader_311_fails(): with pytest.raises(EnvironmentError): from src.configuraptor.loaders.loaders_311 import toml toml() + + +def test_invalid_extension(): + from src.configuraptor.loaders import get + + with pytest.raises(ValueError): + get(".doesntexist") diff --git a/tests/test_json_yaml.py b/tests/test_json_yaml.py new file mode 100644 index 0000000..4044191 --- /dev/null +++ b/tests/test_json_yaml.py @@ -0,0 +1,21 @@ +from src.configuraptor import load_into +from tests.constants import PYTEST_EXAMPLES + + +class SomeRegularClass: + number: int + numbers: list[int] + string: str + + +class Config: + name: str + reference: SomeRegularClass + + +def test_basic_json_and_yaml(): + toml = load_into(Config, PYTEST_EXAMPLES / "example.toml") + json = load_into(Config, PYTEST_EXAMPLES / "example.json") + yaml = load_into(Config, PYTEST_EXAMPLES / "example.yaml") + + assert toml.reference.numbers and toml.reference.numbers == json.reference.numbers == yaml.reference.numbers