From 2ca3010939e299f13981664b17eea67d5387773c Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Wed, 30 Nov 2022 17:47:33 +0100 Subject: [PATCH 01/22] Add Storage protocol --- .../data_storage/in_memory_storage.py | 1 + openfisca_core/data_storage/storage.py | 24 +++++++++++++++++++ openfisca_core/types/_data.py | 17 +++++++++++++ openfisca_core/types/_infrastructure.py | 0 openfisca_tasks/lint.mk | 2 ++ setup.cfg | 3 +++ 6 files changed, 47 insertions(+) create mode 100644 openfisca_core/data_storage/storage.py create mode 100644 openfisca_core/types/_infrastructure.py diff --git a/openfisca_core/data_storage/in_memory_storage.py b/openfisca_core/data_storage/in_memory_storage.py index bd40460a56..5aa8b153cf 100644 --- a/openfisca_core/data_storage/in_memory_storage.py +++ b/openfisca_core/data_storage/in_memory_storage.py @@ -20,6 +20,7 @@ def get(self, period): values = self._arrays.get(period) if values is None: return None + return values def put(self, value, period): diff --git a/openfisca_core/data_storage/storage.py b/openfisca_core/data_storage/storage.py new file mode 100644 index 0000000000..957315795e --- /dev/null +++ b/openfisca_core/data_storage/storage.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Any, Optional +from typing_extensions import Protocol + +import abc + +import numpy + + +class Storage(Protocol): + """Storage protocol.""" + + @abc.abstractmethod + def get(self, period: Any) -> Optional[numpy.ndarray]: + """Abstract method.""" + + @abc.abstractmethod + def put(self, values: Any, period: Any) -> None: + """Abstract method.""" + + @abc.abstractmethod + def delete(self, period: Any = None) -> None: + """Abstract method.""" diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index ff7066d43a..8a9c1d3dc5 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -1,3 +1,20 @@ +"""Data-model. + +The data-model is composed of structures meant to hold and to represent data in +a certain way. Therefore, their identity is equivalent to the sum of their +properties. If two data objects hold the same data represented in the same way, +they are for all purposes equal and fungible. + +Examples: + >>> from openfisca_core import periods + + >>> this = periods.Instant((1234, 5, 6)) + >>> that = periods.Instant((1234, 5, 6)) + + >>> this == that + +""" + from typing import Sequence, TypeVar, Union from nptyping import types, NDArray as Array diff --git a/openfisca_core/types/_infrastructure.py b/openfisca_core/types/_infrastructure.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 115c6267bb..6f3730dd73 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -42,6 +42,8 @@ check-types: ## Run static type checkers for type errors (strict). lint-typing-strict: \ lint-typing-strict-commons \ + lint-typing-strict-data_storage \ + lint-typing-strict-holders \ lint-typing-strict-types \ ; diff --git a/setup.cfg b/setup.cfg index 467e3ede59..e4191445a0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,9 @@ non_interactive = True [mypy-openfisca_core.commons.tests.*] ignore_errors = True +[mypy-openfisca_core.data_storage.tests.*] +ignore_errors = True + [mypy-openfisca_core.holders.tests.*] ignore_errors = True From 0798f16f61212db3aaaccd8ebe7db6b0fcb71471 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Wed, 30 Nov 2022 18:10:13 +0100 Subject: [PATCH 02/22] Document types module --- openfisca_core/data_storage/storage.py | 24 ------------------- openfisca_core/entities/entity.py | 8 ++++++- openfisca_core/types/_data.py | 9 ++++--- openfisca_core/types/_domain.py | 17 ++++++++++++++ openfisca_core/types/_infrastructure.py | 31 +++++++++++++++++++++++++ openfisca_tasks/test_code.mk | 1 + setup.cfg | 2 +- 7 files changed, 61 insertions(+), 31 deletions(-) delete mode 100644 openfisca_core/data_storage/storage.py diff --git a/openfisca_core/data_storage/storage.py b/openfisca_core/data_storage/storage.py deleted file mode 100644 index 957315795e..0000000000 --- a/openfisca_core/data_storage/storage.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import annotations - -from typing import Any, Optional -from typing_extensions import Protocol - -import abc - -import numpy - - -class Storage(Protocol): - """Storage protocol.""" - - @abc.abstractmethod - def get(self, period: Any) -> Optional[numpy.ndarray]: - """Abstract method.""" - - @abc.abstractmethod - def put(self, values: Any, period: Any) -> None: - """Abstract method.""" - - @abc.abstractmethod - def delete(self, period: Any = None) -> None: - """Abstract method.""" diff --git a/openfisca_core/entities/entity.py b/openfisca_core/entities/entity.py index 1be0e89fa1..ce65efa47a 100644 --- a/openfisca_core/entities/entity.py +++ b/openfisca_core/entities/entity.py @@ -20,6 +20,12 @@ def __init__(self, key, plural, label, doc): self.is_person = True self._tax_benefit_system = None + def __eq__(self, other: object) -> bool: + if isinstance(other, Entity): + return self.key == other.key + + return NotImplemented + def set_tax_benefit_system(self, tax_benefit_system: TaxBenefitSystem): self._tax_benefit_system = tax_benefit_system @@ -43,7 +49,7 @@ def check_variable_defined_for_entity(self, variable_name: str) -> None: if variable is not None: entity = variable.entity - if entity.key != self.key: + if self != entity: message = os.linesep.join([ "You tried to compute the variable '{0}' for the entity '{1}';".format(variable_name, self.plural), "however the variable '{0}' is defined for '{1}'.".format(variable_name, entity.plural), diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index 8a9c1d3dc5..3f4185fc94 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -1,17 +1,16 @@ """Data-model. -The data-model is composed of structures meant to hold and to represent data in -a certain way. Therefore, their identity is equivalent to the sum of their -properties. If two data objects hold the same data represented in the same way, -they are for all purposes equal and fungible. +The data-model is composed of structures meant to hold data in a certain way. +Their identity is equivalent to the sum of their properties. If two data +objects hold the same data, they are for all purposes equal and fungible. Examples: >>> from openfisca_core import periods >>> this = periods.Instant((1234, 5, 6)) >>> that = periods.Instant((1234, 5, 6)) - >>> this == that + True """ diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py index 643f27964f..5576273a74 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types/_domain.py @@ -1,3 +1,20 @@ +"""Domain-model. + +The domain-model is composed of structures meant to encapsulate data in a way +that is unique to the context. Therefore, their identity is not equivalent to +the sum of their properties. If two data objects hold the same identifier, even +if the data they hold is different, they are equal but not fungible. + +Examples: + >>> from openfisca_core import entities + + >>> this = entities.Entity(1, "a", "b", "c") + >>> that = entities.Entity(1, "d", "f", "g") + >>> this == that + True + +""" + from __future__ import annotations import numpy diff --git a/openfisca_core/types/_infrastructure.py b/openfisca_core/types/_infrastructure.py index e69de29bb2..356502e0ef 100644 --- a/openfisca_core/types/_infrastructure.py +++ b/openfisca_core/types/_infrastructure.py @@ -0,0 +1,31 @@ +"""Infrastructure-model. + +The infrastructure-model is composed of structures meant to encapsulate the +relationships with layers outside of the domain (memory, disk, etc.). + +""" + +from __future__ import annotations + +from typing import Any, Optional +from typing_extensions import Protocol + +import abc + +import numpy + + +class Storage(Protocol): + """Storage protocol.""" + + @abc.abstractmethod + def get(self, period: Any) -> Optional[numpy.ndarray]: + """Abstract method.""" + + @abc.abstractmethod + def put(self, values: Any, period: Any) -> None: + """Abstract method.""" + + @abc.abstractmethod + def delete(self, period: Any = None) -> None: + """Abstract method.""" diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 63fdd4386a..a934237de5 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -33,6 +33,7 @@ test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 @$(call print_help,$@:) @pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ openfisca_core/commons \ + openfisca_core/data_storage \ openfisca_core/holders \ openfisca_core/types @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ diff --git a/setup.cfg b/setup.cfg index e4191445a0..9b2229f976 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ extend-ignore = D hang-closing = true ignore = E128,E251,F403,F405,E501,RST301,W503,W504 in-place = true -include-in-doctest = openfisca_core/commons openfisca_core/holders openfisca_core/types +include-in-doctest = openfisca_core/commons openfisca_core/data_storage openfisca_core/holders openfisca_core/types rst-directives = attribute, deprecated, seealso, versionadded, versionchanged rst-roles = any, attr, class, exc, func, meth, mod, obj strictness = short From 369638f53a1503d587dce0f4da3079cc0f436e12 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Wed, 30 Nov 2022 18:16:01 +0100 Subject: [PATCH 03/22] Update public exposure --- openfisca_core/types/__init__.py | 14 ++++++++++---- openfisca_core/types/_data.py | 27 +++++++++++++++++++++++++-- openfisca_core/types/_domain.py | 22 ++-------------------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py index 699133aecb..13c0257ed3 100644 --- a/openfisca_core/types/__init__.py +++ b/openfisca_core/types/__init__.py @@ -16,8 +16,9 @@ * :attr:`.Params` * :attr:`.Period` * :attr:`.Population` - * :attr:`.Role`, - * :attr:`.Simulation`, + * :attr:`.Role` + * :attr:`.Simulation` + * :attr:`.Storage` * :attr:`.TaxBenefitSystem` * :attr:`.Variable` @@ -52,16 +53,16 @@ from ._data import ( # noqa: F401 Array, ArrayLike, + Instant, + Period, ) from ._domain import ( # noqa: F401 Entity, Formula, Holder, - Instant, ParameterNodeAtInstant, Params, - Period, Population, Role, Simulation, @@ -69,6 +70,10 @@ Variable, ) +from ._infrastructure import ( # noqa: F401 + Storage, + ) + __all__ = [ "Array", "ArrayLike", @@ -82,6 +87,7 @@ "Population", "Role", "Simulation", + "Storage", "TaxBenefitSystem", "Variable", ] diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index 3f4185fc94..d9d933a48a 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -14,11 +14,16 @@ """ -from typing import Sequence, TypeVar, Union +from __future__ import annotations -from nptyping import types, NDArray as Array +import typing_extensions +from typing import Any, Sequence, TypeVar, Union +from typing_extensions import Protocol + +import abc import numpy +from nptyping import types, NDArray as Array T = TypeVar("T", bool, bytes, float, int, object, str) @@ -65,3 +70,21 @@ https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray """ + + +class Instant(Protocol): + """Instant protocol.""" + + +@typing_extensions.runtime_checkable +class Period(Protocol): + """Period protocol.""" + + @property + @abc.abstractmethod + def start(self) -> Any: + """Abstract method.""" + @property + @abc.abstractmethod + def unit(self) -> Any: + """Abstract method.""" diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py index 5576273a74..afd7891c9a 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types/_domain.py @@ -54,7 +54,7 @@ class Formula(Protocol): def __call__( self, population: Population, - instant: Instant, + instant: Any, params: Params, ) -> numpy.ndarray: """Abstract method.""" @@ -72,10 +72,6 @@ def get_memory_usage(self) -> Any: """Abstract method.""" -class Instant(Protocol): - """Instant protocol.""" - - @typing_extensions.runtime_checkable class ParameterNodeAtInstant(Protocol): """ParameterNodeAtInstant protocol.""" @@ -85,21 +81,7 @@ class Params(Protocol): """Params protocol.""" @abc.abstractmethod - def __call__(self, instant: Instant) -> ParameterNodeAtInstant: - """Abstract method.""" - - -@typing_extensions.runtime_checkable -class Period(Protocol): - """Period protocol.""" - - @property - @abc.abstractmethod - def start(self) -> Any: - """Abstract method.""" - @property - @abc.abstractmethod - def unit(self) -> Any: + def __call__(self, instant: Any) -> ParameterNodeAtInstant: """Abstract method.""" From f983d7716023e5c441a8ac3fcd62ee6e12f22235 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Wed, 30 Nov 2022 19:01:48 +0100 Subject: [PATCH 04/22] Add types to disk storage --- .../data_storage/on_disk_storage.py | 42 ++++++++++++------- openfisca_core/holders/holder.py | 20 ++++++++- openfisca_core/types/_data.py | 4 ++ 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/openfisca_core/data_storage/on_disk_storage.py b/openfisca_core/data_storage/on_disk_storage.py index 10d4696b58..07214208d0 100644 --- a/openfisca_core/data_storage/on_disk_storage.py +++ b/openfisca_core/data_storage/on_disk_storage.py @@ -1,10 +1,13 @@ +from __future__ import annotations + +from typing import Any, Dict, KeysView, Optional, Type + import os import shutil import numpy -from openfisca_core import periods -from openfisca_core.indexed_enums import EnumArray +from openfisca_core import periods, indexed_enums as enums, types class OnDiskStorage: @@ -12,21 +15,31 @@ class OnDiskStorage: Low-level class responsible for storing and retrieving calculated vectors on disk """ - def __init__(self, storage_dir, is_eternal = False, preserve_storage_dir = False): + _files: Dict[types.Period, str] + _enums: Dict[str, Type[enums.Enum]] + + def __init__( + self, + storage_dir: str, + is_eternal: bool = False, + preserve_storage_dir: bool = False, + ) -> None: self._files = {} self._enums = {} self.is_eternal = is_eternal self.preserve_storage_dir = preserve_storage_dir self.storage_dir = storage_dir - def _decode_file(self, file): + def _decode_file(self, file: str) -> Any: + enum: Optional[Type[enums.Enum]] enum = self._enums.get(file) + if enum is not None: - return EnumArray(numpy.load(file), enum) + return enums.EnumArray(numpy.load(file), enum) else: return numpy.load(file) - def get(self, period): + def get(self, period: types.Period) -> Any: if self.is_eternal: period = periods.period(periods.ETERNITY) period = periods.period(period) @@ -34,22 +47,23 @@ def get(self, period): values = self._files.get(period) if values is None: return None + return self._decode_file(values) - def put(self, value, period): + def put(self, value: Any, period: types.Period) -> None: if self.is_eternal: period = periods.period(periods.ETERNITY) period = periods.period(period) filename = str(period) path = os.path.join(self.storage_dir, filename) + '.npy' - if isinstance(value, EnumArray): + if isinstance(value, enums.EnumArray): self._enums[path] = value.possible_values value = value.view(numpy.ndarray) numpy.save(path, value) self._files[period] = path - def delete(self, period = None): + def delete(self, period: Optional[types.Period] = None) -> None: if period is None: self._files = {} return @@ -65,11 +79,11 @@ def delete(self, period = None): if not period.contains(period_item) } - def get_known_periods(self): + def get_known_periods(self) -> KeysView[types.Period]: return self._files.keys() - def restore(self): - self._files = files = {} + def restore(self) -> None: + self._files = {} # Restore self._files from content of storage_dir. for filename in os.listdir(self.storage_dir): if not filename.endswith('.npy'): @@ -77,9 +91,9 @@ def restore(self): path = os.path.join(self.storage_dir, filename) filename_core = filename.rsplit('.', 1)[0] period = periods.period(filename_core) - files[period] = path + self._files[period] = path - def __del__(self): + def __del__(self) -> None: if self.preserve_storage_dir: return shutil.rmtree(self.storage_dir) # Remove the holder temporary files diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index ae7e3fbcec..f079722599 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -26,7 +26,19 @@ class Holder: A holder keeps tracks of a variable values after they have been calculated, or set as an input. """ - def __init__(self, variable, population): + _disk_storage: types.Storage + _do_not_store: bool + _memory_storage: types.Storage + _on_disk_storable: bool + population: types.Population + simulation: types.Simulation + variable: types.Variable + + def __init__( + self, + variable: types.Variable, + population: types.Population, + ) -> None: self.population = population self.variable = variable self.simulation = population.simulation @@ -59,7 +71,11 @@ def clone(self, population): return new - def create_disk_storage(self, directory = None, preserve = False): + def create_disk_storage( + self, + directory: Optional[str] = None, + preserve: bool = False, + ) -> types.Storage: if directory is None: directory = self.simulation.data_storage_dir storage_dir = os.path.join(directory, self.variable.name) diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index d9d933a48a..3883d19c9f 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -88,3 +88,7 @@ def start(self) -> Any: @abc.abstractmethod def unit(self) -> Any: """Abstract method.""" + + @abc.abstractmethod + def contains(self, other: object) -> bool: + """Abstract method.""" From 43e1c66c8e10e9f9b316e079b112c58637babbd8 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Wed, 30 Nov 2022 19:47:51 +0100 Subject: [PATCH 05/22] Fix population types --- CHANGELOG.md | 6 ++ openfisca_core/data_storage/_arrays.py | 15 ++++ openfisca_core/data_storage/_enums.py | 13 ++++ openfisca_core/data_storage/_files.py | 13 ++++ .../data_storage/in_memory_storage.py | 56 ++++++++------ .../data_storage/on_disk_storage.py | 73 +++++++++++++------ openfisca_core/holders/__init__.py | 1 - openfisca_core/holders/holder.py | 8 +- openfisca_core/holders/memory_usage.py | 26 ------- openfisca_core/indexed_enums/enum_array.py | 6 +- openfisca_core/periods/period_.py | 8 +- openfisca_core/populations/population.py | 73 ++++++++++++------- openfisca_core/types/__init__.py | 9 +++ openfisca_core/types/_data.py | 38 +++++++++- openfisca_core/types/_domain.py | 33 +++++++++ openfisca_core/types/_infrastructure.py | 4 + openfisca_tasks/lint.mk | 2 - setup.py | 2 +- 18 files changed, 273 insertions(+), 113 deletions(-) create mode 100644 openfisca_core/data_storage/_arrays.py create mode 100644 openfisca_core/data_storage/_enums.py create mode 100644 openfisca_core/data_storage/_files.py delete mode 100644 openfisca_core/holders/memory_usage.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 38bed40483..1a8bc0606f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +### 37.0.3 [#1167](https://github.com/openfisca/openfisca-core/pull/1167) + +#### Technical changes + +- Document the `data_storage` module. + ### 37.0.2 [#1170](https://github.com/openfisca/openfisca-core/pull/1170) #### Technical changes diff --git a/openfisca_core/data_storage/_arrays.py b/openfisca_core/data_storage/_arrays.py new file mode 100644 index 0000000000..b9fce0b775 --- /dev/null +++ b/openfisca_core/data_storage/_arrays.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from typing import Dict + +import collections +import dataclasses + +import numpy + +from openfisca_core import types + + +@dataclasses.dataclass(frozen = True) +class Arrays(collections.UserDict): + data: Dict[types.Period, numpy.ndarray] diff --git a/openfisca_core/data_storage/_enums.py b/openfisca_core/data_storage/_enums.py new file mode 100644 index 0000000000..4a735cfe13 --- /dev/null +++ b/openfisca_core/data_storage/_enums.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Dict, Type + +import collections +import dataclasses + +from openfisca_core import types + + +@dataclasses.dataclass(frozen = True) +class Enums(collections.UserDict): + data: Dict[str, Type[types.Enum]] diff --git a/openfisca_core/data_storage/_files.py b/openfisca_core/data_storage/_files.py new file mode 100644 index 0000000000..69377e117d --- /dev/null +++ b/openfisca_core/data_storage/_files.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Dict + +import collections +import dataclasses + +from openfisca_core import types + + +@dataclasses.dataclass(frozen = True) +class Files(collections.UserDict): + data: Dict[types.Period, str] diff --git a/openfisca_core/data_storage/in_memory_storage.py b/openfisca_core/data_storage/in_memory_storage.py index 5aa8b153cf..efecbeb8e6 100644 --- a/openfisca_core/data_storage/in_memory_storage.py +++ b/openfisca_core/data_storage/in_memory_storage.py @@ -1,6 +1,12 @@ +from __future__ import annotations + +from typing import Any, Optional, Sequence + import numpy -from openfisca_core import periods +from openfisca_core import periods, types + +from ._arrays import Arrays class InMemoryStorage: @@ -8,11 +14,13 @@ class InMemoryStorage: Low-level class responsible for storing and retrieving calculated vectors in memory """ - def __init__(self, is_eternal = False): - self._arrays = {} + _arrays: Arrays + + def __init__(self, is_eternal: bool = False) -> None: + self._arrays = Arrays({}) self.is_eternal = is_eternal - def get(self, period): + def get(self, period: types.Period) -> Any: if self.is_eternal: period = periods.period(periods.ETERNITY) period = periods.period(period) @@ -23,43 +31,49 @@ def get(self, period): return values - def put(self, value, period): + def put(self, value: Any, period: types.Period) -> None: if self.is_eternal: period = periods.period(periods.ETERNITY) - period = periods.period(period) - self._arrays[period] = value + else: + period = periods.period(period) + + self._arrays = Arrays({period: value, **self._arrays}) - def delete(self, period = None): + def delete(self, period: Optional[types.Period] = None) -> None: if period is None: - self._arrays = {} - return + self._arrays = Arrays({}) + return None if self.is_eternal: period = periods.period(periods.ETERNITY) - period = periods.period(period) - self._arrays = { + else: + period = periods.period(period) + + self._arrays = Arrays({ period_item: value for period_item, value in self._arrays.items() if not period.contains(period_item) - } + }) - def get_known_periods(self): - return self._arrays.keys() + def get_known_periods(self) -> Sequence[types.Period]: + return list(self._arrays.keys()) - def get_memory_usage(self): + def get_memory_usage(self) -> types.MemoryUsage: if not self._arrays: - return dict( + return types.MemoryUsage( + cell_size = numpy.nan, nb_arrays = 0, total_nb_bytes = 0, - cell_size = numpy.nan, ) nb_arrays = len(self._arrays) array = next(iter(self._arrays.values())) - return dict( - nb_arrays = nb_arrays, - total_nb_bytes = array.nbytes * nb_arrays, + total = array.nbytes * nb_arrays + + return types.MemoryUsage( cell_size = array.itemsize, + nb_arrays = nb_arrays, + total_nb_bytes = total, ) diff --git a/openfisca_core/data_storage/on_disk_storage.py b/openfisca_core/data_storage/on_disk_storage.py index 07214208d0..82174d3db6 100644 --- a/openfisca_core/data_storage/on_disk_storage.py +++ b/openfisca_core/data_storage/on_disk_storage.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, KeysView, Optional, Type +from typing import Any, NoReturn, Optional, Sequence import os import shutil @@ -9,14 +9,32 @@ from openfisca_core import periods, indexed_enums as enums, types +from ._enums import Enums +from ._files import Files + class OnDiskStorage: - """ - Low-level class responsible for storing and retrieving calculated vectors on disk + """Class responsible for storing and retrieving calculated vectors on disk. + + Attributes: + _enums: ? + _files: ? + is_eternal: ? + storage_dir: Path to store calculated vectors. + preserve_storage_dir: ? + + Args: + storage_dir: Path to store calculated vectors. + is_eternal: ? + preserve_storage_dir: ? + """ - _files: Dict[types.Period, str] - _enums: Dict[str, Type[enums.Enum]] + _enums: Enums + _files: Files + is_eternal: bool + storage_dir: str + preserve_storage_dir: bool def __init__( self, @@ -24,20 +42,20 @@ def __init__( is_eternal: bool = False, preserve_storage_dir: bool = False, ) -> None: - self._files = {} - self._enums = {} + self._enums = Enums({}) + self._files = Files({}) self.is_eternal = is_eternal - self.preserve_storage_dir = preserve_storage_dir self.storage_dir = storage_dir + self.preserve_storage_dir = preserve_storage_dir def _decode_file(self, file: str) -> Any: - enum: Optional[Type[enums.Enum]] enum = self._enums.get(file) + load = numpy.load(file) - if enum is not None: - return enums.EnumArray(numpy.load(file), enum) - else: - return numpy.load(file) + if enum is None: + return load + + return enums.EnumArray(load, enum) def get(self, period: types.Period) -> Any: if self.is_eternal: @@ -53,19 +71,23 @@ def get(self, period: types.Period) -> Any: def put(self, value: Any, period: types.Period) -> None: if self.is_eternal: period = periods.period(periods.ETERNITY) - period = periods.period(period) + + else: + period = periods.period(period) filename = str(period) - path = os.path.join(self.storage_dir, filename) + '.npy' + path = os.path.join(self.storage_dir, filename) + ".npy" + if isinstance(value, enums.EnumArray): - self._enums[path] = value.possible_values + self._enums = Enums({path: value.possible_values, **self._enums}) value = value.view(numpy.ndarray) + numpy.save(path, value) - self._files[period] = path + self._files = Files({period: path, **self._files}) def delete(self, period: Optional[types.Period] = None) -> None: if period is None: - self._files = {} + self._files = Files({}) return if self.is_eternal: @@ -73,17 +95,20 @@ def delete(self, period: Optional[types.Period] = None) -> None: period = periods.period(period) if period is not None: - self._files = { + self._files = Files({ period_item: value for period_item, value in self._files.items() if not period.contains(period_item) - } + }) + + def get_known_periods(self) -> Sequence[types.Period]: + return list(self._files.keys()) - def get_known_periods(self) -> KeysView[types.Period]: - return self._files.keys() + def get_memory_usage(self) -> NoReturn: + raise NotImplementedError def restore(self) -> None: - self._files = {} + self._files = Files({}) # Restore self._files from content of storage_dir. for filename in os.listdir(self.storage_dir): if not filename.endswith('.npy'): @@ -91,7 +116,7 @@ def restore(self) -> None: path = os.path.join(self.storage_dir, filename) filename_core = filename.rsplit('.', 1)[0] period = periods.period(filename_core) - self._files[period] = path + self._files = Files({period: path, **self._files}) def __del__(self) -> None: if self.preserve_storage_dir: diff --git a/openfisca_core/holders/__init__.py b/openfisca_core/holders/__init__.py index 8bd7722ba3..a7d46e38a6 100644 --- a/openfisca_core/holders/__init__.py +++ b/openfisca_core/holders/__init__.py @@ -23,4 +23,3 @@ from .helpers import set_input_dispatch_by_period, set_input_divide_by_period # noqa: F401 from .holder import Holder # noqa: F401 -from .memory_usage import MemoryUsage # noqa: F401 diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index f079722599..d10eb68212 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -18,15 +18,13 @@ types, ) -from .memory_usage import MemoryUsage - class Holder: """ A holder keeps tracks of a variable values after they have been calculated, or set as an input. """ - _disk_storage: types.Storage + _disk_storage: Optional[types.Storage] _do_not_store: bool _memory_storage: types.Storage _on_disk_storable: bool @@ -112,7 +110,7 @@ def get_array(self, period): if self._disk_storage: return self._disk_storage.get(period) - def get_memory_usage(self) -> MemoryUsage: + def get_memory_usage(self) -> types.MemoryUsage: """Get data about the virtual memory usage of the Holder. Returns: @@ -154,7 +152,7 @@ def get_memory_usage(self) -> MemoryUsage: """ - usage = MemoryUsage( + usage = types.MemoryUsage( nb_cells_by_array = self.population.count, dtype = self.variable.dtype, ) diff --git a/openfisca_core/holders/memory_usage.py b/openfisca_core/holders/memory_usage.py deleted file mode 100644 index 41964f3086..0000000000 --- a/openfisca_core/holders/memory_usage.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing_extensions import TypedDict - -import numpy - - -class MemoryUsage(TypedDict, total = False): - """Virtual memory usage of a Holder. - - Attributes: - cell_size: The amount of bytes assigned to each value. - dtype: The :mod:`numpy.dtype` of any, each, and every value. - nb_arrays: The number of periods for which the Holder contains values. - nb_cells_by_array: The number of entities in the current Simulation. - nb_requests: The number of times the Variable has been computed. - nb_requests_by_array: Average times a stored array has been read. - total_nb_bytes: The total number of bytes used by the Holder. - - """ - - cell_size: int - dtype: numpy.dtype - nb_arrays: int - nb_cells_by_array: int - nb_requests: int - nb_requests_by_array: int - total_nb_bytes: int diff --git a/openfisca_core/indexed_enums/enum_array.py b/openfisca_core/indexed_enums/enum_array.py index 2439c16867..20faa8cebf 100644 --- a/openfisca_core/indexed_enums/enum_array.py +++ b/openfisca_core/indexed_enums/enum_array.py @@ -1,12 +1,10 @@ from __future__ import annotations -import typing from typing import Any, NoReturn, Optional, Type import numpy -if typing.TYPE_CHECKING: - from openfisca_core.indexed_enums import Enum +from openfisca_core import types class EnumArray(numpy.ndarray): @@ -22,7 +20,7 @@ class EnumArray(numpy.ndarray): def __new__( cls, input_array: numpy.int_, - possible_values: Optional[Type[Enum]] = None, + possible_values: Optional[Type[types.Enum]] = None, ) -> EnumArray: obj = numpy.asarray(input_array).view(cls) obj.possible_values = possible_values diff --git a/openfisca_core/periods/period_.py b/openfisca_core/periods/period_.py index 7de0459bdf..5e02c21be6 100644 --- a/openfisca_core/periods/period_.py +++ b/openfisca_core/periods/period_.py @@ -320,11 +320,15 @@ def offset(self, offset, unit = None): """ return self.__class__((self[0], self[1].offset(offset, self[0] if unit is None else unit), self[2])) - def contains(self, other: Period) -> bool: + def contains(self, other: object) -> bool: """ Returns ``True`` if the period contains ``other``. For instance, ``period(2015)`` contains ``period(2015-01)`` """ - return self.start <= other.start and self.stop >= other.stop + + if isinstance(other, Period): + return self.start <= other.start and self.stop >= other.stop + + return NotImplemented @property def size(self): diff --git a/openfisca_core/populations/population.py b/openfisca_core/populations/population.py index cb243aff70..5f76ae3965 100644 --- a/openfisca_core/populations/population.py +++ b/openfisca_core/populations/population.py @@ -7,30 +7,29 @@ import numpy -from openfisca_core import periods, projectors -from openfisca_core.holders import Holder, MemoryUsage +from openfisca_core import errors, periods, projectors, types +from openfisca_core.holders import Holder from openfisca_core.projectors import Projector -from openfisca_core.types import Array, Entity, Period, Role, Simulation from . import config class Population: - simulation: Optional[Simulation] - entity: Entity + simulation: Optional[types.Simulation] + entity: types.Entity _holders: Dict[str, Holder] count: int - ids: Array[str] + ids: types.Array[str] - def __init__(self, entity: Entity) -> None: + def __init__(self, entity: types.Entity) -> None: self.simulation = None self.entity = entity self._holders = {} self.count = 0 self.ids = [] - def clone(self, simulation: Simulation) -> Population: + def clone(self, simulation: types.Simulation) -> Population: result = Population(self.entity) result.simulation = simulation result._holders = {variable: holder.clone(result) for (variable, holder) in self._holders.items()} @@ -38,14 +37,14 @@ def clone(self, simulation: Simulation) -> Population: result.ids = self.ids return result - def empty_array(self) -> Array[float]: + def empty_array(self) -> types.Array[float]: return numpy.zeros(self.count) def filled_array( self, value: Union[float, bool], dtype: Optional[numpy.dtype] = None, - ) -> Union[Array[float], Array[bool]]: + ) -> Union[types.Array[float], types.Array[bool]]: return numpy.full(self.count, value, dtype) def __getattr__(self, attribute: str) -> Projector: @@ -64,7 +63,7 @@ def get_index(self, id: str) -> int: def check_array_compatible_with_entity( self, - array: Array[float], + array: types.Array[float], ) -> None: if self.count == array.size: return None @@ -75,9 +74,9 @@ def check_array_compatible_with_entity( def check_period_validity( self, variable_name: str, - period: Optional[Union[int, str, Period]], + period: Optional[Union[int, str, types.Period]], ) -> None: - if isinstance(period, (int, str, Period)): + if isinstance(period, (int, str, types.Period)): return None stack = traceback.extract_stack() @@ -93,9 +92,9 @@ def check_period_validity( def __call__( self, variable_name: str, - period: Optional[Union[int, str, Period]] = None, + period: Optional[Union[int, str, types.Period]] = None, options: Optional[Sequence[str]] = None, - ) -> Optional[Array[float]]: + ) -> Optional[types.Array[float]]: """ Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. @@ -141,13 +140,35 @@ def __call__( # Helpers def get_holder(self, variable_name: str) -> Holder: + holder: Optional[types.Holder] + variable: Optional[types.Variable] + simulation: Optional[types.Simulation] + tax_benefit_system: Optional[types.TaxBenefitSystem] + self.entity.check_variable_defined_for_entity(variable_name) holder = self._holders.get(variable_name) - if holder: + + if holder is not None: return holder + variable = self.entity.get_variable(variable_name) - self._holders[variable_name] = holder = Holder(variable, self) - return holder + + if variable is not None: + holder = Holder(variable, self) + self._holders[variable_name] = holder + return holder + + simulation = self.simulation + + if simulation is None: + raise TypeError("Simulation can't be None.") + + tax_benefit_system = simulation.tax_benefit_system + + if tax_benefit_system is None: + raise TypeError("TaxBenefitSystem can't be None.") + + raise errors.VariableNotFoundError(variable_name, tax_benefit_system) def get_memory_usage( self, @@ -169,7 +190,7 @@ def get_memory_usage( }) @projectors.projectable - def has_role(self, role: Role) -> Optional[Array[bool]]: + def has_role(self, role: types.Role) -> Optional[types.Array[bool]]: """ Check if a person has a given role within its `GroupEntity` @@ -195,10 +216,10 @@ def has_role(self, role: Role) -> Optional[Array[bool]]: @projectors.projectable def value_from_partner( self, - array: Array[float], + array: types.Array[float], entity: Projector, - role: Role, - ) -> Optional[Array[float]]: + role: types.Role, + ) -> Optional[types.Array[float]]: self.check_array_compatible_with_entity(array) self.entity.check_role_validity(role) @@ -218,9 +239,9 @@ def value_from_partner( def get_rank( self, entity: Population, - criteria: Array[float], + criteria: types.Array[float], condition: bool = True, - ) -> Array[int]: + ) -> types.Array[int]: """ Get the rank of a person within an entity according to a criteria. The person with rank 0 has the minimum value of criteria. @@ -265,10 +286,10 @@ def get_rank( class Calculate(NamedTuple): variable: str - period: Period + period: types.Period option: Optional[Sequence[str]] class MemoryUsageByVariable(TypedDict, total = False): - by_variable: Dict[str, MemoryUsage] + by_variable: Dict[str, types.MemoryUsage] total_nb_bytes: int diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py index 13c0257ed3..e9bc64d08b 100644 --- a/openfisca_core/types/__init__.py +++ b/openfisca_core/types/__init__.py @@ -8,10 +8,13 @@ * :attr:`.Array` * ``ArrayLike`` * :attr:`.Cache` + * :attr:`.Enum` + * :attr:`.EnumArray` * :attr:`.Entity` * :attr:`.Formula` * :attr:`.Holder` * :attr:`.Instant` + * :attr:`.MemoryUsage` * :attr:`.ParameterNodeAtInstant` * :attr:`.Params` * :attr:`.Period` @@ -53,7 +56,10 @@ from ._data import ( # noqa: F401 Array, ArrayLike, + Enum, + EnumArray, Instant, + MemoryUsage, Period, ) @@ -78,9 +84,12 @@ "Array", "ArrayLike", "Entity", + "Enum", + "EnumArray", "Formula", "Holder", "Instant", + "MemoryUsage", "ParameterNodeAtInstant", "Params", "Period", diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index 3883d19c9f..a8a1fbd8fd 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -12,13 +12,18 @@ >>> this == that True + >>> that = periods.Instant((1234, 7, 8)) + >>> this == that + False + + """ from __future__ import annotations import typing_extensions from typing import Any, Sequence, TypeVar, Union -from typing_extensions import Protocol +from typing_extensions import Protocol, TypedDict import abc @@ -72,10 +77,41 @@ """ +class Enum(Protocol): + """Enum protocol.""" + + +class EnumArray(Protocol): + """EnumArray protocol.""" + + class Instant(Protocol): """Instant protocol.""" +class MemoryUsage(TypedDict, total = False): + """Virtual memory usage of a Holder. + + Attributes: + cell_size: The amount of bytes assigned to each value. + dtype: The :mod:`numpy.dtype` of any, each, and every value. + nb_arrays: The number of periods for which the Holder contains values. + nb_cells_by_array: The number of entities in the current Simulation. + nb_requests: The number of times the Variable has been computed. + nb_requests_by_array: Average times a stored array has been read. + total_nb_bytes: The total number of bytes used by the Holder. + + """ + + cell_size: float + dtype: numpy.dtype[Any] + nb_arrays: int + nb_cells_by_array: int + nb_requests: int + nb_requests_by_array: int + total_nb_bytes: int + + @typing_extensions.runtime_checkable class Period(Protocol): """Period protocol.""" diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py index afd7891c9a..09a3474714 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types/_domain.py @@ -6,6 +6,9 @@ if the data they hold is different, they are equal but not fungible. Examples: + If we take entities, they are equal as long as they share the same ``key``. + Let's take the following example: + >>> from openfisca_core import entities >>> this = entities.Entity(1, "a", "b", "c") @@ -13,6 +16,21 @@ >>> this == that True + As you can see, ``this`` and ``that`` are equal because they share the same + ``key``: + + >>> this.key == that.key + True + + The opposite is also true: + + >>> that = entities.Entity(2, "a", "b", "c") + >>> this == that + False + + >>> this.key == that.key + False + """ from __future__ import annotations @@ -88,7 +106,9 @@ def __call__(self, instant: Any) -> ParameterNodeAtInstant: class Population(Protocol): """Population protocol.""" + count: Any entity: Any + simulation: Any @abc.abstractmethod def get_holder(self, variable_name: Any) -> Any: @@ -105,6 +125,12 @@ class Role(Protocol): class Simulation(Protocol): """Simulation protocol.""" + trace: Any + tracer: Any + memory_config: Any + data_storage_dir: Any + tax_benefit_system: Any + @abc.abstractmethod def calculate(self, variable_name: Any, period: Any) -> Any: """Abstract method.""" @@ -135,7 +161,14 @@ def get_variable( """Abstract method.""" +@typing_extensions.runtime_checkable class Variable(Protocol): """Variable protocol.""" + name: Any + dtype: Any entity: Any + set_input: Any + value_type: Any + is_neutralized: Any + definition_period: Any diff --git a/openfisca_core/types/_infrastructure.py b/openfisca_core/types/_infrastructure.py index 356502e0ef..2f8c153940 100644 --- a/openfisca_core/types/_infrastructure.py +++ b/openfisca_core/types/_infrastructure.py @@ -29,3 +29,7 @@ def put(self, values: Any, period: Any) -> None: @abc.abstractmethod def delete(self, period: Any = None) -> None: """Abstract method.""" + + @abc.abstractmethod + def get_memory_usage(self) -> Any: + """Abstract method.""" diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 6f3730dd73..115c6267bb 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -42,8 +42,6 @@ check-types: ## Run static type checkers for type errors (strict). lint-typing-strict: \ lint-typing-strict-commons \ - lint-typing-strict-data_storage \ - lint-typing-strict-holders \ lint-typing-strict-types \ ; diff --git a/setup.py b/setup.py index 3726a3bf4a..2177bec5f5 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup( name = 'OpenFisca-Core', - version = '37.0.2', + version = '37.0.3', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [ From 364a3cdb5a63ed42be7522cb75de1272ecd4cced Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 6 Dec 2022 05:43:30 +0100 Subject: [PATCH 06/22] Add missing documentation --- openfisca_core/data_storage/__init__.py | 53 ++++++----- openfisca_core/data_storage/_arrays.py | 22 ++++- openfisca_core/data_storage/_enums.py | 26 +++++- openfisca_core/data_storage/_files.py | 22 ++++- openfisca_core/data_storage/_funcs.py | 37 ++++++++ .../data_storage/in_memory_storage.py | 62 +++++++++---- .../data_storage/on_disk_storage.py | 93 +++++++++++++------ openfisca_core/types/_infrastructure.py | 7 +- 8 files changed, 246 insertions(+), 76 deletions(-) create mode 100644 openfisca_core/data_storage/_funcs.py diff --git a/openfisca_core/data_storage/__init__.py b/openfisca_core/data_storage/__init__.py index e2b4d8911d..fac44a81c6 100644 --- a/openfisca_core/data_storage/__init__.py +++ b/openfisca_core/data_storage/__init__.py @@ -1,25 +1,34 @@ -# Transitional imports to ensure non-breaking changes. -# Could be deprecated in the next major release. -# -# How imports are being used today: -# -# >>> from openfisca_core.module import symbol -# -# The previous example provokes cyclic dependency problems -# that prevent us from modularizing the different components -# of the library so to make them easier to test and to maintain. -# -# How could them be used after the next major release: -# -# >>> from openfisca_core import module -# >>> module.symbol() -# -# And for classes: -# -# >>> from openfisca_core.module import Symbol -# >>> Symbol() -# -# See: https://www.python.org/dev/peps/pep-0008/#imports +"""Transitional imports to ensure non-breaking changes. + +Could be deprecated in the next major release. + +How imports are being used today:: + + from openfisca_core.module import symbol + +The previous example provokes cyclic dependency problems +that prevent us from modularizing the different components +of the library so to make them easier to test and to maintain. + +How could them be used after the next major release:: + + from openfisca_core import module + module.symbol() + +And for classes:: + + from openfisca_core.module import Symbol + Symbol() + +.. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. + +.. _PEP8#Imports: + https://www.python.org/dev/peps/pep-0008/#imports + +.. _OpenFisca's Styleguide: + https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md + +""" from .in_memory_storage import InMemoryStorage # noqa: F401 from .on_disk_storage import OnDiskStorage # noqa: F401 diff --git a/openfisca_core/data_storage/_arrays.py b/openfisca_core/data_storage/_arrays.py index b9fce0b775..23784fb3dd 100644 --- a/openfisca_core/data_storage/_arrays.py +++ b/openfisca_core/data_storage/_arrays.py @@ -3,13 +3,29 @@ from typing import Dict import collections -import dataclasses import numpy from openfisca_core import types +CalculatedVector = numpy.ndarray + -@dataclasses.dataclass(frozen = True) class Arrays(collections.UserDict): - data: Dict[types.Period, numpy.ndarray] + """Dictionary of calculated vectors by period. + + Examples: + >>> from openfisca_core import periods + + >>> instant = periods.Instant((2023, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + >>> vector = numpy.array([1]) + + >>> Arrays({period: vector}) + {Period(('year', Instant((2023, 1, 1)), 1)): array([1])} + + .. versionadded:: 36.0.1 + + """ + + data: Dict[types.Period, CalculatedVector] diff --git a/openfisca_core/data_storage/_enums.py b/openfisca_core/data_storage/_enums.py index 4a735cfe13..9b98f72393 100644 --- a/openfisca_core/data_storage/_enums.py +++ b/openfisca_core/data_storage/_enums.py @@ -3,11 +3,31 @@ from typing import Dict, Type import collections -import dataclasses from openfisca_core import types +FilePath = str +PossibleValues = Type[types.Enum] + -@dataclasses.dataclass(frozen = True) class Enums(collections.UserDict): - data: Dict[str, Type[types.Enum]] + """Dictionary of an Enum's possible values by file path. + + Examples: + >>> from openfisca_core import indexed_enums as enums + + >>> class Enum(enums.Enum): + ... A = "a" + ... B = "b" + + >>> path = "path/to/file.py" + >>> possible_values = tuple(Enum) + + >>> Enums({path: possible_values}) + {'path/to/file.py': (, )} + + .. versionadded:: 36.0.1 + + """ + + data: Dict[FilePath, PossibleValues] diff --git a/openfisca_core/data_storage/_files.py b/openfisca_core/data_storage/_files.py index 69377e117d..3650625536 100644 --- a/openfisca_core/data_storage/_files.py +++ b/openfisca_core/data_storage/_files.py @@ -3,11 +3,27 @@ from typing import Dict import collections -import dataclasses from openfisca_core import types +FilePath = str + -@dataclasses.dataclass(frozen = True) class Files(collections.UserDict): - data: Dict[types.Period, str] + """Dictionary of file paths by periods. + + Examples: + >>> from openfisca_core import periods + + >>> instant = periods.Instant((2023, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + >>> path = "path/to/file.py" + + >>> Files({period: path}) + {Period(('year', Instant((2023, 1, 1)), 1)): 'path/to/file.py'} + + .. versionadded:: 36.0.1 + + """ + + data: Dict[types.Period, FilePath] diff --git a/openfisca_core/data_storage/_funcs.py b/openfisca_core/data_storage/_funcs.py new file mode 100644 index 0000000000..aee8c73316 --- /dev/null +++ b/openfisca_core/data_storage/_funcs.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from openfisca_core import periods, types + + +def parse_period(value: types.Period, eternity: bool) -> types.Period: + """Return a period. + + Args: + value: Period-like value to be parsed. + eternity: Whether to return the eternity period. + + Returns: + A period. + + + Examples: + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> parse_period(period, True) + Period(('eternity', Instant((1, 1, 1)), inf)) + + >>> parse_period(period, False) + Period(('year', Instant((2017, 1, 1)), 1)) + + .. versionadded:: 36.0.1 + + """ + + if eternity: + return periods.period(periods.ETERNITY) + + if isinstance(value, types.Period): + return value + + return periods.period(value) diff --git a/openfisca_core/data_storage/in_memory_storage.py b/openfisca_core/data_storage/in_memory_storage.py index efecbeb8e6..d0e0b57e47 100644 --- a/openfisca_core/data_storage/in_memory_storage.py +++ b/openfisca_core/data_storage/in_memory_storage.py @@ -6,12 +6,20 @@ from openfisca_core import periods, types +from . import _funcs from ._arrays import Arrays class InMemoryStorage: - """ - Low-level class responsible for storing and retrieving calculated vectors in memory + """Class responsible for storing/retrieving vectors in/from memory. + + Attributes: + _arrays: ? + is_eternal: ? + + Args: + is_eternal: ? + """ _arrays: Arrays @@ -21,23 +29,16 @@ def __init__(self, is_eternal: bool = False) -> None: self.is_eternal = is_eternal def get(self, period: types.Period) -> Any: - if self.is_eternal: - period = periods.period(periods.ETERNITY) - period = periods.period(period) - + period = _funcs.parse_period(period, self.is_eternal) values = self._arrays.get(period) + if values is None: return None return values def put(self, value: Any, period: types.Period) -> None: - if self.is_eternal: - period = periods.period(periods.ETERNITY) - - else: - period = periods.period(period) - + period = _funcs.parse_period(period, self.is_eternal) self._arrays = Arrays({period: value, **self._arrays}) def delete(self, period: Optional[types.Period] = None) -> None: @@ -45,11 +46,7 @@ def delete(self, period: Optional[types.Period] = None) -> None: self._arrays = Arrays({}) return None - if self.is_eternal: - period = periods.period(periods.ETERNITY) - - else: - period = periods.period(period) + period = _funcs.parse_period(period, self.is_eternal) self._arrays = Arrays({ period_item: value @@ -58,9 +55,40 @@ def delete(self, period: Optional[types.Period] = None) -> None: }) def get_known_periods(self) -> Sequence[types.Period]: + """List of storage's known periods. + + Returns: + A list of periods. + + Examples: + >>> storage = InMemoryStorage() + + >>> storage.get_known_periods() + [] + + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + >>> storage.put([], period) + >>> storage.get_known_periods() + [Period(('year', Instant((2017, 1, 1)), 1))] + + """ + return list(self._arrays.keys()) def get_memory_usage(self) -> types.MemoryUsage: + """Memory usage of the storage. + + Returns: + A dictionary representing the memory usage. + + Examples: + >>> storage = InMemoryStorage() + >>> storage.get_memory_usage() + {'cell_size': nan, 'nb_arrays': 0, 'total_nb_bytes': 0} + + """ + if not self._arrays: return types.MemoryUsage( cell_size = numpy.nan, diff --git a/openfisca_core/data_storage/on_disk_storage.py b/openfisca_core/data_storage/on_disk_storage.py index 82174d3db6..4e973f57d3 100644 --- a/openfisca_core/data_storage/on_disk_storage.py +++ b/openfisca_core/data_storage/on_disk_storage.py @@ -3,18 +3,20 @@ from typing import Any, NoReturn, Optional, Sequence import os +import pathlib import shutil import numpy from openfisca_core import periods, indexed_enums as enums, types +from . import _funcs from ._enums import Enums from ._files import Files class OnDiskStorage: - """Class responsible for storing and retrieving calculated vectors on disk. + """Class responsible for storing/retrieving vectors on/from disk. Attributes: _enums: ? @@ -58,25 +60,18 @@ def _decode_file(self, file: str) -> Any: return enums.EnumArray(load, enum) def get(self, period: types.Period) -> Any: - if self.is_eternal: - period = periods.period(periods.ETERNITY) - period = periods.period(period) - + period = _funcs.parse_period(period, self.is_eternal) values = self._files.get(period) + if values is None: return None return self._decode_file(values) def put(self, value: Any, period: types.Period) -> None: - if self.is_eternal: - period = periods.period(periods.ETERNITY) - - else: - period = periods.period(period) - - filename = str(period) - path = os.path.join(self.storage_dir, filename) + ".npy" + period = _funcs.parse_period(period, self.is_eternal) + stem = str(period) + path = os.path.join(self.storage_dir, f"{stem}.npy") if isinstance(value, enums.EnumArray): self._enums = Enums({path: value.possible_values, **self._enums}) @@ -88,24 +83,63 @@ def put(self, value: Any, period: types.Period) -> None: def delete(self, period: Optional[types.Period] = None) -> None: if period is None: self._files = Files({}) - return + return None - if self.is_eternal: - period = periods.period(periods.ETERNITY) - period = periods.period(period) + period = _funcs.parse_period(period, self.is_eternal) - if period is not None: - self._files = Files({ - period_item: value - for period_item, value in self._files.items() - if not period.contains(period_item) - }) + self._files = Files({ + period_item: value + for period_item, value in self._files.items() + if not period.contains(period_item) + }) def get_known_periods(self) -> Sequence[types.Period]: + """List of storage's known periods. + + Returns: + A list of periods. + + Examples: + >>> import tempfile + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.get_known_periods() + [] + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... instant = periods.Instant((2017, 1, 1)) + ... period = periods.Period(("year", instant, 1)) + ... storage = OnDiskStorage(storage_dir) + ... storage.put([], period) + ... storage.get_known_periods() + [Period(('year', Instant((2017, 1, 1)), 1))] + + """ + return list(self._files.keys()) def get_memory_usage(self) -> NoReturn: - raise NotImplementedError + """Memory usage of the storage. + + Raises: + NotImplementedError: Method not implemented for this storage. + + Examples: + >>> import tempfile + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.get_memory_usage() + Traceback (most recent call last): + ... + NotImplementedError: Method not implemented for this storage. + + .. versionadded:: 36.0.1 + + """ + + raise NotImplementedError("Method not implemented for this storage.") def restore(self) -> None: self._files = Files({}) @@ -120,9 +154,16 @@ def restore(self) -> None: def __del__(self) -> None: if self.preserve_storage_dir: - return - shutil.rmtree(self.storage_dir) # Remove the holder temporary files + return None + + path = pathlib.Path(self.storage_dir) + + if path.exists(): + # Remove the holder temporary files + shutil.rmtree(self.storage_dir) + # If the simulation temporary directory is empty, remove it parent_dir = os.path.abspath(os.path.join(self.storage_dir, os.pardir)) + if not os.listdir(parent_dir): shutil.rmtree(parent_dir) diff --git a/openfisca_core/types/_infrastructure.py b/openfisca_core/types/_infrastructure.py index 2f8c153940..35333b1cfd 100644 --- a/openfisca_core/types/_infrastructure.py +++ b/openfisca_core/types/_infrastructure.py @@ -16,7 +16,7 @@ class Storage(Protocol): - """Storage protocol.""" + """Vector storage protocol.""" @abc.abstractmethod def get(self, period: Any) -> Optional[numpy.ndarray]: @@ -30,6 +30,9 @@ def put(self, values: Any, period: Any) -> None: def delete(self, period: Any = None) -> None: """Abstract method.""" + def get_known_periods(self) -> Any: + """Storage's known periods.""" + @abc.abstractmethod def get_memory_usage(self) -> Any: - """Abstract method.""" + """Memory usage of the storage.""" From e26ad104e871d4b66c0d236bd4035b21e61c6bf0 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 6 Dec 2022 07:41:58 +0100 Subject: [PATCH 07/22] Bump version --- CHANGELOG.md | 10 ++++++++-- openfisca_core/types/_data.py | 1 + setup.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a8bc0606f..07f85dcee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,16 @@ # Changelog -### 37.0.3 [#1167](https://github.com/openfisca/openfisca-core/pull/1167) +## 37.1.0 [#1167](https://github.com/openfisca/openfisca-core/pull/1167) + +#### New features + +- Use `UserDict` to encapsulate the data model of the `data_storage` module. #### Technical changes -- Document the `data_storage` module. +- Add tests to `data_storage`. +- Add typing to `data_storage`. +- Add documentation to `data_storage`. ### 37.0.2 [#1170](https://github.com/openfisca/openfisca-core/pull/1170) diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index a8a1fbd8fd..f56665d1d6 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -120,6 +120,7 @@ class Period(Protocol): @abc.abstractmethod def start(self) -> Any: """Abstract method.""" + @property @abc.abstractmethod def unit(self) -> Any: diff --git a/setup.py b/setup.py index 2177bec5f5..de4088904d 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup( name = 'OpenFisca-Core', - version = '37.0.3', + version = '37.1.0', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [ From 1b17e1cee7b9f6a4f20144550b14d3fb25a4b650 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 9 Dec 2022 19:29:01 +0100 Subject: [PATCH 08/22] Complete InMemoryStorage doc --- .../data_storage/in_memory_storage.py | 77 ++++++++++++++++--- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/openfisca_core/data_storage/in_memory_storage.py b/openfisca_core/data_storage/in_memory_storage.py index d0e0b57e47..d6c64cb036 100644 --- a/openfisca_core/data_storage/in_memory_storage.py +++ b/openfisca_core/data_storage/in_memory_storage.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Optional, Sequence +from typing import Optional, Sequence import numpy @@ -14,21 +14,41 @@ class InMemoryStorage: """Class responsible for storing/retrieving vectors in/from memory. Attributes: - _arrays: ? - is_eternal: ? + _arrays: A dictionary containing data that has been stored in memory. + is_eternal: A boolean indicating whether the storage is eternal. Args: - is_eternal: ? + is_eternal: A boolean indicating whether the storage is eternal. """ - _arrays: Arrays + _arrays: Arrays = Arrays({}) def __init__(self, is_eternal: bool = False) -> None: - self._arrays = Arrays({}) self.is_eternal = is_eternal - def get(self, period: types.Period) -> Any: + def get(self, period: types.Period) -> Optional[numpy.ndarray]: + """Retrieve the data for the specified period from memory. + + Args: + period: The period for which data should be retrieved. + + Returns: + The data for the specified period, or None if no data is available. + + Examples: + >>> storage = InMemoryStorage() + >>> value = numpy.array([1, 2, 3]) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> storage.put(value, period) + + >>> storage.get(period) + array([1, 2, 3]) + + """ + period = _funcs.parse_period(period, self.is_eternal) values = self._arrays.get(period) @@ -37,11 +57,50 @@ def get(self, period: types.Period) -> Any: return values - def put(self, value: Any, period: types.Period) -> None: + def put(self, value: numpy.ndarray, period: types.Period) -> None: + """Store the specified data in memory for the specified period. + + Args: + value: The data to store + period: The period for which the data should be stored. + + Examples: + >>> storage = InMemoryStorage() + >>> value = numpy.array([1, 2, 3]) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> storage.put(value, period) + + """ + period = _funcs.parse_period(period, self.is_eternal) self._arrays = Arrays({period: value, **self._arrays}) def delete(self, period: Optional[types.Period] = None) -> None: + """Delete the data for the specified period from memory. + + Args: + period: The period for which data should be deleted. If not + specified, all data will be deleted. + + Examples: + >>> storage = InMemoryStorage() + >>> value = numpy.array([1, 2, 3]) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> storage.put(value, period) + + >>> storage.get(period) + array([1, 2, 3]) + + >>> storage.delete(period) + + >>> storage.get(period) + + """ + if period is None: self._arrays = Arrays({}) return None @@ -58,7 +117,7 @@ def get_known_periods(self) -> Sequence[types.Period]: """List of storage's known periods. Returns: - A list of periods. + A sequence containing the storage's known periods. Examples: >>> storage = InMemoryStorage() From 7820c27fa70da207cf689ef3fb51f1a043aea572 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 9 Dec 2022 20:29:42 +0100 Subject: [PATCH 09/22] Complete OnDiskStorage doc --- openfisca_core/data_storage/__init__.py | 12 +- openfisca_core/data_storage/_arrays.py | 6 +- openfisca_core/data_storage/_enums.py | 2 +- openfisca_core/data_storage/_files.py | 2 +- openfisca_core/data_storage/_funcs.py | 2 +- .../data_storage/in_memory_storage.py | 22 ++- .../data_storage/on_disk_storage.py | 151 +++++++++++++++--- 7 files changed, 159 insertions(+), 38 deletions(-) diff --git a/openfisca_core/data_storage/__init__.py b/openfisca_core/data_storage/__init__.py index fac44a81c6..16f6962bf6 100644 --- a/openfisca_core/data_storage/__init__.py +++ b/openfisca_core/data_storage/__init__.py @@ -1,16 +1,16 @@ """Transitional imports to ensure non-breaking changes. -Could be deprecated in the next major release. +These imports could be deprecated in the next major release. -How imports are being used today:: +Currently, imports are used in the following way:: from openfisca_core.module import symbol -The previous example provokes cyclic dependency problems -that prevent us from modularizing the different components -of the library so to make them easier to test and to maintain. +This example causes cyclic dependency problems, which prevent us from +modularising the different components of the library and make them easier to +test and maintain. -How could them be used after the next major release:: +After the next major release, imports could be used in the following way:: from openfisca_core import module module.symbol() diff --git a/openfisca_core/data_storage/_arrays.py b/openfisca_core/data_storage/_arrays.py index 23784fb3dd..24d02f5c02 100644 --- a/openfisca_core/data_storage/_arrays.py +++ b/openfisca_core/data_storage/_arrays.py @@ -8,8 +8,6 @@ from openfisca_core import types -CalculatedVector = numpy.ndarray - class Arrays(collections.UserDict): """Dictionary of calculated vectors by period. @@ -24,8 +22,8 @@ class Arrays(collections.UserDict): >>> Arrays({period: vector}) {Period(('year', Instant((2023, 1, 1)), 1)): array([1])} - .. versionadded:: 36.0.1 + .. versionadded:: 37.1.0 """ - data: Dict[types.Period, CalculatedVector] + data: Dict[types.Period, numpy.ndarray] diff --git a/openfisca_core/data_storage/_enums.py b/openfisca_core/data_storage/_enums.py index 9b98f72393..2eb08b3b14 100644 --- a/openfisca_core/data_storage/_enums.py +++ b/openfisca_core/data_storage/_enums.py @@ -26,7 +26,7 @@ class Enums(collections.UserDict): >>> Enums({path: possible_values}) {'path/to/file.py': (, )} - .. versionadded:: 36.0.1 + .. versionadded:: 37.1.0 """ diff --git a/openfisca_core/data_storage/_files.py b/openfisca_core/data_storage/_files.py index 3650625536..59de729225 100644 --- a/openfisca_core/data_storage/_files.py +++ b/openfisca_core/data_storage/_files.py @@ -22,7 +22,7 @@ class Files(collections.UserDict): >>> Files({period: path}) {Period(('year', Instant((2023, 1, 1)), 1)): 'path/to/file.py'} - .. versionadded:: 36.0.1 + .. versionadded:: 37.1.0 """ diff --git a/openfisca_core/data_storage/_funcs.py b/openfisca_core/data_storage/_funcs.py index aee8c73316..e4cd2be669 100644 --- a/openfisca_core/data_storage/_funcs.py +++ b/openfisca_core/data_storage/_funcs.py @@ -24,7 +24,7 @@ def parse_period(value: types.Period, eternity: bool) -> types.Period: >>> parse_period(period, False) Period(('year', Instant((2017, 1, 1)), 1)) - .. versionadded:: 36.0.1 + .. versionadded:: 37.1.0 """ diff --git a/openfisca_core/data_storage/in_memory_storage.py b/openfisca_core/data_storage/in_memory_storage.py index d6c64cb036..8309118e5f 100644 --- a/openfisca_core/data_storage/in_memory_storage.py +++ b/openfisca_core/data_storage/in_memory_storage.py @@ -15,10 +15,10 @@ class InMemoryStorage: Attributes: _arrays: A dictionary containing data that has been stored in memory. - is_eternal: A boolean indicating whether the storage is eternal. + is_eternal: Flag indicating if the storage of period eternity. Args: - is_eternal: A boolean indicating whether the storage is eternal. + is_eternal: Flag indicating if the storage of period eternity. """ @@ -72,6 +72,9 @@ def put(self, value: numpy.ndarray, period: types.Period) -> None: >>> storage.put(value, period) + >>> storage.get(period) + array([1, 2, 3]) + """ period = _funcs.parse_period(period, self.is_eternal) @@ -99,6 +102,12 @@ def delete(self, period: Optional[types.Period] = None) -> None: >>> storage.get(period) + >>> storage.put(value, period) + + >>> storage.delete() + + >>> storage.get(period) + """ if period is None: @@ -108,9 +117,9 @@ def delete(self, period: Optional[types.Period] = None) -> None: period = _funcs.parse_period(period, self.is_eternal) self._arrays = Arrays({ - period_item: value - for period_item, value in self._arrays.items() - if not period.contains(period_item) + key: value + for key, value in self._arrays.items() + if not period.contains(key) }) def get_known_periods(self) -> Sequence[types.Period]: @@ -139,10 +148,11 @@ def get_memory_usage(self) -> types.MemoryUsage: """Memory usage of the storage. Returns: - A dictionary representing the memory usage. + A dictionary representing the storage's memory usage. Examples: >>> storage = InMemoryStorage() + >>> storage.get_memory_usage() {'cell_size': nan, 'nb_arrays': 0, 'total_nb_bytes': 0} diff --git a/openfisca_core/data_storage/on_disk_storage.py b/openfisca_core/data_storage/on_disk_storage.py index 4e973f57d3..2c2fa74a7e 100644 --- a/openfisca_core/data_storage/on_disk_storage.py +++ b/openfisca_core/data_storage/on_disk_storage.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, NoReturn, Optional, Sequence +from typing import Any, NoReturn, Optional, Sequence, Union import os import pathlib @@ -19,21 +19,21 @@ class OnDiskStorage: """Class responsible for storing/retrieving vectors on/from disk. Attributes: - _enums: ? - _files: ? - is_eternal: ? + _enums: Mapping of file paths to possible Enum values. + _files: Mapping of periods to file paths for stored vectors. + is_eternal: Flag indicating if the storage of period eternity. + preserve_storage_dir: Flag indicating if folders should be preserved. storage_dir: Path to store calculated vectors. - preserve_storage_dir: ? Args: storage_dir: Path to store calculated vectors. - is_eternal: ? - preserve_storage_dir: ? + is_eternal: Flag indicating if the storage of period eternity. + preserve_storage_dir: Flag indicating if folders should be preserved. """ - _enums: Enums - _files: Files + _enums: Enums = Enums({}) + _files: Files = Files({}) is_eternal: bool storage_dir: str preserve_storage_dir: bool @@ -44,13 +44,44 @@ def __init__( is_eternal: bool = False, preserve_storage_dir: bool = False, ) -> None: - self._enums = Enums({}) - self._files = Files({}) self.is_eternal = is_eternal self.storage_dir = storage_dir self.preserve_storage_dir = preserve_storage_dir def _decode_file(self, file: str) -> Any: + """Decodes a file by loading its contents as a NumPy array. + + If the file is associated with Enum values, the array is converted back + to an EnumArray object. + + Args: + file: Path to the file to be decoded. + + Returns: + NumPy array or EnumArray object representing the data in the file. + + Examples + >>> import tempfile + + >>> class Housing(enums.Enum): + ... OWNER = "Owner" + ... TENANT = "Tenant" + ... FREE_LODGER = "Free lodger" + ... HOMELESS = "Homeless" + + >>> array = numpy.array([1]) + >>> value = enums.EnumArray(array, Housing) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.put(value, period) + ... storage._decode_file(storage._files[period]) + EnumArray([]) + + """ + enum = self._enums.get(file) load = numpy.load(file) @@ -59,7 +90,34 @@ def _decode_file(self, file: str) -> Any: return enums.EnumArray(load, enum) - def get(self, period: types.Period) -> Any: + def get( + self, + period: types.Period, + ) -> Optional[Union[numpy.ndarray, enums.EnumArray]]: + """Retrieve the data for the specified period from disk. + + Args: + period: The period for which data should be retrieved. + + Returns: + A NumPy array or EnumArray object representing the vector for the + specified period, or None if no vector is stored for that period. + + Examples: + >>> import tempfile + + >>> value = numpy.array([1, 2, 3]) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.put(value, period) + ... storage.get(period) + array([1, 2, 3]) + + """ + period = _funcs.parse_period(period, self.is_eternal) values = self._files.get(period) @@ -69,6 +127,27 @@ def get(self, period: types.Period) -> Any: return self._decode_file(values) def put(self, value: Any, period: types.Period) -> None: + """Store the specified data on disk for the specified period. + + Args: + value: The data to store + period: The period for which the data should be stored. + + Examples: + >>> import tempfile + + >>> value = numpy.array([1, 2, 3]) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.put(value, period) + ... storage.get(period) + array([1, 2, 3]) + + """ + period = _funcs.parse_period(period, self.is_eternal) stem = str(period) path = os.path.join(self.storage_dir, f"{stem}.npy") @@ -81,6 +160,39 @@ def put(self, value: Any, period: types.Period) -> None: self._files = Files({period: path, **self._files}) def delete(self, period: Optional[types.Period] = None) -> None: + """Delete the data for the specified period from disk. + + Args: + period: The period for which data should be deleted. If not + specified, all data will be deleted. + + Examples: + >>> import tempfile + + >>> value = numpy.array([1, 2, 3]) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.put(value, period) + ... storage.get(period) + array([1, 2, 3]) + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.put(value, period) + ... storage.delete(period) + ... storage.get(period) + + >>> with tempfile.TemporaryDirectory() as storage_dir: + ... storage = OnDiskStorage(storage_dir) + ... storage.put(value, period) + ... storage.delete() + ... storage.get(period) + + """ + if period is None: self._files = Files({}) return None @@ -88,28 +200,29 @@ def delete(self, period: Optional[types.Period] = None) -> None: period = _funcs.parse_period(period, self.is_eternal) self._files = Files({ - period_item: value - for period_item, value in self._files.items() - if not period.contains(period_item) + key: value + for key, value in self._files.items() + if not period.contains(key) }) def get_known_periods(self) -> Sequence[types.Period]: """List of storage's known periods. Returns: - A list of periods. + A sequence containing the storage's known periods. Examples: >>> import tempfile + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + >>> with tempfile.TemporaryDirectory() as storage_dir: ... storage = OnDiskStorage(storage_dir) ... storage.get_known_periods() [] >>> with tempfile.TemporaryDirectory() as storage_dir: - ... instant = periods.Instant((2017, 1, 1)) - ... period = periods.Period(("year", instant, 1)) ... storage = OnDiskStorage(storage_dir) ... storage.put([], period) ... storage.get_known_periods() @@ -135,7 +248,7 @@ def get_memory_usage(self) -> NoReturn: ... NotImplementedError: Method not implemented for this storage. - .. versionadded:: 36.0.1 + .. versionadded:: 37.1.0 """ From 2729f19546c22a63bbaf7d2463db27770457dbc1 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 09:38:13 +0100 Subject: [PATCH 10/22] Add isort config to setup.cfg --- openfisca_core/data_storage/py.typed | 0 openfisca_core/data_storage/typing.py | 29 +++++++++++++++++++ openfisca_core/types/_infrastructure.py | 38 ------------------------- setup.cfg | 13 +++++++++ 4 files changed, 42 insertions(+), 38 deletions(-) create mode 100644 openfisca_core/data_storage/py.typed create mode 100644 openfisca_core/data_storage/typing.py delete mode 100644 openfisca_core/types/_infrastructure.py diff --git a/openfisca_core/data_storage/py.typed b/openfisca_core/data_storage/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openfisca_core/data_storage/typing.py b/openfisca_core/data_storage/typing.py new file mode 100644 index 0000000000..80d7f571db --- /dev/null +++ b/openfisca_core/data_storage/typing.py @@ -0,0 +1,29 @@ +"""Infrastructure-model. + +The infrastructure-model is composed of structures meant to encapsulate the +relationships with layers outside of the domain (memory, disk, etc.). + +""" + +from __future__ import annotations + +from typing import Any +from typing_extensions import Protocol + +import abc + + +class Storage(Protocol): + @abc.abstractmethod + def get(self, period: Any) -> Any: ... + + @abc.abstractmethod + def put(self, values: Any, period: Any) -> None: ... + + @abc.abstractmethod + def delete(self, period: Any = None) -> None: ... + + def periods(self) -> Any: ... + + @abc.abstractmethod + def usage(self) -> Any: ... diff --git a/openfisca_core/types/_infrastructure.py b/openfisca_core/types/_infrastructure.py deleted file mode 100644 index 35333b1cfd..0000000000 --- a/openfisca_core/types/_infrastructure.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Infrastructure-model. - -The infrastructure-model is composed of structures meant to encapsulate the -relationships with layers outside of the domain (memory, disk, etc.). - -""" - -from __future__ import annotations - -from typing import Any, Optional -from typing_extensions import Protocol - -import abc - -import numpy - - -class Storage(Protocol): - """Vector storage protocol.""" - - @abc.abstractmethod - def get(self, period: Any) -> Optional[numpy.ndarray]: - """Abstract method.""" - - @abc.abstractmethod - def put(self, values: Any, period: Any) -> None: - """Abstract method.""" - - @abc.abstractmethod - def delete(self, period: Any = None) -> None: - """Abstract method.""" - - def get_known_periods(self) -> Any: - """Storage's known periods.""" - - @abc.abstractmethod - def get_memory_usage(self) -> Any: - """Memory usage of the storage.""" diff --git a/setup.cfg b/setup.cfg index 9b2229f976..b8b082198b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,10 +15,23 @@ hang-closing = true ignore = E128,E251,F403,F405,E501,RST301,W503,W504 in-place = true include-in-doctest = openfisca_core/commons openfisca_core/data_storage openfisca_core/holders openfisca_core/types +per-file-ignores = */typing.py:D101,D102,E704, */__init__.py:F401 rst-directives = attribute, deprecated, seealso, versionadded, versionchanged rst-roles = any, attr, class, exc, func, meth, mod, obj strictness = short +[isort] +case_sensitive = true +force_alphabetical_sort_within_sections = true +group_by_package = true +include_trailing_comma = true +known_first_party = openfisca_core +known_openfisca = openfisca_core,openfisca_country_template +known_typing = mypy,mypy_extensions +multi_line_output = 8 +py_version = 37 +sections = FUTURE,TYPING,STDLIB,THIRDPARTY,OPENFISCA,FIRSTPARTY,LOCALFOLDER + [pylint.message_control] disable = all enable = C0115,C0116,R0401 From d4e77704862f34566b711661cfdc76aa827985c2 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 09:43:43 +0100 Subject: [PATCH 11/22] Fix imports in populations --- openfisca_core/populations/population.py | 60 ++++++++++++++---------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/openfisca_core/populations/population.py b/openfisca_core/populations/population.py index 5f76ae3965..9c3817616d 100644 --- a/openfisca_core/populations/population.py +++ b/openfisca_core/populations/population.py @@ -1,5 +1,15 @@ from __future__ import annotations +from openfisca_core.data_storage.typing import MemoryUsage +from openfisca_core.types import ( + Array, + Entity, + Period, + Role, + Simulation, + TaxBenefitSystem, + Variable, + ) from typing import Dict, NamedTuple, Optional, Sequence, Union from typing_extensions import TypedDict @@ -7,7 +17,7 @@ import numpy -from openfisca_core import errors, periods, projectors, types +from openfisca_core import errors, periods, projectors from openfisca_core.holders import Holder from openfisca_core.projectors import Projector @@ -16,20 +26,20 @@ class Population: - simulation: Optional[types.Simulation] - entity: types.Entity + simulation: Optional[Simulation] + entity: Entity _holders: Dict[str, Holder] count: int - ids: types.Array[str] + ids: Array[str] - def __init__(self, entity: types.Entity) -> None: + def __init__(self, entity: Entity) -> None: self.simulation = None self.entity = entity self._holders = {} self.count = 0 self.ids = [] - def clone(self, simulation: types.Simulation) -> Population: + def clone(self, simulation: Simulation) -> Population: result = Population(self.entity) result.simulation = simulation result._holders = {variable: holder.clone(result) for (variable, holder) in self._holders.items()} @@ -37,14 +47,14 @@ def clone(self, simulation: types.Simulation) -> Population: result.ids = self.ids return result - def empty_array(self) -> types.Array[float]: + def empty_array(self) -> Array[float]: return numpy.zeros(self.count) def filled_array( self, value: Union[float, bool], dtype: Optional[numpy.dtype] = None, - ) -> Union[types.Array[float], types.Array[bool]]: + ) -> Union[Array[float], Array[bool]]: return numpy.full(self.count, value, dtype) def __getattr__(self, attribute: str) -> Projector: @@ -63,7 +73,7 @@ def get_index(self, id: str) -> int: def check_array_compatible_with_entity( self, - array: types.Array[float], + array: Array[float], ) -> None: if self.count == array.size: return None @@ -74,9 +84,9 @@ def check_array_compatible_with_entity( def check_period_validity( self, variable_name: str, - period: Optional[Union[int, str, types.Period]], + period: Optional[Union[int, str, Period]], ) -> None: - if isinstance(period, (int, str, types.Period)): + if isinstance(period, (int, str, Period)): return None stack = traceback.extract_stack() @@ -92,9 +102,9 @@ def check_period_validity( def __call__( self, variable_name: str, - period: Optional[Union[int, str, types.Period]] = None, + period: Optional[Union[int, str, Period]] = None, options: Optional[Sequence[str]] = None, - ) -> Optional[types.Array[float]]: + ) -> Optional[Array[float]]: """ Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. @@ -140,10 +150,10 @@ def __call__( # Helpers def get_holder(self, variable_name: str) -> Holder: - holder: Optional[types.Holder] - variable: Optional[types.Variable] - simulation: Optional[types.Simulation] - tax_benefit_system: Optional[types.TaxBenefitSystem] + holder: Optional[Holder] + variable: Optional[Variable] + simulation: Optional[Simulation] + tax_benefit_system: Optional[TaxBenefitSystem] self.entity.check_variable_defined_for_entity(variable_name) holder = self._holders.get(variable_name) @@ -190,7 +200,7 @@ def get_memory_usage( }) @projectors.projectable - def has_role(self, role: types.Role) -> Optional[types.Array[bool]]: + def has_role(self, role: Role) -> Optional[Array[bool]]: """ Check if a person has a given role within its `GroupEntity` @@ -216,10 +226,10 @@ def has_role(self, role: types.Role) -> Optional[types.Array[bool]]: @projectors.projectable def value_from_partner( self, - array: types.Array[float], + array: Array[float], entity: Projector, - role: types.Role, - ) -> Optional[types.Array[float]]: + role: Role, + ) -> Optional[Array[float]]: self.check_array_compatible_with_entity(array) self.entity.check_role_validity(role) @@ -239,9 +249,9 @@ def value_from_partner( def get_rank( self, entity: Population, - criteria: types.Array[float], + criteria: Array[float], condition: bool = True, - ) -> types.Array[int]: + ) -> Array[int]: """ Get the rank of a person within an entity according to a criteria. The person with rank 0 has the minimum value of criteria. @@ -286,10 +296,10 @@ def get_rank( class Calculate(NamedTuple): variable: str - period: types.Period + period: Period option: Optional[Sequence[str]] class MemoryUsageByVariable(TypedDict, total = False): - by_variable: Dict[str, types.MemoryUsage] + by_variable: Dict[str, MemoryUsage] total_nb_bytes: int From 2461453b700451fa866f763fb86b262a2f592956 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 09:45:35 +0100 Subject: [PATCH 12/22] Simplify InMemoryStorage --- .../data_storage/in_memory_storage.py | 24 ++++++++++--------- setup.cfg | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/openfisca_core/data_storage/in_memory_storage.py b/openfisca_core/data_storage/in_memory_storage.py index 8309118e5f..fc52d15c35 100644 --- a/openfisca_core/data_storage/in_memory_storage.py +++ b/openfisca_core/data_storage/in_memory_storage.py @@ -1,13 +1,15 @@ from __future__ import annotations +from openfisca_core.types import Period from typing import Optional, Sequence import numpy -from openfisca_core import periods, types +from openfisca_core import periods from . import _funcs from ._arrays import Arrays +from .typing import MemoryUsage class InMemoryStorage: @@ -27,7 +29,7 @@ class InMemoryStorage: def __init__(self, is_eternal: bool = False) -> None: self.is_eternal = is_eternal - def get(self, period: types.Period) -> Optional[numpy.ndarray]: + def get(self, period: Period) -> Optional[numpy.ndarray]: """Retrieve the data for the specified period from memory. Args: @@ -57,7 +59,7 @@ def get(self, period: types.Period) -> Optional[numpy.ndarray]: return values - def put(self, value: numpy.ndarray, period: types.Period) -> None: + def put(self, value: numpy.ndarray, period: Period) -> None: """Store the specified data in memory for the specified period. Args: @@ -80,7 +82,7 @@ def put(self, value: numpy.ndarray, period: types.Period) -> None: period = _funcs.parse_period(period, self.is_eternal) self._arrays = Arrays({period: value, **self._arrays}) - def delete(self, period: Optional[types.Period] = None) -> None: + def delete(self, period: Optional[Period] = None) -> None: """Delete the data for the specified period from memory. Args: @@ -122,7 +124,7 @@ def delete(self, period: Optional[types.Period] = None) -> None: if not period.contains(key) }) - def get_known_periods(self) -> Sequence[types.Period]: + def periods(self) -> Sequence[Period]: """List of storage's known periods. Returns: @@ -131,20 +133,20 @@ def get_known_periods(self) -> Sequence[types.Period]: Examples: >>> storage = InMemoryStorage() - >>> storage.get_known_periods() + >>> storage.periods() [] >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) >>> storage.put([], period) - >>> storage.get_known_periods() + >>> storage.periods() [Period(('year', Instant((2017, 1, 1)), 1))] """ return list(self._arrays.keys()) - def get_memory_usage(self) -> types.MemoryUsage: + def usage(self) -> MemoryUsage: """Memory usage of the storage. Returns: @@ -153,13 +155,13 @@ def get_memory_usage(self) -> types.MemoryUsage: Examples: >>> storage = InMemoryStorage() - >>> storage.get_memory_usage() + >>> storage.usage() {'cell_size': nan, 'nb_arrays': 0, 'total_nb_bytes': 0} """ if not self._arrays: - return types.MemoryUsage( + return MemoryUsage( cell_size = numpy.nan, nb_arrays = 0, total_nb_bytes = 0, @@ -169,7 +171,7 @@ def get_memory_usage(self) -> types.MemoryUsage: array = next(iter(self._arrays.values())) total = array.nbytes * nb_arrays - return types.MemoryUsage( + return MemoryUsage( cell_size = array.itemsize, nb_arrays = nb_arrays, total_nb_bytes = total, diff --git a/setup.cfg b/setup.cfg index b8b082198b..5ab087d6c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ group_by_package = true include_trailing_comma = true known_first_party = openfisca_core known_openfisca = openfisca_core,openfisca_country_template -known_typing = mypy,mypy_extensions +known_typing = *mypy*,*types*,*typing* multi_line_output = 8 py_version = 37 sections = FUTURE,TYPING,STDLIB,THIRDPARTY,OPENFISCA,FIRSTPARTY,LOCALFOLDER From efb5e05be57fd8a52ee8d049cae069825a760c6c Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 09:46:08 +0100 Subject: [PATCH 13/22] Simplify OnDiskStorage --- openfisca_core/data_storage/on_disk_storage.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openfisca_core/data_storage/on_disk_storage.py b/openfisca_core/data_storage/on_disk_storage.py index 2c2fa74a7e..7797e63d7e 100644 --- a/openfisca_core/data_storage/on_disk_storage.py +++ b/openfisca_core/data_storage/on_disk_storage.py @@ -8,7 +8,8 @@ import numpy -from openfisca_core import periods, indexed_enums as enums, types +from openfisca_core import indexed_enums as enums +from openfisca_core import periods, types from . import _funcs from ._enums import Enums @@ -205,7 +206,7 @@ def delete(self, period: Optional[types.Period] = None) -> None: if not period.contains(key) }) - def get_known_periods(self) -> Sequence[types.Period]: + def periods(self) -> Sequence[types.Period]: """List of storage's known periods. Returns: @@ -219,20 +220,20 @@ def get_known_periods(self) -> Sequence[types.Period]: >>> with tempfile.TemporaryDirectory() as storage_dir: ... storage = OnDiskStorage(storage_dir) - ... storage.get_known_periods() + ... storage.periods() [] >>> with tempfile.TemporaryDirectory() as storage_dir: ... storage = OnDiskStorage(storage_dir) ... storage.put([], period) - ... storage.get_known_periods() + ... storage.periods() [Period(('year', Instant((2017, 1, 1)), 1))] """ return list(self._files.keys()) - def get_memory_usage(self) -> NoReturn: + def usage(self) -> NoReturn: """Memory usage of the storage. Raises: @@ -243,7 +244,7 @@ def get_memory_usage(self) -> NoReturn: >>> with tempfile.TemporaryDirectory() as storage_dir: ... storage = OnDiskStorage(storage_dir) - ... storage.get_memory_usage() + ... storage.usage() Traceback (most recent call last): ... NotImplementedError: Method not implemented for this storage. From 6a9382a0b534fd87ad2a70ddaf01abbb0d481280 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 09:46:55 +0100 Subject: [PATCH 14/22] Move storage typing to storage --- openfisca_core/data_storage/typing.py | 38 ++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/openfisca_core/data_storage/typing.py b/openfisca_core/data_storage/typing.py index 80d7f571db..07cd6d52e8 100644 --- a/openfisca_core/data_storage/typing.py +++ b/openfisca_core/data_storage/typing.py @@ -1,17 +1,12 @@ -"""Infrastructure-model. - -The infrastructure-model is composed of structures meant to encapsulate the -relationships with layers outside of the domain (memory, disk, etc.). - -""" - -from __future__ import annotations +# pylint: disable=missing-class-docstring,missing-function-docstring from typing import Any -from typing_extensions import Protocol +from typing_extensions import Protocol, TypedDict import abc +import numpy + class Storage(Protocol): @abc.abstractmethod @@ -27,3 +22,28 @@ def periods(self) -> Any: ... @abc.abstractmethod def usage(self) -> Any: ... + + +class MemoryUsage(TypedDict, total = False): + """Virtual memory usage of a storage.""" + + #: The amount of bytes assigned to each value. + cell_size: float + + #: The :mod:`numpy.dtype` of any, each, and every value. + dtype: numpy.dtype + + #: The number of arrays for which the storage contains values. + nb_arrays: int + + #: The number of entities in the current Simulation. + nb_cells_by_array: int + + #: The number of times the Variable has been computed. + nb_requests: int + + #: Average times a stored array has been read. + nb_requests_by_array: int + + #: The total number of bytes used by the storage. + total_nb_bytes: int From e4d83f419e6a877654e4964e92bc354d20bb80a8 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 09:50:05 +0100 Subject: [PATCH 15/22] Refactor storage logic in holders --- openfisca_core/holders/__init__.py | 5 +- openfisca_core/holders/helpers.py | 15 +- openfisca_core/holders/holder.py | 237 ++++++++++++++++++----------- 3 files changed, 163 insertions(+), 94 deletions(-) diff --git a/openfisca_core/holders/__init__.py b/openfisca_core/holders/__init__.py index a7d46e38a6..ecfcb3c67c 100644 --- a/openfisca_core/holders/__init__.py +++ b/openfisca_core/holders/__init__.py @@ -21,5 +21,8 @@ # # See: https://www.python.org/dev/peps/pep-0008/#imports -from .helpers import set_input_dispatch_by_period, set_input_divide_by_period # noqa: F401 +from .helpers import ( # noqa: F401 + set_input_dispatch_by_period, + set_input_divide_by_period, + ) from .holder import Holder # noqa: F401 diff --git a/openfisca_core/holders/helpers.py b/openfisca_core/holders/helpers.py index 176f6b6f30..4fe8441664 100644 --- a/openfisca_core/holders/helpers.py +++ b/openfisca_core/holders/helpers.py @@ -3,7 +3,6 @@ import numpy from openfisca_core import periods -from openfisca_core.periods import Period log = logging.getLogger(__name__) @@ -21,14 +20,14 @@ def set_input_dispatch_by_period(holder, period, array): period_size = period.size period_unit = period.unit - if holder.variable.definition_period == periods.ETERNITY: + if holder.eternal: raise ValueError("set_input_dispatch_by_period can't be used for eternal variables.") - cached_period_unit = holder.variable.definition_period + cached_period_unit = holder.period after_instant = period.start.offset(period_size, period_unit) # Cache the input data, skipping the existing cached months - sub_period = Period((cached_period_unit, period.start, 1)) + sub_period = periods.Period((cached_period_unit, period.start, 1)) while sub_period.start < after_instant: existing_array = holder.get_array(sub_period) if existing_array is None: @@ -53,15 +52,15 @@ def set_input_divide_by_period(holder, period, array): period_size = period.size period_unit = period.unit - if holder.variable.definition_period == periods.ETERNITY: + if holder.eternal: raise ValueError("set_input_divide_by_period can't be used for eternal variables.") - cached_period_unit = holder.variable.definition_period + cached_period_unit = holder.period after_instant = period.start.offset(period_size, period_unit) # Count the number of elementary periods to change, and the difference with what is already known. remaining_array = array.copy() - sub_period = Period((cached_period_unit, period.start, 1)) + sub_period = periods.Period((cached_period_unit, period.start, 1)) sub_periods_count = 0 while sub_period.start < after_instant: existing_array = holder.get_array(sub_period) @@ -74,7 +73,7 @@ def set_input_divide_by_period(holder, period, array): # Cache the input data if sub_periods_count > 0: divided_array = remaining_array / sub_periods_count - sub_period = Period((cached_period_unit, period.start, 1)) + sub_period = periods.Period((cached_period_unit, period.start, 1)) while sub_period.start < after_instant: if holder.get_array(sub_period) is None: holder._set(sub_period, divided_array) diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index d10eb68212..1324d8ecf4 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -1,57 +1,108 @@ from __future__ import annotations -from typing import Any, Optional, Sequence, Union +from openfisca_core.data_storage.typing import MemoryUsage, Storage +from openfisca_core.types import Period, Population, Simulation, Variable +from typing import Any, Dict, Optional, Sequence, Union +from typing_extensions import Literal +import itertools import os import warnings import numpy import psutil +from sortedcontainers import sorteddict -from openfisca_core import ( - errors, - commons, - data_storage as storage, - indexed_enums as enums, - periods, - tools, - types, - ) +from openfisca_core import commons +from openfisca_core import data_storage as storage +from openfisca_core import errors, experimental +from openfisca_core import indexed_enums as enums +from openfisca_core import periods, tools class Holder: - """ - A holder keeps tracks of a variable values after they have been calculated, or set as an input. - """ - - _disk_storage: Optional[types.Storage] - _do_not_store: bool - _memory_storage: types.Storage - _on_disk_storable: bool - population: types.Population - simulation: types.Simulation - variable: types.Variable - - def __init__( - self, - variable: types.Variable, - population: types.Population, - ) -> None: - self.population = population + """Caches calculated or input variable values.""" + + variable: Variable + population: Population + simulation: Simulation + __stores__: Dict[Literal['memory', 'disk'], Storage] + + def __init__(self, variable: Variable, population: Population) -> None: self.variable = variable + self.population = population self.simulation = population.simulation - self._memory_storage = storage.InMemoryStorage(is_eternal = (self.variable.definition_period == periods.ETERNITY)) - - # By default, do not activate on-disk storage, or variable dropping - self._disk_storage = None - self._on_disk_storable = False - self._do_not_store = False - if self.simulation and self.simulation.memory_config: - if self.variable.name not in self.simulation.memory_config.priority_variables: - self._disk_storage = self.create_disk_storage() - self._on_disk_storable = True - if self.variable.name in self.simulation.memory_config.variables_to_drop: - self._do_not_store = True + + if self.storable: + self.stores = sorteddict.SortedDict({ + "memory": storage.InMemoryStorage(self.eternal), + "disk": self.create_disk_storage(), + }) + + else: + self.stores = sorteddict.SortedDict({ + "memory": storage.InMemoryStorage(self.eternal), + }) + + @property + def name(self) -> str: + return self.variable.name + + @property + def period(self) -> Period: + return self.variable.definition_period + + @property + def eternal(self) -> bool: + return self.period == periods.ETERNITY + + @property + def neutralised(self) -> bool: + return self.variable.is_neutralized + + @property + def config(self) -> Optional[experimental.MemoryConfig]: + try: + return self.simulation.memory_config + + except AttributeError: + return None + + @property + def durable(self) -> bool: + return bool(self.config) + + @property + def transient(self) -> bool: + return not self.durable + + @property + def storable(self) -> bool: + if self.transient: + return False + + if self.config is None: + return False + + return self.name not in self.config.priority_variables + + @property + def cacheable(self) -> bool: + if self.transient: + return False + + if self.config is None: + return False + + return self.name not in self.config.variables_to_drop + + @property + def stores(self) -> Dict[Literal['memory', 'disk'], Storage]: + return self.__stores__ + + @stores.setter + def stores(self, stores: Dict[Literal['memory', 'disk'], Storage]) -> None: + self.__stores__ = stores def clone(self, population): """ @@ -73,17 +124,13 @@ def create_disk_storage( self, directory: Optional[str] = None, preserve: bool = False, - ) -> types.Storage: + ) -> Storage: if directory is None: directory = self.simulation.data_storage_dir - storage_dir = os.path.join(directory, self.variable.name) + storage_dir = os.path.join(directory, self.name) if not os.path.isdir(storage_dir): os.mkdir(storage_dir) - return storage.OnDiskStorage( - storage_dir, - is_eternal = (self.variable.definition_period == periods.ETERNITY), - preserve_storage_dir = preserve - ) + return storage.OnDiskStorage(storage_dir, self.eternal, preserve) def delete_arrays(self, period = None): """ @@ -92,9 +139,10 @@ def delete_arrays(self, period = None): If ``period`` is not ``None``, only remove all values for any period included in period (e.g. if period is "2017", values for "2017-01", "2017-07", etc. would be removed) """ - self._memory_storage.delete(period) - if self._disk_storage: - self._disk_storage.delete(period) + for store in self.stores.values(): + store.delete(period) + + return None def get_array(self, period): """ @@ -102,15 +150,18 @@ def get_array(self, period): If the value is not known, return ``None``. """ - if self.variable.is_neutralized: + if self.neutralised: return self.default_array() - value = self._memory_storage.get(period) - if value is not None: - return value - if self._disk_storage: - return self._disk_storage.get(period) - def get_memory_usage(self) -> types.MemoryUsage: + for store in self.stores.values(): + value = store.get(period) + + if value is not None: + return value + + return None + + def get_memory_usage(self) -> MemoryUsage: """Get data about the virtual memory usage of the Holder. Returns: @@ -152,15 +203,15 @@ def get_memory_usage(self) -> types.MemoryUsage: """ - usage = types.MemoryUsage( + usage = MemoryUsage( nb_cells_by_array = self.population.count, dtype = self.variable.dtype, ) - usage.update(self._memory_storage.get_memory_usage()) + usage.update(self.stores["memory"].usage()) if self.simulation.trace: - nb_requests = self.simulation.tracer.get_nb_requests(self.variable.name) + nb_requests = self.simulation.tracer.get_nb_requests(self.name) usage.update(dict( nb_requests = nb_requests, nb_requests_by_array = nb_requests / float(usage['nb_arrays']) if usage['nb_arrays'] > 0 else numpy.nan @@ -173,12 +224,13 @@ def get_known_periods(self): Get the list of periods the variable value is known for. """ - return list(self._memory_storage.get_known_periods()) + list(( - self._disk_storage.get_known_periods() if self._disk_storage else [])) + known_periods = (store.periods() for store in self.stores.values()) + + return list(itertools.chain(*known_periods)) def set_input( self, - period: types.Period, + period: Period, array: Union[numpy.ndarray, Sequence[Any]], ) -> Optional[numpy.ndarray]: """Set a Variable's array of values of a given Period. @@ -225,92 +277,107 @@ def set_input( """ period = periods.period(period) - if period.unit == periods.ETERNITY and self.variable.definition_period != periods.ETERNITY: + + if period.unit == periods.ETERNITY and not self.eternal: error_message = os.linesep.join([ 'Unable to set a value for variable {0} for periods.ETERNITY.', '{0} is only defined for {1}s. Please adapt your input.', ]).format( - self.variable.name, - self.variable.definition_period + self.name, + self.period ) raise errors.PeriodMismatchError( - self.variable.name, + self.name, period, - self.variable.definition_period, + self.period, error_message ) - if self.variable.is_neutralized: - warning_message = "You cannot set a value for the variable {}, as it has been neutralized. The value you provided ({}) will be ignored.".format(self.variable.name, array) + + if self.neutralised: + warning_message = "You cannot set a value for the variable {}, as it has been neutralized. The value you provided ({}) will be ignored.".format(self.name, array) return warnings.warn( warning_message, Warning ) + if self.variable.value_type in (float, int) and isinstance(array, str): array = tools.eval_expression(array) + if self.variable.set_input: return self.variable.set_input(self, period, array) + return self._set(period, array) def _to_array(self, value): if not isinstance(value, numpy.ndarray): value = numpy.asarray(value) + if value.ndim == 0: # 0-dim arrays are casted to scalar when they interact with float. We don't want that. value = value.reshape(1) + if len(value) != self.population.count: raise ValueError( 'Unable to set value "{}" for variable "{}", as its length is {} while there are {} {} in the simulation.' - .format(value, self.variable.name, len(value), self.population.count, self.population.entity.plural)) + .format(value, self.name, len(value), self.population.count, self.population.entity.plural)) + if self.variable.value_type == enums.Enum: value = self.variable.possible_values.encode(value) + if value.dtype != self.variable.dtype: try: value = value.astype(self.variable.dtype) + except ValueError: raise ValueError( 'Unable to set value "{}" for variable "{}", as the variable dtype "{}" does not match the value dtype "{}".' - .format(value, self.variable.name, self.variable.dtype, value.dtype)) + .format(value, self.name, self.variable.dtype, value.dtype)) + return value def _set(self, period, value): value = self._to_array(value) - if self.variable.definition_period != periods.ETERNITY: + + if not self.eternal: if period is None: raise ValueError('A period must be specified to set values, except for variables with periods.ETERNITY as as period_definition.') - if (self.variable.definition_period != period.unit or period.size > 1): - name = self.variable.name + + if (self.period != period.unit or period.size > 1): + name = self.name period_size_adj = f'{period.unit}' if (period.size == 1) else f'{period.size}-{period.unit}s' error_message = os.linesep.join([ f'Unable to set a value for variable "{name}" for {period_size_adj}-long period "{period}".', - f'"{name}" can only be set for one {self.variable.definition_period} at a time. Please adapt your input.', + f'"{name}" can only be set for one {self.period} at a time. Please adapt your input.', f'If you are the maintainer of "{name}", you can consider adding it a set_input attribute to enable automatic period casting.' ]) raise errors.PeriodMismatchError( - self.variable.name, + self.name, period, - self.variable.definition_period, + self.period, error_message ) should_store_on_disk = ( - self._on_disk_storable and - self._memory_storage.get(period) is None and # If there is already a value in memory, replace it and don't put a new value in the disk storage + self.storable and + self.stores["memory"].get(period) is None and # If there is already a value in memory, replace it + # and don't put a new value in the disk storage psutil.virtual_memory().percent >= self.simulation.memory_config.max_memory_occupation_pc ) if should_store_on_disk: - self._disk_storage.put(value, period) + self.stores["disk"].put(value, period) + else: - self._memory_storage.put(value, period) + self.stores["memory"].put(value, period) def put_in_cache(self, value, period): - if self._do_not_store: - return + if not self.transient and not self.cacheable: + return None if (self.simulation.opt_out_cache and self.simulation.tax_benefit_system.cache_blacklist and - self.variable.name in self.simulation.tax_benefit_system.cache_blacklist): + self.name in self.simulation.tax_benefit_system.cache_blacklist): return self._set(period, value) From 1b810bb98ffaad904ac521cd7cc7a6d1553e228e Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 09:52:15 +0100 Subject: [PATCH 16/22] Update tests --- openfisca_core/tools/simulation_dumper.py | 4 +- openfisca_core/types/__init__.py | 10 ----- openfisca_core/types/_data.py | 28 ++----------- openfisca_core/types/_domain.py | 3 +- tests/core/test_holders.py | 50 +++++++---------------- tests/fixtures/simulations.py | 22 ++++++++++ 6 files changed, 43 insertions(+), 74 deletions(-) diff --git a/openfisca_core/tools/simulation_dumper.py b/openfisca_core/tools/simulation_dumper.py index 424861bba2..73835299e2 100644 --- a/openfisca_core/tools/simulation_dumper.py +++ b/openfisca_core/tools/simulation_dumper.py @@ -5,9 +5,9 @@ import numpy -from openfisca_core.simulations import Simulation from openfisca_core.data_storage import OnDiskStorage from openfisca_core.periods import ETERNITY +from openfisca_core.simulations import Simulation def dump_simulation(simulation, directory): @@ -126,6 +126,6 @@ def _restore_holder(simulation, variable, directory): holder = simulation.get_holder(variable) - for period in disk_storage.get_known_periods(): + for period in disk_storage.periods(): value = disk_storage.get(period) holder.put_in_cache(value, period) diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py index e9bc64d08b..a24c6d02c3 100644 --- a/openfisca_core/types/__init__.py +++ b/openfisca_core/types/__init__.py @@ -14,14 +14,12 @@ * :attr:`.Formula` * :attr:`.Holder` * :attr:`.Instant` - * :attr:`.MemoryUsage` * :attr:`.ParameterNodeAtInstant` * :attr:`.Params` * :attr:`.Period` * :attr:`.Population` * :attr:`.Role` * :attr:`.Simulation` - * :attr:`.Storage` * :attr:`.TaxBenefitSystem` * :attr:`.Variable` @@ -59,10 +57,8 @@ Enum, EnumArray, Instant, - MemoryUsage, Period, ) - from ._domain import ( # noqa: F401 Entity, Formula, @@ -76,10 +72,6 @@ Variable, ) -from ._infrastructure import ( # noqa: F401 - Storage, - ) - __all__ = [ "Array", "ArrayLike", @@ -89,14 +81,12 @@ "Formula", "Holder", "Instant", - "MemoryUsage", "ParameterNodeAtInstant", "Params", "Period", "Population", "Role", "Simulation", - "Storage", "TaxBenefitSystem", "Variable", ] diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index f56665d1d6..9f6302fc29 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -22,13 +22,14 @@ from __future__ import annotations import typing_extensions +from nptyping import NDArray as Array +from nptyping import types from typing import Any, Sequence, TypeVar, Union -from typing_extensions import Protocol, TypedDict +from typing_extensions import Protocol import abc import numpy -from nptyping import types, NDArray as Array T = TypeVar("T", bool, bytes, float, int, object, str) @@ -89,29 +90,6 @@ class Instant(Protocol): """Instant protocol.""" -class MemoryUsage(TypedDict, total = False): - """Virtual memory usage of a Holder. - - Attributes: - cell_size: The amount of bytes assigned to each value. - dtype: The :mod:`numpy.dtype` of any, each, and every value. - nb_arrays: The number of periods for which the Holder contains values. - nb_cells_by_array: The number of entities in the current Simulation. - nb_requests: The number of times the Variable has been computed. - nb_requests_by_array: Average times a stored array has been read. - total_nb_bytes: The total number of bytes used by the Holder. - - """ - - cell_size: float - dtype: numpy.dtype[Any] - nb_arrays: int - nb_cells_by_array: int - nb_requests: int - nb_requests_by_array: int - total_nb_bytes: int - - @typing_extensions.runtime_checkable class Period(Protocol): """Period protocol.""" diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py index 09a3474714..9dd524baf6 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types/_domain.py @@ -35,13 +35,14 @@ from __future__ import annotations -import numpy import typing_extensions from typing import Any, Optional from typing_extensions import Protocol import abc +import numpy + class Entity(Protocol): """Entity protocol.""" diff --git a/tests/core/test_holders.py b/tests/core/test_holders.py index 907aefceb5..bff4596c1d 100644 --- a/tests/core/test_holders.py +++ b/tests/core/test_holders.py @@ -1,31 +1,12 @@ -import pytest - import numpy +import pytest -from openfisca_country_template import situation_examples from openfisca_country_template.variables import housing from openfisca_core import holders, periods, tools from openfisca_core.errors import PeriodMismatchError -from openfisca_core.memory_config import MemoryConfig -from openfisca_core.simulations import SimulationBuilder from openfisca_core.holders import Holder - -@pytest.fixture -def single(tax_benefit_system): - return \ - SimulationBuilder() \ - .build_from_entities(tax_benefit_system, situation_examples.single) - - -@pytest.fixture -def couple(tax_benefit_system): - return \ - SimulationBuilder(). \ - build_from_entities(tax_benefit_system, situation_examples.couple) - - period = periods.period('2017-12') @@ -152,12 +133,9 @@ def test_set_input_dispatch_by_period(single): assert holder.get_array('2019-01') is holder.get_array('2019-12') # Check that the vectors are the same in memory, to avoid duplication -force_storage_on_disk = MemoryConfig(max_memory_occupation = 0) - - -def test_delete_arrays_on_disk(single): +def test_delete_arrays_on_disk(single, memory_config): simulation = single - simulation.memory_config = force_storage_on_disk + simulation.memory_config = memory_config salary_holder = simulation.person.get_holder('salary') salary_holder.set_input(periods.period(2017), numpy.asarray([30000])) salary_holder.set_input(periods.period(2018), numpy.asarray([60000])) @@ -169,9 +147,9 @@ def test_delete_arrays_on_disk(single): assert simulation.person('salary', '2018-01') == 1250 -def test_cache_disk(couple): +def test_cache_disk(couple, memory_config): simulation = couple - simulation.memory_config = force_storage_on_disk + simulation.memory_config = memory_config month = periods.period('2017-01') holder = simulation.person.get_holder('disposable_income') data = numpy.asarray([2000, 3000]) @@ -180,32 +158,32 @@ def test_cache_disk(couple): tools.assert_near(data, stored_data) -def test_known_periods(couple): +def test_known_periods(couple, memory_config): simulation = couple - simulation.memory_config = force_storage_on_disk + simulation.memory_config = memory_config month = periods.period('2017-01') month_2 = periods.period('2017-02') holder = simulation.person.get_holder('disposable_income') data = numpy.asarray([2000, 3000]) holder.put_in_cache(data, month) - holder._memory_storage.put(data, month_2) - + holder.stores["memory"].put(data, month_2) assert sorted(holder.get_known_periods()), [month == month_2] -def test_cache_enum_on_disk(single): +def test_cache_enum_on_disk(single, memory_config): simulation = single - simulation.memory_config = force_storage_on_disk + simulation.memory_config = memory_config month = periods.period('2017-01') simulation.calculate('housing_occupancy_status', month) # First calculation housing_occupancy_status = simulation.calculate('housing_occupancy_status', month) # Read from cache assert housing_occupancy_status == housing.HousingOccupancyStatus.tenant -def test_set_not_cached_variable(single): - dont_cache_variable = MemoryConfig(max_memory_occupation = 1, variables_to_drop = ['salary']) +def test_set_not_cached_variable(single, memory_config): + memory_config.max_memory_occupation = 1 + memory_config.variables_to_drop = ['salary'] simulation = single - simulation.memory_config = dont_cache_variable + simulation.memory_config = memory_config holder = simulation.person.get_holder('salary') array = numpy.asarray([2000]) holder.set_input('2015-01', array) diff --git a/tests/fixtures/simulations.py b/tests/fixtures/simulations.py index 9d343d5ac0..2ad8093d53 100644 --- a/tests/fixtures/simulations.py +++ b/tests/fixtures/simulations.py @@ -2,9 +2,17 @@ import pytest +from openfisca_country_template import situation_examples + +from openfisca_core.memory_config import MemoryConfig from openfisca_core.simulations import SimulationBuilder +@pytest.fixture +def memory_config(): + return MemoryConfig(max_memory_occupation = 0) + + @pytest.fixture def simulation(tax_benefit_system, request): variables, period = request.param @@ -29,3 +37,17 @@ def _simulation(simulation_builder, tax_benefit_system, variables, period): .build_from_variables(tax_benefit_system, variables) return simulation + + +@pytest.fixture +def single(tax_benefit_system): + return \ + SimulationBuilder() \ + .build_from_entities(tax_benefit_system, situation_examples.single) + + +@pytest.fixture +def couple(tax_benefit_system): + return \ + SimulationBuilder(). \ + build_from_entities(tax_benefit_system, situation_examples.couple) From 4ea5d8a50f84fd9b19098e971106f5608027dc7e Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 10:58:59 +0100 Subject: [PATCH 17/22] Remove default array from holders --- openfisca_core/holders/holder.py | 9 +-------- openfisca_core/simulations/simulation.py | 5 +++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index 1324d8ecf4..439fc1f1c6 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -151,7 +151,7 @@ def get_array(self, period): If the value is not known, return ``None``. """ if self.neutralised: - return self.default_array() + return self.variable.default_array(self.population.count) for store in self.stores.values(): value = store.get(period) @@ -381,10 +381,3 @@ def put_in_cache(self, value, period): return self._set(period, value) - - def default_array(self): - """ - Return a new array of the appropriate length for the entity, filled with the variable default values. - """ - - return self.variable.default_array(self.population.count) diff --git a/openfisca_core/simulations/simulation.py b/openfisca_core/simulations/simulation.py index ef1cbbd869..06df856b47 100644 --- a/openfisca_core/simulations/simulation.py +++ b/openfisca_core/simulations/simulation.py @@ -140,13 +140,14 @@ def _calculate(self, variable_name: str, period: Period): # If no result, use the default value and cache it if array is None: - array = holder.default_array() + array = variable.default_array(population.count) array = self._cast_formula_result(array, variable) + holder.put_in_cache(array, period) except SpiralError: - array = holder.default_array() + array = variable.default_array(population.count) return array From 076571c9a5af27c85effb609753055846afef51c Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 12:03:28 +0100 Subject: [PATCH 18/22] Merge data_storage with holders --- openfisca_core/data_storage/__init__.py | 34 --- openfisca_core/data_storage/_arrays.py | 29 --- openfisca_core/data_storage/_enums.py | 33 --- openfisca_core/data_storage/_files.py | 29 --- openfisca_core/data_storage/_funcs.py | 37 ---- openfisca_core/holders/__init__.py | 63 +++--- openfisca_core/holders/holder.py | 53 +++-- .../{data_storage => holders}/py.typed | 0 openfisca_core/holders/storage/__init__.py | 2 + .../storage/_disk_storage.py} | 209 ++++++++---------- .../storage/_memory_storage.py} | 64 ++---- .../{data_storage => holders}/typing.py | 10 + openfisca_core/populations/population.py | 2 +- openfisca_core/tools/simulation_dumper.py | 10 +- openfisca_core/types/__init__.py | 3 - openfisca_core/types/_domain.py | 12 - openfisca_tasks/test_code.mk | 1 - setup.cfg | 2 +- 18 files changed, 204 insertions(+), 389 deletions(-) delete mode 100644 openfisca_core/data_storage/__init__.py delete mode 100644 openfisca_core/data_storage/_arrays.py delete mode 100644 openfisca_core/data_storage/_enums.py delete mode 100644 openfisca_core/data_storage/_files.py delete mode 100644 openfisca_core/data_storage/_funcs.py rename openfisca_core/{data_storage => holders}/py.typed (100%) create mode 100644 openfisca_core/holders/storage/__init__.py rename openfisca_core/{data_storage/on_disk_storage.py => holders/storage/_disk_storage.py} (58%) rename openfisca_core/{data_storage/in_memory_storage.py => holders/storage/_memory_storage.py} (69%) rename openfisca_core/{data_storage => holders}/typing.py (85%) diff --git a/openfisca_core/data_storage/__init__.py b/openfisca_core/data_storage/__init__.py deleted file mode 100644 index 16f6962bf6..0000000000 --- a/openfisca_core/data_storage/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Transitional imports to ensure non-breaking changes. - -These imports could be deprecated in the next major release. - -Currently, imports are used in the following way:: - - from openfisca_core.module import symbol - -This example causes cyclic dependency problems, which prevent us from -modularising the different components of the library and make them easier to -test and maintain. - -After the next major release, imports could be used in the following way:: - - from openfisca_core import module - module.symbol() - -And for classes:: - - from openfisca_core.module import Symbol - Symbol() - -.. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. - -.. _PEP8#Imports: - https://www.python.org/dev/peps/pep-0008/#imports - -.. _OpenFisca's Styleguide: - https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md - -""" - -from .in_memory_storage import InMemoryStorage # noqa: F401 -from .on_disk_storage import OnDiskStorage # noqa: F401 diff --git a/openfisca_core/data_storage/_arrays.py b/openfisca_core/data_storage/_arrays.py deleted file mode 100644 index 24d02f5c02..0000000000 --- a/openfisca_core/data_storage/_arrays.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from typing import Dict - -import collections - -import numpy - -from openfisca_core import types - - -class Arrays(collections.UserDict): - """Dictionary of calculated vectors by period. - - Examples: - >>> from openfisca_core import periods - - >>> instant = periods.Instant((2023, 1, 1)) - >>> period = periods.Period(("year", instant, 1)) - >>> vector = numpy.array([1]) - - >>> Arrays({period: vector}) - {Period(('year', Instant((2023, 1, 1)), 1)): array([1])} - - .. versionadded:: 37.1.0 - - """ - - data: Dict[types.Period, numpy.ndarray] diff --git a/openfisca_core/data_storage/_enums.py b/openfisca_core/data_storage/_enums.py deleted file mode 100644 index 2eb08b3b14..0000000000 --- a/openfisca_core/data_storage/_enums.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -from typing import Dict, Type - -import collections - -from openfisca_core import types - -FilePath = str -PossibleValues = Type[types.Enum] - - -class Enums(collections.UserDict): - """Dictionary of an Enum's possible values by file path. - - Examples: - >>> from openfisca_core import indexed_enums as enums - - >>> class Enum(enums.Enum): - ... A = "a" - ... B = "b" - - >>> path = "path/to/file.py" - >>> possible_values = tuple(Enum) - - >>> Enums({path: possible_values}) - {'path/to/file.py': (, )} - - .. versionadded:: 37.1.0 - - """ - - data: Dict[FilePath, PossibleValues] diff --git a/openfisca_core/data_storage/_files.py b/openfisca_core/data_storage/_files.py deleted file mode 100644 index 59de729225..0000000000 --- a/openfisca_core/data_storage/_files.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from typing import Dict - -import collections - -from openfisca_core import types - -FilePath = str - - -class Files(collections.UserDict): - """Dictionary of file paths by periods. - - Examples: - >>> from openfisca_core import periods - - >>> instant = periods.Instant((2023, 1, 1)) - >>> period = periods.Period(("year", instant, 1)) - >>> path = "path/to/file.py" - - >>> Files({period: path}) - {Period(('year', Instant((2023, 1, 1)), 1)): 'path/to/file.py'} - - .. versionadded:: 37.1.0 - - """ - - data: Dict[types.Period, FilePath] diff --git a/openfisca_core/data_storage/_funcs.py b/openfisca_core/data_storage/_funcs.py deleted file mode 100644 index e4cd2be669..0000000000 --- a/openfisca_core/data_storage/_funcs.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -from openfisca_core import periods, types - - -def parse_period(value: types.Period, eternity: bool) -> types.Period: - """Return a period. - - Args: - value: Period-like value to be parsed. - eternity: Whether to return the eternity period. - - Returns: - A period. - - - Examples: - >>> instant = periods.Instant((2017, 1, 1)) - >>> period = periods.Period(("year", instant, 1)) - - >>> parse_period(period, True) - Period(('eternity', Instant((1, 1, 1)), inf)) - - >>> parse_period(period, False) - Period(('year', Instant((2017, 1, 1)), 1)) - - .. versionadded:: 37.1.0 - - """ - - if eternity: - return periods.period(periods.ETERNITY) - - if isinstance(value, types.Period): - return value - - return periods.period(value) diff --git a/openfisca_core/holders/__init__.py b/openfisca_core/holders/__init__.py index ecfcb3c67c..e903a54821 100644 --- a/openfisca_core/holders/__init__.py +++ b/openfisca_core/holders/__init__.py @@ -1,28 +1,35 @@ -# Transitional imports to ensure non-breaking changes. -# Could be deprecated in the next major release. -# -# How imports are being used today: -# -# >>> from openfisca_core.module import symbol -# -# The previous example provokes cyclic dependency problems -# that prevent us from modularizing the different components -# of the library so to make them easier to test and to maintain. -# -# How could them be used after the next major release: -# -# >>> from openfisca_core import module -# >>> module.symbol() -# -# And for classes: -# -# >>> from openfisca_core.module import Symbol -# >>> Symbol() -# -# See: https://www.python.org/dev/peps/pep-0008/#imports - -from .helpers import ( # noqa: F401 - set_input_dispatch_by_period, - set_input_divide_by_period, - ) -from .holder import Holder # noqa: F401 +"""Transitional imports to ensure non-breaking changes. + +These imports could be deprecated in the next major release. + +Currently, imports are used in the following way:: + + from openfisca_core.module import symbol + +This example causes cyclic dependency problems, which prevent us from +modularising the different components of the library and make them easier to +test and maintain. + +After the next major release, imports could be used in the following way:: + + from openfisca_core import module + module.symbol() + +And for classes:: + + from openfisca_core.module import Symbol + Symbol() + +.. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. + +.. _PEP8#Imports: + https://www.python.org/dev/peps/pep-0008/#imports + +.. _OpenFisca's Styleguide: + https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md + +""" + +from .helpers import set_input_dispatch_by_period, set_input_divide_by_period +from .holder import Holder +from .storage import DiskStorage, MemoryStorage diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index 439fc1f1c6..3fbf51d6d5 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -1,8 +1,8 @@ from __future__ import annotations -from openfisca_core.data_storage.typing import MemoryUsage, Storage +from openfisca_core.holders.typing import MemoryUsage, Storage from openfisca_core.types import Period, Population, Simulation, Variable -from typing import Any, Dict, Optional, Sequence, Union +from typing import Any, Sequence from typing_extensions import Literal import itertools @@ -13,12 +13,12 @@ import psutil from sortedcontainers import sorteddict -from openfisca_core import commons -from openfisca_core import data_storage as storage -from openfisca_core import errors, experimental +from openfisca_core import commons, errors, experimental from openfisca_core import indexed_enums as enums from openfisca_core import periods, tools +from .storage import DiskStorage, MemoryStorage + class Holder: """Caches calculated or input variable values.""" @@ -26,7 +26,7 @@ class Holder: variable: Variable population: Population simulation: Simulation - __stores__: Dict[Literal['memory', 'disk'], Storage] + __stores__: dict[Literal['memory', 'disk'], Storage] def __init__(self, variable: Variable, population: Population) -> None: self.variable = variable @@ -35,13 +35,13 @@ def __init__(self, variable: Variable, population: Population) -> None: if self.storable: self.stores = sorteddict.SortedDict({ - "memory": storage.InMemoryStorage(self.eternal), + "memory": MemoryStorage(), "disk": self.create_disk_storage(), }) else: self.stores = sorteddict.SortedDict({ - "memory": storage.InMemoryStorage(self.eternal), + "memory": MemoryStorage(), }) @property @@ -61,7 +61,7 @@ def neutralised(self) -> bool: return self.variable.is_neutralized @property - def config(self) -> Optional[experimental.MemoryConfig]: + def config(self) -> experimental.MemoryConfig | None: try: return self.simulation.memory_config @@ -97,11 +97,11 @@ def cacheable(self) -> bool: return self.name not in self.config.variables_to_drop @property - def stores(self) -> Dict[Literal['memory', 'disk'], Storage]: + def stores(self) -> dict[Literal['memory', 'disk'], Storage]: return self.__stores__ @stores.setter - def stores(self, stores: Dict[Literal['memory', 'disk'], Storage]) -> None: + def stores(self, stores: dict[Literal['memory', 'disk'], Storage]) -> None: self.__stores__ = stores def clone(self, population): @@ -122,7 +122,7 @@ def clone(self, population): def create_disk_storage( self, - directory: Optional[str] = None, + directory: str | None = None, preserve: bool = False, ) -> Storage: if directory is None: @@ -130,15 +130,21 @@ def create_disk_storage( storage_dir = os.path.join(directory, self.name) if not os.path.isdir(storage_dir): os.mkdir(storage_dir) - return storage.OnDiskStorage(storage_dir, self.eternal, preserve) + return DiskStorage(storage_dir, preserve) - def delete_arrays(self, period = None): + def delete_arrays(self, period: Any = None): """ If ``period`` is ``None``, remove all known values of the variable. If ``period`` is not ``None``, only remove all values for any period included in period (e.g. if period is "2017", values for "2017-01", "2017-07", etc. would be removed) """ + if self.eternal and period is not None: + period = periods.ETERNITY + + else: + period = periods.period(period) + for store in self.stores.values(): store.delete(period) @@ -153,6 +159,12 @@ def get_array(self, period): if self.neutralised: return self.variable.default_array(self.population.count) + if self.eternal: + period = periods.ETERNITY + + else: + period = periods.period(period) + for store in self.stores.values(): value = store.get(period) @@ -231,8 +243,8 @@ def get_known_periods(self): def set_input( self, period: Period, - array: Union[numpy.ndarray, Sequence[Any]], - ) -> Optional[numpy.ndarray]: + array: numpy.ndarray | Sequence[Any], + ) -> numpy.ndarray | None: """Set a Variable's array of values of a given Period. Args: @@ -294,7 +306,7 @@ def set_input( ) if self.neutralised: - warning_message = "You cannot set a value for the variable {}, as it has been neutralized. The value you provided ({}) will be ignored.".format(self.name, array) + warning_message = f"You cannot set a value for the variable {self.name}, as it has been neutralized. The value you provided ({array}) will be ignored." return warnings.warn( warning_message, Warning @@ -338,11 +350,14 @@ def _to_array(self, value): def _set(self, period, value): value = self._to_array(value) - if not self.eternal: + if self.eternal: + period = periods.ETERNITY + + else: if period is None: raise ValueError('A period must be specified to set values, except for variables with periods.ETERNITY as as period_definition.') - if (self.period != period.unit or period.size > 1): + if self.period != period.unit or period.size > 1: name = self.name period_size_adj = f'{period.unit}' if (period.size == 1) else f'{period.size}-{period.unit}s' error_message = os.linesep.join([ diff --git a/openfisca_core/data_storage/py.typed b/openfisca_core/holders/py.typed similarity index 100% rename from openfisca_core/data_storage/py.typed rename to openfisca_core/holders/py.typed diff --git a/openfisca_core/holders/storage/__init__.py b/openfisca_core/holders/storage/__init__.py new file mode 100644 index 0000000000..402b214bbf --- /dev/null +++ b/openfisca_core/holders/storage/__init__.py @@ -0,0 +1,2 @@ +from ._disk_storage import DiskStorage +from ._memory_storage import MemoryStorage diff --git a/openfisca_core/data_storage/on_disk_storage.py b/openfisca_core/holders/storage/_disk_storage.py similarity index 58% rename from openfisca_core/data_storage/on_disk_storage.py rename to openfisca_core/holders/storage/_disk_storage.py index 7797e63d7e..eb12fc31db 100644 --- a/openfisca_core/data_storage/on_disk_storage.py +++ b/openfisca_core/holders/storage/_disk_storage.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Any, NoReturn, Optional, Sequence, Union +from openfisca_core.types import Enum, Period +from typing import Any, NoReturn, Sequence import os import pathlib @@ -9,92 +10,36 @@ import numpy from openfisca_core import indexed_enums as enums -from openfisca_core import periods, types +from openfisca_core import periods -from . import _funcs -from ._enums import Enums -from ._files import Files - -class OnDiskStorage: +class DiskStorage: """Class responsible for storing/retrieving vectors on/from disk. Attributes: - _enums: Mapping of file paths to possible Enum values. - _files: Mapping of periods to file paths for stored vectors. - is_eternal: Flag indicating if the storage of period eternity. - preserve_storage_dir: Flag indicating if folders should be preserved. - storage_dir: Path to store calculated vectors. - - Args: - storage_dir: Path to store calculated vectors. - is_eternal: Flag indicating if the storage of period eternity. - preserve_storage_dir: Flag indicating if folders should be preserved. + directory: Path to store calculated vectors. + keep: Flag indicating if folders should be preserved. """ - _enums: Enums = Enums({}) - _files: Files = Files({}) is_eternal: bool - storage_dir: str - preserve_storage_dir: bool - - def __init__( - self, - storage_dir: str, - is_eternal: bool = False, - preserve_storage_dir: bool = False, - ) -> None: - self.is_eternal = is_eternal - self.storage_dir = storage_dir - self.preserve_storage_dir = preserve_storage_dir - - def _decode_file(self, file: str) -> Any: - """Decodes a file by loading its contents as a NumPy array. - - If the file is associated with Enum values, the array is converted back - to an EnumArray object. + directory: str + keep: bool - Args: - file: Path to the file to be decoded. - - Returns: - NumPy array or EnumArray object representing the data in the file. + #: Mapping of file paths to possible Enum values. + __enums__: dict[str, type[Enum]] = {} - Examples - >>> import tempfile + #: Mapping of periods to file paths for stored vectors. + __files__: dict[Period, str] = {} - >>> class Housing(enums.Enum): - ... OWNER = "Owner" - ... TENANT = "Tenant" - ... FREE_LODGER = "Free lodger" - ... HOMELESS = "Homeless" - - >>> array = numpy.array([1]) - >>> value = enums.EnumArray(array, Housing) - >>> instant = periods.Instant((2017, 1, 1)) - >>> period = periods.Period(("year", instant, 1)) - - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) - ... storage.put(value, period) - ... storage._decode_file(storage._files[period]) - EnumArray([]) - - """ - - enum = self._enums.get(file) - load = numpy.load(file) - - if enum is None: - return load - - return enums.EnumArray(load, enum) + def __init__(self, directory: str, keep: bool = False) -> None: + self.directory = directory + self.keep = keep def get( self, - period: types.Period, - ) -> Optional[Union[numpy.ndarray, enums.EnumArray]]: + period: Period, + ) -> numpy.ndarray | enums.EnumArray | None: """Retrieve the data for the specified period from disk. Args: @@ -111,23 +56,22 @@ def get( >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.put(value, period) ... storage.get(period) array([1, 2, 3]) """ - period = _funcs.parse_period(period, self.is_eternal) - values = self._files.get(period) + values = self.__files__.get(period) if values is None: return None return self._decode_file(values) - def put(self, value: Any, period: types.Period) -> None: + def put(self, value: Any, period: Period) -> None: """Store the specified data on disk for the specified period. Args: @@ -141,26 +85,25 @@ def put(self, value: Any, period: types.Period) -> None: >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.put(value, period) ... storage.get(period) array([1, 2, 3]) """ - period = _funcs.parse_period(period, self.is_eternal) stem = str(period) - path = os.path.join(self.storage_dir, f"{stem}.npy") + path = os.path.join(self.directory, f"{stem}.npy") if isinstance(value, enums.EnumArray): - self._enums = Enums({path: value.possible_values, **self._enums}) + self.__enums__ = {path: value.possible_values, **self.__enums__} value = value.view(numpy.ndarray) numpy.save(path, value) - self._files = Files({period: path, **self._files}) + self.__files__ = {period: path, **self.__files__} - def delete(self, period: Optional[types.Period] = None) -> None: + def delete(self, period: Period | None = None) -> None: """Delete the data for the specified period from disk. Args: @@ -174,20 +117,20 @@ def delete(self, period: Optional[types.Period] = None) -> None: >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.put(value, period) ... storage.get(period) array([1, 2, 3]) - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.put(value, period) ... storage.delete(period) ... storage.get(period) - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.put(value, period) ... storage.delete() ... storage.get(period) @@ -195,18 +138,16 @@ def delete(self, period: Optional[types.Period] = None) -> None: """ if period is None: - self._files = Files({}) + self.__files__ = {} return None - period = _funcs.parse_period(period, self.is_eternal) - - self._files = Files({ + self.__files__ = { key: value - for key, value in self._files.items() + for key, value in self.__files__.items() if not period.contains(key) - }) + } - def periods(self) -> Sequence[types.Period]: + def periods(self) -> Sequence[Period]: """List of storage's known periods. Returns: @@ -218,20 +159,20 @@ def periods(self) -> Sequence[types.Period]: >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.periods() [] - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.put([], period) ... storage.periods() [Period(('year', Instant((2017, 1, 1)), 1))] """ - return list(self._files.keys()) + return list(self.__files__.keys()) def usage(self) -> NoReturn: """Memory usage of the storage. @@ -242,8 +183,8 @@ def usage(self) -> NoReturn: Examples: >>> import tempfile - >>> with tempfile.TemporaryDirectory() as storage_dir: - ... storage = OnDiskStorage(storage_dir) + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) ... storage.usage() Traceback (most recent call last): ... @@ -256,28 +197,70 @@ def usage(self) -> NoReturn: raise NotImplementedError("Method not implemented for this storage.") def restore(self) -> None: - self._files = Files({}) - # Restore self._files from content of storage_dir. - for filename in os.listdir(self.storage_dir): + self.__files__ = {} + # Restore self.__files__ from content of directory. + for filename in os.listdir(self.directory): if not filename.endswith('.npy'): continue - path = os.path.join(self.storage_dir, filename) + path = os.path.join(self.directory, filename) filename_core = filename.rsplit('.', 1)[0] period = periods.period(filename_core) - self._files = Files({period: path, **self._files}) + self.__files__ = {period: path, **self.__files__} def __del__(self) -> None: - if self.preserve_storage_dir: + if self.keep: return None - path = pathlib.Path(self.storage_dir) + path = pathlib.Path(self.directory) if path.exists(): # Remove the holder temporary files - shutil.rmtree(self.storage_dir) + shutil.rmtree(self.directory) # If the simulation temporary directory is empty, remove it - parent_dir = os.path.abspath(os.path.join(self.storage_dir, os.pardir)) + parent_dir = os.path.abspath(os.path.join(self.directory, os.pardir)) if not os.listdir(parent_dir): shutil.rmtree(parent_dir) + + def _decode_file(self, file: str) -> Any: + """Decodes a file by loading its contents as a NumPy array. + + If the file is associated with Enum values, the array is converted back + to an EnumArray object. + + Args: + file: Path to the file to be decoded. + + Returns: + NumPy array or EnumArray object representing the data in the file. + + Examples + >>> import tempfile + + >>> class Housing(enums.Enum): + ... OWNER = "Owner" + ... TENANT = "Tenant" + ... FREE_LODGER = "Free lodger" + ... HOMELESS = "Homeless" + + >>> array = numpy.array([1]) + >>> value = enums.EnumArray(array, Housing) + >>> instant = periods.Instant((2017, 1, 1)) + >>> period = periods.Period(("year", instant, 1)) + + >>> with tempfile.TemporaryDirectory() as directory: + ... storage = DiskStorage(directory) + ... storage.put(value, period) + ... storage._decode_file(storage.__files__[period]) + EnumArray([]) + + """ + + enum = self.__enums__.get(file) + load = numpy.load(file) + + if enum is None: + return load + + return enums.EnumArray(load, enum) diff --git a/openfisca_core/data_storage/in_memory_storage.py b/openfisca_core/holders/storage/_memory_storage.py similarity index 69% rename from openfisca_core/data_storage/in_memory_storage.py rename to openfisca_core/holders/storage/_memory_storage.py index fc52d15c35..beb9bd6f2d 100644 --- a/openfisca_core/data_storage/in_memory_storage.py +++ b/openfisca_core/holders/storage/_memory_storage.py @@ -1,35 +1,21 @@ from __future__ import annotations +from openfisca_core.holders.typing import MemoryUsage from openfisca_core.types import Period -from typing import Optional, Sequence +from typing import Sequence import numpy from openfisca_core import periods -from . import _funcs -from ._arrays import Arrays -from .typing import MemoryUsage +class MemoryStorage: + """Class responsible for storing/retrieving vectors in/from memory.""" -class InMemoryStorage: - """Class responsible for storing/retrieving vectors in/from memory. + #: A dictionary containing data that has been stored in memory. + __arrays__: dict[Period, numpy.ndarray] = {} - Attributes: - _arrays: A dictionary containing data that has been stored in memory. - is_eternal: Flag indicating if the storage of period eternity. - - Args: - is_eternal: Flag indicating if the storage of period eternity. - - """ - - _arrays: Arrays = Arrays({}) - - def __init__(self, is_eternal: bool = False) -> None: - self.is_eternal = is_eternal - - def get(self, period: Period) -> Optional[numpy.ndarray]: + def get(self, period: Period) -> numpy.ndarray | None: """Retrieve the data for the specified period from memory. Args: @@ -39,7 +25,7 @@ def get(self, period: Period) -> Optional[numpy.ndarray]: The data for the specified period, or None if no data is available. Examples: - >>> storage = InMemoryStorage() + >>> storage = MemoryStorage() >>> value = numpy.array([1, 2, 3]) >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) @@ -51,8 +37,7 @@ def get(self, period: Period) -> Optional[numpy.ndarray]: """ - period = _funcs.parse_period(period, self.is_eternal) - values = self._arrays.get(period) + values = self.__arrays__.get(period) if values is None: return None @@ -67,7 +52,7 @@ def put(self, value: numpy.ndarray, period: Period) -> None: period: The period for which the data should be stored. Examples: - >>> storage = InMemoryStorage() + >>> storage = MemoryStorage() >>> value = numpy.array([1, 2, 3]) >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) @@ -79,10 +64,9 @@ def put(self, value: numpy.ndarray, period: Period) -> None: """ - period = _funcs.parse_period(period, self.is_eternal) - self._arrays = Arrays({period: value, **self._arrays}) + self.__arrays__ = {period: value, **self.__arrays__} - def delete(self, period: Optional[Period] = None) -> None: + def delete(self, period: Period | None = None) -> None: """Delete the data for the specified period from memory. Args: @@ -90,7 +74,7 @@ def delete(self, period: Optional[Period] = None) -> None: specified, all data will be deleted. Examples: - >>> storage = InMemoryStorage() + >>> storage = MemoryStorage() >>> value = numpy.array([1, 2, 3]) >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) @@ -113,16 +97,14 @@ def delete(self, period: Optional[Period] = None) -> None: """ if period is None: - self._arrays = Arrays({}) + self.__arrays__ = {} return None - period = _funcs.parse_period(period, self.is_eternal) - - self._arrays = Arrays({ + self.__arrays__ = { key: value - for key, value in self._arrays.items() + for key, value in self.__arrays__.items() if not period.contains(key) - }) + } def periods(self) -> Sequence[Period]: """List of storage's known periods. @@ -131,7 +113,7 @@ def periods(self) -> Sequence[Period]: A sequence containing the storage's known periods. Examples: - >>> storage = InMemoryStorage() + >>> storage = MemoryStorage() >>> storage.periods() [] @@ -144,7 +126,7 @@ def periods(self) -> Sequence[Period]: """ - return list(self._arrays.keys()) + return list(self.__arrays__.keys()) def usage(self) -> MemoryUsage: """Memory usage of the storage. @@ -153,22 +135,22 @@ def usage(self) -> MemoryUsage: A dictionary representing the storage's memory usage. Examples: - >>> storage = InMemoryStorage() + >>> storage = MemoryStorage() >>> storage.usage() {'cell_size': nan, 'nb_arrays': 0, 'total_nb_bytes': 0} """ - if not self._arrays: + if not self.__arrays__: return MemoryUsage( cell_size = numpy.nan, nb_arrays = 0, total_nb_bytes = 0, ) - nb_arrays = len(self._arrays) - array = next(iter(self._arrays.values())) + nb_arrays = len(self.__arrays__) + array = next(iter(self.__arrays__.values())) total = array.nbytes * nb_arrays return MemoryUsage( diff --git a/openfisca_core/data_storage/typing.py b/openfisca_core/holders/typing.py similarity index 85% rename from openfisca_core/data_storage/typing.py rename to openfisca_core/holders/typing.py index 07cd6d52e8..bec5b1ae19 100644 --- a/openfisca_core/data_storage/typing.py +++ b/openfisca_core/holders/typing.py @@ -1,5 +1,7 @@ # pylint: disable=missing-class-docstring,missing-function-docstring +from __future__ import annotations + from typing import Any from typing_extensions import Protocol, TypedDict @@ -8,6 +10,14 @@ import numpy +class Holder(Protocol): + @abc.abstractmethod + def clone(self, population: Any) -> Holder: ... + + @abc.abstractmethod + def get_memory_usage(self) -> Any: ... + + class Storage(Protocol): @abc.abstractmethod def get(self, period: Any) -> Any: ... diff --git a/openfisca_core/populations/population.py b/openfisca_core/populations/population.py index 9c3817616d..34e3d9e902 100644 --- a/openfisca_core/populations/population.py +++ b/openfisca_core/populations/population.py @@ -1,6 +1,6 @@ from __future__ import annotations -from openfisca_core.data_storage.typing import MemoryUsage +from openfisca_core.holders.typing import MemoryUsage from openfisca_core.types import ( Array, Entity, diff --git a/openfisca_core/tools/simulation_dumper.py b/openfisca_core/tools/simulation_dumper.py index 73835299e2..ac48b9b1e0 100644 --- a/openfisca_core/tools/simulation_dumper.py +++ b/openfisca_core/tools/simulation_dumper.py @@ -5,8 +5,7 @@ import numpy -from openfisca_core.data_storage import OnDiskStorage -from openfisca_core.periods import ETERNITY +from openfisca_core import holders from openfisca_core.simulations import Simulation @@ -116,12 +115,7 @@ def _restore_entity(population, directory): def _restore_holder(simulation, variable, directory): storage_dir = os.path.join(directory, variable) - is_variable_eternal = simulation.tax_benefit_system.get_variable(variable).definition_period == ETERNITY - disk_storage = OnDiskStorage( - storage_dir, - is_eternal = is_variable_eternal, - preserve_storage_dir = True - ) + disk_storage = holders.DiskStorage(storage_dir, keep = True) disk_storage.restore() holder = simulation.get_holder(variable) diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py index a24c6d02c3..1f16c47b89 100644 --- a/openfisca_core/types/__init__.py +++ b/openfisca_core/types/__init__.py @@ -12,7 +12,6 @@ * :attr:`.EnumArray` * :attr:`.Entity` * :attr:`.Formula` - * :attr:`.Holder` * :attr:`.Instant` * :attr:`.ParameterNodeAtInstant` * :attr:`.Params` @@ -62,7 +61,6 @@ from ._domain import ( # noqa: F401 Entity, Formula, - Holder, ParameterNodeAtInstant, Params, Population, @@ -79,7 +77,6 @@ "Enum", "EnumArray", "Formula", - "Holder", "Instant", "ParameterNodeAtInstant", "Params", diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py index 9dd524baf6..7bfbb2eba3 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types/_domain.py @@ -79,18 +79,6 @@ def __call__( """Abstract method.""" -class Holder(Protocol): - """Holder protocol.""" - - @abc.abstractmethod - def clone(self, population: Any) -> Holder: - """Abstract method.""" - - @abc.abstractmethod - def get_memory_usage(self) -> Any: - """Abstract method.""" - - @typing_extensions.runtime_checkable class ParameterNodeAtInstant(Protocol): """ParameterNodeAtInstant protocol.""" diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index a934237de5..63fdd4386a 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -33,7 +33,6 @@ test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 @$(call print_help,$@:) @pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ openfisca_core/commons \ - openfisca_core/data_storage \ openfisca_core/holders \ openfisca_core/types @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ diff --git a/setup.cfg b/setup.cfg index 5ab087d6c8..14ec27e939 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ extend-ignore = D hang-closing = true ignore = E128,E251,F403,F405,E501,RST301,W503,W504 in-place = true -include-in-doctest = openfisca_core/commons openfisca_core/data_storage openfisca_core/holders openfisca_core/types +include-in-doctest = openfisca_core/commons openfisca_core/holders openfisca_core/types per-file-ignores = */typing.py:D101,D102,E704, */__init__.py:F401 rst-directives = attribute, deprecated, seealso, versionadded, versionchanged rst-roles = any, attr, class, exc, func, meth, mod, obj From 0e2fedbd0abe9b02110d46c61af9df00d434d7c5 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 15 Dec 2022 12:53:56 +0100 Subject: [PATCH 19/22] Fix typing --- openfisca_core/holders/__init__.py | 2 +- openfisca_core/holders/helpers/__init__.py | 4 +++ .../{helpers.py => helpers/_set_input.py} | 9 +++-- openfisca_core/holders/holder.py | 34 ++++++++++--------- openfisca_core/holders/repos/__init__.py | 2 ++ .../_disk_storage.py => repos/_disk_repo.py} | 20 +++++------ .../_memory_repo.py} | 12 +++---- openfisca_core/holders/storage/__init__.py | 2 -- openfisca_core/tools/simulation_dumper.py | 4 +-- openfisca_core/types/_data.py | 5 +++ openfisca_core/types/_domain.py | 14 +++++--- setup.cfg | 4 +-- 12 files changed, 67 insertions(+), 45 deletions(-) create mode 100644 openfisca_core/holders/helpers/__init__.py rename openfisca_core/holders/{helpers.py => helpers/_set_input.py} (93%) create mode 100644 openfisca_core/holders/repos/__init__.py rename openfisca_core/holders/{storage/_disk_storage.py => repos/_disk_repo.py} (93%) rename openfisca_core/holders/{storage/_memory_storage.py => repos/_memory_repo.py} (94%) delete mode 100644 openfisca_core/holders/storage/__init__.py diff --git a/openfisca_core/holders/__init__.py b/openfisca_core/holders/__init__.py index e903a54821..f43b503be8 100644 --- a/openfisca_core/holders/__init__.py +++ b/openfisca_core/holders/__init__.py @@ -32,4 +32,4 @@ from .helpers import set_input_dispatch_by_period, set_input_divide_by_period from .holder import Holder -from .storage import DiskStorage, MemoryStorage +from .repos import DiskRepo, MemoryRepo diff --git a/openfisca_core/holders/helpers/__init__.py b/openfisca_core/holders/helpers/__init__.py new file mode 100644 index 0000000000..0b167994de --- /dev/null +++ b/openfisca_core/holders/helpers/__init__.py @@ -0,0 +1,4 @@ +from ._set_input import ( + set_input_dispatch_by_period, + set_input_divide_by_period, + ) diff --git a/openfisca_core/holders/helpers.py b/openfisca_core/holders/helpers/_set_input.py similarity index 93% rename from openfisca_core/holders/helpers.py rename to openfisca_core/holders/helpers/_set_input.py index 4fe8441664..04115212f7 100644 --- a/openfisca_core/holders/helpers.py +++ b/openfisca_core/holders/helpers/_set_input.py @@ -1,13 +1,18 @@ +from openfisca_core.types import Period +from typing import Any + import logging import numpy from openfisca_core import periods +from ..holder import Holder + log = logging.getLogger(__name__) -def set_input_dispatch_by_period(holder, period, array): +def set_input_dispatch_by_period(holder: Holder, period: Period, array: Any) -> None: """ This function can be declared as a ``set_input`` attribute of a variable. @@ -39,7 +44,7 @@ def set_input_dispatch_by_period(holder, period, array): sub_period = sub_period.offset(1) -def set_input_divide_by_period(holder, period, array): +def set_input_divide_by_period(holder: Holder, period: Period, array: Any) -> None: """ This function can be declared as a ``set_input`` attribute of a variable. diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index 3fbf51d6d5..211eed856e 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -17,7 +17,7 @@ from openfisca_core import indexed_enums as enums from openfisca_core import periods, tools -from .storage import DiskStorage, MemoryStorage +from .repos import DiskRepo, MemoryRepo class Holder: @@ -35,13 +35,13 @@ def __init__(self, variable: Variable, population: Population) -> None: if self.storable: self.stores = sorteddict.SortedDict({ - "memory": MemoryStorage(), - "disk": self.create_disk_storage(), + "memory": MemoryRepo(), + "disk": self.create_disk_repo(), }) else: self.stores = sorteddict.SortedDict({ - "memory": MemoryStorage(), + "memory": MemoryRepo(), }) @property @@ -120,7 +120,7 @@ def clone(self, population): return new - def create_disk_storage( + def create_disk_repo( self, directory: str | None = None, preserve: bool = False, @@ -130,9 +130,9 @@ def create_disk_storage( storage_dir = os.path.join(directory, self.name) if not os.path.isdir(storage_dir): os.mkdir(storage_dir) - return DiskStorage(storage_dir, preserve) + return DiskRepo(storage_dir, preserve) - def delete_arrays(self, period: Any = None): + def delete_arrays(self, period: Any = None) -> None: """ If ``period`` is ``None``, remove all known values of the variable. @@ -140,7 +140,7 @@ def delete_arrays(self, period: Any = None): """ if self.eternal and period is not None: - period = periods.ETERNITY + period = periods.period(periods.ETERNITY) else: period = periods.period(period) @@ -150,7 +150,7 @@ def delete_arrays(self, period: Any = None): return None - def get_array(self, period): + def get_array(self, period: Period) -> numpy.ndarray | None: """ Get the value of the variable for the given period. @@ -160,7 +160,7 @@ def get_array(self, period): return self.variable.default_array(self.population.count) if self.eternal: - period = periods.ETERNITY + period = periods.period(periods.ETERNITY) else: period = periods.period(period) @@ -231,7 +231,7 @@ def get_memory_usage(self) -> MemoryUsage: return usage - def get_known_periods(self): + def get_known_periods(self) -> Sequence[Period]: """ Get the list of periods the variable value is known for. """ @@ -318,9 +318,11 @@ def set_input( if self.variable.set_input: return self.variable.set_input(self, period, array) - return self._set(period, array) + self._set(period, array) - def _to_array(self, value): + return None + + def _to_array(self, value: Any) -> numpy.ndarray: if not isinstance(value, numpy.ndarray): value = numpy.asarray(value) @@ -347,11 +349,11 @@ def _to_array(self, value): return value - def _set(self, period, value): + def _set(self, period: Period | None, value: numpy.ndarray | Sequence[Any]) -> None: value = self._to_array(value) if self.eternal: - period = periods.ETERNITY + period = periods.period(periods.ETERNITY) else: if period is None: @@ -386,7 +388,7 @@ def _set(self, period, value): else: self.stores["memory"].put(value, period) - def put_in_cache(self, value, period): + def put_in_cache(self, value: numpy.ndarray, period: Period) -> None: if not self.transient and not self.cacheable: return None diff --git a/openfisca_core/holders/repos/__init__.py b/openfisca_core/holders/repos/__init__.py new file mode 100644 index 0000000000..e54b6d9f98 --- /dev/null +++ b/openfisca_core/holders/repos/__init__.py @@ -0,0 +1,2 @@ +from ._disk_repo import DiskRepo +from ._memory_repo import MemoryRepo diff --git a/openfisca_core/holders/storage/_disk_storage.py b/openfisca_core/holders/repos/_disk_repo.py similarity index 93% rename from openfisca_core/holders/storage/_disk_storage.py rename to openfisca_core/holders/repos/_disk_repo.py index eb12fc31db..785eb5d1d1 100644 --- a/openfisca_core/holders/storage/_disk_storage.py +++ b/openfisca_core/holders/repos/_disk_repo.py @@ -13,7 +13,7 @@ from openfisca_core import periods -class DiskStorage: +class DiskRepo: """Class responsible for storing/retrieving vectors on/from disk. Attributes: @@ -57,7 +57,7 @@ def get( >>> period = periods.Period(("year", instant, 1)) >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.put(value, period) ... storage.get(period) array([1, 2, 3]) @@ -86,7 +86,7 @@ def put(self, value: Any, period: Period) -> None: >>> period = periods.Period(("year", instant, 1)) >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.put(value, period) ... storage.get(period) array([1, 2, 3]) @@ -118,19 +118,19 @@ def delete(self, period: Period | None = None) -> None: >>> period = periods.Period(("year", instant, 1)) >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.put(value, period) ... storage.get(period) array([1, 2, 3]) >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.put(value, period) ... storage.delete(period) ... storage.get(period) >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.put(value, period) ... storage.delete() ... storage.get(period) @@ -160,12 +160,12 @@ def periods(self) -> Sequence[Period]: >>> period = periods.Period(("year", instant, 1)) >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.periods() [] >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.put([], period) ... storage.periods() [Period(('year', Instant((2017, 1, 1)), 1))] @@ -184,7 +184,7 @@ def usage(self) -> NoReturn: >>> import tempfile >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.usage() Traceback (most recent call last): ... @@ -250,7 +250,7 @@ def _decode_file(self, file: str) -> Any: >>> period = periods.Period(("year", instant, 1)) >>> with tempfile.TemporaryDirectory() as directory: - ... storage = DiskStorage(directory) + ... storage = DiskRepo(directory) ... storage.put(value, period) ... storage._decode_file(storage.__files__[period]) EnumArray([]) diff --git a/openfisca_core/holders/storage/_memory_storage.py b/openfisca_core/holders/repos/_memory_repo.py similarity index 94% rename from openfisca_core/holders/storage/_memory_storage.py rename to openfisca_core/holders/repos/_memory_repo.py index beb9bd6f2d..0d083437f7 100644 --- a/openfisca_core/holders/storage/_memory_storage.py +++ b/openfisca_core/holders/repos/_memory_repo.py @@ -9,7 +9,7 @@ from openfisca_core import periods -class MemoryStorage: +class MemoryRepo: """Class responsible for storing/retrieving vectors in/from memory.""" #: A dictionary containing data that has been stored in memory. @@ -25,7 +25,7 @@ def get(self, period: Period) -> numpy.ndarray | None: The data for the specified period, or None if no data is available. Examples: - >>> storage = MemoryStorage() + >>> storage = MemoryRepo() >>> value = numpy.array([1, 2, 3]) >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) @@ -52,7 +52,7 @@ def put(self, value: numpy.ndarray, period: Period) -> None: period: The period for which the data should be stored. Examples: - >>> storage = MemoryStorage() + >>> storage = MemoryRepo() >>> value = numpy.array([1, 2, 3]) >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) @@ -74,7 +74,7 @@ def delete(self, period: Period | None = None) -> None: specified, all data will be deleted. Examples: - >>> storage = MemoryStorage() + >>> storage = MemoryRepo() >>> value = numpy.array([1, 2, 3]) >>> instant = periods.Instant((2017, 1, 1)) >>> period = periods.Period(("year", instant, 1)) @@ -113,7 +113,7 @@ def periods(self) -> Sequence[Period]: A sequence containing the storage's known periods. Examples: - >>> storage = MemoryStorage() + >>> storage = MemoryRepo() >>> storage.periods() [] @@ -135,7 +135,7 @@ def usage(self) -> MemoryUsage: A dictionary representing the storage's memory usage. Examples: - >>> storage = MemoryStorage() + >>> storage = MemoryRepo() >>> storage.usage() {'cell_size': nan, 'nb_arrays': 0, 'total_nb_bytes': 0} diff --git a/openfisca_core/holders/storage/__init__.py b/openfisca_core/holders/storage/__init__.py deleted file mode 100644 index 402b214bbf..0000000000 --- a/openfisca_core/holders/storage/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from ._disk_storage import DiskStorage -from ._memory_storage import MemoryStorage diff --git a/openfisca_core/tools/simulation_dumper.py b/openfisca_core/tools/simulation_dumper.py index ac48b9b1e0..7a1f7a332c 100644 --- a/openfisca_core/tools/simulation_dumper.py +++ b/openfisca_core/tools/simulation_dumper.py @@ -60,7 +60,7 @@ def restore_simulation(directory, tax_benefit_system, **kwargs): def _dump_holder(holder, directory): - disk_storage = holder.create_disk_storage(directory, preserve = True) + disk_storage = holder.create_disk_repo(directory, preserve = True) for period in holder.get_known_periods(): value = holder.get_array(period) disk_storage.put(value, period) @@ -115,7 +115,7 @@ def _restore_entity(population, directory): def _restore_holder(simulation, variable, directory): storage_dir = os.path.join(directory, variable) - disk_storage = holders.DiskStorage(storage_dir, keep = True) + disk_storage = holders.DiskRepo(storage_dir, keep = True) disk_storage.restore() holder = simulation.get_holder(variable) diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index 9f6302fc29..ab52dd21ba 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -94,6 +94,11 @@ class Instant(Protocol): class Period(Protocol): """Period protocol.""" + @property + @abc.abstractmethod + def size(self) -> int: + """Abstract property.""" + @property @abc.abstractmethod def start(self) -> Any: diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py index 7bfbb2eba3..29aa951435 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types/_domain.py @@ -36,7 +36,7 @@ from __future__ import annotations import typing_extensions -from typing import Any, Optional +from typing import Any from typing_extensions import Protocol import abc @@ -62,7 +62,7 @@ def check_variable_defined_for_entity(self, variable_name: Any) -> None: def get_variable( self, variable_name: Any, check_existence: Any = ..., - ) -> Optional[Any]: + ) -> Any | None: """Abstract method.""" @@ -117,6 +117,7 @@ class Simulation(Protocol): trace: Any tracer: Any memory_config: Any + opt_out_cache: Any data_storage_dir: Any tax_benefit_system: Any @@ -133,7 +134,7 @@ def calculate_divide(self, variable_name: Any, period: Any) -> Any: """Abstract method.""" @abc.abstractmethod - def get_population(self, plural: Optional[Any]) -> Any: + def get_population(self, plural: Any | None) -> Any: """Abstract method.""" @@ -146,7 +147,7 @@ class TaxBenefitSystem(Protocol): def get_variable( self, variable_name: Any, check_existence: Any = ..., - ) -> Optional[Any]: + ) -> Any | None: """Abstract method.""" @@ -160,4 +161,9 @@ class Variable(Protocol): set_input: Any value_type: Any is_neutralized: Any + possible_values: Any definition_period: Any + + @abc.abstractmethod + def default_array(self, array_size: Any) -> Any: + """Abstract method.""" diff --git a/setup.cfg b/setup.cfg index 14ec27e939..ebf3dc186f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,8 +26,8 @@ force_alphabetical_sort_within_sections = true group_by_package = true include_trailing_comma = true known_first_party = openfisca_core -known_openfisca = openfisca_core,openfisca_country_template -known_typing = *mypy*,*types*,*typing* +known_openfisca = openfisca_country_template, openfisca_extension_template +known_typing = mypy*, *types*, *typing* multi_line_output = 8 py_version = 37 sections = FUTURE,TYPING,STDLIB,THIRDPARTY,OPENFISCA,FIRSTPARTY,LOCALFOLDER From ca4125b4e6b828d62e6162e730cb88b617a349ad Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Mon, 25 Sep 2023 10:26:02 +0200 Subject: [PATCH 20/22] Remove typo in CHANGELOG --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36272ebfae..6189d3ebf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,7 +113,6 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). - Drop support for OpenAPI specification v2 and prior. - Users relying on OpenAPI v2 can use [Swagger Converter](https://converter.swagger.io/api/convert?url=OAS2_YAML_OR_JSON_URL) to migrate ([example](https://web.archive.org/web/20221103230822/https://converter.swagger.io/api/convert?url=https://api.demo.openfisca.org/latest/spec)). ->>>>>>> master ### 37.0.2 [#1170](https://github.com/openfisca/openfisca-core/pull/1170) From 5216c630ba9683cf3813b1f134a922e34ce2ddd5 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Mon, 25 Sep 2023 20:22:44 +0200 Subject: [PATCH 21/22] Add has_index protocol --- openfisca_core/indexed_enums/enum.py | 12 +++++++++--- openfisca_core/indexed_enums/enum_array.py | 21 ++++++++++++--------- openfisca_core/indexed_enums/typing.py | 7 +++++++ 3 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 openfisca_core/indexed_enums/typing.py diff --git a/openfisca_core/indexed_enums/enum.py b/openfisca_core/indexed_enums/enum.py index 7957ced3a2..2017d56064 100644 --- a/openfisca_core/indexed_enums/enum.py +++ b/openfisca_core/indexed_enums/enum.py @@ -16,12 +16,18 @@ class Enum(enum.Enum): have an index. """ + #: Index of the enum. + index: int + # Tweak enums to add an index attribute to each enum item - def __init__(self, name: str) -> None: - # When the enum item is initialized, self._member_names_ contains the + def __new__(cls, name: str) -> Enum: + # When the enum item is initialized, cls._member_names_ contains the # names of the previously initialized items, so its length is the index # of this item. - self.index = len(self._member_names_) + new = object.__new__(cls) + new._value_ = name + new.index = len(cls._member_names_) + return new # Bypass the slow Enum.__eq__ __eq__ = object.__eq__ diff --git a/openfisca_core/indexed_enums/enum_array.py b/openfisca_core/indexed_enums/enum_array.py index b42df165a6..59f312eb70 100644 --- a/openfisca_core/indexed_enums/enum_array.py +++ b/openfisca_core/indexed_enums/enum_array.py @@ -1,12 +1,14 @@ from __future__ import annotations -from openfisca_core.types import Enum -from typing import Any, NoReturn, Optional, Type +from numpy.typing import NDArray +from typing import Any, Iterable, NoReturn import numpy +from .typing import HasIndex -class EnumArray(numpy.ndarray): + +class EnumArray(NDArray[numpy.int_]): """ NumPy array subclass representing an array of enum items. @@ -18,24 +20,25 @@ class EnumArray(numpy.ndarray): # https://docs.scipy.org/doc/numpy-1.13.0/user/basics.subclassing.html#slightly-more-realistic-example-attribute-added-to-existing-array. def __new__( cls, - input_array: numpy.int_, - possible_values: Optional[Type[Enum]] = None, + input_array: NDArray[numpy.int_], + possible_values: Iterable[HasIndex] | None = None, ) -> EnumArray: obj = numpy.asarray(input_array).view(cls) obj.possible_values = possible_values return obj # See previous comment - def __array_finalize__(self, obj: Optional[numpy.int_]) -> None: + def __array_finalize__(self, obj: NDArray[numpy.int_] | EnumArray | None) -> None: if obj is None: return self.possible_values = getattr(obj, "possible_values", None) - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: HasIndex | Any) -> bool: # When comparing to an item of self.possible_values, use the item index # to speed up the comparison. - if other.__class__.__name__ is self.possible_values.__name__: + + if hasattr(other, "index"): # Use view(ndarray) so that the result is a classic ndarray, not an # EnumArray. return self.view(numpy.ndarray) == other.index @@ -79,7 +82,7 @@ def decode(self) -> numpy.object_: list(self.possible_values), ) - def decode_to_str(self) -> numpy.str_: + def decode_to_str(self) -> NDArray[numpy.str_]: """ Return the array of string identifiers corresponding to self. diff --git a/openfisca_core/indexed_enums/typing.py b/openfisca_core/indexed_enums/typing.py new file mode 100644 index 0000000000..359166e348 --- /dev/null +++ b/openfisca_core/indexed_enums/typing.py @@ -0,0 +1,7 @@ +from typing import Protocol + + +class HasIndex(Protocol): + """Indexable class protocol.""" + + index: int From 53dd5efb31037bbd5044799a55940e5390a2f42d Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 4 Oct 2024 20:34:32 +0200 Subject: [PATCH 22/22] chore: backport changes --- .github/dependabot.yml | 2 +- .github/get_pypi_info.py | 17 +- .github/workflows/_before-conda.yaml | 109 +++++ .github/workflows/_before-pip.yaml | 103 +++++ .github/workflows/_lint-pip.yaml | 57 +++ .github/workflows/_test-conda.yaml | 76 ++++ .github/workflows/_test-pip.yaml | 72 ++++ .github/workflows/_version.yaml | 38 ++ .github/workflows/merge.yaml | 247 ++++++++++++ .github/workflows/push.yaml | 89 +++++ CHANGELOG.md | 240 ++++++++++- openfisca_core/commons/__init__.py | 19 +- openfisca_core/commons/dummy.py | 4 +- openfisca_core/commons/formulas.py | 73 ++-- openfisca_core/commons/misc.py | 58 ++- openfisca_core/commons/py.typed | 0 openfisca_core/commons/rates.py | 56 +-- openfisca_core/commons/tests/test_dummy.py | 2 +- openfisca_core/commons/tests/test_formulas.py | 33 +- openfisca_core/commons/tests/test_rates.py | 14 +- openfisca_core/commons/types.py | 3 + openfisca_core/entities/__init__.py | 70 ++-- openfisca_core/entities/_core_entity.py | 142 +++++++ openfisca_core/entities/_description.py | 55 +++ openfisca_core/entities/entity.py | 74 ++-- openfisca_core/entities/group_entity.py | 81 ++-- openfisca_core/entities/helpers.py | 170 +++++++- openfisca_core/entities/py.typed | 0 openfisca_core/entities/role.py | 92 ++++- openfisca_core/entities/tests/__init__.py | 0 openfisca_core/entities/tests/test_entity.py | 10 + .../entities/tests/test_group_entity.py | 70 ++++ openfisca_core/entities/tests/test_role.py | 11 + openfisca_core/entities/types.py | 42 ++ openfisca_core/holders/helpers/_set_input.py | 3 +- openfisca_core/holders/holder.py | 22 +- openfisca_core/holders/repos/_disk_repo.py | 6 +- openfisca_core/holders/repos/_memory_repo.py | 1 + openfisca_core/holders/typing.py | 24 +- openfisca_core/indexed_enums/__init__.py | 63 ++- openfisca_core/indexed_enums/enum.py | 29 +- openfisca_core/indexed_enums/enum_array.py | 48 +-- openfisca_core/indexed_enums/types.py | 3 + openfisca_core/parameters/at_instant_like.py | 3 +- openfisca_core/parameters/parameter_node.py | 6 +- .../vectorial_parameter_node_at_instant.py | 12 +- openfisca_core/periods/__init__.py | 98 +++-- openfisca_core/periods/_errors.py | 28 ++ openfisca_core/periods/_parsers.py | 121 +++--- openfisca_core/periods/config.py | 20 +- openfisca_core/periods/date_unit.py | 20 +- openfisca_core/periods/helpers.py | 268 ++++++------- openfisca_core/periods/instant_.py | 114 ++++-- openfisca_core/periods/period_.py | 378 +++++++++++------- openfisca_core/periods/py.typed | 0 .../periods/tests/helpers/test_helpers.py | 64 +-- .../periods/tests/helpers/test_instant.py | 109 +++-- .../periods/tests/helpers/test_period.py | 218 +++++----- openfisca_core/periods/tests/test_instant.py | 44 +- openfisca_core/periods/tests/test_parsers.py | 129 ++++++ openfisca_core/periods/tests/test_period.py | 218 +++++----- openfisca_core/periods/types.py | 183 +++++++++ .../taxbenefitsystems/tax_benefit_system.py | 6 +- openfisca_core/taxscales/tax_scale_like.py | 9 +- openfisca_core/types.py | 191 +++++++++ openfisca_core/types/__init__.py | 64 --- openfisca_core/types/_data.py | 65 --- openfisca_core/types/_domain.py | 185 --------- openfisca_core/variables/__init__.py | 2 +- openfisca_core/variables/config.py | 1 - openfisca_core/variables/helpers.py | 23 +- .../variables/tests/test_definition_period.py | 12 +- openfisca_core/variables/variable.py | 165 ++++---- openfisca_web_api/app.py | 41 +- openfisca_web_api/errors.py | 9 +- openfisca_web_api/handlers.py | 54 +-- openfisca_web_api/loader/__init__.py | 12 +- openfisca_web_api/loader/entities.py | 8 +- openfisca_web_api/loader/parameters.py | 17 +- openfisca_web_api/loader/spec.py | 73 ++-- .../loader/tax_benefit_system.py | 10 +- openfisca_web_api/loader/variables.py | 21 +- openfisca_web_api/openAPI.yml | 6 +- openfisca_web_api/scripts/serve.py | 23 +- pyproject.toml | 2 + setup.cfg | 137 ++++--- setup.py | 68 ++-- 87 files changed, 3814 insertions(+), 1751 deletions(-) create mode 100644 .github/workflows/_before-conda.yaml create mode 100644 .github/workflows/_before-pip.yaml create mode 100644 .github/workflows/_lint-pip.yaml create mode 100644 .github/workflows/_test-conda.yaml create mode 100644 .github/workflows/_test-pip.yaml create mode 100644 .github/workflows/_version.yaml create mode 100644 .github/workflows/merge.yaml create mode 100644 .github/workflows/push.yaml create mode 100644 openfisca_core/commons/py.typed create mode 100644 openfisca_core/commons/types.py create mode 100644 openfisca_core/entities/_core_entity.py create mode 100644 openfisca_core/entities/_description.py create mode 100644 openfisca_core/entities/py.typed create mode 100644 openfisca_core/entities/tests/__init__.py create mode 100644 openfisca_core/entities/tests/test_entity.py create mode 100644 openfisca_core/entities/tests/test_group_entity.py create mode 100644 openfisca_core/entities/tests/test_role.py create mode 100644 openfisca_core/entities/types.py create mode 100644 openfisca_core/indexed_enums/types.py create mode 100644 openfisca_core/periods/_errors.py create mode 100644 openfisca_core/periods/py.typed create mode 100644 openfisca_core/periods/tests/test_parsers.py create mode 100644 openfisca_core/periods/types.py create mode 100644 openfisca_core/types.py delete mode 100644 openfisca_core/types/__init__.py delete mode 100644 openfisca_core/types/_data.py delete mode 100644 openfisca_core/types/_domain.py create mode 100644 pyproject.toml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fcb2acc162..71eaf02d67 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,7 @@ version: 2 updates: - package-ecosystem: pip - directory: "/" + directory: / schedule: interval: monthly labels: diff --git a/.github/get_pypi_info.py b/.github/get_pypi_info.py index 03d2f1ab15..70013fbe98 100644 --- a/.github/get_pypi_info.py +++ b/.github/get_pypi_info.py @@ -18,12 +18,14 @@ def get_info(package_name: str = "") -> dict: ::return:: A dict with last_version, url and sha256 """ if package_name == "": - raise ValueError("Package name not provided.") + msg = "Package name not provided." + raise ValueError(msg) url = f"https://pypi.org/pypi/{package_name}/json" print(f"Calling {url}") # noqa: T201 resp = requests.get(url) if resp.status_code != 200: - raise Exception(f"ERROR calling PyPI ({url}) : {resp}") + msg = f"ERROR calling PyPI ({url}) : {resp}" + raise Exception(msg) resp = resp.json() version = resp["info"]["version"] @@ -38,19 +40,19 @@ def get_info(package_name: str = "") -> dict: return {} -def replace_in_file(filepath: str, info: dict): +def replace_in_file(filepath: str, info: dict) -> None: """Replace placeholder in meta.yaml by their values. ::filepath:: Path to meta.yaml, with filename. ::info:: Dict with information to populate. """ - with open(filepath, "rt", encoding="utf-8") as fin: + with open(filepath, encoding="utf-8") as fin: meta = fin.read() # Replace with info from PyPi meta = meta.replace("PYPI_VERSION", info["last_version"]) meta = meta.replace("PYPI_URL", info["url"]) meta = meta.replace("PYPI_SHA256", info["sha256"]) - with open(filepath, "wt", encoding="utf-8") as fout: + with open(filepath, "w", encoding="utf-8") as fout: fout.write(meta) print(f"File {filepath} has been updated with info from PyPi.") # noqa: T201 @@ -69,12 +71,13 @@ def replace_in_file(filepath: str, info: dict): "-f", "--filename", type=str, - default=".conda/meta.yaml", + default=".conda/openfisca-core/meta.yaml", help="Path to meta.yaml, with filename", ) args = parser.parse_args() info = get_info(args.package) print( # noqa: T201 - "Information of the last published PyPi package :", info["last_version"] + "Information of the last published PyPi package :", + info["last_version"], ) replace_in_file(args.filename, info) diff --git a/.github/workflows/_before-conda.yaml b/.github/workflows/_before-conda.yaml new file mode 100644 index 0000000000..7528a6a1c2 --- /dev/null +++ b/.github/workflows/_before-conda.yaml @@ -0,0 +1,109 @@ +name: Setup conda + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + +defaults: + run: + shell: bash -l {0} + +jobs: + setup: + runs-on: ${{ inputs.os }} + name: conda-setup-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + # To colorize output of make tasks. + TERM: xterm-256color + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache conda env + uses: actions/cache@v4 + with: + path: | + /usr/share/miniconda/envs/openfisca + ~/.conda/envs/openfisca + .env.yaml + key: conda-env-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + restore-keys: conda-env-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- + id: cache-env + + - name: Cache conda deps + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: conda-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + restore-keys: conda-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- + id: cache-deps + + - name: Cache release + uses: actions/cache@v4 + with: + path: ~/conda-rel + key: conda-release-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: openfisca + miniforge-version: latest + python-version: ${{ inputs.python }} + use-mamba: true + if: steps.cache-env.outputs.cache-hit != 'true' + + - name: Install dependencies + run: mamba install boa rattler-build anaconda-client + if: steps.cache-env.outputs.cache-hit != 'true' + + - name: Update conda & dependencies + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: openfisca + environment-file: .env.yaml + miniforge-version: latest + use-mamba: true + if: steps.cache-env.outputs.cache-hit == 'true' + + - name: Build pylint plugin package + run: | + rattler-build build \ + --recipe .conda/pylint-per-file-ignores \ + --output-dir ~/conda-rel + + - name: Build core package + run: | + conda mambabuild .conda/openfisca-core \ + --use-local \ + --no-anaconda-upload \ + --output-folder ~/conda-rel \ + --numpy ${{ inputs.numpy }} \ + --python ${{ inputs.python }} + + - name: Build country template package + run: | + rattler-build build \ + --recipe .conda/openfisca-country-template \ + --output-dir ~/conda-rel \ + + - name: Build extension template package + run: | + rattler-build build \ + --recipe .conda/openfisca-extension-template \ + --output-dir ~/conda-rel + + - name: Export env + run: mamba env export --name openfisca > .env.yaml diff --git a/.github/workflows/_before-pip.yaml b/.github/workflows/_before-pip.yaml new file mode 100644 index 0000000000..02554419c8 --- /dev/null +++ b/.github/workflows/_before-pip.yaml @@ -0,0 +1,103 @@ +name: Setup package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + deps: + runs-on: ${{ inputs.os }} + name: pip-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + # To colorize output of make tasks. + TERM: xterm-256color + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: venv + key: pip-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + restore-keys: pip-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- + + - name: Install dependencies + run: | + python -m venv venv + ${{ inputs.activate_command }} + make install-deps install-dist + pip install numpy==${{ inputs.numpy }} + + build: + runs-on: ${{ inputs.os }} + needs: [deps] + name: pip-build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + TERM: xterm-256color + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: venv + key: pip-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v4 + with: + path: venv/**/[Oo]pen[Ff]isca* + key: pip-build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + restore-keys: | + pip-build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}- + pip-build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- + + - name: Cache release + uses: actions/cache@v4 + with: + path: dist + key: pip-release-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Build package + run: | + ${{ inputs.activate_command }} + make install-test clean build diff --git a/.github/workflows/_lint-pip.yaml b/.github/workflows/_lint-pip.yaml new file mode 100644 index 0000000000..e994f473e3 --- /dev/null +++ b/.github/workflows/_lint-pip.yaml @@ -0,0 +1,57 @@ +name: Lint package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + lint: + runs-on: ${{ inputs.os }} + name: pip-lint-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: venv + key: pip-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Lint doc + run: | + ${{ inputs.activate_command }} + make clean check-syntax-errors lint-doc + + - name: Lint styles + run: | + ${{ inputs.activate_command }} + make clean check-syntax-errors check-style diff --git a/.github/workflows/_test-conda.yaml b/.github/workflows/_test-conda.yaml new file mode 100644 index 0000000000..fab88ac1df --- /dev/null +++ b/.github/workflows/_test-conda.yaml @@ -0,0 +1,76 @@ +name: Test conda package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + +defaults: + run: + shell: bash -l {0} + +jobs: + test: + runs-on: ${{ inputs.os }} + name: conda-test-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache conda env + uses: actions/cache@v4 + with: + path: | + /usr/share/miniconda/envs/openfisca + ~/.conda/envs/openfisca + .env.yaml + key: conda-env-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Cache conda deps + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: conda-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Cache release + uses: actions/cache@v4 + with: + path: ~/conda-rel + key: conda-release-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Update conda & dependencies + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: openfisca + environment-file: .env.yaml + miniforge-version: latest + use-mamba: true + + - name: Install packages + run: | + mamba install --channel file:///home/runner/conda-rel \ + openfisca-core-dev \ + openfisca-country-template \ + openfisca-extension-template + + - name: Run core tests + run: make test-core + + - name: Run country tests + run: make test-country + + - name: Run extension tests + run: make test-extension diff --git a/.github/workflows/_test-pip.yaml b/.github/workflows/_test-pip.yaml new file mode 100644 index 0000000000..704c7fbd17 --- /dev/null +++ b/.github/workflows/_test-pip.yaml @@ -0,0 +1,72 @@ +name: Test package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + test: + runs-on: ${{ inputs.os }} + name: pip-test-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: venv + key: pip-deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v4 + with: + path: venv/**/[Oo]pen[Ff]isca* + key: pip-build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Run Openfisca Core tests + run: | + ${{ inputs.activate_command }} + make test-core + python -m coveralls --service=github + + - name: Run Country Template tests + if: ${{ startsWith(inputs.os, 'ubuntu') }} + run: | + ${{ inputs.activate_command }} + make test-country + + - name: Run Extension Template tests + if: ${{ startsWith(inputs.os, 'ubuntu') }} + run: | + ${{ inputs.activate_command }} + make test-extension diff --git a/.github/workflows/_version.yaml b/.github/workflows/_version.yaml new file mode 100644 index 0000000000..27c4737a4f --- /dev/null +++ b/.github/workflows/_version.yaml @@ -0,0 +1,38 @@ +name: Check version + +on: + workflow_call: + inputs: + os: + required: true + type: string + + python: + required: true + type: string + +jobs: + # The idea behind these dependencies is that we want to give feedback to + # contributors on the version number only after they have passed all tests, + # so they don't have to do it twice after changes happened to the main branch + # during the time they took to fix the tests. + check-version: + runs-on: ${{ inputs.os }} + env: + # To colorize output of make tasks. + TERM: xterm-256color + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Fetch all the tags + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python }} + + - name: Check version number has been properly updated + run: ${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh diff --git a/.github/workflows/merge.yaml b/.github/workflows/merge.yaml new file mode 100644 index 0000000000..5c2b4c791d --- /dev/null +++ b/.github/workflows/merge.yaml @@ -0,0 +1,247 @@ +name: OpenFisca-Core / Deploy package to PyPi & Conda + +on: + push: + branches: [master] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + setup-pip: + strategy: + fail-fast: true + matrix: + os: [ubuntu-22.04, windows-2019] + numpy: [1.26.4, 1.24.2] + # Patch version must be specified to avoid any cache confusion, since + # the cache key depends on the full Python version. If left unspecified, + # different patch versions could be allocated between jobs, and any + # such difference would lead to a cache not found error. + python: [3.11.9, 3.9.13] + include: + - os: ubuntu-22.04 + activate_command: source venv/bin/activate + - os: windows-2019 + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_before-pip.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + setup-conda: + uses: ./.github/workflows/_before-conda.yaml + with: + os: ubuntu-22.04 + numpy: 1.26.4 + python: 3.10.6 + + test-pip: + needs: [setup-pip] + strategy: + fail-fast: true + matrix: + os: [ubuntu-22.04, windows-2019] + numpy: [1.26.4, 1.24.2] + python: [3.11.9, 3.9.13] + include: + - os: ubuntu-22.04 + activate_command: source venv/bin/activate + - os: windows-2019 + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_test-pip.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + test-conda: + uses: ./.github/workflows/_test-conda.yaml + needs: [setup-conda] + with: + os: ubuntu-22.04 + numpy: 1.26.4 + python: 3.10.6 + + lint-pip: + needs: [setup-pip] + strategy: + fail-fast: true + matrix: + numpy: [1.24.2] + python: [3.11.9, 3.9.13] + uses: ./.github/workflows/_lint-pip.yaml + with: + os: ubuntu-22.04 + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: source venv/bin/activate + + check-version: + needs: [test-pip, test-conda, lint-pip] + uses: ./.github/workflows/_version.yaml + with: + os: ubuntu-22.04 + python: 3.9.13 + + # GitHub Actions does not have a halt job option, to stop from deploying if + # no functional changes were found. We build a separate job to substitute the + # halt option. The `deploy` job is dependent on the output of the + # `check-for-functional-changes`job. + check-for-functional-changes: + runs-on: ubuntu-22.04 + # Last job to run + needs: [check-version] + outputs: + status: ${{ steps.stop-early.outputs.status }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9.13 + + - id: stop-early + # The `check-for-functional-changes` job should always succeed regardless + # of the `has-functional-changes` script's exit code. Consequently, we do + # not use that exit code to trigger deploy, but rather a dedicated output + # variable `status`, to avoid a job failure if the exit code is different + # from 0. Conversely, if the job fails the entire workflow would be + # marked as `failed` which is disturbing for contributors. + run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo + "::set-output name=status::success" ; fi + + publish-to-pypi: + runs-on: ubuntu-22.04 + needs: [check-for-functional-changes] + if: needs.check-for-functional-changes.outputs.status == 'success' + env: + PYPI_USERNAME: __token__ + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_OPENFISCA_BOT }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9.13 + + - name: Cache deps + uses: actions/cache@v4 + with: + path: venv + key: pip-deps-ubuntu-22.04-np1.24.2-py3.9.13-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v4 + with: + path: venv/**/[oO]pen[fF]isca* + key: pip-build-ubuntu-22.04-np1.24.2-py3.9.13-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Cache release + uses: actions/cache@v4 + with: + path: dist + key: pip-release-ubuntu-22.04-np1.24.2-py3.9.13-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Upload package to PyPi + run: | + source venv/bin/activate + make publish + + - name: Update doc + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.OPENFISCADOC_BOT_ACCESS_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/openfisca/openfisca-doc/actions/workflows/deploy.yaml/dispatches \ + -d '{"ref":"main"}' + + publish-to-conda: + runs-on: ubuntu-22.04 + needs: [publish-to-pypi] + defaults: + run: + shell: bash -l {0} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache conda env + uses: actions/cache@v4 + with: + path: | + /usr/share/miniconda/envs/openfisca + ~/.conda/envs/openfisca + .env.yaml + key: conda-env-ubuntu-22.04-np1.26.4-py3.10.6-${{ hashFiles('setup.py') }} + + - name: Cache conda deps + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: conda-deps-ubuntu-22.04-np1.26.4-py3.10.6-${{ hashFiles('setup.py') }} + + - name: Cache release + uses: actions/cache@v4 + with: + path: ~/conda-rel + key: conda-release-ubuntu-22.04-np1.26.4-py3.10.6-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Update conda & dependencies + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: openfisca + environment-file: .env.yaml + miniforge-version: latest + use-mamba: true + + - name: Publish to conda + shell: bash -l {0} + run: | + anaconda upload ~/conda-rel/noarch/openfisca-core-* \ + --token ${{ secrets.ANACONDA_TOKEN }} + --user openfisca + --force + + test-on-windows: + runs-on: windows-2019 + needs: [publish-to-conda] + defaults: + run: + shell: bash -l {0} + + steps: + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + # See GHA Windows + # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json + python-version: 3.10.6 + channels: conda-forge + activate-environment: true + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install with conda + shell: bash -l {0} + run: conda install -c openfisca openfisca-core + + - name: Test openfisca + shell: bash -l {0} + run: openfisca --help diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 0000000000..7bee48c81c --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,89 @@ +name: OpenFisca-Core / Pull request review + +on: + pull_request: + types: [assigned, opened, reopened, synchronize, ready_for_review] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + setup-pip: + strategy: + fail-fast: true + matrix: + os: [ubuntu-22.04, windows-2019] + numpy: [1.26.4, 1.24.2] + # Patch version must be specified to avoid any cache confusion, since + # the cache key depends on the full Python version. If left unspecified, + # different patch versions could be allocated between jobs, and any + # such difference would lead to a cache not found error. + python: [3.11.9, 3.9.13] + include: + - os: ubuntu-22.04 + activate_command: source venv/bin/activate + - os: windows-2019 + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_before-pip.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + setup-conda: + uses: ./.github/workflows/_before-conda.yaml + with: + os: ubuntu-22.04 + numpy: 1.26.4 + python: 3.10.6 + + test-pip: + needs: [setup-pip] + strategy: + fail-fast: true + matrix: + os: [ubuntu-22.04, windows-2019] + numpy: [1.26.4, 1.24.2] + python: [3.11.9, 3.9.13] + include: + - os: ubuntu-22.04 + activate_command: source venv/bin/activate + - os: windows-2019 + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_test-pip.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + test-conda: + uses: ./.github/workflows/_test-conda.yaml + needs: [setup-conda] + with: + os: ubuntu-22.04 + numpy: 1.26.4 + python: 3.10.6 + + lint-pip: + needs: [setup-pip] + strategy: + fail-fast: true + matrix: + numpy: [1.24.2] + python: [3.11.9, 3.9.13] + uses: ./.github/workflows/_lint-pip.yaml + with: + os: ubuntu-22.04 + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: source venv/bin/activate + + check-version: + needs: [test-pip, test-conda, lint-pip] + uses: ./.github/workflows/_version.yaml + with: + os: ubuntu-22.04 + python: 3.9.13 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6189d3ebf7..0a30914e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 41.1.0 [#1167](https://github.com/openfisca/openfisca-core/pull/1167) +## 42.1.0 [#1167](https://github.com/openfisca/openfisca-core/pull/1167) #### New features @@ -12,6 +12,240 @@ - Add typing to `data_storage`. - Add documentation to `data_storage`. +### 42.0.4 [#1257](https://github.com/openfisca/openfisca-core/pull/1257) + +#### Technical changes + +- Fix conda test and publish +- Add matrix testing to CI + - Now it tests lower and upper bounds of python and numpy versions + +### 42.0.3 [#1234](https://github.com/openfisca/openfisca-core/pull/1234) + +#### Technical changes + +- Add matrix testing to CI + - Now it tests lower and upper bounds of python and numpy versions + +> Note: Version `42.0.3` has been unpublished as was deployed by mistake. +> Please use versions `42.0.4` and subsequents. + +### 42.0.2 [#1256](https://github.com/openfisca/openfisca-core/pull/1256) + +#### Documentation + +- Fix bad indent + +### 42.0.1 [#1253](https://github.com/openfisca/openfisca-core/pull/1253) + +#### Documentation + +- Fix documentation of `entities` + +# 42.0.0 [#1223](https://github.com/openfisca/openfisca-core/pull/1223) + +#### Breaking changes + +- Changes to `eternity` instants and periods + - Eternity instants are now `` instead of + `` + - Eternity periods are now `, -1))>` + instead of `, inf))>` + - The reason is to avoid mixing data types: `inf` is a float, periods and + instants are integers. Mixed data types make memory optimisations impossible. + - Migration should be straightforward. If you have a test that checks for + `inf`, you should update it to check for `-1` or use the `is_eternal` method. +- `periods.instant` no longer returns `None` + - Now, it raises `periods.InstantError` + +#### New features + +- Introduce `Instant.eternity()` + - This behaviour was duplicated across + - Now it is encapsulated in a single method +- Introduce `Instant.is_eternal` and `Period.is_eternal` + - These methods check if the instant or period are eternity (`bool`). +- Now `periods.instant` parses also ISO calendar strings (weeks) + - For instance, `2022-W01` is now a valid input + +#### Technical changes + +- Update `pendulum` +- Reduce code complexity +- Remove run-time type-checks +- Add typing to the periods module + +### 41.5.7 [#1225](https://github.com/openfisca/openfisca-core/pull/1225) + +#### Technical changes + +- Refactor & test `eval_expression` + +### 41.5.6 [#1185](https://github.com/openfisca/openfisca-core/pull/1185) + +#### Technical changes + +- Remove pre Python 3.9 syntax. + +### 41.5.5 [#1220](https://github.com/openfisca/openfisca-core/pull/1220) + +#### Technical changes + +- Fix doc & type definitions in the entities module + +### 41.5.4 [#1219](https://github.com/openfisca/openfisca-core/pull/1219) + +#### Technical changes + +- Fix doc & type definitions in the commons module + +### 41.5.3 [#1218](https://github.com/openfisca/openfisca-core/pull/1218) + +#### Technical changes + +- Fix `flake8` doc linting: + - Add format "google" + - Fix per-file skips +- Fix failing lints + +### 41.5.2 [#1217](https://github.com/openfisca/openfisca-core/pull/1217) + +#### Technical changes + +- Fix styles by applying `isort`. +- Add a `isort` dry-run check to `make lint` + +### 41.5.1 [#1216](https://github.com/openfisca/openfisca-core/pull/1216) + +#### Technical changes + +- Fix styles by applying `black`. +- Add a `black` dry-run check to `make lint` + +## 41.5.0 [#1212](https://github.com/openfisca/openfisca-core/pull/1212) + +#### New features + +- Introduce `VectorialAsofDateParameterNodeAtInstant` + - It is a parameter node of the legislation at a given instant which has been vectorized along some date. + - Vectorized parameters allow requests such as parameters.housing_benefit[date], where date is a `numpy.datetime64` vector + +### 41.4.7 [#1211](https://github.com/openfisca/openfisca-core/pull/1211) + +#### Technical changes + +- Update documentation continuous deployment method to reflect OpenFisca-Doc [process updates](https://github.com/openfisca/openfisca-doc/pull/308) + +### 41.4.6 [#1210](https://github.com/openfisca/openfisca-core/pull/1210) + +#### Technical changes + +- Abide by OpenAPI v3.0.0 instead of v3.1.0 + - Drop support for `propertyNames` in `Values` definition + +### 41.4.5 [#1209](https://github.com/openfisca/openfisca-core/pull/1209) + +#### Technical changes + +- Support loading metadata from both `setup.py` and `pyproject.toml` package description files. + +### ~41.4.4~ [#1208](https://github.com/openfisca/openfisca-core/pull/1208) + +_Unpublished due to introduced backwards incompatibilities._ + +#### Technical changes + +- Adapt testing pipeline to Country Template [v7](https://github.com/openfisca/country-template/pull/139). + +### 41.4.3 [#1206](https://github.com/openfisca/openfisca-core/pull/1206) + +#### Technical changes + +- Increase spiral and cycle tests robustness. + - The current test is ambiguous, as it hides a failure at the first spiral + occurrence (from 2017 to 2016). + +### 41.4.2 [#1203](https://github.com/openfisca/openfisca-core/pull/1203) + +#### Technical changes + +- Changes the Pypi's deployment authentication way to use token API following Pypi's 2FA enforcement starting 2024/01/01. + +### 41.4.1 [#1202](https://github.com/openfisca/openfisca-core/pull/1202) + +#### Technical changes + +- Check that entities are fully specified when expanding over axes. + +## 41.4.0 [#1197](https://github.com/openfisca/openfisca-core/pull/1197) + +#### New features + +- Add `entities.find_role()` to find roles by key and `max`. + +#### Technical changes + +- Document `projectors.get_projector_from_shortcut()`. + +## 41.3.0 [#1200](https://github.com/openfisca/openfisca-core/pull/1200) + +> As `TracingParameterNodeAtInstant` is a wrapper for `ParameterNodeAtInstant` +> which allows iteration and the use of `contains`, it was not possible +> to use those on a `TracingParameterNodeAtInstant` + +#### New features + +- Allows iterations on `TracingParameterNodeAtInstant` +- Allows keyword `contains` on `TracingParameterNodeAtInstant` + +## 41.2.0 [#1199](https://github.com/openfisca/openfisca-core/pull/1199) + +#### Technical changes + +- Fix `openfisca-core` Web API error triggered by `Gunicorn` < 22.0. + - Bump `Gunicorn` major revision to fix error on Web API. + Source: https://github.com/benoitc/gunicorn/issues/2564 + +### 41.1.2 [#1192](https://github.com/openfisca/openfisca-core/pull/1192) + +#### Technical changes + +- Add tests to `entities`. + +### 41.1.1 [#1186](https://github.com/openfisca/openfisca-core/pull/1186) + +#### Technical changes + +- Skip type-checking tasks + - Before their definition was commented out but still run with `make test` + - Now they're skipped but not commented, which is needed to fix the + underlying issues + +## 41.1.0 [#1195](https://github.com/openfisca/openfisca-core/pull/1195) + +#### Technical changes + +- Make `Role` explicitly hashable. +- Details: + - By introducing `__eq__`, naturally `Role` became unhashable, because + equality was calculated based on a property of `Role` + (`role.key == another_role.key`), and no longer structurally + (`"1" == "1"`). + - This changeset removes `__eq__`, as `Role` is being used downstream as a + hashable object, and adds a test to ensure `Role`'s hashability. + +### 41.0.2 [#1194](https://github.com/openfisca/openfisca-core/pull/1194) + +#### Technical changes + +- Add `__hash__` method to `Role`. + +### 41.0.1 [#1187](https://github.com/openfisca/openfisca-core/pull/1187) + +#### Technical changes + +- Document `Role`. + # 41.0.0 [#1189](https://github.com/openfisca/openfisca-core/pull/1189) #### Breaking changes @@ -24,8 +258,8 @@ The Web API was very prone to crashing, timeouting at startup because of the tim #### New Features -* Allows for dispatching and dividing inputs over a broader range. - * For example, divide a monthly variable by week. +- Allows for dispatching and dividing inputs over a broader range. + - For example, divide a monthly variable by week. ### 40.0.1 [#1184](https://github.com/openfisca/openfisca-core/pull/1184) diff --git a/openfisca_core/commons/__init__.py b/openfisca_core/commons/__init__.py index ce0be7d377..1a3d065ee1 100644 --- a/openfisca_core/commons/__init__.py +++ b/openfisca_core/commons/__init__.py @@ -8,6 +8,7 @@ * :func:`.average_rate` * :func:`.concat` * :func:`.empty_clone` + * :func:`.eval_expression` * :func:`.marginal_rate` * :func:`.stringify_array` * :func:`.switch` @@ -32,7 +33,7 @@ from modularizing the different components of the library, which would make them easier to test and to maintain. - How they could be used in a future release: + How they could be used in a future release:: from openfisca_core import commons from openfisca_core.commons import deprecated @@ -50,7 +51,21 @@ """ +from . import types from .dummy import Dummy from .formulas import apply_thresholds, concat, switch -from .misc import empty_clone, stringify_array +from .misc import empty_clone, eval_expression, stringify_array from .rates import average_rate, marginal_rate + +__all__ = [ + "Dummy", + "apply_thresholds", + "average_rate", + "concat", + "empty_clone", + "eval_expression", + "marginal_rate", + "stringify_array", + "switch", + "types", +] diff --git a/openfisca_core/commons/dummy.py b/openfisca_core/commons/dummy.py index 3788e48705..b9fc31d89f 100644 --- a/openfisca_core/commons/dummy.py +++ b/openfisca_core/commons/dummy.py @@ -20,4 +20,6 @@ def __init__(self) -> None: "and will be removed in the future.", ] warnings.warn(" ".join(message), DeprecationWarning, stacklevel=2) - pass + + +__all__ = ["Dummy"] diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index 77720d9bac..a184ad2dc4 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,16 +1,17 @@ -from openfisca_core.types import Array -from typing import Any, Sequence, TypeVar +from __future__ import annotations + +from collections.abc import Mapping import numpy -T = TypeVar("T") +from . import types as t def apply_thresholds( - input: Array[float], - thresholds: Array[float], - choices: Array[float], -) -> Array[float]: + input: t.Array[numpy.float32], + thresholds: t.ArrayLike[float], + choices: t.ArrayLike[float], +) -> t.Array[numpy.float32]: """Makes a choice based on an input and thresholds. From a list of ``choices``, this function selects one of these values @@ -23,12 +24,10 @@ def apply_thresholds( choices: A list of the possible values to choose from. Returns: - :obj:`numpy.ndarray` of :obj:`float`: - A list of the values chosen. + Array[numpy.float32]: A list of the values chosen. Raises: - :exc:`AssertionError`: When the number of ``thresholds`` (t) and the - number of choices (c) are not either t == c or t == c - 1. + AssertionError: When thresholds and choices are incompatible. Examples: >>> input = numpy.array([4, 5, 6, 7, 8]) @@ -39,7 +38,7 @@ def apply_thresholds( """ - condlist: Sequence[Array[bool]] + condlist: list[t.Array[numpy.bool_] | bool] condlist = [input <= threshold for threshold in thresholds] if len(condlist) == len(choices) - 1: @@ -47,26 +46,27 @@ def apply_thresholds( # must be true to return it. condlist += [True] - assert len(condlist) == len(choices), " ".join( - [ - "'apply_thresholds' must be called with the same number of", - "thresholds than choices, or one more choice.", - ] + msg = ( + "'apply_thresholds' must be called with the same number of thresholds " + "than choices, or one more choice." ) + assert len(condlist) == len(choices), msg return numpy.select(condlist, choices) -def concat(this: Array[str], that: Array[str]) -> Array[str]: - """Concatenates the values of two arrays. +def concat( + this: t.Array[numpy.str_] | t.ArrayLike[object], + that: t.Array[numpy.str_] | t.ArrayLike[object], +) -> t.Array[numpy.str_]: + """Concatenate the values of two arrays. Args: this: An array to concatenate. that: Another array to concatenate. Returns: - :obj:`numpy.ndarray` of :obj:`float`: - An array with the concatenated values. + Array[numpy.str_]: An array with the concatenated values. Examples: >>> this = ["this", "that"] @@ -76,20 +76,26 @@ def concat(this: Array[str], that: Array[str]) -> Array[str]: """ - if isinstance(this, numpy.ndarray) and not numpy.issubdtype(this.dtype, numpy.str_): + if not isinstance(this, numpy.ndarray): + this = numpy.array(this) + + if not numpy.issubdtype(this.dtype, numpy.str_): this = this.astype("str") - if isinstance(that, numpy.ndarray) and not numpy.issubdtype(that.dtype, numpy.str_): + if not isinstance(that, numpy.ndarray): + that = numpy.array(that) + + if not numpy.issubdtype(that.dtype, numpy.str_): that = that.astype("str") return numpy.char.add(this, that) def switch( - conditions: Array[Any], - value_by_condition: dict[float, T], -) -> Array[T]: - """Mimicks a switch statement. + conditions: t.Array[numpy.float32] | t.ArrayLike[float], + value_by_condition: Mapping[float, float], +) -> t.Array[numpy.float32]: + """Mimick a switch statement. Given an array of conditions, returns an array of the same size, replacing each condition item with the matching given value. @@ -99,11 +105,10 @@ def switch( value_by_condition: Values to replace for each condition. Returns: - :obj:`numpy.ndarray`: - An array with the replaced values. + Array: An array with the replaced values. Raises: - :exc:`AssertionError`: When ``value_by_condition`` is empty. + AssertionError: When ``value_by_condition`` is empty. Examples: >>> conditions = numpy.array([1, 1, 1, 2]) @@ -112,11 +117,13 @@ def switch( array([80, 80, 80, 90]) """ - assert ( len(value_by_condition) > 0 ), "'switch' must be called with at least one value." - condlist = [conditions == condition for condition in value_by_condition.keys()] + condlist = [conditions == condition for condition in value_by_condition] + + return numpy.select(condlist, tuple(value_by_condition.values())) + - return numpy.select(condlist, value_by_condition.values()) +__all__ = ["apply_thresholds", "concat", "switch"] diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index 0966fa1dce..ba9687619c 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,11 +1,13 @@ -from openfisca_core.types import Array -from typing import TypeVar +from __future__ import annotations -T = TypeVar("T") +import numexpr +import numpy +from openfisca_core import types as t -def empty_clone(original: T) -> T: - """Creates an empty instance of the same class of the original object. + +def empty_clone(original: object) -> object: + """Create an empty instance of the same class of the original object. Args: original: An object to clone. @@ -28,13 +30,12 @@ def empty_clone(original: T) -> T: """ - Dummy: object - new: T + def __init__(_: object) -> None: ... Dummy = type( "Dummy", (original.__class__,), - {"__init__": lambda self: None}, + {"__init__": __init__}, ) new = Dummy() @@ -42,22 +43,22 @@ def empty_clone(original: T) -> T: return new -def stringify_array(array: Array) -> str: - """Generates a clean string representation of a numpy array. +def stringify_array(array: None | t.Array[numpy.generic]) -> str: + """Generate a clean string representation of a numpy array. Args: array: An array. Returns: - :obj:`str`: - "None" if the ``array`` is None, the stringified ``array`` otherwise. + str: "None" if the ``array`` is None. + str: The stringified ``array`` otherwise. Examples: >>> import numpy >>> stringify_array(None) 'None' - >>> array = numpy.array([10, 20.]) + >>> array = numpy.array([10, 20.0]) >>> stringify_array(array) '[10.0, 20.0]' @@ -75,3 +76,34 @@ def stringify_array(array: Array) -> str: return "None" return f"[{', '.join(str(cell) for cell in array)}]" + + +def eval_expression( + expression: str, +) -> str | t.Array[numpy.bool_] | t.Array[numpy.int32] | t.Array[numpy.float32]: + """Evaluate a string expression to a numpy array. + + Args: + expression: An expression to evaluate. + + Returns: + Array: The result of the evaluation. + str: The expression if it couldn't be evaluated. + + Examples: + >>> eval_expression("1 + 2") + array(3, dtype=int32) + + >>> eval_expression("salary") + 'salary' + + """ + + try: + return numexpr.evaluate(expression) + + except (KeyError, TypeError): + return expression + + +__all__ = ["empty_clone", "eval_expression", "stringify_array"] diff --git a/openfisca_core/commons/py.typed b/openfisca_core/commons/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openfisca_core/commons/rates.py b/openfisca_core/commons/rates.py index 3b9094b64b..cefc65406e 100644 --- a/openfisca_core/commons/rates.py +++ b/openfisca_core/commons/rates.py @@ -1,15 +1,16 @@ -from openfisca_core.types import Array -from typing import Optional +from __future__ import annotations import numpy +from . import types as t + def average_rate( - target: Array[float], - varying: Array[float], - trim: Optional[Array[float]] = None, -) -> Array[float]: - """Computes the average rate of a target net income. + target: t.Array[numpy.float32], + varying: t.Array[numpy.float32] | t.ArrayLike[float], + trim: None | t.ArrayLike[float] = None, +) -> t.Array[numpy.float32]: + """Compute the average rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross income. Optionally, a ``trim`` can be applied consisting of the lower and @@ -24,23 +25,21 @@ def average_rate( trim: The lower and upper bounds of the average rate. Returns: - :obj:`numpy.ndarray` of :obj:`float`: - - The average rate for each target. - - When ``trim`` is provided, values that are out of the provided bounds - are replaced by :obj:`numpy.nan`. + Array[numpy.float32]: The average rate for each target. When ``trim`` + is provided, values that are out of the provided bounds are + replaced by :any:`numpy.nan`. Examples: >>> target = numpy.array([1, 2, 3]) >>> varying = [2, 2, 2] - >>> trim = [-1, .25] + >>> trim = [-1, 0.25] >>> average_rate(target, varying, trim) array([ nan, 0. , -0.5]) """ - average_rate: Array[float] + if not isinstance(varying, numpy.ndarray): + varying = numpy.array(varying, dtype=numpy.float32) average_rate = 1 - target / varying @@ -61,11 +60,11 @@ def average_rate( def marginal_rate( - target: Array[float], - varying: Array[float], - trim: Optional[Array[float]] = None, -) -> Array[float]: - """Computes the marginal rate of a target net income. + target: t.Array[numpy.float32], + varying: t.Array[numpy.float32] | t.ArrayLike[float], + trim: None | t.ArrayLike[float] = None, +) -> t.Array[numpy.float32]: + """Compute the marginal rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross income. Optionally, a ``trim`` can be applied consisting of the lower and @@ -80,23 +79,21 @@ def marginal_rate( trim: The lower and upper bounds of the marginal rate. Returns: - :obj:`numpy.ndarray` of :obj:`float`: - - The marginal rate for each target. - - When ``trim`` is provided, values that are out of the provided bounds - are replaced by :obj:`numpy.nan`. + Array[numpy.float32]: The marginal rate for each target. When ``trim`` + is provided, values that are out of the provided bounds are replaced by + :any:`numpy.nan`. Examples: >>> target = numpy.array([1, 2, 3]) >>> varying = numpy.array([1, 2, 4]) - >>> trim = [.25, .75] + >>> trim = [0.25, 0.75] >>> marginal_rate(target, varying, trim) array([nan, 0.5]) """ - marginal_rate: Array[float] + if not isinstance(varying, numpy.ndarray): + varying = numpy.array(varying, dtype=numpy.float32) marginal_rate = +1 - (target[:-1] - target[1:]) / (varying[:-1] - varying[1:]) @@ -114,3 +111,6 @@ def marginal_rate( ) return marginal_rate + + +__all__ = ["average_rate", "marginal_rate"] diff --git a/openfisca_core/commons/tests/test_dummy.py b/openfisca_core/commons/tests/test_dummy.py index d4ecec3842..dfe04b3e44 100644 --- a/openfisca_core/commons/tests/test_dummy.py +++ b/openfisca_core/commons/tests/test_dummy.py @@ -3,7 +3,7 @@ from openfisca_core.commons import Dummy -def test_dummy_deprecation(): +def test_dummy_deprecation() -> None: """Dummy throws a deprecation warning when instantiated.""" with pytest.warns(DeprecationWarning): diff --git a/openfisca_core/commons/tests/test_formulas.py b/openfisca_core/commons/tests/test_formulas.py index 82755583e6..130df9505b 100644 --- a/openfisca_core/commons/tests/test_formulas.py +++ b/openfisca_core/commons/tests/test_formulas.py @@ -5,8 +5,8 @@ from openfisca_core import commons -def test_apply_thresholds_when_several_inputs(): - """Makes a choice for any given input.""" +def test_apply_thresholds_when_several_inputs() -> None: + """Make a choice for any given input.""" input_ = numpy.array([4, 5, 6, 7, 8, 9, 10]) thresholds = [5, 7, 9] @@ -17,8 +17,8 @@ def test_apply_thresholds_when_several_inputs(): assert_array_equal(result, [10, 10, 15, 15, 20, 20, 25]) -def test_apply_thresholds_when_too_many_thresholds(): - """Raises an AssertionError when thresholds > choices.""" +def test_apply_thresholds_when_too_many_thresholds() -> None: + """Raise an AssertionError when thresholds > choices.""" input_ = numpy.array([6]) thresholds = [5, 7, 9, 11] @@ -28,8 +28,8 @@ def test_apply_thresholds_when_too_many_thresholds(): assert commons.apply_thresholds(input_, thresholds, choices) -def test_apply_thresholds_when_too_many_choices(): - """Raises an AssertionError when thresholds < choices - 1.""" +def test_apply_thresholds_when_too_many_choices() -> None: + """Raise an AssertionError when thresholds < choices - 1.""" input_ = numpy.array([6]) thresholds = [5, 7] @@ -39,8 +39,8 @@ def test_apply_thresholds_when_too_many_choices(): assert commons.apply_thresholds(input_, thresholds, choices) -def test_concat_when_this_is_array_not_str(): - """Casts ``this`` to ``str`` when it is a NumPy array other than string.""" +def test_concat_when_this_is_array_not_str() -> None: + """Cast ``this`` to ``str`` when it is a NumPy array other than string.""" this = numpy.array([1, 2]) that = numpy.array(["la", "o"]) @@ -50,8 +50,8 @@ def test_concat_when_this_is_array_not_str(): assert_array_equal(result, ["1la", "2o"]) -def test_concat_when_that_is_array_not_str(): - """Casts ``that`` to ``str`` when it is a NumPy array other than string.""" +def test_concat_when_that_is_array_not_str() -> None: + """Cast ``that`` to ``str`` when it is a NumPy array other than string.""" this = numpy.array(["ho", "cha"]) that = numpy.array([1, 2]) @@ -61,18 +61,19 @@ def test_concat_when_that_is_array_not_str(): assert_array_equal(result, ["ho1", "cha2"]) -def test_concat_when_args_not_str_array_like(): - """Raises a TypeError when args are not a string array-like object.""" +def test_concat_when_args_not_str_array_like() -> None: + """Cast ``this`` and ``that`` to a NumPy array or strings.""" this = (1, 2) that = (3, 4) - with pytest.raises(TypeError): - commons.concat(this, that) + result = commons.concat(this, that) + + assert_array_equal(result, ["13", "24"]) -def test_switch_when_values_are_empty(): - """Raises an AssertionError when the values are empty.""" +def test_switch_when_values_are_empty() -> None: + """Raise an AssertionError when the values are empty.""" conditions = [1, 1, 1, 2] value_by_condition = {} diff --git a/openfisca_core/commons/tests/test_rates.py b/openfisca_core/commons/tests/test_rates.py index 01565d9527..c266582fc5 100644 --- a/openfisca_core/commons/tests/test_rates.py +++ b/openfisca_core/commons/tests/test_rates.py @@ -1,26 +1,28 @@ +import math + import numpy from numpy.testing import assert_array_equal from openfisca_core import commons -def test_average_rate_when_varying_is_zero(): - """Yields infinity when the varying gross income crosses zero.""" +def test_average_rate_when_varying_is_zero() -> None: + """Yield infinity when the varying gross income crosses zero.""" target = numpy.array([1, 2, 3]) varying = [0, 0, 0] result = commons.average_rate(target, varying) - assert_array_equal(result, [-numpy.inf, -numpy.inf, -numpy.inf]) + assert_array_equal(result, numpy.array([-math.inf, -math.inf, -math.inf])) -def test_marginal_rate_when_varying_is_zero(): - """Yields infinity when the varying gross income crosses zero.""" +def test_marginal_rate_when_varying_is_zero() -> None: + """Yield infinity when the varying gross income crosses zero.""" target = numpy.array([1, 2, 3]) varying = numpy.array([0, 0, 0]) result = commons.marginal_rate(target, varying) - assert_array_equal(result, [numpy.inf, numpy.inf]) + assert_array_equal(result, numpy.array([math.inf, math.inf])) diff --git a/openfisca_core/commons/types.py b/openfisca_core/commons/types.py new file mode 100644 index 0000000000..39c067f455 --- /dev/null +++ b/openfisca_core/commons/types.py @@ -0,0 +1,3 @@ +from openfisca_core.types import Array, ArrayLike + +__all__ = ["Array", "ArrayLike"] diff --git a/openfisca_core/entities/__init__.py b/openfisca_core/entities/__init__.py index 172a207111..9546773cb8 100644 --- a/openfisca_core/entities/__init__.py +++ b/openfisca_core/entities/__init__.py @@ -1,36 +1,40 @@ -"""Transitional imports to ensure non-breaking changes. - -These imports could be deprecated in the next major release. - -Currently, imports are used in the following way:: - - from openfisca_core.module import symbol - -This example causes cyclic dependency problems, which prevent us from -modularising the different components of the library and make them easier to -test and maintain. - -After the next major release, imports could be used in the following way:: - - from openfisca_core import module - module.symbol() - -And for classes:: - - from openfisca_core.module import Symbol - Symbol() - -.. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. - -.. _PEP8#Imports: - https://www.python.org/dev/peps/pep-0008/#imports - -.. _OpenFisca's Styleguide: - https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md - -""" - +# Transitional imports to ensure non-breaking changes. +# Could be deprecated in the next major release. +# +# How imports are being used today: +# +# >>> from openfisca_core.module import symbol +# +# The previous example provokes cyclic dependency problems +# that prevent us from modularizing the different components +# of the library so to make them easier to test and to maintain. +# +# How could them be used after the next major release: +# +# >>> from openfisca_core import module +# >>> module.symbol() +# +# And for classes: +# +# >>> from openfisca_core import module +# >>> module.Symbol() +# +# See: https://www.python.org/dev/peps/pep-0008/#imports + +from . import types from .entity import Entity from .group_entity import GroupEntity -from .helpers import build_entity +from .helpers import build_entity, find_role from .role import Role + +SingleEntity = Entity + +__all__ = [ + "Entity", + "SingleEntity", + "GroupEntity", + "Role", + "build_entity", + "find_role", + "types", +] diff --git a/openfisca_core/entities/_core_entity.py b/openfisca_core/entities/_core_entity.py new file mode 100644 index 0000000000..f44353e112 --- /dev/null +++ b/openfisca_core/entities/_core_entity.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from typing import ClassVar + +import abc +import os + +from . import types as t +from .role import Role + + +class _CoreEntity: + """Base class to build entities from. + + Args: + __key: A key to identify the ``_CoreEntity``. + __plural: The ``key`` pluralised. + __label: A summary description. + __doc: A full description. + *__args: Additional arguments. + + """ + + #: A key to identify the ``_CoreEntity``. + key: t.EntityKey + + #: The ``key`` pluralised. + plural: t.EntityPlural + + #: A summary description. + label: str + + #: A full description. + doc: str + + #: Whether the ``_CoreEntity`` is a person or not. + is_person: ClassVar[bool] + + #: A ``TaxBenefitSystem`` instance. + _tax_benefit_system: None | t.TaxBenefitSystem = None + + @abc.abstractmethod + def __init__( + self, + __key: str, + __plural: str, + __label: str, + __doc: str, + *__args: object, + ) -> None: ... + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.key})" + + def set_tax_benefit_system(self, tax_benefit_system: t.TaxBenefitSystem) -> None: + """A ``_CoreEntity`` belongs to a ``TaxBenefitSystem``.""" + self._tax_benefit_system = tax_benefit_system + + def get_variable( + self, + variable_name: t.VariableName, + check_existence: bool = False, + ) -> t.Variable | None: + """Get ``variable_name`` from ``variables``. + + Args: + variable_name: The ``Variable`` to be found. + check_existence: Was the ``Variable`` found? + + Returns: + Variable: When the ``Variable`` exists. + None: When the ``Variable`` doesn't exist. + + Raises: + ValueError: When ``check_existence`` is ``True`` and + the ``Variable`` doesn't exist. + + """ + + if self._tax_benefit_system is None: + msg = "You must set 'tax_benefit_system' before calling this method." + raise ValueError( + msg, + ) + return self._tax_benefit_system.get_variable(variable_name, check_existence) + + def check_variable_defined_for_entity(self, variable_name: t.VariableName) -> None: + """Check if ``variable_name`` is defined for ``self``. + + Args: + variable_name: The ``Variable`` to be found. + + Returns: + Variable: When the ``Variable`` exists. + None: When the :attr:`_tax_benefit_system` is not set. + + Raises: + ValueError: When the ``Variable`` exists but is defined + for another ``Entity``. + + """ + + entity: None | t.CoreEntity = None + variable: None | t.Variable = self.get_variable( + variable_name, + check_existence=True, + ) + + if variable is not None: + entity = variable.entity + + if entity is None: + return + + if entity.key != self.key: + message = ( + f"You tried to compute the variable '{variable_name}' for", + f"the entity '{self.plural}'; however the variable", + f"'{variable_name}' is defined for '{entity.plural}'.", + "Learn more about entities in our documentation:", + ".", + ) + raise ValueError(os.linesep.join(message)) + + @staticmethod + def check_role_validity(role: object) -> None: + """Check if ``role`` is an instance of ``Role``. + + Args: + role: Any object. + + Raises: + ValueError: When ``role`` is not a ``Role``. + + """ + + if role is not None and not isinstance(role, Role): + msg = f"{role} is not a valid role" + raise ValueError(msg) + + +__all__ = ["_CoreEntity"] diff --git a/openfisca_core/entities/_description.py b/openfisca_core/entities/_description.py new file mode 100644 index 0000000000..78634ca270 --- /dev/null +++ b/openfisca_core/entities/_description.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import dataclasses +import textwrap + + +@dataclasses.dataclass(frozen=True) +class _Description: + """A ``Role``'s description. + + Examples: + >>> data = { + ... "key": "parent", + ... "label": "Parents", + ... "plural": "parents", + ... "doc": "\t\t\tThe one/two adults in charge of the household.", + ... } + + >>> description = _Description(**data) + + >>> repr(_Description) + "" + + >>> repr(description) + "_Description(key='parent', plural='parents', label='Parents', ...)" + + >>> str(description) + "_Description(key='parent', plural='parents', label='Parents', ...)" + + >>> {description} + {_Description(key='parent', plural='parents', label='Parents', doc=...} + + >>> description.key + 'parent' + + """ + + #: A key to identify the ``Role``. + key: str + + #: The ``key`` pluralised. + plural: None | str = None + + #: A summary description. + label: None | str = None + + #: A full description, non-indented. + doc: None | str = None + + def __post_init__(self) -> None: + if self.doc is not None: + object.__setattr__(self, "doc", textwrap.dedent(self.doc)) + + +__all__ = ["_Description"] diff --git a/openfisca_core/entities/entity.py b/openfisca_core/entities/entity.py index 05c29bae9d..c001918168 100644 --- a/openfisca_core/entities/entity.py +++ b/openfisca_core/entities/entity.py @@ -1,62 +1,42 @@ -from openfisca_core.types import TaxBenefitSystem, Variable -from typing import Any, Optional +from typing import ClassVar -import os import textwrap -from .role import Role +from . import types as t +from ._core_entity import _CoreEntity -class Entity: - """ - Represents an entity (e.g. a person, a household, etc.) on which calculations can be run. - """ - - def __init__(self, key, plural, label, doc): - self.key = key - self.label = label - self.plural = plural - self.doc = textwrap.dedent(doc) - self.is_person = True - self._tax_benefit_system = None +class Entity(_CoreEntity): + """An entity (e.g. a person, a household) on which calculations can be run. - def __eq__(self, other: object) -> bool: - if isinstance(other, Entity): - return self.key == other.key + Args: + key: A key to identify the ``Entity``. + plural: The ``key`` pluralised. + label: A summary description. + doc: A full description. - return NotImplemented + """ - def set_tax_benefit_system(self, tax_benefit_system: TaxBenefitSystem): - self._tax_benefit_system = tax_benefit_system + #: A key to identify the ``Entity``. + key: t.EntityKey - def check_role_validity(self, role: Any) -> None: - if role is not None and not isinstance(role, Role): - raise ValueError(f"{role} is not a valid role") + #: The ``key`` pluralised. + plural: t.EntityPlural - def get_variable( - self, - variable_name: str, - check_existence: bool = False, - ) -> Optional[Variable]: - return self._tax_benefit_system.get_variable(variable_name, check_existence) + #: A summary description. + label: str - def check_variable_defined_for_entity(self, variable_name: str) -> None: - variable: Optional[Variable] - entity: Entity + #: A full description. + doc: str - variable = self.get_variable(variable_name, check_existence=True) + #: Whether the ``Entity`` is a person or not. + is_person: ClassVar[bool] = True - if variable is not None: - entity = variable.entity + def __init__(self, key: str, plural: str, label: str, doc: str) -> None: + self.key = t.EntityKey(key) + self.plural = t.EntityPlural(plural) + self.label = label + self.doc = textwrap.dedent(doc) - if self != entity: - message = os.linesep.join( - [ - f"You tried to compute the variable '{variable_name}' for the entity '{self.plural}';", - f"however the variable '{variable_name}' is defined for '{entity.plural}'.", - "Learn more about entities in our documentation:", - ".", - ] - ) - raise ValueError(message) +__all__ = ["Entity"] diff --git a/openfisca_core/entities/group_entity.py b/openfisca_core/entities/group_entity.py index beb5172d82..4b588567ad 100644 --- a/openfisca_core/entities/group_entity.py +++ b/openfisca_core/entities/group_entity.py @@ -1,46 +1,81 @@ -from .entity import Entity +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import ClassVar + +import textwrap +from itertools import chain + +from . import types as t +from ._core_entity import _CoreEntity from .role import Role -class GroupEntity(Entity): +class GroupEntity(_CoreEntity): """Represents an entity containing several others with different roles. - A :class:`.GroupEntity` represents an :class:`.Entity` containing - several other :class:`.Entity` with different :class:`.Role`, and on - which calculations can be run. + A ``GroupEntity`` represents an ``Entity`` containing several other entities, + with different roles, and on which calculations can be run. Args: - key: A key to identify the group entity. - plural: The ``key``, pluralised. + key: A key to identify the ``GroupEntity``. + plural: The ``key`` pluralised. label: A summary description. doc: A full description. - roles: The list of :class:`.Role` of the group entity. + roles: The list of roles of the group entity. containing_entities: The list of keys of group entities whose members are guaranteed to be a superset of this group's entities. - .. versionchanged:: 35.7.0 - Added ``containing_entities``, that allows the defining of group - entities which entirely contain other group entities. - """ - def __init__(self, key, plural, label, doc, roles, containing_entities=()): - super().__init__(key, plural, label, doc) + #: A key to identify the ``Entity``. + key: t.EntityKey + + #: The ``key`` pluralised. + plural: t.EntityPlural + + #: A summary description. + label: str + + #: A full description. + doc: str + + #: The list of roles of the ``GroupEntity``. + roles: Iterable[Role] + + #: Whether the entity is a person or not. + is_person: ClassVar[bool] = False + + def __init__( + self, + key: str, + plural: str, + label: str, + doc: str, + roles: Sequence[t.RoleParams], + containing_entities: Iterable[str] = (), + ) -> None: + self.key = t.EntityKey(key) + self.plural = t.EntityPlural(plural) + self.label = label + self.doc = textwrap.dedent(doc) self.roles_description = roles - self.roles = [] + self.roles: Iterable[Role] = () for role_description in roles: role = Role(role_description, self) setattr(self, role.key.upper(), role) - self.roles.append(role) - if role_description.get("subroles"): - role.subroles = [] - for subrole_key in role_description["subroles"]: + self.roles = (*self.roles, role) + if subroles := role_description.get("subroles"): + role.subroles = () + for subrole_key in subroles: subrole = Role({"key": subrole_key, "max": 1}, self) setattr(self, subrole.key.upper(), subrole) - role.subroles.append(subrole) + role.subroles = (*role.subroles, subrole) role.max = len(role.subroles) - self.flattened_roles = sum( - [role2.subroles or [role2] for role2 in self.roles], [] + self.flattened_roles = tuple( + chain.from_iterable(role.subroles or [role] for role in self.roles), ) - self.is_person = False self.containing_entities = containing_entities + + +__all__ = ["GroupEntity"] diff --git a/openfisca_core/entities/helpers.py b/openfisca_core/entities/helpers.py index 734d385b5e..146ab6d25b 100644 --- a/openfisca_core/entities/helpers.py +++ b/openfisca_core/entities/helpers.py @@ -1,21 +1,165 @@ -from .entity import Entity +from __future__ import annotations + +from collections.abc import Iterable, Sequence + +from . import types as t +from .entity import Entity as SingleEntity from .group_entity import GroupEntity def build_entity( - key, - plural, - label, - doc="", - roles=None, - is_person=False, - class_override=None, - containing_entities=(), -): + key: str, + plural: str, + label: str, + doc: str = "", + roles: None | Sequence[t.RoleParams] = None, + is_person: bool = False, + *, + class_override: object = None, + containing_entities: Sequence[str] = (), +) -> t.SingleEntity | t.GroupEntity: + """Build an ``Entity`` or a ``GroupEntity``. + + Args: + key: Key to identify the ``Entity`` or ``GroupEntity``. + plural: The ``key`` pluralised. + label: A summary description. + doc: A full description. + roles: A list of roles —if it's a ``GroupEntity``. + is_person: If is an individual, or not. + class_override: ? + containing_entities: Keys of contained entities. + + Returns: + Entity: When ``is_person`` is ``True``. + GroupEntity: When ``is_person`` is ``False``. + + Raises: + NotImplementedError: If ``roles`` is ``None``. + + Examples: + >>> from openfisca_core import entities + + >>> entity = build_entity( + ... "syndicate", + ... "syndicates", + ... "Banks loaning jointly.", + ... roles=[], + ... containing_entities=(), + ... ) + >>> entity + GroupEntity(syndicate) + + >>> build_entity( + ... "company", + ... "companies", + ... "A small or medium company.", + ... is_person=True, + ... ) + Entity(company) + + >>> role = entities.Role({"key": "key"}, entity) + + >>> build_entity( + ... "syndicate", + ... "syndicates", + ... "Banks loaning jointly.", + ... roles=[role], + ... ) + Traceback (most recent call last): + TypeError: 'Role' object is not subscriptable + + """ + if is_person: - return Entity(key, plural, label, doc) + return SingleEntity(key, plural, label, doc) - else: + if roles is not None: return GroupEntity( - key, plural, label, doc, roles, containing_entities=containing_entities + key, + plural, + label, + doc, + roles, + containing_entities=containing_entities, ) + + raise NotImplementedError + + +def find_role( + roles: Iterable[t.Role], + key: t.RoleKey, + *, + total: None | int = None, +) -> None | t.Role: + """Find a ``Role`` in a ``GroupEntity``. + + Args: + roles: The roles to search. + key: The key of the role to find. + total: The ``max`` attribute of the role to find. + + Returns: + Role: The role if found + None: Else ``None``. + + Examples: + >>> from openfisca_core.entities.types import RoleParams + + >>> principal = RoleParams( + ... key="principal", + ... label="Principal", + ... doc="Person focus of a calculation in a family context.", + ... max=1, + ... ) + + >>> partner = RoleParams( + ... key="partner", + ... plural="partners", + ... label="Partners", + ... doc="Persons partners of the principal.", + ... ) + + >>> parent = RoleParams( + ... key="parent", + ... plural="parents", + ... label="Parents", + ... doc="Persons parents of children of the principal", + ... subroles=["first_parent", "second_parent"], + ... ) + + >>> group_entity = build_entity( + ... key="family", + ... plural="families", + ... label="Family", + ... doc="A Family represents a collection of related persons.", + ... roles=[principal, partner, parent], + ... ) + + >>> find_role(group_entity.roles, "principal", total=1) + Role(principal) + + >>> find_role(group_entity.roles, "partner") + Role(partner) + + >>> find_role(group_entity.roles, "parent", total=2) + Role(parent) + + >>> find_role(group_entity.roles, "first_parent", total=1) + Role(first_parent) + + """ + for role in roles: + if role.subroles: + for subrole in role.subroles: + if (subrole.max == total) and (subrole.key == key): + return subrole + + if (role.max == total) and (role.key == key): + return role + + return None + + +__all__ = ["build_entity", "find_role"] diff --git a/openfisca_core/entities/py.typed b/openfisca_core/entities/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openfisca_core/entities/role.py b/openfisca_core/entities/role.py index 7c85656fb4..e687b2604c 100644 --- a/openfisca_core/entities/role.py +++ b/openfisca_core/entities/role.py @@ -1,15 +1,91 @@ -import textwrap +from __future__ import annotations + +from collections.abc import Iterable + +from . import types as t +from ._description import _Description class Role: - def __init__(self, description, entity): + """The role of an ``Entity`` within a ``GroupEntity``. + + Each ``Entity`` related to a ``GroupEntity`` has a ``Role``. For example, + if you have a family, its roles could include a parent, a child, and so on. + Or if you have a tax household, its roles could include the taxpayer, a + spouse, several dependents, and the like. + + Args: + description: A description of the Role. + entity: The Entity to which the Role belongs. + + Examples: + >>> from openfisca_core import entities + >>> entity = entities.GroupEntity("key", "plural", "label", "doc", []) + >>> role = entities.Role({"key": "parent"}, entity) + + >>> repr(Role) + "" + + >>> repr(role) + 'Role(parent)' + + >>> str(role) + 'Role(parent)' + + >>> {role} + {Role(parent)} + + >>> role.key + 'parent' + + """ + + #: The ``GroupEntity`` the Role belongs to. + entity: t.GroupEntity + + #: A description of the ``Role``. + description: _Description + + #: Max number of members. + max: None | int = None + + #: A list of subroles. + subroles: None | Iterable[Role] = None + + @property + def key(self) -> t.RoleKey: + """A key to identify the ``Role``.""" + return t.RoleKey(self.description.key) + + @property + def plural(self) -> None | t.RolePlural: + """The ``key`` pluralised.""" + if (plural := self.description.plural) is None: + return None + return t.RolePlural(plural) + + @property + def label(self) -> None | str: + """A summary description.""" + return self.description.label + + @property + def doc(self) -> None | str: + """A full description, non-indented.""" + return self.description.doc + + def __init__(self, description: t.RoleParams, entity: t.GroupEntity) -> None: + self.description = _Description( + key=description["key"], + plural=description.get("plural"), + label=description.get("label"), + doc=description.get("doc"), + ) self.entity = entity - self.key = description["key"] - self.label = description.get("label") - self.plural = description.get("plural") - self.doc = textwrap.dedent(description.get("doc", "")) self.max = description.get("max") - self.subroles = None - def __repr__(self): + def __repr__(self) -> str: return f"Role({self.key})" + + +__all__ = ["Role"] diff --git a/openfisca_core/entities/tests/__init__.py b/openfisca_core/entities/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openfisca_core/entities/tests/test_entity.py b/openfisca_core/entities/tests/test_entity.py new file mode 100644 index 0000000000..b3cb813ddc --- /dev/null +++ b/openfisca_core/entities/tests/test_entity.py @@ -0,0 +1,10 @@ +from openfisca_core import entities + + +def test_init_when_doc_indented() -> None: + """De-indent the ``doc`` attribute if it is passed at initialisation.""" + key = "\tkey" + doc = "\tdoc" + entity = entities.Entity(key, "label", "plural", doc) + assert entity.key == key + assert entity.doc == doc.lstrip() diff --git a/openfisca_core/entities/tests/test_group_entity.py b/openfisca_core/entities/tests/test_group_entity.py new file mode 100644 index 0000000000..092c9d3575 --- /dev/null +++ b/openfisca_core/entities/tests/test_group_entity.py @@ -0,0 +1,70 @@ +from collections.abc import Mapping +from typing import Any + +import pytest + +from openfisca_core import entities + + +@pytest.fixture +def parent() -> str: + return "parent" + + +@pytest.fixture +def uncle() -> str: + return "uncle" + + +@pytest.fixture +def first_parent() -> str: + return "first_parent" + + +@pytest.fixture +def second_parent() -> str: + return "second_parent" + + +@pytest.fixture +def third_parent() -> str: + return "third_parent" + + +@pytest.fixture +def role(parent: str, first_parent: str, third_parent: str) -> Mapping[str, Any]: + return {"key": parent, "subroles": {first_parent, third_parent}} + + +@pytest.fixture +def group_entity(role: Mapping[str, Any]) -> entities.GroupEntity: + return entities.GroupEntity("key", "label", "plural", "doc", (role,)) + + +def test_init_when_doc_indented() -> None: + """De-indent the ``doc`` attribute if it is passed at initialisation.""" + key = "\tkey" + doc = "\tdoc" + group_entity = entities.GroupEntity(key, "label", "plural", doc, ()) + assert group_entity.key == key + assert group_entity.doc == doc.lstrip() + + +def test_group_entity_with_roles( + group_entity: entities.GroupEntity, + parent: str, + uncle: str, +) -> None: + """Assign a Role for each role-like passed as argument.""" + assert hasattr(group_entity, parent.upper()) + assert not hasattr(group_entity, uncle.upper()) + + +def test_group_entity_with_subroles( + group_entity: entities.GroupEntity, + first_parent: str, + second_parent: str, +) -> None: + """Assign a Role for each subrole-like passed as argument.""" + assert hasattr(group_entity, first_parent.upper()) + assert not hasattr(group_entity, second_parent.upper()) diff --git a/openfisca_core/entities/tests/test_role.py b/openfisca_core/entities/tests/test_role.py new file mode 100644 index 0000000000..454d862c70 --- /dev/null +++ b/openfisca_core/entities/tests/test_role.py @@ -0,0 +1,11 @@ +from openfisca_core import entities + + +def test_init_when_doc_indented() -> None: + """De-indent the ``doc`` attribute if it is passed at initialisation.""" + key = "\tkey" + doc = "\tdoc" + entity = entities.GroupEntity("key", "plural", "label", "doc", []) + role = entities.Role({"key": key, "doc": doc}, entity) + assert role.key == key + assert role.doc == doc.lstrip() diff --git a/openfisca_core/entities/types.py b/openfisca_core/entities/types.py new file mode 100644 index 0000000000..ef6af9024f --- /dev/null +++ b/openfisca_core/entities/types.py @@ -0,0 +1,42 @@ +from typing_extensions import Required, TypedDict + +from openfisca_core.types import ( + CoreEntity, + EntityKey, + EntityPlural, + GroupEntity, + Role, + RoleKey, + RolePlural, + SingleEntity, + TaxBenefitSystem, + Variable, + VariableName, +) + +# Entities + + +class RoleParams(TypedDict, total=False): + key: Required[str] + plural: str + label: str + doc: str + max: int + subroles: list[str] + + +__all__ = [ + "CoreEntity", + "EntityKey", + "EntityPlural", + "GroupEntity", + "Role", + "RoleKey", + "RoleParams", + "RolePlural", + "SingleEntity", + "TaxBenefitSystem", + "Variable", + "VariableName", +] diff --git a/openfisca_core/holders/helpers/_set_input.py b/openfisca_core/holders/helpers/_set_input.py index a946ecf7f9..ae325aa7e2 100644 --- a/openfisca_core/holders/helpers/_set_input.py +++ b/openfisca_core/holders/helpers/_set_input.py @@ -1,6 +1,7 @@ -from openfisca_core.types import Period from typing import Any +from openfisca_core.types import Period + import logging import numpy diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index ad062ab3bc..33c00d7096 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -1,10 +1,11 @@ from __future__ import annotations from collections.abc import Sequence -from openfisca_core.types import Period, Population, Simulation, Variable from typing import Any from typing_extensions import Literal +from openfisca_core.types import Period, Population, Simulation, Variable + import itertools import os import warnings @@ -13,9 +14,14 @@ import psutil from sortedcontainers import sorteddict -from openfisca_core import commons, errors, experimental -from openfisca_core import indexed_enums as enums -from openfisca_core import periods, tools +from openfisca_core import ( + commons, + errors, + experimental, + indexed_enums as enums, + periods, + tools, +) from .repos import DiskRepo, MemoryRepo from .typing import MemoryUsage, Storage @@ -235,9 +241,11 @@ def get_memory_usage(self) -> MemoryUsage: usage.update( dict( nb_requests=nb_requests, - nb_requests_by_array=nb_requests / float(usage["nb_arrays"]) - if usage["nb_arrays"] > 0 - else numpy.nan, + nb_requests_by_array=( + nb_requests / float(usage["nb_arrays"]) + if usage["nb_arrays"] > 0 + else numpy.nan + ), ) ) diff --git a/openfisca_core/holders/repos/_disk_repo.py b/openfisca_core/holders/repos/_disk_repo.py index f331a88708..57c24297d0 100644 --- a/openfisca_core/holders/repos/_disk_repo.py +++ b/openfisca_core/holders/repos/_disk_repo.py @@ -1,17 +1,17 @@ from __future__ import annotations from collections.abc import Sequence -from openfisca_core.types import Enum, Period from typing import Any, NoReturn +from openfisca_core.types import Enum, Period + import os import pathlib import shutil import numpy -from openfisca_core import indexed_enums as enums -from openfisca_core import periods +from openfisca_core import indexed_enums as enums, periods class DiskRepo: diff --git a/openfisca_core/holders/repos/_memory_repo.py b/openfisca_core/holders/repos/_memory_repo.py index a5cc13ea5d..0f84063cd5 100644 --- a/openfisca_core/holders/repos/_memory_repo.py +++ b/openfisca_core/holders/repos/_memory_repo.py @@ -2,6 +2,7 @@ from collections.abc import Sequence from openfisca_core.holders.typing import MemoryUsage + from openfisca_core.types import Period import numpy diff --git a/openfisca_core/holders/typing.py b/openfisca_core/holders/typing.py index 7cb83e47e9..2f08c3d73a 100644 --- a/openfisca_core/holders/typing.py +++ b/openfisca_core/holders/typing.py @@ -2,42 +2,36 @@ from __future__ import annotations -import abc from typing import Any from typing_extensions import Protocol, TypedDict +import abc + import numpy class Holder(Protocol): @abc.abstractmethod - def clone(self, population: Any) -> Holder: - ... + def clone(self, population: Any) -> Holder: ... @abc.abstractmethod - def get_memory_usage(self) -> Any: - ... + def get_memory_usage(self) -> Any: ... class Storage(Protocol): @abc.abstractmethod - def get(self, period: Any) -> Any: - ... + def get(self, period: Any) -> Any: ... @abc.abstractmethod - def put(self, values: Any, period: Any) -> None: - ... + def put(self, values: Any, period: Any) -> None: ... @abc.abstractmethod - def delete(self, period: Any = None) -> None: - ... + def delete(self, period: Any = None) -> None: ... - def periods(self) -> Any: - ... + def periods(self) -> Any: ... @abc.abstractmethod - def usage(self) -> Any: - ... + def usage(self) -> Any: ... class MemoryUsage(TypedDict, total=False): diff --git a/openfisca_core/indexed_enums/__init__.py b/openfisca_core/indexed_enums/__init__.py index 932cbb8a9b..9c4ff7dd6d 100644 --- a/openfisca_core/indexed_enums/__init__.py +++ b/openfisca_core/indexed_enums/__init__.py @@ -1,35 +1,34 @@ -"""Transitional imports to ensure non-breaking changes. - -These imports could be deprecated in the next major release. - -Currently, imports are used in the following way:: - - from openfisca_core.module import symbol - -This example causes cyclic dependency problems, which prevent us from -modularising the different components of the library and make them easier to -test and maintain. - -After the next major release, imports could be used in the following way:: - - from openfisca_core import module - module.symbol() - -And for classes:: - - from openfisca_core.module import Symbol - Symbol() - -.. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. - -.. _PEP8#Imports: - https://www.python.org/dev/peps/pep-0008/#imports - -.. _OpenFisca's Styleguide: - https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md - -""" - +# Transitional imports to ensure non-breaking changes. +# Could be deprecated in the next major release. +# +# How imports are being used today: +# +# >>> from openfisca_core.module import symbol +# +# The previous example provokes cyclic dependency problems +# that prevent us from modularizing the different components +# of the library so to make them easier to test and to maintain. +# +# How could them be used after the next major release: +# +# >>> from openfisca_core import module +# >>> module.symbol() +# +# And for classes: +# +# >>> from openfisca_core.module import Symbol +# >>> Symbol() +# +# See: https://www.python.org/dev/peps/pep-0008/#imports + +from . import types from .config import ENUM_ARRAY_DTYPE from .enum import Enum from .enum_array import EnumArray + +__all__ = [ + "ENUM_ARRAY_DTYPE", + "Enum", + "EnumArray", + "types", +] diff --git a/openfisca_core/indexed_enums/enum.py b/openfisca_core/indexed_enums/enum.py index 2017d56064..ec1afa45a9 100644 --- a/openfisca_core/indexed_enums/enum.py +++ b/openfisca_core/indexed_enums/enum.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Union - import enum import numpy @@ -11,23 +9,16 @@ class Enum(enum.Enum): - """ - Enum based on `enum34 `_, whose items + """Enum based on `enum34 `_, whose items have an index. """ - #: Index of the enum. - index: int - # Tweak enums to add an index attribute to each enum item - def __new__(cls, name: str) -> Enum: - # When the enum item is initialized, cls._member_names_ contains the + def __init__(self, name: str) -> None: + # When the enum item is initialized, self._member_names_ contains the # names of the previously initialized items, so its length is the index # of this item. - new = object.__new__(cls) - new._value_ = name - new.index = len(cls._member_names_) - return new + self.index = len(self._member_names_) # Bypass the slow Enum.__eq__ __eq__ = object.__eq__ @@ -39,15 +30,9 @@ def __new__(cls, name: str) -> Enum: @classmethod def encode( cls, - array: Union[ - EnumArray, - numpy.int_, - numpy.float_, - numpy.object_, - ], + array: EnumArray | numpy.int32 | numpy.float32 | numpy.object_, ) -> EnumArray: - """ - Encode a string numpy array, an enum item numpy array, or an int numpy + """Encode a string numpy array, an enum item numpy array, or an int numpy array into an :any:`EnumArray`. See :any:`EnumArray.decode` for decoding. @@ -59,7 +44,7 @@ def encode( For instance: - >>> string_identifier_array = asarray(['free_lodger', 'owner']) + >>> string_identifier_array = asarray(["free_lodger", "owner"]) >>> encoded_array = HousingOccupancyStatus.encode(string_identifier_array) >>> encoded_array[0] 2 # Encoded value diff --git a/openfisca_core/indexed_enums/enum_array.py b/openfisca_core/indexed_enums/enum_array.py index 59f312eb70..1b6c512b8e 100644 --- a/openfisca_core/indexed_enums/enum_array.py +++ b/openfisca_core/indexed_enums/enum_array.py @@ -1,16 +1,18 @@ from __future__ import annotations -from numpy.typing import NDArray -from typing import Any, Iterable, NoReturn +import typing +from typing import Any, NoReturn import numpy -from .typing import HasIndex +from . import types as t +if typing.TYPE_CHECKING: + from openfisca_core.indexed_enums import Enum -class EnumArray(NDArray[numpy.int_]): - """ - NumPy array subclass representing an array of enum items. + +class EnumArray(numpy.ndarray): + """NumPy array subclass representing an array of enum items. EnumArrays are encoded as ``int`` arrays to improve performance """ @@ -20,38 +22,40 @@ class EnumArray(NDArray[numpy.int_]): # https://docs.scipy.org/doc/numpy-1.13.0/user/basics.subclassing.html#slightly-more-realistic-example-attribute-added-to-existing-array. def __new__( cls, - input_array: NDArray[numpy.int_], - possible_values: Iterable[HasIndex] | None = None, + input_array: t.Array[numpy.int16], + possible_values: type[Enum] | None = None, ) -> EnumArray: obj = numpy.asarray(input_array).view(cls) obj.possible_values = possible_values return obj # See previous comment - def __array_finalize__(self, obj: NDArray[numpy.int_] | EnumArray | None) -> None: + def __array_finalize__(self, obj: numpy.int32 | None) -> None: if obj is None: return self.possible_values = getattr(obj, "possible_values", None) - def __eq__(self, other: HasIndex | Any) -> bool: + def __eq__(self, other: object) -> bool: # When comparing to an item of self.possible_values, use the item index # to speed up the comparison. - - if hasattr(other, "index"): + if other.__class__.__name__ is self.possible_values.__name__: # Use view(ndarray) so that the result is a classic ndarray, not an # EnumArray. return self.view(numpy.ndarray) == other.index return self.view(numpy.ndarray) == other - def __ne__(self, other: Any) -> bool: + def __ne__(self, other: object) -> bool: return numpy.logical_not(self == other) def _forbidden_operation(self, other: Any) -> NoReturn: - raise TypeError( + msg = ( "Forbidden operation. The only operations allowed on EnumArrays " - "are '==' and '!='.", + "are '==' and '!='." + ) + raise TypeError( + msg, ) __add__ = _forbidden_operation @@ -64,12 +68,11 @@ def _forbidden_operation(self, other: Any) -> NoReturn: __or__ = _forbidden_operation def decode(self) -> numpy.object_: - """ - Return the array of enum items corresponding to self. + """Return the array of enum items corresponding to self. For instance: - >>> enum_array = household('housing_occupancy_status', period) + >>> enum_array = household("housing_occupancy_status", period) >>> enum_array[0] >>> 2 # Encoded value >>> enum_array.decode()[0] @@ -82,13 +85,12 @@ def decode(self) -> numpy.object_: list(self.possible_values), ) - def decode_to_str(self) -> NDArray[numpy.str_]: - """ - Return the array of string identifiers corresponding to self. + def decode_to_str(self) -> numpy.str_: + """Return the array of string identifiers corresponding to self. For instance: - >>> enum_array = household('housing_occupancy_status', period) + >>> enum_array = household("housing_occupancy_status", period) >>> enum_array[0] >>> 2 # Encoded value >>> enum_array.decode_to_str()[0] @@ -100,7 +102,7 @@ def decode_to_str(self) -> NDArray[numpy.str_]: ) def __repr__(self) -> str: - return f"{self.__class__.__name__}({str(self.decode())})" + return f"{self.__class__.__name__}({self.decode()!s})" def __str__(self) -> str: return str(self.decode_to_str()) diff --git a/openfisca_core/indexed_enums/types.py b/openfisca_core/indexed_enums/types.py new file mode 100644 index 0000000000..43c38780ff --- /dev/null +++ b/openfisca_core/indexed_enums/types.py @@ -0,0 +1,3 @@ +from openfisca_core.types import Array + +__all__ = ["Array"] diff --git a/openfisca_core/parameters/at_instant_like.py b/openfisca_core/parameters/at_instant_like.py index 1a1db34beb..5bb482fd3a 100644 --- a/openfisca_core/parameters/at_instant_like.py +++ b/openfisca_core/parameters/at_instant_like.py @@ -16,5 +16,4 @@ def get_at_instant(self, instant): return self._get_at_instant(instant) @abc.abstractmethod - def _get_at_instant(self, instant): - ... + def _get_at_instant(self, instant): ... diff --git a/openfisca_core/parameters/parameter_node.py b/openfisca_core/parameters/parameter_node.py index 83a7d35d30..987c35d4e8 100644 --- a/openfisca_core/parameters/parameter_node.py +++ b/openfisca_core/parameters/parameter_node.py @@ -13,9 +13,9 @@ class ParameterNode(AtInstantLike): A node in the legislation `parameter tree `_. """ - _allowed_keys: typing.Optional[ - typing.Iterable[str] - ] = None # By default, no restriction on the keys + _allowed_keys: typing.Optional[typing.Iterable[str]] = ( + None # By default, no restriction on the keys + ) def __init__(self, name="", directory_path=None, data=None, file_path=None): """ diff --git a/openfisca_core/parameters/vectorial_parameter_node_at_instant.py b/openfisca_core/parameters/vectorial_parameter_node_at_instant.py index 0681848cfa..f34ddfe76b 100644 --- a/openfisca_core/parameters/vectorial_parameter_node_at_instant.py +++ b/openfisca_core/parameters/vectorial_parameter_node_at_instant.py @@ -19,11 +19,13 @@ def build_from_node(node): # Recursively vectorize the children of the node vectorial_subnodes = tuple( [ - VectorialParameterNodeAtInstant.build_from_node( - node[subnode_name] - ).vector - if isinstance(node[subnode_name], parameters.ParameterNodeAtInstant) - else node[subnode_name] + ( + VectorialParameterNodeAtInstant.build_from_node( + node[subnode_name] + ).vector + if isinstance(node[subnode_name], parameters.ParameterNodeAtInstant) + else node[subnode_name] + ) for subnode_name in subnodes_name ] ) diff --git a/openfisca_core/periods/__init__.py b/openfisca_core/periods/__init__.py index 225404d20b..2335f1792a 100644 --- a/openfisca_core/periods/__init__.py +++ b/openfisca_core/periods/__init__.py @@ -1,43 +1,30 @@ -"""Transitional imports to ensure non-breaking changes. - -These imports could be deprecated in the next major release. - -Currently, imports are used in the following way:: - - from openfisca_core.module import symbol - -This example causes cyclic dependency problems, which prevent us from -modularising the different components of the library and make them easier to -test and maintain. - -After the next major release, imports could be used in the following way:: - - from openfisca_core import module - module.symbol() - -And for classes:: - - from openfisca_core.module import Symbol - Symbol() - -.. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. - -.. _PEP8#Imports: - https://www.python.org/dev/peps/pep-0008/#imports - -.. _OpenFisca's Styleguide: - https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md - -""" +# Transitional imports to ensure non-breaking changes. +# Could be deprecated in the next major release. +# +# How imports are being used today: +# +# >>> from openfisca_core.module import symbol +# +# The previous example provokes cyclic dependency problems +# that prevent us from modularizing the different components +# of the library so to make them easier to test and to maintain. +# +# How could them be used after the next major release: +# +# >>> from openfisca_core import module +# >>> module.symbol() +# +# And for classes: +# +# >>> from openfisca_core.module import Symbol +# >>> Symbol() +# +# See: https://www.python.org/dev/peps/pep-0008/#imports +from . import types +from ._errors import InstantError, ParserError, PeriodError from .config import ( - DAY, - ETERNITY, INSTANT_PATTERN, - MONTH, - WEEK, - WEEKDAY, - YEAR, date_by_instant_cache, str_by_instant_cache, year_or_month_or_day_re, @@ -53,3 +40,40 @@ ) from .instant_ import Instant from .period_ import Period + +WEEKDAY = DateUnit.WEEKDAY +WEEK = DateUnit.WEEK +DAY = DateUnit.DAY +MONTH = DateUnit.MONTH +YEAR = DateUnit.YEAR +ETERNITY = DateUnit.ETERNITY +ISOFORMAT = DateUnit.isoformat +ISOCALENDAR = DateUnit.isocalendar + +__all__ = [ + "DAY", + "DateUnit", + "ETERNITY", + "INSTANT_PATTERN", + "ISOCALENDAR", + "ISOFORMAT", + "Instant", + "InstantError", + "MONTH", + "ParserError", + "Period", + "PeriodError", + "WEEK", + "WEEKDAY", + "YEAR", + "date_by_instant_cache", + "instant", + "instant_date", + "key_period_size", + "period", + "str_by_instant_cache", + "types", + "unit_weight", + "unit_weights", + "year_or_month_or_day_re", +] diff --git a/openfisca_core/periods/_errors.py b/openfisca_core/periods/_errors.py new file mode 100644 index 0000000000..733d03ce2a --- /dev/null +++ b/openfisca_core/periods/_errors.py @@ -0,0 +1,28 @@ +from pendulum.parsing.exceptions import ParserError + + +class InstantError(ValueError): + """Raised when an invalid instant-like is provided.""" + + def __init__(self, value: str) -> None: + msg = ( + f"'{value}' is not a valid instant string. Instants are described " + "using either the 'YYYY-MM-DD' format, for instance '2015-06-15', " + "or the 'YYYY-Www-D' format, for instance '2015-W24-1'." + ) + super().__init__(msg) + + +class PeriodError(ValueError): + """Raised when an invalid period-like is provided.""" + + def __init__(self, value: str) -> None: + msg = ( + "Expected a period (eg. '2017', 'month:2017-01', 'week:2017-W01-1:3', " + f"...); got: '{value}'. Learn more about legal period formats in " + "OpenFisca: ." + ) + super().__init__(msg) + + +__all__ = ["InstantError", "ParserError", "PeriodError"] diff --git a/openfisca_core/periods/_parsers.py b/openfisca_core/periods/_parsers.py index 64b2077831..9973b890a0 100644 --- a/openfisca_core/periods/_parsers.py +++ b/openfisca_core/periods/_parsers.py @@ -1,63 +1,92 @@ -from typing import Optional +"""To parse periods and instants from strings.""" -import re +from __future__ import annotations + +import datetime import pendulum -from pendulum.datetime import Date -from pendulum.parsing import ParserError +from . import types as t +from ._errors import InstantError, ParserError, PeriodError from .date_unit import DateUnit from .instant_ import Instant from .period_ import Period -invalid_week = re.compile(r".*(W[1-9]|W[1-9]-[0-9]|W[0-5][0-9]-0)$") +def parse_instant(value: str) -> t.Instant: + """Parse a string into an instant. + + Args: + value (str): The string to parse. + + Returns: + An InstantStr. + + Raises: + InstantError: When the string is not a valid ISO Calendar/Format. + ParserError: When the string couldn't be parsed. + + Examples: + >>> parse_instant("2022") + Instant((2022, 1, 1)) + + >>> parse_instant("2022-02") + Instant((2022, 2, 1)) + + >>> parse_instant("2022-W02-7") + Instant((2022, 1, 16)) + + >>> parse_instant("2022-W013") + Traceback (most recent call last): + openfisca_core.periods._errors.InstantError: '2022-W013' is not a va... + + >>> parse_instant("2022-02-29") + Traceback (most recent call last): + pendulum.parsing.exceptions.ParserError: Unable to parse string [202... + + """ -def _parse_period(value: str) -> Optional[Period]: + if not isinstance(value, t.InstantStr): + raise InstantError(str(value)) + + date = pendulum.parse(value, exact=True) + + if not isinstance(date, datetime.date): + msg = f"Unable to parse string [{value}]" + raise ParserError(msg) + + return Instant((date.year, date.month, date.day)) + + +def parse_period(value: str) -> t.Period: """Parses ISO format/calendar periods. Such as "2012" or "2015-03". Examples: - >>> _parse_period("2022") + >>> parse_period("2022") Period((, Instant((2022, 1, 1)), 1)) - >>> _parse_period("2022-02") + >>> parse_period("2022-02") Period((, Instant((2022, 2, 1)), 1)) - >>> _parse_period("2022-W02-7") + >>> parse_period("2022-W02-7") Period((, Instant((2022, 1, 16)), 1)) """ - # If it's a complex period, next! - if len(value.split(":")) != 1: - return None - - # Check for a non-empty string. - if not (value and isinstance(value, str)): - raise AttributeError + try: + instant = parse_instant(value) - # If it's negative, next! - if value[0] == "-": - raise ValueError + except InstantError as error: + raise PeriodError(value) from error - # If it's an invalid week, next! - if invalid_week.match(value): - raise ParserError - - unit = _parse_unit(value) - date = pendulum.parse(value, exact=True) - - if not isinstance(date, Date): - raise ValueError - - instant = Instant((date.year, date.month, date.day)) + unit = parse_unit(value) return Period((unit, instant, 1)) -def _parse_unit(value: str) -> DateUnit: +def parse_unit(value: str) -> t.DateUnit: """Determine the date unit of a date string. Args: @@ -67,36 +96,26 @@ def _parse_unit(value: str) -> DateUnit: A DateUnit. Raises: - ValueError when no DateUnit can be determined. + InstantError: when no DateUnit can be determined. Examples: - >>> _parse_unit("2022") + >>> parse_unit("2022") - >>> _parse_unit("2022-W03-01") + >>> parse_unit("2022-W03-1") """ - length = len(value.split("-")) - isweek = value.find("W") != -1 - - if length == 1: - return DateUnit.YEAR + if not isinstance(value, t.InstantStr): + raise InstantError(str(value)) - elif length == 2: - if isweek: - return DateUnit.WEEK + length = len(value.split("-")) - else: - return DateUnit.MONTH + if isinstance(value, t.ISOCalendarStr): + return DateUnit.isocalendar[-length] - elif length == 3: - if isweek: - return DateUnit.WEEKDAY + return DateUnit.isoformat[-length] - else: - return DateUnit.DAY - else: - raise ValueError +__all__ = ["parse_instant", "parse_period", "parse_unit"] diff --git a/openfisca_core/periods/config.py b/openfisca_core/periods/config.py index 17807160e4..4486a5caf0 100644 --- a/openfisca_core/periods/config.py +++ b/openfisca_core/periods/config.py @@ -1,22 +1,20 @@ import re -from .date_unit import DateUnit +import pendulum -WEEKDAY = DateUnit.WEEKDAY -WEEK = DateUnit.WEEK -DAY = DateUnit.DAY -MONTH = DateUnit.MONTH -YEAR = DateUnit.YEAR -ETERNITY = DateUnit.ETERNITY +from . import types as t # Matches "2015", "2015-01", "2015-01-01" # Does not match "2015-13", "2015-12-32" INSTANT_PATTERN = re.compile( - r"^\d{4}(-(0[1-9]|1[012]))?(-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01]))?$" + r"^\d{4}(-(0[1-9]|1[012]))?(-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01]))?$", ) -date_by_instant_cache: dict = {} -str_by_instant_cache: dict = {} +date_by_instant_cache: dict[t.Instant, pendulum.Date] = {} +str_by_instant_cache: dict[t.Instant, t.InstantStr] = {} year_or_month_or_day_re = re.compile( - r"(18|19|20)\d{2}(-(0?[1-9]|1[0-2])(-([0-2]?\d|3[0-1]))?)?$" + r"(18|19|20)\d{2}(-(0?[1-9]|1[0-2])(-([0-2]?\d|3[0-1]))?)?$", ) + + +__all__ = ["INSTANT_PATTERN", "date_by_instant_cache", "str_by_instant_cache"] diff --git a/openfisca_core/periods/date_unit.py b/openfisca_core/periods/date_unit.py index a813211495..c66346c3c2 100644 --- a/openfisca_core/periods/date_unit.py +++ b/openfisca_core/periods/date_unit.py @@ -4,10 +4,12 @@ from strenum import StrEnum +from . import types as t + class DateUnitMeta(EnumMeta): @property - def isoformat(self) -> tuple[DateUnit, ...]: + def isoformat(self) -> tuple[t.DateUnit, ...]: """Creates a :obj:`tuple` of ``key`` with isoformat items. Returns: @@ -24,11 +26,10 @@ def isoformat(self) -> tuple[DateUnit, ...]: False """ - return DateUnit.DAY, DateUnit.MONTH, DateUnit.YEAR @property - def isocalendar(self) -> tuple[DateUnit, ...]: + def isocalendar(self) -> tuple[t.DateUnit, ...]: """Creates a :obj:`tuple` of ``key`` with isocalendar items. Returns: @@ -45,7 +46,6 @@ def isocalendar(self) -> tuple[DateUnit, ...]: False """ - return DateUnit.WEEKDAY, DateUnit.WEEK, DateUnit.YEAR @@ -66,7 +66,7 @@ class DateUnit(StrEnum, metaclass=DateUnitMeta): {: 'day'} >>> list(DateUnit) - [, , , ...] + [, , >> len(DateUnit) 6 @@ -92,13 +92,19 @@ class DateUnit(StrEnum, metaclass=DateUnitMeta): >>> DateUnit.DAY.value 'day' - .. versionadded:: 35.9.0 - """ + def __contains__(self, other: object) -> bool: + if isinstance(other, str): + return super().__contains__(other) + return NotImplemented + WEEKDAY = "weekday" WEEK = "week" DAY = "day" MONTH = "month" YEAR = "year" ETERNITY = "eternity" + + +__all__ = ["DateUnit"] diff --git a/openfisca_core/periods/helpers.py b/openfisca_core/periods/helpers.py index 2ce4e0cd35..fab26c48ab 100644 --- a/openfisca_core/periods/helpers.py +++ b/openfisca_core/periods/helpers.py @@ -1,26 +1,29 @@ -from typing import NoReturn, Optional +from __future__ import annotations + +from typing import NoReturn import datetime -import os +import functools import pendulum -from pendulum.parsing import ParserError -from . import _parsers, config +from . import config, types as t +from ._errors import InstantError, PeriodError +from ._parsers import parse_instant, parse_period from .date_unit import DateUnit from .instant_ import Instant from .period_ import Period -def instant(instant) -> Optional[Instant]: +@functools.singledispatch +def instant(value: object) -> t.Instant: """Build a new instant, aka a triple of integers (year, month, day). Args: - instant: An ``instant-like`` object. + value(object): An ``instant-like`` object. Returns: - None: When ``instant`` is None. - :obj:`.Instant`: Otherwise. + :obj:`.Instant`: A new instant. Raises: :exc:`ValueError`: When the arguments were invalid, like "2021-32-13". @@ -47,53 +50,69 @@ def instant(instant) -> Optional[Instant]: >>> instant("2021") Instant((2021, 1, 1)) + >>> instant([2021]) + Instant((2021, 1, 1)) + + >>> instant([2021, 9]) + Instant((2021, 9, 1)) + + >>> instant(None) + Traceback (most recent call last): + openfisca_core.periods._errors.InstantError: 'None' is not a valid i... + """ - if instant is None: - return None - if isinstance(instant, Instant): - return instant - if isinstance(instant, str): - if not config.INSTANT_PATTERN.match(instant): - raise ValueError( - f"'{instant}' is not a valid instant. Instants are described using the 'YYYY-MM-DD' format, for instance '2015-06-15'." - ) - instant = Instant(int(fragment) for fragment in instant.split("-", 2)[:3]) - elif isinstance(instant, datetime.date): - instant = Instant((instant.year, instant.month, instant.day)) - elif isinstance(instant, int): - instant = (instant,) - elif isinstance(instant, list): - assert 1 <= len(instant) <= 3 - instant = tuple(instant) - elif isinstance(instant, Period): - instant = instant.start - else: - assert isinstance(instant, tuple), instant - assert 1 <= len(instant) <= 3 - if len(instant) == 1: - return Instant((instant[0], 1, 1)) - if len(instant) == 2: - return Instant((instant[0], instant[1], 1)) - return Instant(instant) - - -def instant_date(instant: Optional[Instant]) -> Optional[datetime.date]: - """Returns the date representation of an :class:`.Instant`. + if isinstance(value, t.SeqInt): + return Instant((list(value) + [1] * 3)[:3]) + + raise InstantError(str(value)) + + +@instant.register +def _(value: None) -> NoReturn: + raise InstantError(str(value)) + + +@instant.register +def _(value: int) -> t.Instant: + return Instant((value, 1, 1)) + + +@instant.register +def _(value: Period) -> t.Instant: + return value.start + + +@instant.register +def _(value: t.Instant) -> t.Instant: + return value + + +@instant.register +def _(value: datetime.date) -> t.Instant: + return Instant((value.year, value.month, value.day)) + + +@instant.register +def _(value: str) -> t.Instant: + return parse_instant(value) + + +def instant_date(instant: None | t.Instant) -> None | datetime.date: + """Returns the date representation of an ``Instant``. Args: - instant (:obj:`.Instant`, optional): + instant: An ``Instant``. Returns: None: When ``instant`` is None. - :obj:`datetime.date`: Otherwise. + datetime.date: Otherwise. Examples: >>> instant_date(Instant((2021, 1, 1))) Date(2021, 1, 1) """ - if instant is None: return None @@ -105,7 +124,8 @@ def instant_date(instant: Optional[Instant]) -> Optional[datetime.date]: return instant_date -def period(value) -> Period: +@functools.singledispatch +def period(value: object) -> t.Period: """Build a new period, aka a triple (unit, start_instant, size). Args: @@ -125,7 +145,7 @@ def period(value) -> Period: Period((, Instant((2021, 1, 1)), 1)) >>> period(DateUnit.ETERNITY) - Period((, Instant((1, 1, 1)), inf)) + Period((, Instant((-1, -1, -1)), -1)) >>> period(2021) Period((, Instant((2021, 1, 1)), 1)) @@ -148,126 +168,86 @@ def period(value) -> Period: >>> period("day:2014-02-02:3") Period((, Instant((2014, 2, 2)), 3)) - """ - if isinstance(value, Period): - return value - - # We return a "day-period", for example - # ``, 1))>``. - if isinstance(value, Instant): - return Period((DateUnit.DAY, value, 1)) - - # For example ``datetime.date(2021, 9, 16)``. - if isinstance(value, datetime.date): - return Period((DateUnit.DAY, instant(value), 1)) + one, two, three = 1, 2, 3 # We return an "eternity-period", for example - # ``, inf))>``. + # ``, -1))>``. if str(value).lower() == DateUnit.ETERNITY: - return Period( - ( - DateUnit.ETERNITY, - instant(datetime.date.min), - float("inf"), - ) - ) - - # For example ``2021`` gives - # ``, 1))>``. - if isinstance(value, int): - return Period((DateUnit.YEAR, instant(value), 1)) - - # Up to this point, if ``value`` is not a :obj:`str`, we desist. - if not isinstance(value, str): - _raise_error(value) + return Period.eternity() - # There can't be empty strings. - if not value: - _raise_error(value) + # We try to parse from an ISO format/calendar period. + if isinstance(value, t.InstantStr): + return parse_period(value) - # Try to parse from an ISO format/calendar period. - try: - period = _parsers._parse_period(value) + # A complex period has a ':' in its string. + if isinstance(value, t.PeriodStr): + components = value.split(":") - except (AttributeError, ParserError, ValueError): - _raise_error(value) + # The left-most component must be a valid unit + unit = components[0] - if period is not None: - return period + if unit not in list(DateUnit) or unit == DateUnit.ETERNITY: + raise PeriodError(str(value)) - # A complex period has a ':' in its string. - if ":" not in value: - _raise_error(value) + # Cast ``unit`` to DateUnit. + unit = DateUnit(unit) - components = value.split(":") + # The middle component must be a valid iso period + period = parse_period(components[1]) - # left-most component must be a valid unit - unit = components[0] + # Periods like year:2015-03 have a size of 1 + if len(components) == two: + size = one - if unit not in list(DateUnit) or unit == DateUnit.ETERNITY: - _raise_error(value) + # if provided, make sure the size is an integer + elif len(components) == three: + try: + size = int(components[2]) - # Cast ``unit`` to DateUnit. - unit = DateUnit(unit) + except ValueError as error: + raise PeriodError(str(value)) from error - # middle component must be a valid iso period - try: - base_period = _parsers._parse_period(components[1]) + # If there are more than 2 ":" in the string, the period is invalid + else: + raise PeriodError(str(value)) - except (AttributeError, ParserError, ValueError): - _raise_error(value) + # Reject ambiguous periods such as month:2014 + if unit_weight(period.unit) > unit_weight(unit): + raise PeriodError(str(value)) - if not base_period: - _raise_error(value) + return Period((unit, period.start, size)) - # period like year:2015-03 have a size of 1 - if len(components) == 2: - size = 1 + raise PeriodError(str(value)) - # if provided, make sure the size is an integer - elif len(components) == 3: - try: - size = int(components[2]) - except ValueError: - _raise_error(value) +@period.register +def _(value: None) -> NoReturn: + raise PeriodError(str(value)) - # if there is more than 2 ":" in the string, the period is invalid - else: - _raise_error(value) - # reject ambiguous periods such as month:2014 - if unit_weight(base_period.unit) > unit_weight(unit): - _raise_error(value) +@period.register +def _(value: int) -> t.Period: + return Period((DateUnit.YEAR, instant(value), 1)) - return Period((unit, base_period.start, size)) +@period.register +def _(value: t.Period) -> t.Period: + return value -def _raise_error(value: str) -> NoReturn: - """Raise an error. - Examples: - >>> _raise_error("Oi mate!") - Traceback (most recent call last): - ValueError: Expected a period (eg. '2017', '2017-01', '2017-01-01', ... - Learn more about legal period formats in OpenFisca: - t.Period: + return Period((DateUnit.DAY, value, 1)) - """ - message = os.linesep.join( - [ - f"Expected a period (eg. '2017', '2017-01', '2017-01-01', ...); got: '{value}'.", - "Learn more about legal period formats in OpenFisca:", - ".", - ] - ) - raise ValueError(message) +@period.register +def _(value: datetime.date) -> t.Period: + return Period((DateUnit.DAY, instant(value), 1)) -def key_period_size(period: Period) -> str: +def key_period_size(period: t.Period) -> str: """Define a key in order to sort periods by length. It uses two aspects: first, ``unit``, then, ``size``. @@ -291,12 +271,10 @@ def key_period_size(period: Period) -> str: """ - unit, start, size = period + return f"{unit_weight(period.unit)}_{period.size}" - return f"{unit_weight(unit)}_{size}" - -def unit_weights() -> dict[str, int]: +def unit_weights() -> dict[t.DateUnit, int]: """Assign weights to date units. Examples: @@ -304,7 +282,6 @@ def unit_weights() -> dict[str, int]: {: 100, ...ETERNITY: 'eternity'>: 400} """ - return { DateUnit.WEEKDAY: 100, DateUnit.WEEK: 200, @@ -315,7 +292,7 @@ def unit_weights() -> dict[str, int]: } -def unit_weight(unit: str) -> int: +def unit_weight(unit: t.DateUnit) -> int: """Retrieves a specific date unit weight. Examples: @@ -323,5 +300,14 @@ def unit_weight(unit: str) -> int: 100 """ - return unit_weights()[unit] + + +__all__ = [ + "instant", + "instant_date", + "key_period_size", + "period", + "unit_weight", + "unit_weights", +] diff --git a/openfisca_core/periods/instant_.py b/openfisca_core/periods/instant_.py index 9d0893ba41..f71dbb3222 100644 --- a/openfisca_core/periods/instant_.py +++ b/openfisca_core/periods/instant_.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import pendulum -from . import config +from . import config, types as t from .date_unit import DateUnit -class Instant(tuple): +class Instant(tuple[int, int, int]): """An instant in time (year, month, day). An :class:`.Instant` represents the most atomic and indivisible @@ -13,10 +15,6 @@ class Instant(tuple): Current implementation considers this unit to be a day, so :obj:`instants <.Instant>` can be thought of as "day dates". - Args: - (tuple(tuple(int, int, int))): - The ``year``, ``month``, and ``day``, accordingly. - Examples: >>> instant = Instant((2021, 9, 13)) @@ -78,35 +76,57 @@ class Instant(tuple): """ - def __repr__(self): + __slots__ = () + + def __repr__(self) -> str: return f"{self.__class__.__name__}({super().__repr__()})" - def __str__(self): + def __str__(self) -> t.InstantStr: instant_str = config.str_by_instant_cache.get(self) if instant_str is None: - config.str_by_instant_cache[self] = instant_str = self.date.isoformat() + instant_str = t.InstantStr(self.date.isoformat()) + config.str_by_instant_cache[self] = instant_str return instant_str + def __lt__(self, other: object) -> bool: + if isinstance(other, Instant): + return super().__lt__(other) + return NotImplemented + + def __le__(self, other: object) -> bool: + if isinstance(other, Instant): + return super().__le__(other) + return NotImplemented + @property - def date(self): + def date(self) -> pendulum.Date: instant_date = config.date_by_instant_cache.get(self) if instant_date is None: - config.date_by_instant_cache[self] = instant_date = pendulum.date(*self) + instant_date = pendulum.date(*self) + config.date_by_instant_cache[self] = instant_date return instant_date @property - def day(self): + def day(self) -> int: return self[2] @property - def month(self): + def month(self) -> int: return self[1] - def offset(self, offset, unit): + @property + def year(self) -> int: + return self[0] + + @property + def is_eternal(self) -> bool: + return self == self.eternity() + + def offset(self, offset: str | int, unit: t.DateUnit) -> t.Instant | None: """Increments/decrements the given instant with offset units. Args: @@ -135,8 +155,7 @@ def offset(self, offset, unit): Instant((2019, 12, 29)) """ - - year, month, day = self + year, month, _ = self assert unit in ( DateUnit.isoformat + DateUnit.isocalendar @@ -146,53 +165,60 @@ def offset(self, offset, unit): if unit == DateUnit.YEAR: return self.__class__((year, 1, 1)) - elif unit == DateUnit.MONTH: + if unit == DateUnit.MONTH: return self.__class__((year, month, 1)) - elif unit == DateUnit.WEEK: + if unit == DateUnit.WEEK: date = self.date date = date.start_of("week") return self.__class__((date.year, date.month, date.day)) + return None - elif offset == "last-of": + if offset == "last-of": if unit == DateUnit.YEAR: return self.__class__((year, 12, 31)) - elif unit == DateUnit.MONTH: + if unit == DateUnit.MONTH: date = self.date date = date.end_of("month") return self.__class__((date.year, date.month, date.day)) - elif unit == DateUnit.WEEK: + if unit == DateUnit.WEEK: date = self.date date = date.end_of("week") return self.__class__((date.year, date.month, date.day)) + return None - else: - assert isinstance( - offset, int - ), f"Invalid offset: {offset} of type {type(offset)}" + assert isinstance( + offset, + int, + ), f"Invalid offset: {offset} of type {type(offset)}" - if unit == DateUnit.YEAR: - date = self.date - date = date.add(years=offset) - return self.__class__((date.year, date.month, date.day)) + if unit == DateUnit.YEAR: + date = self.date + date = date.add(years=offset) + return self.__class__((date.year, date.month, date.day)) - elif unit == DateUnit.MONTH: - date = self.date - date = date.add(months=offset) - return self.__class__((date.year, date.month, date.day)) + if unit == DateUnit.MONTH: + date = self.date + date = date.add(months=offset) + return self.__class__((date.year, date.month, date.day)) - elif unit == DateUnit.WEEK: - date = self.date - date = date.add(weeks=offset) - return self.__class__((date.year, date.month, date.day)) + if unit == DateUnit.WEEK: + date = self.date + date = date.add(weeks=offset) + return self.__class__((date.year, date.month, date.day)) - elif unit in (DateUnit.DAY, DateUnit.WEEKDAY): - date = self.date - date = date.add(days=offset) - return self.__class__((date.year, date.month, date.day)) + if unit in (DateUnit.DAY, DateUnit.WEEKDAY): + date = self.date + date = date.add(days=offset) + return self.__class__((date.year, date.month, date.day)) + return None - @property - def year(self): - return self[0] + @classmethod + def eternity(cls) -> t.Instant: + """Return an eternity instant.""" + return cls((-1, -1, -1)) + + +__all__ = ["Instant"] diff --git a/openfisca_core/periods/period_.py b/openfisca_core/periods/period_.py index 11a9c02132..00e833d861 100644 --- a/openfisca_core/periods/period_.py +++ b/openfisca_core/periods/period_.py @@ -1,6 +1,5 @@ from __future__ import annotations -import typing from collections.abc import Sequence import calendar @@ -8,15 +7,12 @@ import pendulum -from . import helpers +from . import helpers, types as t from .date_unit import DateUnit from .instant_ import Instant -if typing.TYPE_CHECKING: - from pendulum.datetime import Date - -class Period(tuple): +class Period(tuple[t.DateUnit, t.Instant, int]): """Toolbox to handle date intervals. A :class:`.Period` is a triple (``unit``, ``start``, ``size``). @@ -121,14 +117,16 @@ class Period(tuple): """ + __slots__ = () + def __repr__(self) -> str: return f"{self.__class__.__name__}({super().__repr__()})" - def __str__(self) -> str: + def __str__(self) -> t.PeriodStr: unit, start_instant, size = self if unit == DateUnit.ETERNITY: - return unit.upper() + return t.PeriodStr(unit.upper()) # ISO format date units. f_year, month, day = start_instant @@ -140,58 +138,56 @@ def __str__(self) -> str: if unit == DateUnit.MONTH and size == 12 or unit == DateUnit.YEAR and size == 1: if month == 1: # civil year starting from january - return str(f_year) - else: - # rolling year - return f"{DateUnit.YEAR}:{f_year}-{month:02d}" + return t.PeriodStr(str(f_year)) + # rolling year + return t.PeriodStr(f"{DateUnit.YEAR}:{f_year}-{month:02d}") # simple month if unit == DateUnit.MONTH and size == 1: - return f"{f_year}-{month:02d}" + return t.PeriodStr(f"{f_year}-{month:02d}") # several civil years if unit == DateUnit.YEAR and month == 1: - return f"{unit}:{f_year}:{size}" + return t.PeriodStr(f"{unit}:{f_year}:{size}") if unit == DateUnit.DAY: if size == 1: - return f"{f_year}-{month:02d}-{day:02d}" - else: - return f"{unit}:{f_year}-{month:02d}-{day:02d}:{size}" + return t.PeriodStr(f"{f_year}-{month:02d}-{day:02d}") + return t.PeriodStr(f"{unit}:{f_year}-{month:02d}-{day:02d}:{size}") # 1 week if unit == DateUnit.WEEK and size == 1: if week < 10: - return f"{c_year}-W0{week}" + return t.PeriodStr(f"{c_year}-W0{week}") - return f"{c_year}-W{week}" + return t.PeriodStr(f"{c_year}-W{week}") # several weeks if unit == DateUnit.WEEK and size > 1: if week < 10: - return f"{unit}:{c_year}-W0{week}:{size}" + return t.PeriodStr(f"{unit}:{c_year}-W0{week}:{size}") - return f"{unit}:{c_year}-W{week}:{size}" + return t.PeriodStr(f"{unit}:{c_year}-W{week}:{size}") # 1 weekday if unit == DateUnit.WEEKDAY and size == 1: if week < 10: - return f"{c_year}-W0{week}-{weekday}" + return t.PeriodStr(f"{c_year}-W0{week}-{weekday}") - return f"{c_year}-W{week}-{weekday}" + return t.PeriodStr(f"{c_year}-W{week}-{weekday}") # several weekdays if unit == DateUnit.WEEKDAY and size > 1: if week < 10: - return f"{unit}:{c_year}-W0{week}-{weekday}:{size}" + return t.PeriodStr(f"{unit}:{c_year}-W0{week}-{weekday}:{size}") - return f"{unit}:{c_year}-W{week}-{weekday}:{size}" + return t.PeriodStr(f"{unit}:{c_year}-W{week}-{weekday}:{size}") # complex period - return f"{unit}:{f_year}-{month:02d}:{size}" + return t.PeriodStr(f"{unit}:{f_year}-{month:02d}:{size}") @property - def unit(self) -> str: + def unit(self) -> t.DateUnit: """The ``unit`` of the ``Period``. Example: @@ -201,11 +197,10 @@ def unit(self) -> str: """ - return self[0] @property - def start(self) -> Instant: + def start(self) -> t.Instant: """The ``Instant`` at which the ``Period`` starts. Example: @@ -215,7 +210,6 @@ def start(self) -> Instant: Instant((2021, 10, 1)) """ - return self[1] @property @@ -229,11 +223,10 @@ def size(self) -> int: 3 """ - return self[2] @property - def date(self) -> Date: + def date(self) -> pendulum.Date: """The date representation of the ``Period`` start date. Examples: @@ -249,9 +242,9 @@ def date(self) -> Date: ValueError: "date" is undefined for a period of size > 1: year:2021-10:3. """ - if self.size != 1: - raise ValueError(f'"date" is undefined for a period of size > 1: {self}.') + msg = f'"date" is undefined for a period of size > 1: {self}.' + raise ValueError(msg) return self.start.date @@ -272,11 +265,11 @@ def size_in_years(self) -> int: ValueError: Can't calculate number of years in a month. """ - if self.unit == DateUnit.YEAR: return self.size - raise ValueError(f"Can't calculate number of years in a {self.unit}.") + msg = f"Can't calculate number of years in a {self.unit}." + raise ValueError(msg) @property def size_in_months(self) -> int: @@ -295,14 +288,14 @@ def size_in_months(self) -> int: ValueError: Can't calculate number of months in a day. """ - if self.unit == DateUnit.YEAR: return self.size * 12 if self.unit == DateUnit.MONTH: return self.size - raise ValueError(f"Can't calculate number of months in a {self.unit}.") + msg = f"Can't calculate number of months in a {self.unit}." + raise ValueError(msg) @property def size_in_days(self) -> int: @@ -320,9 +313,13 @@ def size_in_days(self) -> int: 92 """ - if self.unit in (DateUnit.YEAR, DateUnit.MONTH): - last_day = self.start.offset(self.size, self.unit).offset(-1, DateUnit.DAY) + last = self.start.offset(self.size, self.unit) + if last is None: + raise NotImplementedError + last_day = last.offset(-1, DateUnit.DAY) + if last_day is None: + raise NotImplementedError return (last_day.date - self.start.date).days + 1 if self.unit == DateUnit.WEEK: @@ -331,10 +328,11 @@ def size_in_days(self) -> int: if self.unit in (DateUnit.DAY, DateUnit.WEEKDAY): return self.size - raise ValueError(f"Can't calculate number of days in a {self.unit}.") + msg = f"Can't calculate number of days in a {self.unit}." + raise ValueError(msg) @property - def size_in_weeks(self): + def size_in_weeks(self) -> int: """The ``size`` of the ``Period`` in weeks. Examples: @@ -349,26 +347,26 @@ def size_in_weeks(self): 261 """ - if self.unit == DateUnit.YEAR: start = self.start.date cease = start.add(years=self.size) - delta = pendulum.period(start, cease) - return delta.as_interval().weeks + delta = start.diff(cease) + return delta.in_weeks() if self.unit == DateUnit.MONTH: start = self.start.date cease = start.add(months=self.size) - delta = pendulum.period(start, cease) - return delta.as_interval().weeks + delta = start.diff(cease) + return delta.in_weeks() if self.unit == DateUnit.WEEK: return self.size - raise ValueError(f"Can't calculate number of weeks in a {self.unit}.") + msg = f"Can't calculate number of weeks in a {self.unit}." + raise ValueError(msg) @property - def size_in_weekdays(self): + def size_in_weekdays(self) -> int: """The ``size`` of the ``Period`` in weekdays. Examples: @@ -383,12 +381,16 @@ def size_in_weekdays(self): 21 """ - if self.unit == DateUnit.YEAR: return self.size_in_weeks * 7 - if self.unit in DateUnit.MONTH: - last_day = self.start.offset(self.size, self.unit).offset(-1, DateUnit.DAY) + if DateUnit.MONTH in self.unit: + last = self.start.offset(self.size, self.unit) + if last is None: + raise NotImplementedError + last_day = last.offset(-1, DateUnit.DAY) + if last_day is None: + raise NotImplementedError return (last_day.date - self.start.date).days + 1 if self.unit == DateUnit.WEEK: @@ -397,14 +399,17 @@ def size_in_weekdays(self): if self.unit in (DateUnit.DAY, DateUnit.WEEKDAY): return self.size - raise ValueError(f"Can't calculate number of weekdays in a {self.unit}.") + msg = f"Can't calculate number of weekdays in a {self.unit}." + raise ValueError(msg) @property - def days(self): + def days(self) -> int: """Same as ``size_in_days``.""" return (self.stop.date - self.start.date).days + 1 - def intersection(self, start, stop): + def intersection( + self, start: t.Instant | None, stop: t.Instant | None + ) -> t.Period | None: if start is None and stop is None: return self period_start = self[1] @@ -430,7 +435,7 @@ def intersection(self, start, stop): DateUnit.YEAR, intersection_start, intersection_stop.year - intersection_start.year + 1, - ) + ), ) if ( intersection_start.day == 1 @@ -447,32 +452,32 @@ def intersection(self, start, stop): - intersection_start.month + 1 ), - ) + ), ) return self.__class__( ( DateUnit.DAY, intersection_start, (intersection_stop.date - intersection_start.date).days + 1, - ) + ), ) - def get_subperiods(self, unit: DateUnit) -> Sequence[Period]: + def get_subperiods(self, unit: t.DateUnit) -> Sequence[t.Period]: """Return the list of periods of unit ``unit`` contained in self. Examples: >>> period = Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)) >>> period.get_subperiods(DateUnit.MONTH) - [Period((, Instant((2021, 1, 1)), 1)),...2021, 12, 1)), 1))] + [Period((, Instant((2021, 1, 1)), 1)),...] >>> period = Period((DateUnit.YEAR, Instant((2021, 1, 1)), 2)) >>> period.get_subperiods(DateUnit.YEAR) - [Period((, Instant((2021, 1, 1)), 1)),...((2022, 1, 1)), 1))] + [Period((, Instant((2021, 1, 1)), 1)), P...] """ - if helpers.unit_weight(self.unit) < helpers.unit_weight(unit): - raise ValueError(f"Cannot subdivide {self.unit} into {unit}") + msg = f"Cannot subdivide {self.unit} into {unit}" + raise ValueError(msg) if unit == DateUnit.YEAR: return [self.this_year.offset(i, DateUnit.YEAR) for i in range(self.size)] @@ -500,46 +505,65 @@ def get_subperiods(self, unit: DateUnit) -> Sequence[Period]: for i in range(self.size_in_weekdays) ] - raise ValueError(f"Cannot subdivide {self.unit} into {unit}") + msg = f"Cannot subdivide {self.unit} into {unit}" + raise ValueError(msg) - def offset(self, offset, unit=None): + def offset(self, offset: str | int, unit: t.DateUnit | None = None) -> t.Period: """Increment (or decrement) the given period with offset units. Examples: >>> Period((DateUnit.DAY, Instant((2021, 1, 1)), 365)).offset(1) Period((, Instant((2021, 1, 2)), 365)) - >>> Period((DateUnit.DAY, Instant((2021, 1, 1)), 365)).offset(1, DateUnit.DAY) + >>> Period((DateUnit.DAY, Instant((2021, 1, 1)), 365)).offset( + ... 1, DateUnit.DAY + ... ) Period((, Instant((2021, 1, 2)), 365)) - >>> Period((DateUnit.DAY, Instant((2021, 1, 1)), 365)).offset(1, DateUnit.MONTH) + >>> Period((DateUnit.DAY, Instant((2021, 1, 1)), 365)).offset( + ... 1, DateUnit.MONTH + ... ) Period((, Instant((2021, 2, 1)), 365)) - >>> Period((DateUnit.DAY, Instant((2021, 1, 1)), 365)).offset(1, DateUnit.YEAR) + >>> Period((DateUnit.DAY, Instant((2021, 1, 1)), 365)).offset( + ... 1, DateUnit.YEAR + ... ) Period((, Instant((2022, 1, 1)), 365)) >>> Period((DateUnit.MONTH, Instant((2021, 1, 1)), 12)).offset(1) Period((, Instant((2021, 2, 1)), 12)) - >>> Period((DateUnit.MONTH, Instant((2021, 1, 1)), 12)).offset(1, DateUnit.DAY) + >>> Period((DateUnit.MONTH, Instant((2021, 1, 1)), 12)).offset( + ... 1, DateUnit.DAY + ... ) Period((, Instant((2021, 1, 2)), 12)) - >>> Period((DateUnit.MONTH, Instant((2021, 1, 1)), 12)).offset(1, DateUnit.MONTH) + >>> Period((DateUnit.MONTH, Instant((2021, 1, 1)), 12)).offset( + ... 1, DateUnit.MONTH + ... ) Period((, Instant((2021, 2, 1)), 12)) - >>> Period((DateUnit.MONTH, Instant((2021, 1, 1)), 12)).offset(1, DateUnit.YEAR) + >>> Period((DateUnit.MONTH, Instant((2021, 1, 1)), 12)).offset( + ... 1, DateUnit.YEAR + ... ) Period((, Instant((2022, 1, 1)), 12)) >>> Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)).offset(1) Period((, Instant((2022, 1, 1)), 1)) - >>> Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)).offset(1, DateUnit.DAY) + >>> Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)).offset( + ... 1, DateUnit.DAY + ... ) Period((, Instant((2021, 1, 2)), 1)) - >>> Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)).offset(1, DateUnit.MONTH) + >>> Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)).offset( + ... 1, DateUnit.MONTH + ... ) Period((, Instant((2021, 2, 1)), 1)) - >>> Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)).offset(1, DateUnit.YEAR) + >>> Period((DateUnit.YEAR, Instant((2021, 1, 1)), 1)).offset( + ... 1, DateUnit.YEAR + ... ) Period((, Instant((2022, 1, 1)), 1)) >>> Period((DateUnit.DAY, Instant((2011, 2, 28)), 1)).offset(1) @@ -578,124 +602,177 @@ def offset(self, offset, unit=None): >>> Period((DateUnit.YEAR, Instant((2014, 1, 1)), 1)).offset(-3) Period((, Instant((2011, 1, 1)), 1)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset("first-of", DateUnit.MONTH) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset( + ... "first-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 1)), 1)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset("first-of", DateUnit.YEAR) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset( + ... "first-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 1, 1)), 1)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset("first-of", DateUnit.MONTH) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset( + ... "first-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 1)), 4)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset("first-of", DateUnit.YEAR) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset( + ... "first-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 1, 1)), 4)) >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset("first-of") Period((, Instant((2014, 2, 1)), 1)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset("first-of", DateUnit.MONTH) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset( + ... "first-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 1)), 1)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset("first-of", DateUnit.YEAR) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset( + ... "first-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 1, 1)), 1)) >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset("first-of") Period((, Instant((2014, 2, 1)), 4)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset("first-of", DateUnit.MONTH) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset( + ... "first-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 1)), 4)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset("first-of", DateUnit.YEAR) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset( + ... "first-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 1, 1)), 4)) >>> Period((DateUnit.YEAR, Instant((2014, 1, 30)), 1)).offset("first-of") Period((, Instant((2014, 1, 1)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 1, 30)), 1)).offset("first-of", DateUnit.MONTH) + >>> Period((DateUnit.YEAR, Instant((2014, 1, 30)), 1)).offset( + ... "first-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 1, 1)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 1, 30)), 1)).offset("first-of", DateUnit.YEAR) + >>> Period((DateUnit.YEAR, Instant((2014, 1, 30)), 1)).offset( + ... "first-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 1, 1)), 1)) >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("first-of") Period((, Instant((2014, 1, 1)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("first-of", DateUnit.MONTH) + >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset( + ... "first-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 1)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("first-of", DateUnit.YEAR) + >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset( + ... "first-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 1, 1)), 1)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset("last-of", DateUnit.MONTH) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset( + ... "last-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 28)), 1)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset("last-of", DateUnit.YEAR) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 1)).offset( + ... "last-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 12, 31)), 1)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset("last-of", DateUnit.MONTH) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset( + ... "last-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 28)), 4)) - >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset("last-of", DateUnit.YEAR) + >>> Period((DateUnit.DAY, Instant((2014, 2, 3)), 4)).offset( + ... "last-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 12, 31)), 4)) >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset("last-of") Period((, Instant((2014, 2, 28)), 1)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset("last-of", DateUnit.MONTH) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset( + ... "last-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 28)), 1)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset("last-of", DateUnit.YEAR) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 1)).offset( + ... "last-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 12, 31)), 1)) >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset("last-of") Period((, Instant((2014, 2, 28)), 4)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset("last-of", DateUnit.MONTH) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset( + ... "last-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 28)), 4)) - >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset("last-of", DateUnit.YEAR) + >>> Period((DateUnit.MONTH, Instant((2014, 2, 3)), 4)).offset( + ... "last-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 12, 31)), 4)) >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("last-of") Period((, Instant((2014, 12, 31)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 1, 1)), 1)).offset("last-of", DateUnit.MONTH) + >>> Period((DateUnit.YEAR, Instant((2014, 1, 1)), 1)).offset( + ... "last-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 1, 31)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("last-of", DateUnit.YEAR) + >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset( + ... "last-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 12, 31)), 1)) >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("last-of") Period((, Instant((2014, 12, 31)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("last-of", DateUnit.MONTH) + >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset( + ... "last-of", DateUnit.MONTH + ... ) Period((, Instant((2014, 2, 28)), 1)) - >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset("last-of", DateUnit.YEAR) + >>> Period((DateUnit.YEAR, Instant((2014, 2, 3)), 1)).offset( + ... "last-of", DateUnit.YEAR + ... ) Period((, Instant((2014, 12, 31)), 1)) """ + start: None | t.Instant = self[1].offset( + offset, self[0] if unit is None else unit + ) + + if start is None: + raise NotImplementedError + return self.__class__( ( self[0], - self[1].offset(offset, self[0] if unit is None else unit), + start, self[2], - ) + ), ) - def contains(self, other: object) -> bool: - """ - Returns ``True`` if the period contains ``other``. For instance, ``period(2015)`` contains ``period(2015-01)`` - """ + def contains(self, other: t.Period) -> bool: + """Returns ``True`` if the period contains ``other``. - if isinstance(other, Period): - return self.start <= other.start and self.stop >= other.stop + For instance, ``period(2015)`` contains ``period(2015-01)``. - return NotImplemented + """ + return self.start <= other.start and self.stop >= other.stop @property - def stop(self) -> Instant: + def stop(self) -> t.Instant: """Return the last day of the period as an Instant instance. Examples: @@ -727,96 +804,115 @@ def stop(self) -> Instant: Instant((2012, 3, 1)) """ - unit, start_instant, size = self - year, month, day = start_instant if unit == DateUnit.ETERNITY: - return Instant((float("inf"), float("inf"), float("inf"))) + return Instant.eternity() - elif unit == DateUnit.YEAR: + if unit == DateUnit.YEAR: date = start_instant.date.add(years=size, days=-1) return Instant((date.year, date.month, date.day)) - elif unit == DateUnit.MONTH: + if unit == DateUnit.MONTH: date = start_instant.date.add(months=size, days=-1) return Instant((date.year, date.month, date.day)) - elif unit == DateUnit.WEEK: + if unit == DateUnit.WEEK: date = start_instant.date.add(weeks=size, days=-1) return Instant((date.year, date.month, date.day)) - elif unit in (DateUnit.DAY, DateUnit.WEEKDAY): + if unit in (DateUnit.DAY, DateUnit.WEEKDAY): date = start_instant.date.add(days=size - 1) return Instant((date.year, date.month, date.day)) - else: - raise ValueError + raise ValueError + + @property + def is_eternal(self) -> bool: + return self == self.eternity() # Reference periods @property - def last_week(self) -> Period: + def last_week(self) -> t.Period: return self.first_week.offset(-1) @property - def last_fortnight(self) -> Period: - start: Instant = self.first_week.start + def last_fortnight(self) -> t.Period: + start: t.Instant = self.first_week.start return self.__class__((DateUnit.WEEK, start, 1)).offset(-2) @property - def last_2_weeks(self) -> Period: - start: Instant = self.first_week.start + def last_2_weeks(self) -> t.Period: + start: t.Instant = self.first_week.start return self.__class__((DateUnit.WEEK, start, 2)).offset(-2) @property - def last_26_weeks(self) -> Period: - start: Instant = self.first_week.start + def last_26_weeks(self) -> t.Period: + start: t.Instant = self.first_week.start return self.__class__((DateUnit.WEEK, start, 26)).offset(-26) @property - def last_52_weeks(self) -> Period: - start: Instant = self.first_week.start + def last_52_weeks(self) -> t.Period: + start: t.Instant = self.first_week.start return self.__class__((DateUnit.WEEK, start, 52)).offset(-52) @property - def last_month(self) -> Period: + def last_month(self) -> t.Period: return self.first_month.offset(-1) @property - def last_3_months(self) -> Period: - start: Instant = self.first_month.start + def last_3_months(self) -> t.Period: + start: t.Instant = self.first_month.start return self.__class__((DateUnit.MONTH, start, 3)).offset(-3) @property - def last_year(self) -> Period: - start: Instant = self.start.offset("first-of", DateUnit.YEAR) + def last_year(self) -> t.Period: + start: None | t.Instant = self.start.offset("first-of", DateUnit.YEAR) + if start is None: + raise NotImplementedError return self.__class__((DateUnit.YEAR, start, 1)).offset(-1) @property - def n_2(self) -> Period: - start: Instant = self.start.offset("first-of", DateUnit.YEAR) + def n_2(self) -> t.Period: + start: None | t.Instant = self.start.offset("first-of", DateUnit.YEAR) + if start is None: + raise NotImplementedError return self.__class__((DateUnit.YEAR, start, 1)).offset(-2) @property - def this_year(self) -> Period: - start: Instant = self.start.offset("first-of", DateUnit.YEAR) + def this_year(self) -> t.Period: + start: None | t.Instant = self.start.offset("first-of", DateUnit.YEAR) + if start is None: + raise NotImplementedError return self.__class__((DateUnit.YEAR, start, 1)) @property - def first_month(self) -> Period: - start: Instant = self.start.offset("first-of", DateUnit.MONTH) + def first_month(self) -> t.Period: + start: None | t.Instant = self.start.offset("first-of", DateUnit.MONTH) + if start is None: + raise NotImplementedError return self.__class__((DateUnit.MONTH, start, 1)) @property - def first_day(self) -> Period: + def first_day(self) -> t.Period: return self.__class__((DateUnit.DAY, self.start, 1)) @property - def first_week(self) -> Period: - start: Instant = self.start.offset("first-of", DateUnit.WEEK) + def first_week(self) -> t.Period: + start: None | t.Instant = self.start.offset("first-of", DateUnit.WEEK) + if start is None: + raise NotImplementedError return self.__class__((DateUnit.WEEK, start, 1)) @property - def first_weekday(self) -> Period: + def first_weekday(self) -> t.Period: return self.__class__((DateUnit.WEEKDAY, self.start, 1)) + + @classmethod + def eternity(cls) -> t.Period: + """Return an eternity period.""" + return cls((DateUnit.ETERNITY, Instant.eternity(), -1)) + + +__all__ = ["Period"] diff --git a/openfisca_core/periods/py.typed b/openfisca_core/periods/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openfisca_core/periods/tests/helpers/test_helpers.py b/openfisca_core/periods/tests/helpers/test_helpers.py index bb409323d1..175ea8c873 100644 --- a/openfisca_core/periods/tests/helpers/test_helpers.py +++ b/openfisca_core/periods/tests/helpers/test_helpers.py @@ -7,49 +7,59 @@ @pytest.mark.parametrize( - "arg, expected", + ("arg", "expected"), [ - [None, None], - [Instant((1, 1, 1)), datetime.date(1, 1, 1)], - [Instant((4, 2, 29)), datetime.date(4, 2, 29)], - [(1, 1, 1), datetime.date(1, 1, 1)], + (None, None), + (Instant((1, 1, 1)), datetime.date(1, 1, 1)), + (Instant((4, 2, 29)), datetime.date(4, 2, 29)), + ((1, 1, 1), datetime.date(1, 1, 1)), ], ) -def test_instant_date(arg, expected): +def test_instant_date(arg, expected) -> None: assert periods.instant_date(arg) == expected @pytest.mark.parametrize( - "arg, error", + ("arg", "error"), [ - [Instant((-1, 1, 1)), ValueError], - [Instant((1, -1, 1)), ValueError], - [Instant((1, 13, -1)), ValueError], - [Instant((1, 1, -1)), ValueError], - [Instant((1, 1, 32)), ValueError], - [Instant((1, 2, 29)), ValueError], - [Instant(("1", 1, 1)), TypeError], - [(1,), TypeError], - [(1, 1), TypeError], + (Instant((-1, 1, 1)), ValueError), + (Instant((1, -1, 1)), ValueError), + (Instant((1, 13, -1)), ValueError), + (Instant((1, 1, -1)), ValueError), + (Instant((1, 1, 32)), ValueError), + (Instant((1, 2, 29)), ValueError), + (Instant(("1", 1, 1)), TypeError), + ((1,), TypeError), + ((1, 1), TypeError), ], ) -def test_instant_date_with_an_invalid_argument(arg, error): +def test_instant_date_with_an_invalid_argument(arg, error) -> None: with pytest.raises(error): periods.instant_date(arg) @pytest.mark.parametrize( - "arg, expected", + ("arg", "expected"), [ - [Period((DateUnit.WEEKDAY, Instant((1, 1, 1)), 5)), "100_5"], - [Period((DateUnit.WEEK, Instant((1, 1, 1)), 26)), "200_26"], - [Period((DateUnit.DAY, Instant((1, 1, 1)), 365)), "100_365"], - [Period((DateUnit.MONTH, Instant((1, 1, 1)), 12)), "200_12"], - [Period((DateUnit.YEAR, Instant((1, 1, 1)), 2)), "300_2"], - [Period((DateUnit.ETERNITY, Instant((1, 1, 1)), 1)), "400_1"], - [(DateUnit.DAY, None, 1), "100_1"], - [(DateUnit.MONTH, None, -1000), "200_-1000"], + (Period((DateUnit.WEEKDAY, Instant((1, 1, 1)), 5)), "100_5"), + (Period((DateUnit.WEEK, Instant((1, 1, 1)), 26)), "200_26"), + (Period((DateUnit.DAY, Instant((1, 1, 1)), 365)), "100_365"), + (Period((DateUnit.MONTH, Instant((1, 1, 1)), 12)), "200_12"), + (Period((DateUnit.YEAR, Instant((1, 1, 1)), 2)), "300_2"), + (Period((DateUnit.ETERNITY, Instant((1, 1, 1)), 1)), "400_1"), ], ) -def test_key_period_size(arg, expected): +def test_key_period_size(arg, expected) -> None: assert periods.key_period_size(arg) == expected + + +@pytest.mark.parametrize( + ("arg", "error"), + [ + ((DateUnit.DAY, None, 1), AttributeError), + ((DateUnit.MONTH, None, -1000), AttributeError), + ], +) +def test_key_period_size_when_an_invalid_argument(arg, error): + with pytest.raises(error): + periods.key_period_size(arg) diff --git a/openfisca_core/periods/tests/helpers/test_instant.py b/openfisca_core/periods/tests/helpers/test_instant.py index cb74c55ca4..fb4472814b 100644 --- a/openfisca_core/periods/tests/helpers/test_instant.py +++ b/openfisca_core/periods/tests/helpers/test_instant.py @@ -3,74 +3,71 @@ import pytest from openfisca_core import periods -from openfisca_core.periods import DateUnit, Instant, Period +from openfisca_core.periods import DateUnit, Instant, InstantError, Period @pytest.mark.parametrize( - "arg, expected", + ("arg", "expected"), [ - [None, None], - [datetime.date(1, 1, 1), Instant((1, 1, 1))], - [Instant((1, 1, 1)), Instant((1, 1, 1))], - [Period((DateUnit.DAY, Instant((1, 1, 1)), 365)), Instant((1, 1, 1))], - [-1, Instant((-1, 1, 1))], - [0, Instant((0, 1, 1))], - [1, Instant((1, 1, 1))], - [999, Instant((999, 1, 1))], - [1000, Instant((1000, 1, 1))], - ["1000", Instant((1000, 1, 1))], - ["1000-01", Instant((1000, 1, 1))], - ["1000-01-01", Instant((1000, 1, 1))], - [(None,), Instant((None, 1, 1))], - [(None, None), Instant((None, None, 1))], - [(None, None, None), Instant((None, None, None))], - [(datetime.date(1, 1, 1),), Instant((datetime.date(1, 1, 1), 1, 1))], - [(Instant((1, 1, 1)),), Instant((Instant((1, 1, 1)), 1, 1))], - [ - (Period((DateUnit.DAY, Instant((1, 1, 1)), 365)),), - Instant((Period((DateUnit.DAY, Instant((1, 1, 1)), 365)), 1, 1)), - ], - [(-1,), Instant((-1, 1, 1))], - [(-1, -1), Instant((-1, -1, 1))], - [(-1, -1, -1), Instant((-1, -1, -1))], - [("-1",), Instant(("-1", 1, 1))], - [("-1", "-1"), Instant(("-1", "-1", 1))], - [("-1", "-1", "-1"), Instant(("-1", "-1", "-1"))], - [("1-1",), Instant(("1-1", 1, 1))], - [("1-1-1",), Instant(("1-1-1", 1, 1))], + (datetime.date(1, 1, 1), Instant((1, 1, 1))), + (Instant((1, 1, 1)), Instant((1, 1, 1))), + (Period((DateUnit.DAY, Instant((1, 1, 1)), 365)), Instant((1, 1, 1))), + (-1, Instant((-1, 1, 1))), + (0, Instant((0, 1, 1))), + (1, Instant((1, 1, 1))), + (999, Instant((999, 1, 1))), + (1000, Instant((1000, 1, 1))), + ("1000", Instant((1000, 1, 1))), + ("1000-01", Instant((1000, 1, 1))), + ("1000-01-01", Instant((1000, 1, 1))), + ((-1,), Instant((-1, 1, 1))), + ((-1, -1), Instant((-1, -1, 1))), + ((-1, -1, -1), Instant((-1, -1, -1))), ], ) -def test_instant(arg, expected): +def test_instant(arg, expected) -> None: assert periods.instant(arg) == expected @pytest.mark.parametrize( - "arg, error", + ("arg", "error"), [ - [DateUnit.YEAR, ValueError], - [DateUnit.ETERNITY, ValueError], - ["1000-0", ValueError], - ["1000-0-0", ValueError], - ["1000-1", ValueError], - ["1000-1-1", ValueError], - ["1", ValueError], - ["a", ValueError], - ["year", ValueError], - ["eternity", ValueError], - ["999", ValueError], - ["1:1000-01-01", ValueError], - ["a:1000-01-01", ValueError], - ["year:1000-01-01", ValueError], - ["year:1000-01-01:1", ValueError], - ["year:1000-01-01:3", ValueError], - ["1000-01-01:a", ValueError], - ["1000-01-01:1", ValueError], - [(), AssertionError], - [{}, AssertionError], - ["", ValueError], - [(None, None, None, None), AssertionError], + (None, InstantError), + (DateUnit.YEAR, ValueError), + (DateUnit.ETERNITY, ValueError), + ("1000-0", ValueError), + ("1000-0-0", ValueError), + ("1000-1", ValueError), + ("1000-1-1", ValueError), + ("1", ValueError), + ("a", ValueError), + ("year", ValueError), + ("eternity", ValueError), + ("999", ValueError), + ("1:1000-01-01", ValueError), + ("a:1000-01-01", ValueError), + ("year:1000-01-01", ValueError), + ("year:1000-01-01:1", ValueError), + ("year:1000-01-01:3", ValueError), + ("1000-01-01:a", ValueError), + ("1000-01-01:1", ValueError), + ((), InstantError), + ({}, InstantError), + ("", InstantError), + ((None,), InstantError), + ((None, None), InstantError), + ((None, None, None), InstantError), + ((None, None, None, None), InstantError), + (("-1",), InstantError), + (("-1", "-1"), InstantError), + (("-1", "-1", "-1"), InstantError), + (("1-1",), InstantError), + (("1-1-1",), InstantError), + ((datetime.date(1, 1, 1),), InstantError), + ((Instant((1, 1, 1)),), InstantError), + ((Period((DateUnit.DAY, Instant((1, 1, 1)), 365)),), InstantError), ], ) -def test_instant_with_an_invalid_argument(arg, error): +def test_instant_with_an_invalid_argument(arg, error) -> None: with pytest.raises(error): periods.instant(arg) diff --git a/openfisca_core/periods/tests/helpers/test_period.py b/openfisca_core/periods/tests/helpers/test_period.py index 7d50abe102..d2d5c6679a 100644 --- a/openfisca_core/periods/tests/helpers/test_period.py +++ b/openfisca_core/periods/tests/helpers/test_period.py @@ -3,132 +3,132 @@ import pytest from openfisca_core import periods -from openfisca_core.periods import DateUnit, Instant, Period +from openfisca_core.periods import DateUnit, Instant, Period, PeriodError @pytest.mark.parametrize( - "arg, expected", + ("arg", "expected"), [ - ["eternity", Period((DateUnit.ETERNITY, Instant((1, 1, 1)), float("inf")))], - ["ETERNITY", Period((DateUnit.ETERNITY, Instant((1, 1, 1)), float("inf")))], - [ + ("eternity", Period((DateUnit.ETERNITY, Instant((-1, -1, -1)), -1))), + ("ETERNITY", Period((DateUnit.ETERNITY, Instant((-1, -1, -1)), -1))), + ( DateUnit.ETERNITY, - Period((DateUnit.ETERNITY, Instant((1, 1, 1)), float("inf"))), - ], - [datetime.date(1, 1, 1), Period((DateUnit.DAY, Instant((1, 1, 1)), 1))], - [Instant((1, 1, 1)), Period((DateUnit.DAY, Instant((1, 1, 1)), 1))], - [ + Period((DateUnit.ETERNITY, Instant((-1, -1, -1)), -1)), + ), + (datetime.date(1, 1, 1), Period((DateUnit.DAY, Instant((1, 1, 1)), 1))), + (Instant((1, 1, 1)), Period((DateUnit.DAY, Instant((1, 1, 1)), 1))), + ( Period((DateUnit.DAY, Instant((1, 1, 1)), 365)), Period((DateUnit.DAY, Instant((1, 1, 1)), 365)), - ], - [-1, Period((DateUnit.YEAR, Instant((-1, 1, 1)), 1))], - [0, Period((DateUnit.YEAR, Instant((0, 1, 1)), 1))], - [1, Period((DateUnit.YEAR, Instant((1, 1, 1)), 1))], - [999, Period((DateUnit.YEAR, Instant((999, 1, 1)), 1))], - [1000, Period((DateUnit.YEAR, Instant((1000, 1, 1)), 1))], - ["1001", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))], - ["1001-01", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))], - ["1001-01-01", Period((DateUnit.DAY, Instant((1001, 1, 1)), 1))], - ["1004-02-29", Period((DateUnit.DAY, Instant((1004, 2, 29)), 1))], - ["1001-W01", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))], - ["1001-W01-1", Period((DateUnit.WEEKDAY, Instant((1000, 12, 29)), 1))], - ["year:1001", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))], - ["year:1001-01", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))], - ["year:1001-01-01", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))], - ["year:1001-W01", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))], - ["year:1001-W01-1", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))], - ["year:1001:1", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))], - ["year:1001-01:1", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))], - ["year:1001-01-01:1", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))], - ["year:1001-W01:1", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))], - ["year:1001-W01-1:1", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))], - ["year:1001:3", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 3))], - ["year:1001-01:3", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 3))], - ["year:1001-01-01:3", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 3))], - ["year:1001-W01:3", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 3))], - ["year:1001-W01-1:3", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 3))], - ["month:1001-01", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))], - ["month:1001-01-01", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))], - ["week:1001-W01", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))], - ["week:1001-W01-1", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))], - ["month:1001-01:1", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))], - ["month:1001-01:3", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 3))], - ["month:1001-01-01:3", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 3))], - ["week:1001-W01:1", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))], - ["week:1001-W01:3", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 3))], - ["week:1001-W01-1:3", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 3))], - ["day:1001-01-01", Period((DateUnit.DAY, Instant((1001, 1, 1)), 1))], - ["day:1001-01-01:3", Period((DateUnit.DAY, Instant((1001, 1, 1)), 3))], - ["weekday:1001-W01-1", Period((DateUnit.WEEKDAY, Instant((1000, 12, 29)), 1))], - [ + ), + (-1, Period((DateUnit.YEAR, Instant((-1, 1, 1)), 1))), + (0, Period((DateUnit.YEAR, Instant((0, 1, 1)), 1))), + (1, Period((DateUnit.YEAR, Instant((1, 1, 1)), 1))), + (999, Period((DateUnit.YEAR, Instant((999, 1, 1)), 1))), + (1000, Period((DateUnit.YEAR, Instant((1000, 1, 1)), 1))), + ("1001", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("1001-01", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))), + ("1001-01-01", Period((DateUnit.DAY, Instant((1001, 1, 1)), 1))), + ("1004-02-29", Period((DateUnit.DAY, Instant((1004, 2, 29)), 1))), + ("1001-W01", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))), + ("1001-W01-1", Period((DateUnit.WEEKDAY, Instant((1000, 12, 29)), 1))), + ("year:1001", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("year:1001-01", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("year:1001-01-01", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("year:1001-W01", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))), + ("year:1001-W01-1", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))), + ("year:1001:1", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("year:1001-01:1", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("year:1001-01-01:1", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("year:1001-W01:1", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))), + ("year:1001-W01-1:1", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 1))), + ("year:1001:3", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 3))), + ("year:1001-01:3", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 3))), + ("year:1001-01-01:3", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 3))), + ("year:1001-W01:3", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 3))), + ("year:1001-W01-1:3", Period((DateUnit.YEAR, Instant((1000, 12, 29)), 3))), + ("month:1001-01", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))), + ("month:1001-01-01", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))), + ("week:1001-W01", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))), + ("week:1001-W01-1", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))), + ("month:1001-01:1", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))), + ("month:1001-01:3", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 3))), + ("month:1001-01-01:3", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 3))), + ("week:1001-W01:1", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))), + ("week:1001-W01:3", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 3))), + ("week:1001-W01-1:3", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 3))), + ("day:1001-01-01", Period((DateUnit.DAY, Instant((1001, 1, 1)), 1))), + ("day:1001-01-01:3", Period((DateUnit.DAY, Instant((1001, 1, 1)), 3))), + ("weekday:1001-W01-1", Period((DateUnit.WEEKDAY, Instant((1000, 12, 29)), 1))), + ( "weekday:1001-W01-1:3", Period((DateUnit.WEEKDAY, Instant((1000, 12, 29)), 3)), - ], + ), ], ) -def test_period(arg, expected): +def test_period(arg, expected) -> None: assert periods.period(arg) == expected @pytest.mark.parametrize( - "arg, error", + ("arg", "error"), [ - [None, ValueError], - [DateUnit.YEAR, ValueError], - ["1", ValueError], - ["999", ValueError], - ["1000-0", ValueError], - ["1000-13", ValueError], - ["1000-W0", ValueError], - ["1000-W54", ValueError], - ["1000-0-0", ValueError], - ["1000-1-0", ValueError], - ["1000-2-31", ValueError], - ["1000-W0-0", ValueError], - ["1000-W1-0", ValueError], - ["1000-W1-8", ValueError], - ["a", ValueError], - ["year", ValueError], - ["1:1000", ValueError], - ["a:1000", ValueError], - ["month:1000", ValueError], - ["week:1000", ValueError], - ["day:1000-01", ValueError], - ["weekday:1000-W1", ValueError], - ["1000:a", ValueError], - ["1000:1", ValueError], - ["1000-01:1", ValueError], - ["1000-01-01:1", ValueError], - ["1000-W1:1", ValueError], - ["1000-W1-1:1", ValueError], - ["month:1000:1", ValueError], - ["week:1000:1", ValueError], - ["day:1000:1", ValueError], - ["day:1000-01:1", ValueError], - ["weekday:1000:1", ValueError], - ["weekday:1000-W1:1", ValueError], - [(), ValueError], - [{}, ValueError], - ["", ValueError], - [(None,), ValueError], - [(None, None), ValueError], - [(None, None, None), ValueError], - [(None, None, None, None), ValueError], - [(Instant((1, 1, 1)),), ValueError], - [(Period((DateUnit.DAY, Instant((1, 1, 1)), 365)),), ValueError], - [(1,), ValueError], - [(1, 1), ValueError], - [(1, 1, 1), ValueError], - [(-1,), ValueError], - [(-1, -1), ValueError], - [(-1, -1, -1), ValueError], - [("-1",), ValueError], - [("-1", "-1"), ValueError], - [("-1", "-1", "-1"), ValueError], - [("1-1",), ValueError], - [("1-1-1",), ValueError], + (None, PeriodError), + (DateUnit.YEAR, PeriodError), + ("1", PeriodError), + ("999", PeriodError), + ("1000-0", PeriodError), + ("1000-13", PeriodError), + ("1000-W0", PeriodError), + ("1000-W54", PeriodError), + ("1000-0-0", PeriodError), + ("1000-1-0", PeriodError), + ("1000-2-31", PeriodError), + ("1000-W0-0", PeriodError), + ("1000-W1-0", PeriodError), + ("1000-W1-8", PeriodError), + ("a", PeriodError), + ("year", PeriodError), + ("1:1000", PeriodError), + ("a:1000", PeriodError), + ("month:1000", PeriodError), + ("week:1000", PeriodError), + ("day:1000-01", PeriodError), + ("weekday:1000-W1", PeriodError), + ("1000:a", PeriodError), + ("1000:1", PeriodError), + ("1000-01:1", PeriodError), + ("1000-01-01:1", PeriodError), + ("1000-W1:1", PeriodError), + ("1000-W1-1:1", PeriodError), + ("month:1000:1", PeriodError), + ("week:1000:1", PeriodError), + ("day:1000:1", PeriodError), + ("day:1000-01:1", PeriodError), + ("weekday:1000:1", PeriodError), + ("weekday:1000-W1:1", PeriodError), + ((), PeriodError), + ({}, PeriodError), + ("", PeriodError), + ((None,), PeriodError), + ((None, None), PeriodError), + ((None, None, None), PeriodError), + ((None, None, None, None), PeriodError), + ((Instant((1, 1, 1)),), PeriodError), + ((Period((DateUnit.DAY, Instant((1, 1, 1)), 365)),), PeriodError), + ((1,), PeriodError), + ((1, 1), PeriodError), + ((1, 1, 1), PeriodError), + ((-1,), PeriodError), + ((-1, -1), PeriodError), + ((-1, -1, -1), PeriodError), + (("-1",), PeriodError), + (("-1", "-1"), PeriodError), + (("-1", "-1", "-1"), PeriodError), + (("1-1",), PeriodError), + (("1-1-1",), PeriodError), ], ) -def test_period_with_an_invalid_argument(arg, error): +def test_period_with_an_invalid_argument(arg, error) -> None: with pytest.raises(error): periods.period(arg) diff --git a/openfisca_core/periods/tests/test_instant.py b/openfisca_core/periods/tests/test_instant.py index 21549008f4..e9c73ef6aa 100644 --- a/openfisca_core/periods/tests/test_instant.py +++ b/openfisca_core/periods/tests/test_instant.py @@ -4,29 +4,29 @@ @pytest.mark.parametrize( - "instant, offset, unit, expected", + ("instant", "offset", "unit", "expected"), [ - [Instant((2020, 2, 29)), "first-of", DateUnit.YEAR, Instant((2020, 1, 1))], - [Instant((2020, 2, 29)), "first-of", DateUnit.MONTH, Instant((2020, 2, 1))], - [Instant((2020, 2, 29)), "first-of", DateUnit.WEEK, Instant((2020, 2, 24))], - [Instant((2020, 2, 29)), "first-of", DateUnit.DAY, None], - [Instant((2020, 2, 29)), "first-of", DateUnit.WEEKDAY, None], - [Instant((2020, 2, 29)), "last-of", DateUnit.YEAR, Instant((2020, 12, 31))], - [Instant((2020, 2, 29)), "last-of", DateUnit.MONTH, Instant((2020, 2, 29))], - [Instant((2020, 2, 29)), "last-of", DateUnit.WEEK, Instant((2020, 3, 1))], - [Instant((2020, 2, 29)), "last-of", DateUnit.DAY, None], - [Instant((2020, 2, 29)), "last-of", DateUnit.WEEKDAY, None], - [Instant((2020, 2, 29)), -3, DateUnit.YEAR, Instant((2017, 2, 28))], - [Instant((2020, 2, 29)), -3, DateUnit.MONTH, Instant((2019, 11, 29))], - [Instant((2020, 2, 29)), -3, DateUnit.WEEK, Instant((2020, 2, 8))], - [Instant((2020, 2, 29)), -3, DateUnit.DAY, Instant((2020, 2, 26))], - [Instant((2020, 2, 29)), -3, DateUnit.WEEKDAY, Instant((2020, 2, 26))], - [Instant((2020, 2, 29)), 3, DateUnit.YEAR, Instant((2023, 2, 28))], - [Instant((2020, 2, 29)), 3, DateUnit.MONTH, Instant((2020, 5, 29))], - [Instant((2020, 2, 29)), 3, DateUnit.WEEK, Instant((2020, 3, 21))], - [Instant((2020, 2, 29)), 3, DateUnit.DAY, Instant((2020, 3, 3))], - [Instant((2020, 2, 29)), 3, DateUnit.WEEKDAY, Instant((2020, 3, 3))], + (Instant((2020, 2, 29)), "first-of", DateUnit.YEAR, Instant((2020, 1, 1))), + (Instant((2020, 2, 29)), "first-of", DateUnit.MONTH, Instant((2020, 2, 1))), + (Instant((2020, 2, 29)), "first-of", DateUnit.WEEK, Instant((2020, 2, 24))), + (Instant((2020, 2, 29)), "first-of", DateUnit.DAY, None), + (Instant((2020, 2, 29)), "first-of", DateUnit.WEEKDAY, None), + (Instant((2020, 2, 29)), "last-of", DateUnit.YEAR, Instant((2020, 12, 31))), + (Instant((2020, 2, 29)), "last-of", DateUnit.MONTH, Instant((2020, 2, 29))), + (Instant((2020, 2, 29)), "last-of", DateUnit.WEEK, Instant((2020, 3, 1))), + (Instant((2020, 2, 29)), "last-of", DateUnit.DAY, None), + (Instant((2020, 2, 29)), "last-of", DateUnit.WEEKDAY, None), + (Instant((2020, 2, 29)), -3, DateUnit.YEAR, Instant((2017, 2, 28))), + (Instant((2020, 2, 29)), -3, DateUnit.MONTH, Instant((2019, 11, 29))), + (Instant((2020, 2, 29)), -3, DateUnit.WEEK, Instant((2020, 2, 8))), + (Instant((2020, 2, 29)), -3, DateUnit.DAY, Instant((2020, 2, 26))), + (Instant((2020, 2, 29)), -3, DateUnit.WEEKDAY, Instant((2020, 2, 26))), + (Instant((2020, 2, 29)), 3, DateUnit.YEAR, Instant((2023, 2, 28))), + (Instant((2020, 2, 29)), 3, DateUnit.MONTH, Instant((2020, 5, 29))), + (Instant((2020, 2, 29)), 3, DateUnit.WEEK, Instant((2020, 3, 21))), + (Instant((2020, 2, 29)), 3, DateUnit.DAY, Instant((2020, 3, 3))), + (Instant((2020, 2, 29)), 3, DateUnit.WEEKDAY, Instant((2020, 3, 3))), ], ) -def test_offset(instant, offset, unit, expected): +def test_offset(instant, offset, unit, expected) -> None: assert instant.offset(offset, unit) == expected diff --git a/openfisca_core/periods/tests/test_parsers.py b/openfisca_core/periods/tests/test_parsers.py new file mode 100644 index 0000000000..c9131414b2 --- /dev/null +++ b/openfisca_core/periods/tests/test_parsers.py @@ -0,0 +1,129 @@ +import pytest + +from openfisca_core.periods import ( + DateUnit, + Instant, + InstantError, + ParserError, + Period, + PeriodError, + _parsers, +) + + +@pytest.mark.parametrize( + ("arg", "expected"), + [ + ("1001", Instant((1001, 1, 1))), + ("1001-01", Instant((1001, 1, 1))), + ("1001-12", Instant((1001, 12, 1))), + ("1001-01-01", Instant((1001, 1, 1))), + ("2028-02-29", Instant((2028, 2, 29))), + ("1001-W01", Instant((1000, 12, 29))), + ("1001-W52", Instant((1001, 12, 21))), + ("1001-W01-1", Instant((1000, 12, 29))), + ], +) +def test_parse_instant(arg, expected) -> None: + assert _parsers.parse_instant(arg) == expected + + +@pytest.mark.parametrize( + ("arg", "error"), + [ + (None, InstantError), + ({}, InstantError), + ((), InstantError), + ([], InstantError), + (1, InstantError), + ("", InstantError), + ("à", InstantError), + ("1", InstantError), + ("-1", InstantError), + ("999", InstantError), + ("1000-0", InstantError), + ("1000-1", ParserError), + ("1000-1-1", InstantError), + ("1000-00", InstantError), + ("1000-13", InstantError), + ("1000-01-00", InstantError), + ("1000-01-99", InstantError), + ("2029-02-29", ParserError), + ("1000-W0", InstantError), + ("1000-W1", InstantError), + ("1000-W99", InstantError), + ("1000-W1-0", InstantError), + ("1000-W1-1", InstantError), + ("1000-W1-99", InstantError), + ("1000-W01-0", InstantError), + ("1000-W01-00", InstantError), + ], +) +def test_parse_instant_with_invalid_argument(arg, error) -> None: + with pytest.raises(error): + _parsers.parse_instant(arg) + + +@pytest.mark.parametrize( + ("arg", "expected"), + [ + ("1001", Period((DateUnit.YEAR, Instant((1001, 1, 1)), 1))), + ("1001-01", Period((DateUnit.MONTH, Instant((1001, 1, 1)), 1))), + ("1001-12", Period((DateUnit.MONTH, Instant((1001, 12, 1)), 1))), + ("1001-01-01", Period((DateUnit.DAY, Instant((1001, 1, 1)), 1))), + ("1001-W01", Period((DateUnit.WEEK, Instant((1000, 12, 29)), 1))), + ("1001-W52", Period((DateUnit.WEEK, Instant((1001, 12, 21)), 1))), + ("1001-W01-1", Period((DateUnit.WEEKDAY, Instant((1000, 12, 29)), 1))), + ], +) +def test_parse_period(arg, expected) -> None: + assert _parsers.parse_period(arg) == expected + + +@pytest.mark.parametrize( + ("arg", "error"), + [ + (None, PeriodError), + ({}, PeriodError), + ((), PeriodError), + ([], PeriodError), + (1, PeriodError), + ("", PeriodError), + ("à", PeriodError), + ("1", PeriodError), + ("-1", PeriodError), + ("999", PeriodError), + ("1000-0", PeriodError), + ("1000-1", ParserError), + ("1000-1-1", PeriodError), + ("1000-00", PeriodError), + ("1000-13", PeriodError), + ("1000-01-00", PeriodError), + ("1000-01-99", PeriodError), + ("1000-W0", PeriodError), + ("1000-W1", PeriodError), + ("1000-W99", PeriodError), + ("1000-W1-0", PeriodError), + ("1000-W1-1", PeriodError), + ("1000-W1-99", PeriodError), + ("1000-W01-0", PeriodError), + ("1000-W01-00", PeriodError), + ], +) +def test_parse_period_with_invalid_argument(arg, error) -> None: + with pytest.raises(error): + _parsers.parse_period(arg) + + +@pytest.mark.parametrize( + ("arg", "expected"), + [ + ("2022", DateUnit.YEAR), + ("2022-01", DateUnit.MONTH), + ("2022-01-01", DateUnit.DAY), + ("2022-W01", DateUnit.WEEK), + ("2022-W01-1", DateUnit.WEEKDAY), + ], +) +def test_parse_unit(arg, expected) -> None: + assert _parsers.parse_unit(arg) == expected diff --git a/openfisca_core/periods/tests/test_period.py b/openfisca_core/periods/tests/test_period.py index 6553c4fd9b..9e53bf7d12 100644 --- a/openfisca_core/periods/tests/test_period.py +++ b/openfisca_core/periods/tests/test_period.py @@ -4,278 +4,278 @@ @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.YEAR, Instant((2022, 1, 1)), 1, "2022"], - [DateUnit.MONTH, Instant((2022, 1, 1)), 12, "2022"], - [DateUnit.YEAR, Instant((2022, 3, 1)), 1, "year:2022-03"], - [DateUnit.MONTH, Instant((2022, 3, 1)), 12, "year:2022-03"], - [DateUnit.YEAR, Instant((2022, 1, 1)), 3, "year:2022:3"], - [DateUnit.YEAR, Instant((2022, 1, 3)), 3, "year:2022:3"], + (DateUnit.YEAR, Instant((2022, 1, 1)), 1, "2022"), + (DateUnit.MONTH, Instant((2022, 1, 1)), 12, "2022"), + (DateUnit.YEAR, Instant((2022, 3, 1)), 1, "year:2022-03"), + (DateUnit.MONTH, Instant((2022, 3, 1)), 12, "year:2022-03"), + (DateUnit.YEAR, Instant((2022, 1, 1)), 3, "year:2022:3"), + (DateUnit.YEAR, Instant((2022, 1, 3)), 3, "year:2022:3"), ], ) -def test_str_with_years(date_unit, instant, size, expected): +def test_str_with_years(date_unit, instant, size, expected) -> None: assert str(Period((date_unit, instant, size))) == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.MONTH, Instant((2022, 1, 1)), 1, "2022-01"], - [DateUnit.MONTH, Instant((2022, 1, 1)), 3, "month:2022-01:3"], - [DateUnit.MONTH, Instant((2022, 3, 1)), 3, "month:2022-03:3"], + (DateUnit.MONTH, Instant((2022, 1, 1)), 1, "2022-01"), + (DateUnit.MONTH, Instant((2022, 1, 1)), 3, "month:2022-01:3"), + (DateUnit.MONTH, Instant((2022, 3, 1)), 3, "month:2022-03:3"), ], ) -def test_str_with_months(date_unit, instant, size, expected): +def test_str_with_months(date_unit, instant, size, expected) -> None: assert str(Period((date_unit, instant, size))) == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.DAY, Instant((2022, 1, 1)), 1, "2022-01-01"], - [DateUnit.DAY, Instant((2022, 1, 1)), 3, "day:2022-01-01:3"], - [DateUnit.DAY, Instant((2022, 3, 1)), 3, "day:2022-03-01:3"], + (DateUnit.DAY, Instant((2022, 1, 1)), 1, "2022-01-01"), + (DateUnit.DAY, Instant((2022, 1, 1)), 3, "day:2022-01-01:3"), + (DateUnit.DAY, Instant((2022, 3, 1)), 3, "day:2022-03-01:3"), ], ) -def test_str_with_days(date_unit, instant, size, expected): +def test_str_with_days(date_unit, instant, size, expected) -> None: assert str(Period((date_unit, instant, size))) == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.WEEK, Instant((2022, 1, 1)), 1, "2021-W52"], - [DateUnit.WEEK, Instant((2022, 1, 1)), 3, "week:2021-W52:3"], - [DateUnit.WEEK, Instant((2022, 3, 1)), 1, "2022-W09"], - [DateUnit.WEEK, Instant((2022, 3, 1)), 3, "week:2022-W09:3"], + (DateUnit.WEEK, Instant((2022, 1, 1)), 1, "2021-W52"), + (DateUnit.WEEK, Instant((2022, 1, 1)), 3, "week:2021-W52:3"), + (DateUnit.WEEK, Instant((2022, 3, 1)), 1, "2022-W09"), + (DateUnit.WEEK, Instant((2022, 3, 1)), 3, "week:2022-W09:3"), ], ) -def test_str_with_weeks(date_unit, instant, size, expected): +def test_str_with_weeks(date_unit, instant, size, expected) -> None: assert str(Period((date_unit, instant, size))) == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.WEEKDAY, Instant((2022, 1, 1)), 1, "2021-W52-6"], - [DateUnit.WEEKDAY, Instant((2022, 1, 1)), 3, "weekday:2021-W52-6:3"], - [DateUnit.WEEKDAY, Instant((2022, 3, 1)), 1, "2022-W09-2"], - [DateUnit.WEEKDAY, Instant((2022, 3, 1)), 3, "weekday:2022-W09-2:3"], + (DateUnit.WEEKDAY, Instant((2022, 1, 1)), 1, "2021-W52-6"), + (DateUnit.WEEKDAY, Instant((2022, 1, 1)), 3, "weekday:2021-W52-6:3"), + (DateUnit.WEEKDAY, Instant((2022, 3, 1)), 1, "2022-W09-2"), + (DateUnit.WEEKDAY, Instant((2022, 3, 1)), 3, "weekday:2022-W09-2:3"), ], ) -def test_str_with_weekdays(date_unit, instant, size, expected): +def test_str_with_weekdays(date_unit, instant, size, expected) -> None: assert str(Period((date_unit, instant, size))) == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.YEAR, Instant((2022, 12, 1)), 1, 1], - [DateUnit.YEAR, Instant((2022, 1, 1)), 2, 2], + (DateUnit.YEAR, Instant((2022, 12, 1)), 1, 1), + (DateUnit.YEAR, Instant((2022, 1, 1)), 2, 2), ], ) -def test_size_in_years(date_unit, instant, size, expected): +def test_size_in_years(date_unit, instant, size, expected) -> None: period = Period((date_unit, instant, size)) assert period.size_in_years == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.YEAR, Instant((2020, 1, 1)), 1, 12], - [DateUnit.YEAR, Instant((2022, 1, 1)), 2, 24], - [DateUnit.MONTH, Instant((2012, 1, 3)), 3, 3], + (DateUnit.YEAR, Instant((2020, 1, 1)), 1, 12), + (DateUnit.YEAR, Instant((2022, 1, 1)), 2, 24), + (DateUnit.MONTH, Instant((2012, 1, 3)), 3, 3), ], ) -def test_size_in_months(date_unit, instant, size, expected): +def test_size_in_months(date_unit, instant, size, expected) -> None: period = Period((date_unit, instant, size)) assert period.size_in_months == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.YEAR, Instant((2022, 12, 1)), 1, 365], - [DateUnit.YEAR, Instant((2020, 1, 1)), 1, 366], - [DateUnit.YEAR, Instant((2022, 1, 1)), 2, 730], - [DateUnit.MONTH, Instant((2022, 12, 1)), 1, 31], - [DateUnit.MONTH, Instant((2020, 2, 3)), 1, 29], - [DateUnit.MONTH, Instant((2022, 1, 3)), 3, 31 + 28 + 31], - [DateUnit.MONTH, Instant((2012, 1, 3)), 3, 31 + 29 + 31], - [DateUnit.DAY, Instant((2022, 12, 31)), 1, 1], - [DateUnit.DAY, Instant((2022, 12, 31)), 3, 3], - [DateUnit.WEEK, Instant((2022, 12, 31)), 1, 7], - [DateUnit.WEEK, Instant((2022, 12, 31)), 3, 21], - [DateUnit.WEEKDAY, Instant((2022, 12, 31)), 1, 1], - [DateUnit.WEEKDAY, Instant((2022, 12, 31)), 3, 3], + (DateUnit.YEAR, Instant((2022, 12, 1)), 1, 365), + (DateUnit.YEAR, Instant((2020, 1, 1)), 1, 366), + (DateUnit.YEAR, Instant((2022, 1, 1)), 2, 730), + (DateUnit.MONTH, Instant((2022, 12, 1)), 1, 31), + (DateUnit.MONTH, Instant((2020, 2, 3)), 1, 29), + (DateUnit.MONTH, Instant((2022, 1, 3)), 3, 31 + 28 + 31), + (DateUnit.MONTH, Instant((2012, 1, 3)), 3, 31 + 29 + 31), + (DateUnit.DAY, Instant((2022, 12, 31)), 1, 1), + (DateUnit.DAY, Instant((2022, 12, 31)), 3, 3), + (DateUnit.WEEK, Instant((2022, 12, 31)), 1, 7), + (DateUnit.WEEK, Instant((2022, 12, 31)), 3, 21), + (DateUnit.WEEKDAY, Instant((2022, 12, 31)), 1, 1), + (DateUnit.WEEKDAY, Instant((2022, 12, 31)), 3, 3), ], ) -def test_size_in_days(date_unit, instant, size, expected): +def test_size_in_days(date_unit, instant, size, expected) -> None: period = Period((date_unit, instant, size)) assert period.size_in_days == expected assert period.size_in_days == period.days @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.YEAR, Instant((2022, 12, 1)), 1, 52], - [DateUnit.YEAR, Instant((2020, 1, 1)), 5, 261], - [DateUnit.MONTH, Instant((2022, 12, 1)), 1, 4], - [DateUnit.MONTH, Instant((2020, 2, 3)), 1, 4], - [DateUnit.MONTH, Instant((2022, 1, 3)), 3, 12], - [DateUnit.MONTH, Instant((2012, 1, 3)), 3, 13], - [DateUnit.WEEK, Instant((2022, 12, 31)), 1, 1], - [DateUnit.WEEK, Instant((2022, 12, 31)), 3, 3], + (DateUnit.YEAR, Instant((2022, 12, 1)), 1, 52), + (DateUnit.YEAR, Instant((2020, 1, 1)), 5, 261), + (DateUnit.MONTH, Instant((2022, 12, 1)), 1, 4), + (DateUnit.MONTH, Instant((2020, 2, 3)), 1, 4), + (DateUnit.MONTH, Instant((2022, 1, 3)), 3, 12), + (DateUnit.MONTH, Instant((2012, 1, 3)), 3, 13), + (DateUnit.WEEK, Instant((2022, 12, 31)), 1, 1), + (DateUnit.WEEK, Instant((2022, 12, 31)), 3, 3), ], ) -def test_size_in_weeks(date_unit, instant, size, expected): +def test_size_in_weeks(date_unit, instant, size, expected) -> None: period = Period((date_unit, instant, size)) assert period.size_in_weeks == expected @pytest.mark.parametrize( - "date_unit, instant, size, expected", + ("date_unit", "instant", "size", "expected"), [ - [DateUnit.YEAR, Instant((2022, 12, 1)), 1, 364], - [DateUnit.YEAR, Instant((2020, 1, 1)), 1, 364], - [DateUnit.YEAR, Instant((2022, 1, 1)), 2, 728], - [DateUnit.MONTH, Instant((2022, 12, 1)), 1, 31], - [DateUnit.MONTH, Instant((2020, 2, 3)), 1, 29], - [DateUnit.MONTH, Instant((2022, 1, 3)), 3, 31 + 28 + 31], - [DateUnit.MONTH, Instant((2012, 1, 3)), 3, 31 + 29 + 31], - [DateUnit.DAY, Instant((2022, 12, 31)), 1, 1], - [DateUnit.DAY, Instant((2022, 12, 31)), 3, 3], - [DateUnit.WEEK, Instant((2022, 12, 31)), 1, 7], - [DateUnit.WEEK, Instant((2022, 12, 31)), 3, 21], - [DateUnit.WEEKDAY, Instant((2022, 12, 31)), 1, 1], - [DateUnit.WEEKDAY, Instant((2022, 12, 31)), 3, 3], + (DateUnit.YEAR, Instant((2022, 12, 1)), 1, 364), + (DateUnit.YEAR, Instant((2020, 1, 1)), 1, 364), + (DateUnit.YEAR, Instant((2022, 1, 1)), 2, 728), + (DateUnit.MONTH, Instant((2022, 12, 1)), 1, 31), + (DateUnit.MONTH, Instant((2020, 2, 3)), 1, 29), + (DateUnit.MONTH, Instant((2022, 1, 3)), 3, 31 + 28 + 31), + (DateUnit.MONTH, Instant((2012, 1, 3)), 3, 31 + 29 + 31), + (DateUnit.DAY, Instant((2022, 12, 31)), 1, 1), + (DateUnit.DAY, Instant((2022, 12, 31)), 3, 3), + (DateUnit.WEEK, Instant((2022, 12, 31)), 1, 7), + (DateUnit.WEEK, Instant((2022, 12, 31)), 3, 21), + (DateUnit.WEEKDAY, Instant((2022, 12, 31)), 1, 1), + (DateUnit.WEEKDAY, Instant((2022, 12, 31)), 3, 3), ], ) -def test_size_in_weekdays(date_unit, instant, size, expected): +def test_size_in_weekdays(date_unit, instant, size, expected) -> None: period = Period((date_unit, instant, size)) assert period.size_in_weekdays == expected @pytest.mark.parametrize( - "period_unit, sub_unit, instant, start, cease, count", + ("period_unit", "sub_unit", "instant", "start", "cease", "count"), [ - [ + ( DateUnit.YEAR, DateUnit.YEAR, Instant((2022, 12, 31)), Instant((2022, 1, 1)), Instant((2024, 1, 1)), 3, - ], - [ + ), + ( DateUnit.YEAR, DateUnit.MONTH, Instant((2022, 12, 31)), Instant((2022, 12, 1)), Instant((2025, 11, 1)), 36, - ], - [ + ), + ( DateUnit.YEAR, DateUnit.DAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2025, 12, 30)), 1096, - ], - [ + ), + ( DateUnit.YEAR, DateUnit.WEEK, Instant((2022, 12, 31)), Instant((2022, 12, 26)), Instant((2025, 12, 15)), 156, - ], - [ + ), + ( DateUnit.YEAR, DateUnit.WEEKDAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2025, 12, 26)), 1092, - ], - [ + ), + ( DateUnit.MONTH, DateUnit.MONTH, Instant((2022, 12, 31)), Instant((2022, 12, 1)), Instant((2023, 2, 1)), 3, - ], - [ + ), + ( DateUnit.MONTH, DateUnit.DAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2023, 3, 30)), 90, - ], - [ + ), + ( DateUnit.DAY, DateUnit.DAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2023, 1, 2)), 3, - ], - [ + ), + ( DateUnit.DAY, DateUnit.WEEKDAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2023, 1, 2)), 3, - ], - [ + ), + ( DateUnit.WEEK, DateUnit.DAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2023, 1, 20)), 21, - ], - [ + ), + ( DateUnit.WEEK, DateUnit.WEEK, Instant((2022, 12, 31)), Instant((2022, 12, 26)), Instant((2023, 1, 9)), 3, - ], - [ + ), + ( DateUnit.WEEK, DateUnit.WEEKDAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2023, 1, 20)), 21, - ], - [ + ), + ( DateUnit.WEEKDAY, DateUnit.DAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2023, 1, 2)), 3, - ], - [ + ), + ( DateUnit.WEEKDAY, DateUnit.WEEKDAY, Instant((2022, 12, 31)), Instant((2022, 12, 31)), Instant((2023, 1, 2)), 3, - ], + ), ], ) -def test_subperiods(period_unit, sub_unit, instant, start, cease, count): +def test_subperiods(period_unit, sub_unit, instant, start, cease, count) -> None: period = Period((period_unit, instant, 3)) subperiods = period.get_subperiods(sub_unit) assert len(subperiods) == count diff --git a/openfisca_core/periods/types.py b/openfisca_core/periods/types.py new file mode 100644 index 0000000000..092509c621 --- /dev/null +++ b/openfisca_core/periods/types.py @@ -0,0 +1,183 @@ +# TODO(): Properly resolve metaclass types. +# https://github.com/python/mypy/issues/14033 + +from collections.abc import Sequence + +from openfisca_core.types import DateUnit, Instant, Period + +import re + +#: Matches "2015", "2015-01", "2015-01-01" but not "2015-13", "2015-12-32". +iso_format = re.compile(r"^\d{4}(-(?:0[1-9]|1[0-2])(-(?:0[1-9]|[12]\d|3[01]))?)?$") + +#: Matches "2015", "2015-W01", "2015-W53-1" but not "2015-W54", "2015-W10-8". +iso_calendar = re.compile(r"^\d{4}(-W(0[1-9]|[1-4][0-9]|5[0-3]))?(-[1-7])?$") + + +class _SeqIntMeta(type): + def __instancecheck__(self, arg: object) -> bool: + return ( + bool(arg) + and isinstance(arg, Sequence) + and all(isinstance(item, int) for item in arg) + ) + + +class SeqInt(list[int], metaclass=_SeqIntMeta): # type: ignore[misc] + """A sequence of integers. + + Examples: + >>> isinstance([1, 2, 3], SeqInt) + True + + >>> isinstance((1, 2, 3), SeqInt) + True + + >>> isinstance({1, 2, 3}, SeqInt) + False + + >>> isinstance([1, 2, "3"], SeqInt) + False + + >>> isinstance(1, SeqInt) + False + + >>> isinstance([], SeqInt) + False + + """ + + +class _InstantStrMeta(type): + def __instancecheck__(self, arg: object) -> bool: + return isinstance(arg, (ISOFormatStr, ISOCalendarStr)) + + +class InstantStr(str, metaclass=_InstantStrMeta): # type: ignore[misc] + """A string representing an instant in string format. + + Examples: + >>> isinstance("2015", InstantStr) + True + + >>> isinstance("2015-01", InstantStr) + True + + >>> isinstance("2015-W01", InstantStr) + True + + >>> isinstance("2015-W01-12", InstantStr) + False + + >>> isinstance("week:2015-W01:3", InstantStr) + False + + """ + + __slots__ = () + + +class _ISOFormatStrMeta(type): + def __instancecheck__(self, arg: object) -> bool: + return isinstance(arg, str) and bool(iso_format.match(arg)) + + +class ISOFormatStr(str, metaclass=_ISOFormatStrMeta): # type: ignore[misc] + """A string representing an instant in ISO format. + + Examples: + >>> isinstance("2015", ISOFormatStr) + True + + >>> isinstance("2015-01", ISOFormatStr) + True + + >>> isinstance("2015-01-01", ISOFormatStr) + True + + >>> isinstance("2015-13", ISOFormatStr) + False + + >>> isinstance("2015-W01", ISOFormatStr) + False + + """ + + __slots__ = () + + +class _ISOCalendarStrMeta(type): + def __instancecheck__(self, arg: object) -> bool: + return isinstance(arg, str) and bool(iso_calendar.match(arg)) + + +class ISOCalendarStr(str, metaclass=_ISOCalendarStrMeta): # type: ignore[misc] + """A string representing an instant in ISO calendar. + + Examples: + >>> isinstance("2015", ISOCalendarStr) + True + + >>> isinstance("2015-W01", ISOCalendarStr) + True + + >>> isinstance("2015-W11-7", ISOCalendarStr) + True + + >>> isinstance("2015-W010", ISOCalendarStr) + False + + >>> isinstance("2015-01", ISOCalendarStr) + False + + """ + + __slots__ = () + + +class _PeriodStrMeta(type): + def __instancecheck__(self, arg: object) -> bool: + return ( + isinstance(arg, str) + and ":" in arg + and isinstance(arg.split(":")[1], InstantStr) + ) + + +class PeriodStr(str, metaclass=_PeriodStrMeta): # type: ignore[misc] + """A string representing a period. + + Examples: + >>> isinstance("year", PeriodStr) + False + + >>> isinstance("2015", PeriodStr) + False + + >>> isinstance("year:2015", PeriodStr) + True + + >>> isinstance("month:2015-01", PeriodStr) + True + + >>> isinstance("weekday:2015-W01-1:365", PeriodStr) + True + + >>> isinstance("2015-W01:1", PeriodStr) + False + + """ + + __slots__ = () + + +__all__ = [ + "DateUnit", + "ISOCalendarStr", + "ISOFormatStr", + "Instant", + "InstantStr", + "Period", + "PeriodStr", + "SeqInt", +] diff --git a/openfisca_core/taxbenefitsystems/tax_benefit_system.py b/openfisca_core/taxbenefitsystems/tax_benefit_system.py index 812448ae1d..b636d05f09 100644 --- a/openfisca_core/taxbenefitsystems/tax_benefit_system.py +++ b/openfisca_core/taxbenefitsystems/tax_benefit_system.py @@ -76,9 +76,9 @@ def base_tax_benefit_system(self): baseline = self.baseline if baseline is None: return self - self._base_tax_benefit_system = ( - base_tax_benefit_system - ) = baseline.base_tax_benefit_system + self._base_tax_benefit_system = base_tax_benefit_system = ( + baseline.base_tax_benefit_system + ) return base_tax_benefit_system def instantiate_entities(self): diff --git a/openfisca_core/taxscales/tax_scale_like.py b/openfisca_core/taxscales/tax_scale_like.py index 99b7887f67..0220e0ec39 100644 --- a/openfisca_core/taxscales/tax_scale_like.py +++ b/openfisca_core/taxscales/tax_scale_like.py @@ -46,20 +46,17 @@ def __ne__(self, _other: object) -> typing.NoReturn: ) @abc.abstractmethod - def __repr__(self) -> str: - ... + def __repr__(self) -> str: ... @abc.abstractmethod def calc( self, tax_base: NumericalArray, right: bool, - ) -> numpy.float_: - ... + ) -> numpy.float_: ... @abc.abstractmethod - def to_dict(self) -> dict: - ... + def to_dict(self) -> dict: ... def copy(self) -> typing.Any: new = commons.empty_clone(self) diff --git a/openfisca_core/types.py b/openfisca_core/types.py new file mode 100644 index 0000000000..711e6c512f --- /dev/null +++ b/openfisca_core/types.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from collections.abc import Iterable, Sequence, Sized +from numpy.typing import NDArray +from typing import Any, NewType, TypeVar, Union +from typing_extensions import Protocol, TypeAlias + +import numpy +import pendulum + +_N_co = TypeVar("_N_co", bound=numpy.generic, covariant=True) + +#: Type representing an numpy array. +Array: TypeAlias = NDArray[_N_co] + +_L = TypeVar("_L") + +#: Type representing an array-like object. +ArrayLike: TypeAlias = Sequence[_L] + +#: Generic type vars. +_T_co = TypeVar("_T_co", covariant=True) + + +# Entities + +#: For example "person". +EntityKey = NewType("EntityKey", str) + +#: For example "persons". +EntityPlural = NewType("EntityPlural", str) + +#: For example "principal". +RoleKey = NewType("RoleKey", str) + +#: For example "parents". +RolePlural = NewType("RolePlural", str) + + +class CoreEntity(Protocol): + key: EntityKey + plural: EntityPlural + + def check_role_validity(self, role: object, /) -> None: ... + def check_variable_defined_for_entity( + self, + variable_name: VariableName, + /, + ) -> None: ... + def get_variable( + self, + variable_name: VariableName, + check_existence: bool = ..., + /, + ) -> None | Variable: ... + + +class SingleEntity(CoreEntity, Protocol): ... + + +class GroupEntity(CoreEntity, Protocol): ... + + +class Role(Protocol): + entity: GroupEntity + max: int | None + subroles: None | Iterable[Role] + + @property + def key(self, /) -> RoleKey: ... + @property + def plural(self, /) -> None | RolePlural: ... + + +# Holders + + +class Holder(Protocol): + def clone(self, population: Any, /) -> Holder: ... + def get_memory_usage(self, /) -> Any: ... + + +# Parameters + + +class ParameterNodeAtInstant(Protocol): ... + + +# Periods + +#: For example "2000-01". +InstantStr = NewType("InstantStr", str) + +#: For example "1:2000-01-01:day". +PeriodStr = NewType("PeriodStr", str) + + +class Container(Protocol[_T_co]): + def __contains__(self, item: object, /) -> bool: ... + + +class Indexable(Protocol[_T_co]): + def __getitem__(self, index: int, /) -> _T_co: ... + + +class DateUnit(Container[str], Protocol): + def upper(self, /) -> str: ... + + +class Instant(Indexable[int], Iterable[int], Sized, Protocol): + @property + def year(self, /) -> int: ... + @property + def month(self, /) -> int: ... + @property + def day(self, /) -> int: ... + @property + def date(self, /) -> pendulum.Date: ... + def __lt__(self, other: object, /) -> bool: ... + def __le__(self, other: object, /) -> bool: ... + def offset(self, offset: str | int, unit: DateUnit, /) -> None | Instant: ... + + +class Period(Indexable[Union[DateUnit, Instant, int]], Protocol): + @property + def unit(self, /) -> DateUnit: ... + @property + def start(self, /) -> Instant: ... + @property + def size(self, /) -> int: ... + @property + def stop(self, /) -> Instant: ... + def offset(self, offset: str | int, unit: None | DateUnit = None, /) -> Period: ... + + +# Populations + + +class Population(Protocol): + entity: Any + + def get_holder(self, variable_name: VariableName, /) -> Any: ... + + +# Simulations + + +class Simulation(Protocol): + def calculate(self, variable_name: VariableName, period: Any, /) -> Any: ... + def calculate_add(self, variable_name: VariableName, period: Any, /) -> Any: ... + def calculate_divide(self, variable_name: VariableName, period: Any, /) -> Any: ... + def get_population(self, plural: None | str, /) -> Any: ... + + +# Tax-Benefit systems + + +class TaxBenefitSystem(Protocol): + person_entity: Any + + def get_variable( + self, + variable_name: VariableName, + check_existence: bool = ..., + /, + ) -> None | Variable: ... + + +# Variables + +#: For example "salary". +VariableName = NewType("VariableName", str) + + +class Variable(Protocol): + entity: Any + name: VariableName + + +class Formula(Protocol): + def __call__( + self, + population: Population, + instant: Instant, + params: Params, + /, + ) -> Array[Any]: ... + + +class Params(Protocol): + def __call__(self, instant: Instant, /) -> ParameterNodeAtInstant: ... diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py deleted file mode 100644 index a774b448b8..0000000000 --- a/openfisca_core/types/__init__.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Data types and protocols used by OpenFisca Core. - -The type definitions included in this sub-package are intended for -contributors, to help them better understand and document contracts -and expected behaviours. - -Official Public API: - * :attr:`.Array` - * ``ArrayLike`` - * :attr:`.Cache` - * :attr:`.Enum` - * :attr:`.EnumArray` - * :attr:`.Entity` - * :attr:`.Formula` - * :attr:`.Instant` - * :attr:`.ParameterNodeAtInstant` - * :attr:`.Params` - * :attr:`.Period` - * :attr:`.Population` - * :attr:`.Role` - * :attr:`.Simulation` - * :attr:`.TaxBenefitSystem` - * :attr:`.Variable` - -Note: - How imports are being used today:: - - from openfisca_core.types import * # Bad - from openfisca_core.types.data_types.arrays import ArrayLike # Bad - - The previous examples provoke cyclic dependency problems, that prevents us - from modularizing the different components of the library, so as to make - them easier to test and to maintain. - - How could them be used after the next major release:: - - from openfisca_core.types import ArrayLike - - ArrayLike # Good: import types as publicly exposed - - .. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. - - .. _PEP8#Imports: - https://www.python.org/dev/peps/pep-0008/#imports - - .. _OpenFisca's Styleguide: - https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md - -""" - -# Official Public API - -from ._data import Array, Enum, EnumArray, Instant, Period # noqa: F401 -from ._domain import ( # noqa: F401 - Entity, - Formula, - ParameterNodeAtInstant, - Params, - Population, - Role, - Simulation, - TaxBenefitSystem, - Variable, -) diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py deleted file mode 100644 index 74e878ec03..0000000000 --- a/openfisca_core/types/_data.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Data-model. - -The data-model is composed of structures meant to hold data in a certain way. -Their identity is equivalent to the sum of their properties. If two data -objects hold the same data, they are for all purposes equal and fungible. - -Examples: - >>> from openfisca_core import periods - - >>> this = periods.Instant((1234, 5, 6)) - >>> that = periods.Instant((1234, 5, 6)) - >>> this == that - True - - >>> that = periods.Instant((1234, 7, 8)) - >>> this == that - False - -""" - -from __future__ import annotations - -import abc -import typing_extensions -from numpy.typing import NDArray as Array # noqa: F401 -from typing import Any, TypeVar -from typing_extensions import Protocol - -T = TypeVar("T", bool, bytes, float, int, object, str) - - -class Enum(Protocol): - """Enum protocol.""" - - -class EnumArray(Protocol): - """EnumArray protocol.""" - - -class Instant(Protocol): - """Instant protocol.""" - - -@typing_extensions.runtime_checkable -class Period(Protocol): - """Period protocol.""" - - @property - @abc.abstractmethod - def size(self) -> int: - """Abstract property.""" - - @property - @abc.abstractmethod - def start(self) -> Any: - """Abstract method.""" - - @property - @abc.abstractmethod - def unit(self) -> Any: - """Abstract method.""" - - @abc.abstractmethod - def contains(self, other: object) -> bool: - """Abstract method.""" diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py deleted file mode 100644 index 103a5d5abc..0000000000 --- a/openfisca_core/types/_domain.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Domain-model. - -The domain-model is composed of structures meant to encapsulate data in a way -that is unique to the context. Therefore, their identity is not equivalent to -the sum of their properties. If two data objects hold the same identifier, even -if the data they hold is different, they are equal but not fungible. - -Examples: - If we take entities, they are equal as long as they share the same ``key``. - Let's take the following example: - - >>> from openfisca_core import entities - - >>> this = entities.Entity(1, "a", "b", "c") - >>> that = entities.Entity(1, "d", "f", "g") - >>> this == that - True - - As you can see, ``this`` and ``that`` are equal because they share the same - ``key``: - - >>> this.key == that.key - True - - The opposite is also true: - - >>> that = entities.Entity(2, "a", "b", "c") - >>> this == that - False - - >>> this.key == that.key - False - -""" - -from __future__ import annotations - -import abc -import typing_extensions -from typing import Any -from typing_extensions import Protocol - -import numpy - - -class Entity(Protocol): - """Entity protocol.""" - - key: Any - plural: Any - - @abc.abstractmethod - def check_role_validity(self, role: Any) -> None: - """Abstract method.""" - - @abc.abstractmethod - def check_variable_defined_for_entity(self, variable_name: Any) -> None: - """Abstract method.""" - - @abc.abstractmethod - def get_variable( - self, - variable_name: Any, - check_existence: Any = ..., - ) -> Any | None: - """Abstract method.""" - - -class Formula(Protocol): - """Formula protocol.""" - - @abc.abstractmethod - def __call__( - self, - population: Population, - instant: Any, - params: Params, - ) -> numpy.ndarray: - """Abstract method.""" - - -@typing_extensions.runtime_checkable -class ParameterNodeAtInstant(Protocol): - """ParameterNodeAtInstant protocol.""" - - -class Params(Protocol): - """Params protocol.""" - - @abc.abstractmethod - def __call__(self, instant: Any) -> ParameterNodeAtInstant: - """Abstract method.""" - - -@typing_extensions.runtime_checkable -class Period(Protocol): - """Period protocol.""" - - @property - @abc.abstractmethod - def start(self) -> Any: - """Abstract method.""" - - @property - @abc.abstractmethod - def unit(self) -> Any: - """Abstract method.""" - - -class Population(Protocol): - """Population protocol.""" - - count: Any - entity: Any - simulation: Any - - @abc.abstractmethod - def get_holder(self, variable_name: Any) -> Any: - """Abstract method.""" - - -class Role(Protocol): - """Role protocol.""" - - entity: Any - subroles: Any - - -class Simulation(Protocol): - """Simulation protocol.""" - - trace: Any - tracer: Any - memory_config: Any - opt_out_cache: Any - data_storage_dir: Any - tax_benefit_system: Any - - @abc.abstractmethod - def calculate(self, variable_name: Any, period: Any) -> Any: - """Abstract method.""" - - @abc.abstractmethod - def calculate_add(self, variable_name: Any, period: Any) -> Any: - """Abstract method.""" - - @abc.abstractmethod - def calculate_divide(self, variable_name: Any, period: Any) -> Any: - """Abstract method.""" - - @abc.abstractmethod - def get_population(self, plural: Any | None) -> Any: - """Abstract method.""" - - -class TaxBenefitSystem(Protocol): - """TaxBenefitSystem protocol.""" - - person_entity: Any - - @abc.abstractmethod - def get_variable( - self, - variable_name: Any, - check_existence: Any = ..., - ) -> Any | None: - """Abstract method.""" - - -@typing_extensions.runtime_checkable -class Variable(Protocol): - """Variable protocol.""" - - name: Any - dtype: Any - entity: Any - set_input: Any - value_type: Any - is_neutralized: Any - possible_values: Any - definition_period: Any - - @abc.abstractmethod - def default_array(self, array_size: Any) -> Any: - """Abstract method.""" diff --git a/openfisca_core/variables/__init__.py b/openfisca_core/variables/__init__.py index 3decaf8f42..1ab191c5ce 100644 --- a/openfisca_core/variables/__init__.py +++ b/openfisca_core/variables/__init__.py @@ -21,6 +21,6 @@ # # See: https://www.python.org/dev/peps/pep-0008/#imports -from .config import VALUE_TYPES, FORMULA_NAME_PREFIX # noqa: F401 +from .config import FORMULA_NAME_PREFIX, VALUE_TYPES # noqa: F401 from .helpers import get_annualized_variable, get_neutralized_variable # noqa: F401 from .variable import Variable # noqa: F401 diff --git a/openfisca_core/variables/config.py b/openfisca_core/variables/config.py index 81a09c4ecc..54270145bf 100644 --- a/openfisca_core/variables/config.py +++ b/openfisca_core/variables/config.py @@ -5,7 +5,6 @@ from openfisca_core import indexed_enums from openfisca_core.indexed_enums import Enum - VALUE_TYPES = { bool: { "dtype": numpy.bool_, diff --git a/openfisca_core/variables/helpers.py b/openfisca_core/variables/helpers.py index ee36d7a303..5038a78240 100644 --- a/openfisca_core/variables/helpers.py +++ b/openfisca_core/variables/helpers.py @@ -1,18 +1,16 @@ from __future__ import annotations import sortedcontainers -from typing import Optional +from openfisca_core import variables from openfisca_core.periods import Period -from .. import variables - def get_annualized_variable( - variable: variables.Variable, annualization_period: Optional[Period] = None + variable: variables.Variable, + annualization_period: Period | None = None, ) -> variables.Variable: - """ - Returns a clone of ``variable`` that is annualized for the period ``annualization_period``. + """Returns a clone of ``variable`` that is annualized for the period ``annualization_period``. When annualized, a variable's formula is only called for a January calculation, and the results for other months are assumed to be identical. """ @@ -33,23 +31,24 @@ def annual_formula(population, period, parameters): { key: make_annual_formula(formula, annualization_period) for key, formula in variable.formulas.items() - } + }, ) return new_variable def get_neutralized_variable(variable): - """ - Return a new neutralized variable (to be used by reforms). + """Return a new neutralized variable (to be used by reforms). A neutralized variable always returns its default value, and does not cache anything. """ result = variable.clone() result.is_neutralized = True result.label = ( - "[Neutralized]" - if variable.label is None - else "[Neutralized] {}".format(variable.label), + ( + "[Neutralized]" + if variable.label is None + else f"[Neutralized] {variable.label}" + ), ) return result diff --git a/openfisca_core/variables/tests/test_definition_period.py b/openfisca_core/variables/tests/test_definition_period.py index 7938aaeaef..8ef9bfaa87 100644 --- a/openfisca_core/variables/tests/test_definition_period.py +++ b/openfisca_core/variables/tests/test_definition_period.py @@ -13,31 +13,31 @@ class TestVariable(Variable): return TestVariable -def test_weekday_variable(variable): +def test_weekday_variable(variable) -> None: variable.definition_period = periods.WEEKDAY assert variable() -def test_week_variable(variable): +def test_week_variable(variable) -> None: variable.definition_period = periods.WEEK assert variable() -def test_day_variable(variable): +def test_day_variable(variable) -> None: variable.definition_period = periods.DAY assert variable() -def test_month_variable(variable): +def test_month_variable(variable) -> None: variable.definition_period = periods.MONTH assert variable() -def test_year_variable(variable): +def test_year_variable(variable) -> None: variable.definition_period = periods.YEAR assert variable() -def test_eternity_variable(variable): +def test_eternity_variable(variable) -> None: variable.definition_period = periods.ETERNITY assert variable() diff --git a/openfisca_core/variables/variable.py b/openfisca_core/variables/variable.py index 33cae3b20d..926e4c59c1 100644 --- a/openfisca_core/variables/variable.py +++ b/openfisca_core/variables/variable.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Optional, Union +from typing import NoReturn import datetime import re @@ -9,18 +9,16 @@ import numpy import sortedcontainers -from openfisca_core import periods, tools -from openfisca_core.entities import Entity +from openfisca_core import commons, periods, types as t +from openfisca_core.entities import Entity, GroupEntity from openfisca_core.indexed_enums import Enum, EnumArray from openfisca_core.periods import DateUnit, Period -from openfisca_core.types import Formula, Instant from . import config, helpers class Variable: - """ - A `variable `_ of the legislation. + """A `variable `_ of the legislation. Main attributes: @@ -101,7 +99,7 @@ class Variable: __name__: str - def __init__(self, baseline_variable=None): + def __init__(self, baseline_variable=None) -> None: self.name = self.__class__.__name__ attr = { name: value @@ -110,21 +108,30 @@ def __init__(self, baseline_variable=None): } self.baseline_variable = baseline_variable self.value_type = self.set( - attr, "value_type", required=True, allowed_values=config.VALUE_TYPES.keys() + attr, + "value_type", + required=True, + allowed_values=config.VALUE_TYPES.keys(), ) self.dtype = config.VALUE_TYPES[self.value_type]["dtype"] self.json_type = config.VALUE_TYPES[self.value_type]["json_type"] if self.value_type == Enum: self.possible_values = self.set( - attr, "possible_values", required=True, setter=self.set_possible_values + attr, + "possible_values", + required=True, + setter=self.set_possible_values, ) if self.value_type == str: self.max_length = self.set(attr, "max_length", allowed_type=int) if self.max_length: - self.dtype = "|S{}".format(self.max_length) + self.dtype = f"|S{self.max_length}" if self.value_type == Enum: self.default_value = self.set( - attr, "default_value", allowed_type=self.possible_values, required=True + attr, + "default_value", + allowed_type=self.possible_values, + required=True, ) else: self.default_value = self.set( @@ -135,7 +142,10 @@ def __init__(self, baseline_variable=None): ) self.entity = self.set(attr, "entity", required=True, setter=self.set_entity) self.definition_period = self.set( - attr, "definition_period", required=True, allowed_values=DateUnit + attr, + "definition_period", + required=True, + allowed_values=DateUnit, ) self.label = self.set(attr, "label", allowed_type=str, setter=self.set_label) self.end = self.set(attr, "end", allowed_type=str, setter=self.set_end) @@ -143,11 +153,14 @@ def __init__(self, baseline_variable=None): self.cerfa_field = self.set(attr, "cerfa_field", allowed_type=(str, dict)) self.unit = self.set(attr, "unit", allowed_type=str) self.documentation = self.set( - attr, "documentation", allowed_type=str, setter=self.set_documentation + attr, + "documentation", + allowed_type=str, + setter=self.set_documentation, ) self.set_input = self.set_set_input(attr.pop("set_input", None)) self.calculate_output = self.set_calculate_output( - attr.pop("calculate_output", None) + attr.pop("calculate_output", None), ) self.is_period_size_independent = self.set( attr, @@ -162,15 +175,18 @@ def __init__(self, baseline_variable=None): ) formulas_attr, unexpected_attrs = helpers._partition( - attr, lambda name, value: name.startswith(config.FORMULA_NAME_PREFIX) + attr, + lambda name, value: name.startswith(config.FORMULA_NAME_PREFIX), ) self.formulas = self.set_formulas(formulas_attr) if unexpected_attrs: + msg = 'Unexpected attributes in definition of variable "{}": {!r}'.format( + self.name, + ", ".join(sorted(unexpected_attrs.keys())), + ) raise ValueError( - 'Unexpected attributes in definition of variable "{}": {!r}'.format( - self.name, ", ".join(sorted(unexpected_attrs.keys())) - ) + msg, ) self.is_neutralized = False @@ -191,16 +207,14 @@ def set( if value is None and self.baseline_variable: return getattr(self.baseline_variable, attribute_name) if required and value is None: + msg = f"Missing attribute '{attribute_name}' in definition of variable '{self.name}'." raise ValueError( - "Missing attribute '{}' in definition of variable '{}'.".format( - attribute_name, self.name - ) + msg, ) if allowed_values is not None and value not in allowed_values: + msg = f"Invalid value '{value}' for attribute '{attribute_name}' in variable '{self.name}'. Allowed values are '{allowed_values}'." raise ValueError( - "Invalid value '{}' for attribute '{}' in variable '{}'. Allowed values are '{}'.".format( - value, attribute_name, self.name, allowed_values - ) + msg, ) if ( allowed_type is not None @@ -210,10 +224,9 @@ def set( if allowed_type == float and isinstance(value, int): value = float(value) else: + msg = f"Invalid value '{value}' for attribute '{attribute_name}' in variable '{self.name}'. Must be of type '{allowed_type}'." raise ValueError( - "Invalid value '{}' for attribute '{}' in variable '{}'. Must be of type '{}'.".format( - value, attribute_name, self.name, allowed_type - ) + msg, ) if setter is not None: value = setter(value) @@ -222,35 +235,39 @@ def set( return value def set_entity(self, entity): - if not isinstance(entity, Entity): + if not isinstance(entity, (Entity, GroupEntity)): + msg = ( + f"Invalid value '{entity}' for attribute 'entity' in variable " + f"'{self.name}'. Must be an instance of Entity or GroupEntity." + ) raise ValueError( - f"Invalid value '{entity}' for attribute 'entity' in variable '{self.name}'. Must be an instance of Entity." + msg, ) return entity def set_possible_values(self, possible_values): if not issubclass(possible_values, Enum): + msg = f"Invalid value '{possible_values}' for attribute 'possible_values' in variable '{self.name}'. Must be a subclass of {Enum}." raise ValueError( - "Invalid value '{}' for attribute 'possible_values' in variable '{}'. Must be a subclass of {}.".format( - possible_values, self.name, Enum - ) + msg, ) return possible_values def set_label(self, label): if label: return label + return None def set_end(self, end): if end: try: return datetime.datetime.strptime(end, "%Y-%m-%d").date() except ValueError: + msg = f"Incorrect 'end' attribute format in '{self.name}'. 'YYYY-MM-DD' expected where YYYY, MM and DD are year, month and day. Found: {end}" raise ValueError( - "Incorrect 'end' attribute format in '{}'. 'YYYY-MM-DD' expected where YYYY, MM and DD are year, month and day. Found: {}".format( - self.name, end - ) + msg, ) + return None def set_reference(self, reference): if reference: @@ -261,18 +278,16 @@ def set_reference(self, reference): elif isinstance(reference, tuple): reference = list(reference) else: + msg = f"The reference of the variable {self.name} is a {type(reference)} instead of a String or a List of Strings." raise TypeError( - "The reference of the variable {} is a {} instead of a String or a List of Strings.".format( - self.name, type(reference) - ) + msg, ) for element in reference: if not isinstance(element, str): + msg = f"The reference of the variable {self.name} is a {type(reference)} instead of a String or a List of Strings." raise TypeError( - "The reference of the variable {} is a {} instead of a String or a List of Strings.".format( - self.name, type(reference) - ) + msg, ) return reference @@ -280,6 +295,7 @@ def set_reference(self, reference): def set_documentation(self, documentation): if documentation: return textwrap.dedent(documentation) + return None def set_set_input(self, set_input): if not set_input and self.baseline_variable: @@ -297,10 +313,9 @@ def set_formulas(self, formulas_attr): starting_date = self.parse_formula_name(formula_name) if self.end is not None and starting_date > self.end: + msg = f'You declared that "{self.name}" ends on "{self.end}", but you wrote a formula to calculate it from "{starting_date}" ({formula_name}). The "end" attribute of a variable must be posterior to the start dates of all its formulas.' raise ValueError( - 'You declared that "{}" ends on "{}", but you wrote a formula to calculate it from "{}" ({}). The "end" attribute of a variable must be posterior to the start dates of all its formulas.'.format( - self.name, self.end, starting_date, formula_name - ) + msg, ) formulas[str(starting_date)] = formula @@ -314,14 +329,13 @@ def set_formulas(self, formulas_attr): for baseline_start_date, baseline_formula in self.baseline_variable.formulas.items() if first_reform_formula_date is None or baseline_start_date < first_reform_formula_date - } + }, ) return formulas def parse_formula_name(self, attribute_name): - """ - Returns the starting date of a formula based on its name. + """Returns the starting date of a formula based on its name. Valid dated name formats are : 'formula', 'formula_YYYY', 'formula_YYYY_MM' and 'formula_YYYY_MM_DD' where YYYY, MM and DD are a year, month and day. @@ -331,11 +345,10 @@ def parse_formula_name(self, attribute_name): - `formula_YYYY_MM` is `YYYY-MM-01` """ - def raise_error(): + def raise_error() -> NoReturn: + msg = f'Unrecognized formula name in variable "{self.name}". Expecting "formula_YYYY" or "formula_YYYY_MM" or "formula_YYYY_MM_DD where YYYY, MM and DD are year, month and day. Found: "{attribute_name}".' raise ValueError( - 'Unrecognized formula name in variable "{}". Expecting "formula_YYYY" or "formula_YYYY_MM" or "formula_YYYY_MM_DD where YYYY, MM and DD are year, month and day. Found: "{}".'.format( - self.name, attribute_name - ) + msg, ) if attribute_name == config.FORMULA_NAME_PREFIX: @@ -347,7 +360,7 @@ def raise_error(): if not match: raise_error() date_str = "-".join( - [match.group(1), match.group(2) or "01", match.group(3) or "01"] + [match.group(1), match.group(2) or "01", match.group(3) or "01"], ) try: @@ -358,9 +371,7 @@ def raise_error(): # ----- Methods ----- # def is_input_variable(self): - """ - Returns True if the variable is an input variable. - """ + """Returns True if the variable is an input variable.""" return len(self.formulas) == 0 @classmethod @@ -372,8 +383,8 @@ def get_introspection_data(cls): def get_formula( self, - period: Union[Instant, Period, str, int] = None, - ) -> Optional[Formula]: + period: None | t.Instant | t.Period | str | int = None, + ) -> None | t.Formula: """Returns the formula to compute the variable at the given period. If no period is given and the variable has several formulas, the method @@ -386,14 +397,15 @@ def get_formula( Formula used to compute the variable. """ - - instant: Optional[Instant] + instant: None | t.Instant if not self.formulas: return None if period is None: - return self.formulas.peekitem(index=0)[ + return self.formulas.peekitem( + index=0, + )[ 1 ] # peekitem gets the 1st key-value tuple (the oldest start_date and formula). Return the formula. @@ -420,8 +432,7 @@ def get_formula( return None def clone(self): - clone = self.__class__() - return clone + return self.__class__() def check_set_value(self, value): if self.value_type == Enum and isinstance(value, str): @@ -429,39 +440,33 @@ def check_set_value(self, value): value = self.possible_values[value].index except KeyError: possible_values = [item.name for item in self.possible_values] + msg = "'{}' is not a known value for '{}'. Possible values are ['{}'].".format( + value, + self.name, + "', '".join(possible_values), + ) raise ValueError( - "'{}' is not a known value for '{}'. Possible values are ['{}'].".format( - value, self.name, "', '".join(possible_values) - ) + msg, ) if self.value_type in (float, int) and isinstance(value, str): try: - value = tools.eval_expression(value) + value = commons.eval_expression(value) except SyntaxError: + msg = f"I couldn't understand '{value}' as a value for '{self.name}'" raise ValueError( - "I couldn't understand '{}' as a value for '{}'".format( - value, self.name - ) + msg, ) try: value = numpy.array([value], dtype=self.dtype)[0] except (TypeError, ValueError): if self.value_type == datetime.date: - error_message = "Can't deal with date: '{}'.".format(value) + error_message = f"Can't deal with date: '{value}'." else: - error_message = ( - "Can't deal with value: expected type {}, received '{}'.".format( - self.json_type, value - ) - ) + error_message = f"Can't deal with value: expected type {self.json_type}, received '{value}'." raise ValueError(error_message) except OverflowError: - error_message = ( - "Can't deal with value: '{}', it's too large for type '{}'.".format( - value, self.json_type - ) - ) + error_message = f"Can't deal with value: '{value}', it's too large for type '{self.json_type}'." raise ValueError(error_message) return value diff --git a/openfisca_web_api/app.py b/openfisca_web_api/app.py index 4d54207a55..a76f255a0c 100644 --- a/openfisca_web_api/app.py +++ b/openfisca_web_api/app.py @@ -1,19 +1,17 @@ -# -*- coding: utf-8 -*- - import logging import os import traceback -from openfisca_core.errors import SituationParsingError, PeriodMismatchError -from openfisca_web_api.loader import build_data -from openfisca_web_api.errors import handle_import_error +from openfisca_core.errors import PeriodMismatchError, SituationParsingError from openfisca_web_api import handlers +from openfisca_web_api.errors import handle_import_error +from openfisca_web_api.loader import build_data try: - from flask import Flask, jsonify, abort, request, make_response, redirect + import werkzeug.exceptions + from flask import Flask, abort, jsonify, make_response, redirect, request from flask_cors import CORS from werkzeug.middleware.proxy_fix import ProxyFix - import werkzeug.exceptions except ImportError as error: handle_import_error(error) @@ -31,7 +29,7 @@ def init_tracker(url, idsite, tracker_token): "You chose to activate the `tracker` module. ", "Tracking data will be sent to: " + url, "For more information, see .", - ] + ], ) log.info(info) return tracker @@ -42,9 +40,9 @@ def init_tracker(url, idsite, tracker_token): traceback.format_exc(), "You chose to activate the `tracker` module, but it is not installed.", "For more information, see .", - ] + ], ) - log.warn(message) + log.warning(message) def create_app( @@ -77,6 +75,7 @@ def create_app( def before_request(): if request.path != "/" and request.path.endswith("/"): return redirect(request.path[:-1]) + return None @app.route("/") def get_root(): @@ -84,8 +83,8 @@ def get_root(): jsonify( { "welcome": welcome_message - or DEFAULT_WELCOME_MESSAGE.format(request.host_url) - } + or DEFAULT_WELCOME_MESSAGE.format(request.host_url), + }, ), 300, ) @@ -95,7 +94,7 @@ def get_parameters(): parameters = { parameter["id"]: { "description": parameter["description"], - "href": "{}parameter/{}".format(request.host_url, name), + "href": f"{request.host_url}parameter/{name}", } for name, parameter in data["parameters"].items() if parameter.get("subparams") @@ -120,7 +119,7 @@ def get_variables(): variables = { name: { "description": variable["description"], - "href": "{}variable/{}".format(request.host_url, name), + "href": f"{request.host_url}variable/{name}", } for name, variable in data["variables"].items() } @@ -146,15 +145,15 @@ def get_spec(): return jsonify( { **data["openAPI_spec"], - **{"servers": [{"url": url}]}, - } + "servers": [{"url": url}], + }, ) - def handle_invalid_json(error): + def handle_invalid_json(error) -> None: json_response = jsonify( { - "error": "Invalid JSON: {}".format(error.args[0]), - } + "error": f"Invalid JSON: {error.args[0]}", + }, ) abort(make_response(json_response, 400)) @@ -173,7 +172,7 @@ def calculate(): make_response( jsonify({"error": "'" + e[1] + "' is not a valid ASCII value."}), 400, - ) + ), ) return jsonify(result) @@ -194,7 +193,7 @@ def apply_headers(response): { "Country-Package": data["country_package_metadata"]["name"], "Country-Package-Version": data["country_package_metadata"]["version"], - } + }, ) return response diff --git a/openfisca_web_api/errors.py b/openfisca_web_api/errors.py index ba804a7b08..ac93ebd833 100644 --- a/openfisca_web_api/errors.py +++ b/openfisca_web_api/errors.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- +from typing import NoReturn import logging log = logging.getLogger("gunicorn.error") -def handle_import_error(error): +def handle_import_error(error) -> NoReturn: + msg = f"OpenFisca is missing some dependencies to run the Web API: '{error}'. To install them, run `pip install openfisca_core[web-api]`." raise ImportError( - "OpenFisca is missing some dependencies to run the Web API: '{}'. To install them, run `pip install openfisca_core[web-api]`.".format( - error - ) + msg, ) diff --git a/openfisca_web_api/handlers.py b/openfisca_web_api/handlers.py index 47b99338a3..2f6fc4403a 100644 --- a/openfisca_web_api/handlers.py +++ b/openfisca_web_api/handlers.py @@ -1,17 +1,17 @@ -# -*- coding: utf-8 -*- - import dpath.util -from openfisca_core.simulation_builder import SimulationBuilder + from openfisca_core.indexed_enums import Enum +from openfisca_core.simulation_builder import SimulationBuilder def calculate(tax_benefit_system, input_data: dict) -> dict: - """ - Returns the input_data where the None values are replaced by the calculated values. - """ + """Returns the input_data where the None values are replaced by the calculated values.""" simulation = SimulationBuilder().build_from_entities(tax_benefit_system, input_data) requested_computations = dpath.util.search( - input_data, "*/*/*/*", afilter=lambda t: t is None, yielded=True + input_data, + "*/*/*/*", + afilter=lambda t: t is None, + yielded=True, ) computation_results: dict = {} for computation in requested_computations: @@ -28,7 +28,7 @@ def calculate(tax_benefit_system, input_data: dict) -> dict: entity_result = result.decode()[entity_index].name elif variable.value_type == float: entity_result = float( - str(result[entity_index]) + str(result[entity_index]), ) # To turn the float32 into a regular float without adding confusing extra decimals. There must be a better way. elif variable.value_type == str: entity_result = str(result[entity_index]) @@ -39,27 +39,26 @@ def calculate(tax_benefit_system, input_data: dict) -> dict: # See https://github.com/dpath-maintainers/dpath-python/issues/160 if computation_results == {}: computation_results = { - entity_plural: {entity_id: {variable_name: {period: entity_result}}} + entity_plural: {entity_id: {variable_name: {period: entity_result}}}, } - else: - if entity_plural in computation_results: - if entity_id in computation_results[entity_plural]: - if variable_name in computation_results[entity_plural][entity_id]: - computation_results[entity_plural][entity_id][variable_name][ - period - ] = entity_result - else: - computation_results[entity_plural][entity_id][variable_name] = { - period: entity_result - } + elif entity_plural in computation_results: + if entity_id in computation_results[entity_plural]: + if variable_name in computation_results[entity_plural][entity_id]: + computation_results[entity_plural][entity_id][variable_name][ + period + ] = entity_result else: - computation_results[entity_plural][entity_id] = { - variable_name: {period: entity_result} + computation_results[entity_plural][entity_id][variable_name] = { + period: entity_result, } else: - computation_results[entity_plural] = { - entity_id: {variable_name: {period: entity_result}} + computation_results[entity_plural][entity_id] = { + variable_name: {period: entity_result}, } + else: + computation_results[entity_plural] = { + entity_id: {variable_name: {period: entity_result}}, + } dpath.util.merge(input_data, computation_results) return input_data @@ -71,12 +70,15 @@ def trace(tax_benefit_system, input_data): requested_calculations = [] requested_computations = dpath.util.search( - input_data, "*/*/*/*", afilter=lambda t: t is None, yielded=True + input_data, + "*/*/*/*", + afilter=lambda t: t is None, + yielded=True, ) for computation in requested_computations: path = computation[0] entity_plural, entity_id, variable_name, period = path.split("/") - requested_calculations.append(f"{variable_name}<{str(period)}>") + requested_calculations.append(f"{variable_name}<{period!s}>") simulation.calculate(variable_name, period) trace = simulation.tracer.get_serialized_flat_trace() diff --git a/openfisca_web_api/loader/__init__.py b/openfisca_web_api/loader/__init__.py index 169bc58b4e..8d9318d9ae 100644 --- a/openfisca_web_api/loader/__init__.py +++ b/openfisca_web_api/loader/__init__.py @@ -1,23 +1,21 @@ -# -*- coding: utf-8 -*- - - -from openfisca_web_api.loader.parameters import build_parameters -from openfisca_web_api.loader.variables import build_variables from openfisca_web_api.loader.entities import build_entities +from openfisca_web_api.loader.parameters import build_parameters from openfisca_web_api.loader.spec import build_openAPI_specification +from openfisca_web_api.loader.variables import build_variables def build_data(tax_benefit_system): country_package_metadata = tax_benefit_system.get_package_metadata() parameters = build_parameters(tax_benefit_system, country_package_metadata) variables = build_variables(tax_benefit_system, country_package_metadata) + entities = build_entities(tax_benefit_system) data = { "tax_benefit_system": tax_benefit_system, - "country_package_metadata": tax_benefit_system.get_package_metadata(), + "country_package_metadata": country_package_metadata, "openAPI_spec": None, "parameters": parameters, "variables": variables, - "entities": build_entities(tax_benefit_system), + "entities": entities, } data["openAPI_spec"] = build_openAPI_specification(data) diff --git a/openfisca_web_api/loader/entities.py b/openfisca_web_api/loader/entities.py index 683537aa0e..98ce4e6fb9 100644 --- a/openfisca_web_api/loader/entities.py +++ b/openfisca_web_api/loader/entities.py @@ -1,11 +1,5 @@ -# -*- coding: utf-8 -*- - - def build_entities(tax_benefit_system): - entities = { - entity.key: build_entity(entity) for entity in tax_benefit_system.entities - } - return entities + return {entity.key: build_entity(entity) for entity in tax_benefit_system.entities} def build_entity(entity): diff --git a/openfisca_web_api/loader/parameters.py b/openfisca_web_api/loader/parameters.py index 8841f7ebe8..193f12915f 100644 --- a/openfisca_web_api/loader/parameters.py +++ b/openfisca_web_api/loader/parameters.py @@ -1,4 +1,5 @@ -# -*- coding: utf-8 -*- +import functools +import operator from openfisca_core.parameters import Parameter, ParameterNode, Scale @@ -24,8 +25,7 @@ def get_value(date, values): if candidates: return candidates[0][1] - else: - return None + return None def build_api_scale(scale, value_key_name): @@ -39,13 +39,14 @@ def build_api_scale(scale, value_key_name): ] dates = set( - sum( + functools.reduce( + operator.iadd, [ list(bracket["thresholds"].keys()) + list(bracket["values"].keys()) for bracket in brackets ], [], - ) + ), ) # flatten the dates and remove duplicates # We iterate on all dates as we need to build the whole scale for each of them @@ -87,7 +88,8 @@ def build_api_parameter(parameter, country_package_metadata): } if parameter.file_path: api_parameter["source"] = build_source_url( - parameter.file_path, country_package_metadata + parameter.file_path, + country_package_metadata, ) if isinstance(parameter, Parameter): if parameter.documentation: @@ -113,7 +115,8 @@ def build_api_parameter(parameter, country_package_metadata): def build_parameters(tax_benefit_system, country_package_metadata): return { parameter.name.replace(".", "/"): build_api_parameter( - parameter, country_package_metadata + parameter, + country_package_metadata, ) for parameter in tax_benefit_system.parameters.get_descendants() } diff --git a/openfisca_web_api/loader/spec.py b/openfisca_web_api/loader/spec.py index 47948c02bb..4a163bd91f 100644 --- a/openfisca_web_api/loader/spec.py +++ b/openfisca_web_api/loader/spec.py @@ -1,23 +1,22 @@ -# -*- coding: utf-8 -*- - import os -import yaml from copy import deepcopy import dpath.util +import yaml from openfisca_core.indexed_enums import Enum from openfisca_web_api import handlers - OPEN_API_CONFIG_FILE = os.path.join( - os.path.dirname(os.path.abspath(__file__)), os.path.pardir, "openAPI.yml" + os.path.dirname(os.path.abspath(__file__)), + os.path.pardir, + "openAPI.yml", ) def build_openAPI_specification(api_data): tax_benefit_system = api_data["tax_benefit_system"] - file = open(OPEN_API_CONFIG_FILE, "r") + file = open(OPEN_API_CONFIG_FILE) spec = yaml.safe_load(file) country_package_name = api_data["country_package_metadata"]["name"].title() country_package_version = api_data["country_package_metadata"]["version"] @@ -30,21 +29,24 @@ def build_openAPI_specification(api_data): spec, "info/description", spec["info"]["description"].replace( - "{COUNTRY_PACKAGE_NAME}", country_package_name + "{COUNTRY_PACKAGE_NAME}", + country_package_name, ), ) dpath.util.new( spec, "info/version", spec["info"]["version"].replace( - "{COUNTRY_PACKAGE_VERSION}", country_package_version + "{COUNTRY_PACKAGE_VERSION}", + country_package_version, ), ) for entity in tax_benefit_system.entities: name = entity.key.title() spec["components"]["schemas"][name] = get_entity_json_schema( - entity, tax_benefit_system + entity, + tax_benefit_system, ) situation_schema = get_situation_json_schema(tax_benefit_system) @@ -80,7 +82,9 @@ def build_openAPI_specification(api_data): if tax_benefit_system.open_api_config.get("simulation_example"): simulation_example = tax_benefit_system.open_api_config["simulation_example"] dpath.util.new( - spec, "components/schemas/SituationInput/example", simulation_example + spec, + "components/schemas/SituationInput/example", + simulation_example, ) dpath.util.new( spec, @@ -93,9 +97,7 @@ def build_openAPI_specification(api_data): handlers.trace(tax_benefit_system, simulation_example), ) else: - message = "No simulation example has been defined for this tax and benefit system. If you are the maintainer of {}, you can define an example by following this documentation: https://openfisca.org/doc/openfisca-web-api/config-openapi.html".format( - country_package_name - ) + message = f"No simulation example has been defined for this tax and benefit system. If you are the maintainer of {country_package_name}, you can define an example by following this documentation: https://openfisca.org/doc/openfisca-web-api/config-openapi.html" dpath.util.new(spec, "components/schemas/SituationInput/example", message) dpath.util.new(spec, "components/schemas/SituationOutput/example", message) dpath.util.new(spec, "components/schemas/Trace/example", message) @@ -123,32 +125,31 @@ def get_entity_json_schema(entity, tax_benefit_system): "properties": { variable_name: get_variable_json_schema(variable) for variable_name, variable in tax_benefit_system.get_variables( - entity + entity, ).items() }, "additionalProperties": False, } - else: - properties = {} - properties.update( - { - role.plural or role.key: {"type": "array", "items": {"type": "string"}} - for role in entity.roles - } - ) - properties.update( - { - variable_name: get_variable_json_schema(variable) - for variable_name, variable in tax_benefit_system.get_variables( - entity - ).items() - } - ) - return { - "type": "object", - "properties": properties, - "additionalProperties": False, - } + properties = {} + properties.update( + { + role.plural or role.key: {"type": "array", "items": {"type": "string"}} + for role in entity.roles + }, + ) + properties.update( + { + variable_name: get_variable_json_schema(variable) + for variable_name, variable in tax_benefit_system.get_variables( + entity, + ).items() + }, + ) + return { + "type": "object", + "properties": properties, + "additionalProperties": False, + } def get_situation_json_schema(tax_benefit_system): @@ -159,7 +160,7 @@ def get_situation_json_schema(tax_benefit_system): entity.plural: { "type": "object", "additionalProperties": { - "$ref": "#/components/schemas/{}".format(entity.key.title()) + "$ref": f"#/components/schemas/{entity.key.title()}", }, } for entity in tax_benefit_system.entities diff --git a/openfisca_web_api/loader/tax_benefit_system.py b/openfisca_web_api/loader/tax_benefit_system.py index 53a2c47107..358f960501 100644 --- a/openfisca_web_api/loader/tax_benefit_system.py +++ b/openfisca_web_api/loader/tax_benefit_system.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- - import importlib -import traceback import logging +import traceback from os import linesep log = logging.getLogger(__name__) @@ -15,15 +13,15 @@ def build_tax_benefit_system(country_package_name): message = linesep.join( [ traceback.format_exc(), - "Could not import module `{}`.".format(country_package_name), + f"Could not import module `{country_package_name}`.", "Are you sure it is installed in your environment? If so, look at the stack trace above to determine the origin of this error.", "See more at .", linesep, - ] + ], ) raise ValueError(message) try: return country_package.CountryTaxBenefitSystem() except NameError: # Gunicorn swallows NameErrors. Force printing the stack trace. - log.error(traceback.format_exc()) + log.exception(traceback.format_exc()) raise diff --git a/openfisca_web_api/loader/variables.py b/openfisca_web_api/loader/variables.py index f9b6e05887..6730dc0811 100644 --- a/openfisca_web_api/loader/variables.py +++ b/openfisca_web_api/loader/variables.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import datetime import inspect import textwrap @@ -26,7 +24,10 @@ def get_default_value(variable): def build_source_url( - country_package_metadata, source_file_path, start_line_number, source_code + country_package_metadata, + source_file_path, + start_line_number, + source_code, ): nb_lines = source_code.count("\n") return "{}/blob/{}{}#L{}-L{}".format( @@ -45,7 +46,10 @@ def build_formula(formula, country_package_metadata, source_file_path): api_formula = { "source": build_source_url( - country_package_metadata, source_file_path, start_line_number, source_code + country_package_metadata, + source_file_path, + start_line_number, + source_code, ), "content": source_code, } @@ -80,7 +84,10 @@ def build_variable(variable, country_package_metadata): if source_code: result["source"] = build_source_url( - country_package_metadata, source_file_path, start_line_number, source_code + country_package_metadata, + source_file_path, + start_line_number, + source_code, ) if variable.documentation: @@ -91,7 +98,9 @@ def build_variable(variable, country_package_metadata): if len(variable.formulas) > 0: result["formulas"] = build_formulas( - variable.formulas, country_package_metadata, source_file_path + variable.formulas, + country_package_metadata, + source_file_path, ) if variable.end: diff --git a/openfisca_web_api/openAPI.yml b/openfisca_web_api/openAPI.yml index 5eb1d8cf5c..ce935e5596 100644 --- a/openfisca_web_api/openAPI.yml +++ b/openfisca_web_api/openAPI.yml @@ -1,4 +1,4 @@ -openapi: "3.1.0" +openapi: "3.0.0" info: title: "{COUNTRY_PACKAGE_NAME} Web API" @@ -137,8 +137,8 @@ components: type: "object" additionalProperties: $ref: "#/components/schemas/Value" - propertyNames: # this keyword is part of JSON Schema but is not supported in OpenAPI Specification at the time of writing, see https://swagger.io/docs/specification/data-models/keywords/#unsupported - pattern: "^[12][0-9]{3}-[01][0-9]-[0-3][0-9]$" # all keys are ISO dates + # propertyNames: # this keyword is part of JSON Schema but is not supported in OpenAPI v3.0.0 + # pattern: "^[12][0-9]{3}-[01][0-9]-[0-3][0-9]$" # all keys are ISO dates Value: oneOf: diff --git a/openfisca_web_api/scripts/serve.py b/openfisca_web_api/scripts/serve.py index 7522175059..6ba89f440a 100644 --- a/openfisca_web_api/scripts/serve.py +++ b/openfisca_web_api/scripts/serve.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- - -import sys import logging +import sys from openfisca_core.scripts import build_tax_benefit_system from openfisca_web_api.app import create_app from openfisca_web_api.errors import handle_import_error try: - from gunicorn.app.base import BaseApplication from gunicorn import config + from gunicorn.app.base import BaseApplication except ImportError as error: handle_import_error(error) @@ -33,7 +31,7 @@ def read_user_configuration(default_configuration, command_line_parser): if args.configuration_file: file_configuration = {} - with open(args.configuration_file, "r") as file: + with open(args.configuration_file) as file: exec(file.read(), {}, file_configuration) # Configuration file overloads default configuration @@ -43,7 +41,8 @@ def read_user_configuration(default_configuration, command_line_parser): gunicorn_parser = config.Config().parser() configuration = update(configuration, vars(args)) configuration = update( - configuration, vars(gunicorn_parser.parse_args(unknown_args)) + configuration, + vars(gunicorn_parser.parse_args(unknown_args)), ) if configuration["args"]: command_line_parser.print_help() @@ -59,17 +58,17 @@ def update(configuration, new_options): configuration[key] = value if key == "port": configuration["bind"] = configuration["bind"][:-4] + str( - configuration["port"] + configuration["port"], ) return configuration class OpenFiscaWebAPIApplication(BaseApplication): - def __init__(self, options): + def __init__(self, options) -> None: self.options = options - super(OpenFiscaWebAPIApplication, self).__init__() + super().__init__() - def load_config(self): + def load_config(self) -> None: for key, value in self.options.items(): if key in self.cfg.settings: self.cfg.set(key.lower(), value) @@ -89,10 +88,10 @@ def load(self): ) -def main(parser): +def main(parser) -> None: configuration = { "port": DEFAULT_PORT, - "bind": "{}:{}".format(HOST, DEFAULT_PORT), + "bind": f"{HOST}:{DEFAULT_PORT}", "workers": DEFAULT_WORKERS_NUMBER, "timeout": DEFAULT_TIMEOUT, } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..ce4ba3779f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +target-version = ["py39", "py310", "py311", "py312"] diff --git a/setup.cfg b/setup.cfg index e88ce1dd98..9b8ce699bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,75 +1,98 @@ -# C011X: We (progressively) document the code base. -# D10X: We (progressively) check docstrings (see https://www.pydocstyle.org/en/2.1.1/error_codes.html#grouping). -# DARXXX: We (progressively) check docstrings (see https://github.com/terrencepreilly/darglint#error-codes). -# E203: See https://github.com/PyCQA/pycodestyle/issues/373. -# F403/405: We ignore * imports. -# R0401: We avoid cyclic imports —required for unit/doc tests. -# RST301: We use Google Python Style (see https://pypi.org/project/flake8-rst-docstrings/). -# W503/504: We break lines before binary operators (Knuth's style). +# C011X: We (progressively) document the code base. +# D10X: We (progressively) check docstrings (see https://www.pydocstyle.org/en/2.1.1/error_codes.html#grouping). +# DARXXX: We (progressively) check docstrings (see https://github.com/terrencepreilly/darglint#error-codes). +# E203: We ignore a false positive in whitespace before ":" (see https://github.com/PyCQA/pycodestyle/issues/373). +# F403/405: We ignore * imports. +# R0401: We avoid cyclic imports —required for unit/doc tests. +# RST301: We use Google Python Style (see https://pypi.org/project/flake8-rst-docstrings/). +# W503/504: We break lines before binary operators (Knuth's style). [flake8] -extend-ignore = D -ignore = E203,E501,F405,RST301,W503 -in-place = true -include-in-doctest = openfisca_core/commons openfisca_core/holders openfisca_core/periods openfisca_core/types -max-line-length = 88 -per-file-ignores = */typing.py:D101,D102,E704, */__init__.py:F401 -rst-directives = attribute, deprecated, seealso, versionadded, versionchanged -rst-roles = any, attr, class, exc, func, meth, mod, obj -strictness = short +convention = google +docstring_style = google +extend-ignore = D +ignore = B019, E203, E501, F405, E701, E704, RST212, RST213, RST301, RST306, W503 +in-place = true +include-in-doctest = + openfisca_core/commons + openfisca_core/entities + openfisca_core/holders + openfisca_core/periods + openfisca_core/projectors +max-line-length = 88 +per-file-ignores = + */types.py:D101,D102,E301,E704,W504 + */test_*.py:D101,D102,D103 + */__init__.py:F401 + */__init__.pyi:E302,E704 +rst-directives = attribute, deprecated, seealso, versionadded, versionchanged +rst-roles = any, attr, class, exc, func, meth, mod, obj +strictness = short + +[pylint.MASTER] +load-plugins = pylint_per_file_ignores [pylint.message_control] -disable = all -enable = C0115,C0116,R0401 -score = no +disable = all +enable = C0115, C0116, R0401 +per-file-ignores = + types.py:C0115,C0116 + /tests/:C0116 +score = no [isort] -case_sensitive = true +case_sensitive = true +combine_as_imports = true force_alphabetical_sort_within_sections = false -group_by_package = true -include_trailing_comma = true -known_first_party = openfisca_core -known_openfisca = openfisca_country_template, openfisca_extension_template -known_typing = *abc*, *mypy*, *types*, *typing* -profile = black -py_version = 39 -sections = FUTURE,TYPING,STDLIB,THIRDPARTY,OPENFISCA,FIRSTPARTY,LOCALFOLDER +group_by_package = true +honor_noqa = true +include_trailing_comma = true +known_first_party = openfisca_core +known_openfisca = openfisca_country_template, openfisca_extension_template +known_typing = *collections.abc*, *typing*, *typing_extensions* +known_types = *types* +multi_line_output = 3 +profile = black +py_version = 39 +sections = FUTURE, TYPING, TYPES, STDLIB, THIRDPARTY, OPENFISCA, FIRSTPARTY, LOCALFOLDER [coverage:paths] -source = . */site-packages +source = . */site-packages [coverage:run] -branch = true -source = openfisca_core, openfisca_web_api +branch = true +source = openfisca_core, openfisca_web_api [coverage:report] -fail_under = 75 -show_missing = true -skip_covered = true -skip_empty = true +fail_under = 75 +show_missing = true +skip_covered = true +skip_empty = true [tool:pytest] -addopts = --doctest-modules --disable-pytest-warnings --showlocals -doctest_optionflags = ELLIPSIS IGNORE_EXCEPTION_DETAIL NUMBER NORMALIZE_WHITESPACE -python_files = **/*.py -testpaths = tests +addopts = --disable-pytest-warnings --doctest-modules --showlocals +doctest_optionflags = ELLIPSIS IGNORE_EXCEPTION_DETAIL NUMBER NORMALIZE_WHITESPACE +python_files = **/*.py +testpaths = tests [mypy] -ignore_missing_imports = True -install_types = True -non_interactive = True - -[mypy-openfisca_core.commons.tests.*] -ignore_errors = True - -[mypy-openfisca_core.data_storage.tests.*] -ignore_errors = True - -[mypy-openfisca_core.holders.tests.*] -ignore_errors = True - -[mypy-openfisca_core.periods.tests.*] -ignore_errors = True +check_untyped_defs = false +disallow_any_decorated = false +disallow_any_explicit = false +disallow_any_expr = false +disallow_any_unimported = false +follow_imports = skip +ignore_missing_imports = true +implicit_reexport = false +install_types = true +mypy_path = stubs +non_interactive = true +plugins = numpy.typing.mypy_plugin +pretty = true +python_version = 3.9 +strict = false +warn_no_return = true +warn_unreachable = true -[mypy-openfisca_core.scripts.*] -ignore_errors = True +[mypy-openfisca_core.*.tests.*] +ignore_errors = True diff --git a/setup.py b/setup.py index 15e58543ce..a2be85005e 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,10 @@ """ -from setuptools import setup, find_packages from pathlib import Path +from setuptools import find_packages, setup + # Read the contents of our README file for PyPi this_directory = Path(__file__).parent long_description = (this_directory / "README.md").read_text() @@ -27,46 +28,49 @@ # DO NOT add space between '>=' and version number as it break conda build. general_requirements = [ "PyYAML >=6.0, <7.0", + "StrEnum >=0.4.8, <0.5.0", # 3.11.x backport "dpath >=2.1.4, <3.0", - "importlib-metadata >=6.1.0, <7.0", "numexpr >=2.8.4, <3.0", - "numpy >=1.24.2, <1.25", - "pendulum >=2.1.2, <3.0.0", + "numpy >=1.24.2, <2.0", + "pendulum >=3.0.0, <4.0.0", "psutil >=5.9.4, <6.0", - "pytest >=7.2.2, <8.0", + "pytest >=8.3.3, <9.0", "sortedcontainers >=2.4.0, <3.0", "typing_extensions >=4.5.0, <5.0", - "StrEnum >=0.4.8, <0.5.0", # 3.11.x backport ] api_requirements = [ - "Flask >=2.2.3, < 3.0", - "Flask-Cors >=3.0.10, < 4.0", - "gunicorn >=20.1.0, < 21.0", - "Werkzeug >=2.2.3, < 3.0", + "Flask >=2.2.3, <3.0", + "Flask-Cors >=3.0.10, <4.0", + "gunicorn >=21.0, <22.0", + "Werkzeug >=2.2.3, <3.0", ] dev_requirements = [ - "black >=23.1.0, < 24.0", - "coverage >=6.5.0, < 7.0", - "darglint >=1.8.1, < 2.0", - "flake8 >=6.0.0, < 7.0.0", - "flake8-bugbear >=23.3.23, < 24.0", - "flake8-docstrings >=1.7.0, < 2.0", - "flake8-print >=5.0.0, < 6.0", - "flake8-rst-docstrings >=0.3.0, < 0.4.0", - "idna >=3.4, < 4.0", - "isort >=5.12.0, < 6.0", - "mypy >=1.1.1, < 2.0", - "openapi-spec-validator >=0.5.6, < 0.6.0", - "pycodestyle >=2.10.0, < 3.0", - "pylint >=2.17.1, < 3.0", - "xdoctest >=1.1.1, < 2.0", -] + api_requirements + "black >=24.8.0, <25.0", + "coverage >=7.6.1, <8.0", + "darglint >=1.8.1, <2.0", + "flake8 >=7.1.1, <8.0.0", + "flake8-bugbear >=24.8.19, <25.0", + "flake8-docstrings >=1.7.0, <2.0", + "flake8-print >=5.0.0, <6.0", + "flake8-rst-docstrings >=0.3.0, <0.4.0", + "idna >=3.10, <4.0", + "isort >=5.13.2, <6.0", + "mypy >=1.11.2, <2.0", + "openapi-spec-validator >=0.7.1, <0.8.0", + "pylint >=3.3.1, <4.0", + "pylint-per-file-ignores >=1.3.2, <2.0", + "pyright >=1.1.382, <2.0", + "ruff >=0.6.7, <1.0", + "ruff-lsp >=0.0.57, <1.0", + "xdoctest >=1.2.0, <2.0", + *api_requirements, +] setup( name="OpenFisca-Core", - version="41.1.0", + version="42.1.0", author="OpenFisca Team", author_email="contact@openfisca.org", classifiers=[ @@ -102,12 +106,12 @@ "web-api": api_requirements, "dev": dev_requirements, "ci": [ - "build >=0.10.0, < 0.11.0", - "coveralls >=3.3.1, < 4.0", - "twine >=4.0.2, < 5.0", - "wheel >=0.40.0, < 0.41.0", + "build >=0.10.0, <0.11.0", + "coveralls >=4.0.1, <5.0", + "twine >=5.1.1, <6.0", + "wheel >=0.40.0, <0.41.0", ], - "tracker": ["OpenFisca-Tracker >=0.4.0, < 0.5.0"], + "tracker": ["OpenFisca-Tracker >=0.4.0, <0.5.0"], }, include_package_data=True, # Will read MANIFEST.in install_requires=general_requirements,