diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec1b68d0..bd341a5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,8 @@ jobs: uses: "actions/checkout@v2" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v5" + env: + POETRY_VERSION: 1.5.1 with: python-version: "${{ matrix.python-version }}" - name: "Install redis" diff --git a/CHANGELOG.md b/CHANGELOG.md index 84ad77e7..3b9b4114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## v1.9.0 - 2023-10-16 + +### Added + +- #220 - Implement DiffSyncModelFlags.NATURAL_DELETION_ORDER. + +### Changed + +- #219 - Type hinting overhaul + ## v1.8.0 - 2023-04-18 ### Added diff --git a/diffsync/__init__.py b/diffsync/__init__.py index 3fbe7186..412270d8 100644 --- a/diffsync/__init__.py +++ b/diffsync/__init__.py @@ -15,7 +15,8 @@ limitations under the License. """ from inspect import isclass -from typing import Callable, ClassVar, Dict, List, Mapping, Optional, Text, Tuple, Type, Union +from typing import Callable, ClassVar, Dict, List, Optional, Tuple, Type, Union, Any, Set +from typing_extensions import Self from pydantic import BaseModel, PrivateAttr import structlog # type: ignore @@ -28,6 +29,10 @@ from diffsync.store.local import LocalStore from diffsync.utils import get_path, set_key, tree_string +# This workaround is used because we are defining a method called `str` in our class definition, which therefore renders +# the builtin `str` type unusable. +StrType = str + class DiffSyncModel(BaseModel): """Base class for all DiffSync object models. @@ -72,7 +77,7 @@ class DiffSyncModel(BaseModel): Note: inclusion in `_attributes` is mutually exclusive from inclusion in `_identifiers`; a field cannot be in both! """ - _children: ClassVar[Mapping[str, str]] = {} + _children: ClassVar[Dict[str, str]] = {} """Optional: dict of `{_modelname: field_name}` entries describing how to store "child" models in this model. When calculating a Diff or performing a sync, DiffSync will automatically recurse into these child models. @@ -101,7 +106,7 @@ class Config: # pylint: disable=too-few-public-methods # Let us have a DiffSync as an instance variable even though DiffSync is not a Pydantic model itself. arbitrary_types_allowed = True - def __init_subclass__(cls): + def __init_subclass__(cls) -> None: """Validate that the various class attribute declarations correspond to actual instance fields. Called automatically on subclass declaration. @@ -132,19 +137,19 @@ def __init_subclass__(cls): if attr_child_overlap: raise AttributeError(f"Fields {attr_child_overlap} are included in both _attributes and _children.") - def __repr__(self): + def __repr__(self) -> str: return f'{self.get_type()} "{self.get_unique_id()}"' - def __str__(self): + def __str__(self) -> str: return self.get_unique_id() - def dict(self, **kwargs) -> dict: + def dict(self, **kwargs: Any) -> Dict: """Convert this DiffSyncModel to a dict, excluding the diffsync field by default as it is not serializable.""" if "exclude" not in kwargs: kwargs["exclude"] = {"diffsync"} return super().dict(**kwargs) - def json(self, **kwargs) -> str: + def json(self, **kwargs: Any) -> StrType: """Convert this DiffSyncModel to a JSON string, excluding the diffsync field by default as it is not serializable.""" if "exclude" not in kwargs: kwargs["exclude"] = {"diffsync"} @@ -152,7 +157,7 @@ def json(self, **kwargs) -> str: kwargs["exclude_defaults"] = True return super().json(**kwargs) - def str(self, include_children: bool = True, indent: int = 0) -> str: + def str(self, include_children: bool = True, indent: int = 0) -> StrType: """Build a detailed string representation of this DiffSyncModel and optionally its children.""" margin = " " * indent output = f"{margin}{self.get_type()}: {self.get_unique_id()}: {self.get_attrs()}" @@ -172,13 +177,13 @@ def str(self, include_children: bool = True, indent: int = 0) -> str: output += f"\n{margin} {child_id} (ERROR: details unavailable)" return output - def set_status(self, status: DiffSyncStatus, message: Text = ""): + def set_status(self, status: DiffSyncStatus, message: StrType = "") -> None: """Update the status (and optionally status message) of this model in response to a create/update/delete call.""" self._status = status self._status_message = message @classmethod - def create_base(cls, diffsync: "DiffSync", ids: Mapping, attrs: Mapping) -> Optional["DiffSyncModel"]: + def create_base(cls, diffsync: "DiffSync", ids: Dict, attrs: Dict) -> Optional[Self]: """Instantiate this class, along with any platform-specific data creation. This method is not meant to be subclassed, users should redefine create() instead. @@ -196,7 +201,7 @@ def create_base(cls, diffsync: "DiffSync", ids: Mapping, attrs: Mapping) -> Opti return model @classmethod - def create(cls, diffsync: "DiffSync", ids: Mapping, attrs: Mapping) -> Optional["DiffSyncModel"]: + def create(cls, diffsync: "DiffSync", ids: Dict, attrs: Dict) -> Optional[Self]: """Instantiate this class, along with any platform-specific data creation. Subclasses must call `super().create()` or `self.create_base()`; they may wish to then override the default status information @@ -216,7 +221,7 @@ def create(cls, diffsync: "DiffSync", ids: Mapping, attrs: Mapping) -> Optional[ """ return cls.create_base(diffsync=diffsync, ids=ids, attrs=attrs) - def update_base(self, attrs: Mapping) -> Optional["DiffSyncModel"]: + def update_base(self, attrs: Dict) -> Optional[Self]: """Base Update method to update the attributes of this instance, along with any platform-specific data updates. This method is not meant to be subclassed, users should redefine update() instead. @@ -234,7 +239,7 @@ def update_base(self, attrs: Mapping) -> Optional["DiffSyncModel"]: self.set_status(DiffSyncStatus.SUCCESS, "Updated successfully") return self - def update(self, attrs: Mapping) -> Optional["DiffSyncModel"]: + def update(self, attrs: Dict) -> Optional[Self]: """Update the attributes of this instance, along with any platform-specific data updates. Subclasses must call `super().update()` or `self.update_base()`; they may wish to then override the default status information @@ -252,7 +257,7 @@ def update(self, attrs: Mapping) -> Optional["DiffSyncModel"]: """ return self.update_base(attrs=attrs) - def delete_base(self) -> Optional["DiffSyncModel"]: + def delete_base(self) -> Optional[Self]: """Base delete method Delete any platform-specific data corresponding to this instance. This method is not meant to be subclassed, users should redefine delete() instead. @@ -263,7 +268,7 @@ def delete_base(self) -> Optional["DiffSyncModel"]: self.set_status(DiffSyncStatus.SUCCESS, "Deleted successfully") return self - def delete(self) -> Optional["DiffSyncModel"]: + def delete(self) -> Optional[Self]: """Delete any platform-specific data corresponding to this instance. Subclasses must call `super().delete()` or `self.delete_base()`; they may wish to then override the default status information @@ -279,7 +284,7 @@ def delete(self) -> Optional["DiffSyncModel"]: return self.delete_base() @classmethod - def get_type(cls) -> Text: + def get_type(cls) -> StrType: """Return the type AKA modelname of the object or the class Returns: @@ -288,7 +293,7 @@ def get_type(cls) -> Text: return cls._modelname @classmethod - def create_unique_id(cls, **identifiers) -> Text: + def create_unique_id(cls, **identifiers: Dict[StrType, Any]) -> StrType: """Construct a unique identifier for this model class. Args: @@ -297,11 +302,11 @@ def create_unique_id(cls, **identifiers) -> Text: return "__".join(str(identifiers[key]) for key in cls._identifiers) @classmethod - def get_children_mapping(cls) -> Mapping[Text, Text]: + def get_children_mapping(cls) -> Dict[StrType, StrType]: """Get the mapping of types to fieldnames for child models of this model.""" return cls._children - def get_identifiers(self) -> Mapping: + def get_identifiers(self) -> Dict: """Get a dict of all identifiers (primary keys) and their values for this object. Returns: @@ -309,7 +314,7 @@ def get_identifiers(self) -> Mapping: """ return self.dict(include=set(self._identifiers)) - def get_attrs(self) -> Mapping: + def get_attrs(self) -> Dict: """Get all the non-primary-key attributes or parameters for this object. Similar to Pydantic's `BaseModel.dict()` method, with the following key differences: @@ -322,7 +327,7 @@ def get_attrs(self) -> Mapping: """ return self.dict(include=set(self._attributes)) - def get_unique_id(self) -> Text: + def get_unique_id(self) -> StrType: """Get the unique ID of an object. By default the unique ID is built based on all the primary keys defined in `_identifiers`. @@ -332,7 +337,7 @@ def get_unique_id(self) -> Text: """ return self.create_unique_id(**self.get_identifiers()) - def get_shortname(self) -> Text: + def get_shortname(self) -> StrType: """Get the (not guaranteed-unique) shortname of an object, if any. By default the shortname is built based on all the keys defined in `_shortname`. @@ -345,11 +350,11 @@ def get_shortname(self) -> Text: return "__".join([str(getattr(self, key)) for key in self._shortname]) return self.get_unique_id() - def get_status(self) -> Tuple[DiffSyncStatus, Text]: + def get_status(self) -> Tuple[DiffSyncStatus, StrType]: """Get the status of the last create/update/delete operation on this object, and any associated message.""" - return (self._status, self._status_message) + return self._status, self._status_message - def add_child(self, child: "DiffSyncModel"): + def add_child(self, child: "DiffSyncModel") -> None: """Add a child reference to an object. The child object isn't stored, only its unique id. @@ -373,7 +378,7 @@ def add_child(self, child: "DiffSyncModel"): raise ObjectAlreadyExists(f"Already storing a {child_type} with unique_id {child.get_unique_id()}", child) childs.append(child.get_unique_id()) - def remove_child(self, child: "DiffSyncModel"): + def remove_child(self, child: "DiffSyncModel") -> None: """Remove a child reference from an object. The name of the storage attribute is defined in `_children` per object type. @@ -404,13 +409,15 @@ class DiffSync: # pylint: disable=too-many-public-methods # modelname1 = MyModelClass1 # modelname2 = MyModelClass2 - type: ClassVar[Optional[str]] = None + type: Optional[str] = None """Type of the object, will default to the name of the class if not provided.""" top_level: ClassVar[List[str]] = [] """List of top-level modelnames to begin from when diffing or synchronizing.""" - def __init__(self, name=None, internal_storage_engine=LocalStore): + def __init__( + self, name: Optional[str] = None, internal_storage_engine: Union[Type[BaseStore], BaseStore] = LocalStore + ) -> None: """Generic initialization function. Subclasses should be careful to call super().__init__() if they override this method. @@ -429,7 +436,7 @@ def __init__(self, name=None, internal_storage_engine=LocalStore): # If the name has not been provided, use the type as the name self.name = name if name else self.type - def __init_subclass__(cls): + def __init_subclass__(cls) -> None: """Validate that references to specific DiffSyncModels use the correct modelnames. Called automatically on subclass declaration. @@ -448,16 +455,16 @@ def __init_subclass__(cls): if not isclass(value) or not issubclass(value, DiffSyncModel): raise AttributeError(f'top_level references attribute "{name}" but it is not a DiffSyncModel subclass!') - def __str__(self): + def __str__(self) -> StrType: """String representation of a DiffSync.""" if self.type != self.name: return f'{self.type} "{self.name}"' return self.type - def __repr__(self): + def __repr__(self) -> StrType: return f"<{str(self)}>" - def __len__(self): + def __len__(self) -> int: """Total number of elements stored.""" return self.store.count() @@ -481,11 +488,11 @@ def _get_initial_value_order(cls) -> List[str]: value_order.append(item) return value_order - def load(self): + def load(self) -> None: """Load all desired data from whatever backend data source into this instance.""" # No-op in this generic class - def dict(self, exclude_defaults: bool = True, **kwargs) -> Mapping: + def dict(self, exclude_defaults: bool = True, **kwargs: Any) -> Dict[str, Dict[str, Dict]]: """Represent the DiffSync contents as a dict, as if it were a Pydantic model.""" data: Dict[str, Dict[str, Dict]] = {} for modelname in self.store.get_all_model_names(): @@ -494,7 +501,7 @@ def dict(self, exclude_defaults: bool = True, **kwargs) -> Mapping: data[obj.get_type()][obj.get_unique_id()] = obj.dict(exclude_defaults=exclude_defaults, **kwargs) return data - def str(self, indent: int = 0) -> str: + def str(self, indent: int = 0) -> StrType: """Build a detailed string representation of this DiffSync.""" margin = " " * indent output = "" @@ -510,7 +517,7 @@ def str(self, indent: int = 0) -> str: output += "\n" + model.str(indent=indent + 2) return output - def load_from_dict(self, data: Dict): + def load_from_dict(self, data: Dict) -> None: """The reverse of `dict` method, taking a dictionary and loading into the inventory. Args: @@ -531,20 +538,20 @@ def sync_from( # pylint: disable=too-many-arguments source: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE, - callback: Optional[Callable[[Text, int, int], None]] = None, + callback: Optional[Callable[[StrType, int, int], None]] = None, diff: Optional[Diff] = None, ) -> Diff: """Synchronize data from the given source DiffSync object into the current DiffSync object. Args: - source (DiffSync): object to sync data from into this one - diff_class (class): Diff or subclass thereof to use to calculate the diffs to use for synchronization - flags (DiffSyncFlags): Flags influencing the behavior of this sync. - callback (function): Function with parameters (stage, current, total), to be called at intervals as the - calculation of the diff and subsequent sync proceed. - diff (Diff): An existing diff to be used rather than generating a completely new diff. + source: object to sync data from into this one + diff_class: Diff or subclass thereof to use to calculate the diffs to use for synchronization + flags: Flags influencing the behavior of this sync. + callback: Function with parameters (stage, current, total), to be called at intervals as the calculation of + the diff and subsequent sync proceed. + diff: An existing diff to be used rather than generating a completely new diff. Returns: - Diff: Diff between origin object and source + Diff between origin object and source Raises: DiffClassMismatch: The provided diff's class does not match the diff_class """ @@ -569,20 +576,20 @@ def sync_to( # pylint: disable=too-many-arguments target: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE, - callback: Optional[Callable[[Text, int, int], None]] = None, + callback: Optional[Callable[[StrType, int, int], None]] = None, diff: Optional[Diff] = None, ) -> Diff: """Synchronize data from the current DiffSync object into the given target DiffSync object. Args: - target (DiffSync): object to sync data into from this one. - diff_class (class): Diff or subclass thereof to use to calculate the diffs to use for synchronization - flags (DiffSyncFlags): Flags influencing the behavior of this sync. - callback (function): Function with parameters (stage, current, total), to be called at intervals as the - calculation of the diff and subsequent sync proceed. - diff (Diff): An existing diff that will be used when determining what needs to be synced. + target: object to sync data into from this one. + diff_class: Diff or subclass thereof to use to calculate the diffs to use for synchronization + flags: Flags influencing the behavior of this sync. + callback: Function with parameters (stage, current, total), to be called at intervals as the calculation of + the diff and subsequent sync proceed. + diff: An existing diff that will be used when determining what needs to be synced. Returns: - Diff: Diff between origin object and target + Diff between origin object and target Raises: DiffClassMismatch: The provided diff's class does not match the diff_class """ @@ -594,7 +601,7 @@ def sync_complete( diff: Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE, logger: Optional[structlog.BoundLogger] = None, - ): + ) -> None: """Callback triggered after a `sync_from` operation has completed and updated the model data of this instance. Note that this callback is **only** triggered if the sync actually resulted in data changes. If there are no @@ -619,15 +626,15 @@ def diff_from( source: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE, - callback: Optional[Callable[[Text, int, int], None]] = None, + callback: Optional[Callable[[StrType, int, int], None]] = None, ) -> Diff: """Generate a Diff describing the difference from the other DiffSync to this one. Args: - source (DiffSync): Object to diff against. - diff_class (class): Diff or subclass thereof to use for diff calculation and storage. - flags (DiffSyncFlags): Flags influencing the behavior of this diff operation. - callback (function): Function with parameters (stage, current, total), to be called at intervals as the + source: Object to diff against. + diff_class: Diff or subclass thereof to use for diff calculation and storage. + flags: Flags influencing the behavior of this diff operation. + callback: Function with parameters (stage, current, total), to be called at intervals as the calculation of the diff proceeds. """ differ = DiffSyncDiffer( @@ -640,15 +647,15 @@ def diff_to( target: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE, - callback: Optional[Callable[[Text, int, int], None]] = None, + callback: Optional[Callable[[StrType, int, int], None]] = None, ) -> Diff: """Generate a Diff describing the difference from this DiffSync to another one. Args: - target (DiffSync): Object to diff against. - diff_class (class): Diff or subclass thereof to use for diff calculation and storage. - flags (DiffSyncFlags): Flags influencing the behavior of this diff operation. - callback (function): Function with parameters (stage, current, total), to be called at intervals as the + target: Object to diff against. + diff_class: Diff or subclass thereof to use for diff calculation and storage. + flags: Flags influencing the behavior of this diff operation. + callback: Function with parameters (stage, current, total), to be called at intervals as the calculation of the diff proceeds. """ return target.diff_from(self, diff_class=diff_class, flags=flags, callback=callback) @@ -657,16 +664,16 @@ def diff_to( # Object Storage Management # ------------------------------------------------------------------------------ - def get_all_model_names(self): + def get_all_model_names(self) -> Set[StrType]: """Get all model names. Returns: - List[str]: List of model names + List of model names """ return self.store.get_all_model_names() def get( - self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]], identifier: Union[Text, Mapping] + self, obj: Union[StrType, DiffSyncModel, Type[DiffSyncModel]], identifier: Union[StrType, Dict] ) -> DiffSyncModel: """Get one object from the data store based on its unique id. @@ -681,7 +688,7 @@ def get( return self.store.get(model=obj, identifier=identifier) def get_or_none( - self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]], identifier: Union[Text, Mapping] + self, obj: Union[StrType, DiffSyncModel, Type[DiffSyncModel]], identifier: Union[StrType, Dict] ) -> Optional[DiffSyncModel]: """Get one object from the data store based on its unique id or get a None @@ -700,19 +707,19 @@ def get_or_none( except ObjectNotFound: return None - def get_all(self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]) -> List[DiffSyncModel]: + def get_all(self, obj: Union[StrType, DiffSyncModel, Type[DiffSyncModel]]) -> List[DiffSyncModel]: """Get all objects of a given type. Args: obj: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve Returns: - List[DiffSyncModel]: List of Object + List of Object """ return self.store.get_all(model=obj) def get_by_uids( - self, uids: List[Text], obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]] + self, uids: List[StrType], obj: Union[StrType, DiffSyncModel, Type[DiffSyncModel]] ) -> List[DiffSyncModel]: """Get multiple objects from the store by their unique IDs/Keys and type. @@ -726,11 +733,11 @@ def get_by_uids( return self.store.get_by_uids(uids=uids, model=obj) @classmethod - def get_tree_traversal(cls, as_dict: bool = False) -> Union[Text, Mapping]: + def get_tree_traversal(cls, as_dict: bool = False) -> Union[StrType, Dict]: """Get a string describing the tree traversal for the diffsync object. Args: - as_dict: Whether or not to return as a dictionary + as_dict: Whether to return as a dictionary Returns: A string or dictionary representation of tree @@ -751,34 +758,34 @@ def get_tree_traversal(cls, as_dict: bool = False) -> Union[Text, Mapping]: return output_dict return tree_string(output_dict, cls.__name__) - def add(self, obj: DiffSyncModel): + def add(self, obj: DiffSyncModel) -> None: """Add a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to store + obj: Object to store Raises: ObjectAlreadyExists: if a different object with the same uid is already present. """ return self.store.add(obj=obj) - def update(self, obj: DiffSyncModel): + def update(self, obj: DiffSyncModel) -> None: """Update a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to store + obj: Object to store Raises: ObjectAlreadyExists: if a different object with the same uid is already present. """ return self.store.update(obj=obj) - def remove(self, obj: DiffSyncModel, remove_children: bool = False): + def remove(self, obj: DiffSyncModel, remove_children: bool = False) -> None: """Remove a DiffSyncModel object from the store. Args: - obj (DiffSyncModel): object to remove - remove_children (bool): If True, also recursively remove any children of this object + obj: object to remove + remove_children: If True, also recursively remove any children of this object Raises: ObjectNotFound: if the object is not present @@ -791,12 +798,12 @@ def get_or_instantiate( """Attempt to get the object with provided identifiers or instantiate it with provided identifiers and attrs. Args: - model (DiffSyncModel): The DiffSyncModel to get or create. - ids (Mapping): Identifiers for the DiffSyncModel to get or create with. - attrs (Mapping, optional): Attributes when creating an object if it doesn't exist. Defaults to None. + model: The DiffSyncModel to get or create. + ids: Identifiers for the DiffSyncModel to get or create with. + attrs: Attributes when creating an object if it doesn't exist. Defaults to None. Returns: - Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not. + Provides the existing or new object and whether it was created or not. """ return self.store.get_or_instantiate(model=model, ids=ids, attrs=attrs) @@ -815,12 +822,12 @@ def update_or_instantiate(self, model: Type[DiffSyncModel], ids: Dict, attrs: Di """Attempt to update an existing object with provided ids/attrs or instantiate it with provided identifiers and attrs. Args: - model (DiffSyncModel): The DiffSyncModel to update or create. - ids (Dict): Identifiers for the DiffSyncModel to update or create with. - attrs (Dict): Attributes when creating/updating an object if it doesn't exist. Pass in empty dict, if no specific attrs. + model: The DiffSyncModel to update or create. + ids: Identifiers for the DiffSyncModel to update or create with. + attrs: Attributes when creating/updating an object if it doesn't exist. Pass in empty dict, if no specific attrs. Returns: - Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not. + Provides the existing or new object and whether it was created or not. """ return self.store.update_or_instantiate(model=model, ids=ids, attrs=attrs) @@ -828,21 +835,21 @@ def update_or_add_model_instance(self, obj: DiffSyncModel) -> Tuple[DiffSyncMode """Attempt to update an existing object with provided obj ids/attrs or instantiate obj. Args: - instance: An instance of the DiffSyncModel to update or create. + obj: An instance of the DiffSyncModel to update or create. Returns: Provides the existing or new object and whether it was created or not. """ return self.store.update_or_add_model_instance(obj=obj) - def count(self, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"], None] = None): + def count(self, model: Union[StrType, "DiffSyncModel", Type["DiffSyncModel"], None] = None) -> int: """Count how many objects of one model type exist in the backend store. Args: - model (DiffSyncModel): The DiffSyncModel to check the number of elements. If not provided, default to all. + model: The DiffSyncModel to check the number of elements. If not provided, default to all. Returns: - Int: Number of elements of the model type + Number of elements of the model type """ return self.store.count(model=model) diff --git a/diffsync/diff.py b/diffsync/diff.py index afeea436..c85117a6 100644 --- a/diffsync/diff.py +++ b/diffsync/diff.py @@ -16,40 +16,44 @@ """ from functools import total_ordering -from typing import Any, Iterator, Iterable, Mapping, Optional, Text, Type +from typing import Any, Iterator, Optional, Type, List, Dict, Iterable from .exceptions import ObjectAlreadyExists from .utils import intersection, OrderedDefaultDict from .enum import DiffSyncActions +# This workaround is used because we are defining a method called `str` in our class definition, which therefore renders +# the builtin `str` type unusable. +StrType = str + class Diff: """Diff Object, designed to store multiple DiffElement object and organize them in a group.""" - def __init__(self): + def __init__(self) -> None: """Initialize a new, empty Diff object.""" - self.children = OrderedDefaultDict(dict) + self.children = OrderedDefaultDict[StrType, Dict[StrType, DiffElement]](dict) """DefaultDict for storing DiffElement objects. `self.children[group][unique_id] == DiffElement(...)` """ self.models_processed = 0 - def __len__(self): + def __len__(self) -> int: """Total number of DiffElements stored herein.""" total = 0 for child in self.get_children(): total += len(child) return total - def complete(self): + def complete(self) -> None: """Method to call when this Diff has been fully populated with data and is "complete". The default implementation does nothing, but a subclass could use this, for example, to save the completed Diff to a file or database record. """ - def add(self, element: "DiffElement"): + def add(self, element: "DiffElement") -> None: """Add a new DiffElement to the changeset of this Diff. Raises: @@ -61,15 +65,15 @@ def add(self, element: "DiffElement"): self.children[element.type][element.name] = element - def groups(self): + def groups(self) -> List[StrType]: """Get the list of all group keys in self.children.""" - return self.children.keys() + return list(self.children.keys()) def has_diffs(self) -> bool: """Indicate if at least one of the child elements contains some diff. Returns: - bool: True if at least one child element contains some diff + True if at least one child element contains some diff """ for group in self.groups(): for child in self.children[group].values(): @@ -96,7 +100,7 @@ def get_children(self) -> Iterator["DiffElement"]: yield from order_method(self.children[group]) @classmethod - def order_children_default(cls, children: Mapping) -> Iterator["DiffElement"]: + def order_children_default(cls, children: Dict[StrType, "DiffElement"]) -> Iterator["DiffElement"]: """Default method to an Iterator for children. Since children is already an OrderedDefaultDict, this method is not doing anything special. @@ -104,7 +108,7 @@ def order_children_default(cls, children: Mapping) -> Iterator["DiffElement"]: for child in children.values(): yield child - def summary(self) -> Mapping[Text, int]: + def summary(self) -> Dict[StrType, int]: """Build a dict summary of this Diff and its child DiffElements.""" summary = { DiffSyncActions.CREATE: 0, @@ -127,7 +131,7 @@ def summary(self) -> Mapping[Text, int]: ) return summary - def str(self, indent: int = 0): + def str(self, indent: int = 0) -> StrType: """Build a detailed string representation of this Diff and its child DiffElements.""" margin = " " * indent output = [] @@ -144,9 +148,9 @@ def str(self, indent: int = 0): result = "(no diffs)" return result - def dict(self) -> Mapping[Text, Mapping[Text, Mapping]]: + def dict(self) -> Dict[StrType, Dict[StrType, Dict]]: """Build a dictionary representation of this Diff.""" - result = OrderedDefaultDict(dict) + result = OrderedDefaultDict[str, Dict](dict) for child in self.get_children(): if child.has_diffs(include_children=True): result[child.type][child.name] = child.dict() @@ -159,11 +163,11 @@ class DiffElement: # pylint: disable=too-many-instance-attributes def __init__( self, - obj_type: Text, - name: Text, - keys: Mapping, - source_name: Text = "source", - dest_name: Text = "dest", + obj_type: StrType, + name: StrType, + keys: Dict, + source_name: StrType = "source", + dest_name: StrType = "dest", diff_class: Type[Diff] = Diff, ): # pylint: disable=too-many-arguments """Instantiate a DiffElement. @@ -177,10 +181,10 @@ def __init__( dest_name: Name of the destination DiffSync object diff_class: Diff or subclass thereof to use to calculate the diffs to use for synchronization """ - if not isinstance(obj_type, str): + if not isinstance(obj_type, StrType): raise ValueError(f"obj_type must be a string (not {type(obj_type)})") - if not isinstance(name, str): + if not isinstance(name, StrType): raise ValueError(f"name must be a string (not {type(name)})") self.type = obj_type @@ -189,18 +193,18 @@ def __init__( self.source_name = source_name self.dest_name = dest_name # Note: *_attrs == None if no target object exists; it'll be an empty dict if it exists but has no _attributes - self.source_attrs: Optional[Mapping] = None - self.dest_attrs: Optional[Mapping] = None + self.source_attrs: Optional[Dict] = None + self.dest_attrs: Optional[Dict] = None self.child_diff = diff_class() - def __lt__(self, other): + def __lt__(self, other: "DiffElement") -> bool: """Logical ordering of DiffElements. Other comparison methods (__gt__, __le__, __ge__, etc.) are created by our use of the @total_ordering decorator. """ return (self.type, self.name) < (other.type, other.name) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: """Logical equality of DiffElements. Other comparison methods (__gt__, __le__, __ge__, etc.) are created by our use of the @total_ordering decorator. @@ -216,14 +220,14 @@ def __eq__(self, other): # TODO also check that self.child_diff == other.child_diff, needs Diff to implement __eq__(). ) - def __str__(self): + def __str__(self) -> StrType: """Basic string representation of a DiffElement.""" return ( f'{self.type} "{self.name}" : {self.keys} : ' f"{self.source_name} → {self.dest_name} : {self.get_attrs_diffs()}" ) - def __len__(self): + def __len__(self) -> int: """Total number of DiffElements in this one, including itself.""" total = 1 # self for child in self.get_children(): @@ -231,11 +235,11 @@ def __len__(self): return total @property - def action(self) -> Optional[Text]: + def action(self) -> Optional[StrType]: """Action, if any, that should be taken to remediate the diffs described by this element. Returns: - str: DiffSyncActions ("create", "update", "delete", or None) + "create", "update", "delete", or None) """ if self.source_attrs is not None and self.dest_attrs is None: return DiffSyncActions.CREATE @@ -251,7 +255,7 @@ def action(self) -> Optional[Text]: return None # TODO: separate into set_source_attrs() and set_dest_attrs() methods, or just use direct property access instead? - def add_attrs(self, source: Optional[Mapping] = None, dest: Optional[Mapping] = None): + def add_attrs(self, source: Optional[Dict] = None, dest: Optional[Dict] = None) -> None: """Set additional attributes of a source and/or destination item that may result in diffs.""" # TODO: should source_attrs and dest_attrs be "write-once" properties, or is it OK to overwrite them once set? if source is not None: @@ -260,7 +264,7 @@ def add_attrs(self, source: Optional[Mapping] = None, dest: Optional[Mapping] = if dest is not None: self.dest_attrs = dest - def get_attrs_keys(self) -> Iterable[Text]: + def get_attrs_keys(self) -> Iterable[StrType]: """Get the list of shared attrs between source and dest, or the attrs of source or dest if only one is present. - If source_attrs is not set, return the keys of dest_attrs @@ -268,18 +272,18 @@ def get_attrs_keys(self) -> Iterable[Text]: - If both are defined, return the intersection of both keys """ if self.source_attrs is not None and self.dest_attrs is not None: - return intersection(self.dest_attrs.keys(), self.source_attrs.keys()) + return intersection(list(self.dest_attrs.keys()), list(self.source_attrs.keys())) if self.source_attrs is None and self.dest_attrs is not None: return self.dest_attrs.keys() if self.source_attrs is not None and self.dest_attrs is None: return self.source_attrs.keys() return [] - def get_attrs_diffs(self) -> Mapping[Text, Mapping[Text, Any]]: + def get_attrs_diffs(self) -> Dict[StrType, Dict[StrType, Any]]: """Get the dict of actual attribute diffs between source_attrs and dest_attrs. Returns: - dict: of the form `{"-": {key1: , key2: ...}, "+": {key1: , key2: ...}}`, + Dictionary of the form `{"-": {key1: , key2: ...}, "+": {key1: , key2: ...}}`, where the `"-"` or `"+"` dicts may be absent. """ if self.source_attrs is not None and self.dest_attrs is not None: @@ -301,13 +305,10 @@ def get_attrs_diffs(self) -> Mapping[Text, Mapping[Text, Any]]: return {"+": {key: self.source_attrs[key] for key in self.get_attrs_keys()}} return {} - def add_child(self, element: "DiffElement"): + def add_child(self, element: "DiffElement") -> None: """Attach a child object of type DiffElement. Childs are saved in a Diff object and are organized by type and name. - - Args: - element: DiffElement """ self.child_diff.add(element) @@ -336,7 +337,7 @@ def has_diffs(self, include_children: bool = True) -> bool: return False - def summary(self) -> Mapping[Text, int]: + def summary(self) -> Dict[StrType, int]: """Build a summary of this DiffElement and its children.""" summary = { DiffSyncActions.CREATE: 0, @@ -353,7 +354,7 @@ def summary(self) -> Mapping[Text, int]: summary[key] += child_summary[key] return summary - def str(self, indent: int = 0): + def str(self, indent: int = 0) -> StrType: """Build a detailed string representation of this DiffElement and its children.""" margin = " " * indent result = f"{margin}{self.type}: {self.name}" @@ -377,7 +378,7 @@ def str(self, indent: int = 0): result += " (no diffs)" return result - def dict(self) -> Mapping[Text, Mapping[Text, Any]]: + def dict(self) -> Dict[StrType, Dict[StrType, Any]]: """Build a dictionary representation of this DiffElement and its children.""" attrs_diffs = self.get_attrs_diffs() result = {} diff --git a/diffsync/enum.py b/diffsync/enum.py index 57179c97..cb56532f 100644 --- a/diffsync/enum.py +++ b/diffsync/enum.py @@ -47,6 +47,13 @@ class DiffSyncModelFlags(enum.Flag): If this flag is set, the model will not be deleted from the target/"to" DiffSync. """ + NATURAL_DELETION_ORDER = 0b10000 + """When deleting, delete children before instances of this this element. + + If this flag is set, the models children will be deleted from the target/"to" DiffSync before the models instances + themselves. + """ + SKIP_UNMATCHED_BOTH = SKIP_UNMATCHED_SRC | SKIP_UNMATCHED_DST diff --git a/diffsync/exceptions.py b/diffsync/exceptions.py index b604c74e..fd02c0c8 100644 --- a/diffsync/exceptions.py +++ b/diffsync/exceptions.py @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. """ +from typing import TYPE_CHECKING, Union, Any + +if TYPE_CHECKING: + from diffsync import DiffSyncModel + from diffsync.diff import DiffElement class ObjectCrudException(Exception): @@ -39,7 +44,7 @@ class ObjectStoreException(Exception): class ObjectAlreadyExists(ObjectStoreException): """Exception raised when trying to store a DiffSyncModel or DiffElement that is already being stored.""" - def __init__(self, message, existing_object, *args, **kwargs): + def __init__(self, message: str, existing_object: Union["DiffSyncModel", "DiffElement"], *args: Any, **kwargs: Any): """Add existing_object to the exception to provide user with existing object.""" self.existing_object = existing_object super().__init__(message, existing_object, *args, **kwargs) diff --git a/diffsync/helpers.py b/diffsync/helpers.py index e9404105..96882bae 100644 --- a/diffsync/helpers.py +++ b/diffsync/helpers.py @@ -15,7 +15,7 @@ limitations under the License. """ from collections.abc import Iterable as ABCIterable, Mapping as ABCMapping -from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Type, TYPE_CHECKING +from typing import Callable, List, Optional, Tuple, Type, TYPE_CHECKING, Dict, Iterable import structlog # type: ignore @@ -57,7 +57,7 @@ def __init__( # pylint: disable=too-many-arguments self.total_models = len(src_diffsync) + len(dst_diffsync) self.logger.debug(f"Diff calculation between these two datasets will involve {self.total_models} models") - def incr_models_processed(self, delta: int = 1): + def incr_models_processed(self, delta: int = 1) -> None: """Increment self.models_processed, then call self.callback if present.""" if delta: self.models_processed += delta @@ -136,7 +136,9 @@ def diff_object_list(self, src: List["DiffSyncModel"], dst: List["DiffSyncModel" return diff_elements @staticmethod - def validate_objects_for_diff(object_pairs: Iterable[Tuple[Optional["DiffSyncModel"], Optional["DiffSyncModel"]]]): + def validate_objects_for_diff( + object_pairs: Iterable[Tuple[Optional["DiffSyncModel"], Optional["DiffSyncModel"]]] + ) -> None: """Check whether all DiffSyncModels in the given dictionary are valid for comparison to one another. Helper method for `diff_object_list`. @@ -234,7 +236,7 @@ def diff_child_objects( diff_element: DiffElement, src_obj: Optional["DiffSyncModel"], dst_obj: Optional["DiffSyncModel"], - ): + ) -> DiffElement: """For all children of the given DiffSyncModel pair, diff recursively, adding diffs to the given diff_element. Helper method to `calculate_diffs`, usually doesn't need to be called directly. @@ -242,7 +244,7 @@ def diff_child_objects( These helper methods work in a recursive cycle: diff_object_list -> diff_object_pair -> diff_child_objects -> diff_object_list -> etc. """ - children_mapping: Mapping[str, str] + children_mapping: Dict[str, str] if src_obj and dst_obj: # Get the subset of child types common to both src_obj and dst_obj src_mapping = src_obj.get_children_mapping() @@ -308,7 +310,7 @@ def __init__( # pylint: disable=too-many-arguments self.model_class: Type["DiffSyncModel"] self.action: Optional[str] = None - def incr_elements_processed(self, delta: int = 1): + def incr_elements_processed(self, delta: int = 1) -> None: """Increment self.elements_processed, then call self.callback if present.""" if delta: self.elements_processed += delta @@ -319,7 +321,7 @@ def perform_sync(self) -> bool: """Perform data synchronization based on the provided diff. Returns: - bool: True if any changes were actually performed, else False. + True if any changes were actually performed, else False. """ changed = False self.base_logger.info("Beginning sync") @@ -350,11 +352,7 @@ def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSy attrs = diffs.get("+", {}) # Retrieve Source Object to get its flags - src_model: Optional["DiffSyncModel"] - try: - src_model = self.src_diffsync.get(self.model_class, ids) - except ObjectNotFound: - src_model = None + src_model = self.src_diffsync.get_or_none(self.model_class, ids) # Retrieve Dest (and primary) Object dst_model: Optional["DiffSyncModel"] @@ -364,6 +362,18 @@ def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSy except ObjectNotFound: dst_model = None + natural_deletion_order = False + skip_children = False + # Set up flag booleans + if dst_model: + natural_deletion_order = bool(dst_model.model_flags & DiffSyncModelFlags.NATURAL_DELETION_ORDER) + skip_children = bool(dst_model.model_flags & DiffSyncModelFlags.SKIP_CHILDREN_ON_DELETE) + + changed = False + if natural_deletion_order and self.action == DiffSyncActions.DELETE and not skip_children: + for child in element.get_children(): + changed |= self.sync_diff_element(child, parent_model=dst_model) + changed, modified_model = self.sync_model(src_model=src_model, dst_model=dst_model, ids=ids, attrs=attrs) dst_model = modified_model or dst_model @@ -371,7 +381,7 @@ def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSy self.logger.warning("No object resulted from sync, will not process child objects.") return changed - if self.action == DiffSyncActions.CREATE: # type: ignore + if self.action == DiffSyncActions.CREATE: if parent_model: parent_model.add_child(dst_model) self.dst_diffsync.add(dst_model) @@ -379,7 +389,6 @@ def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSy if parent_model: parent_model.remove_child(dst_model) - skip_children = bool(dst_model.model_flags & DiffSyncModelFlags.SKIP_CHILDREN_ON_DELETE) self.dst_diffsync.remove(dst_model, remove_children=skip_children) if skip_children: @@ -387,20 +396,21 @@ def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSy self.incr_elements_processed() - for child in element.get_children(): - changed |= self.sync_diff_element(child, parent_model=dst_model) + if not natural_deletion_order: + for child in element.get_children(): + changed |= self.sync_diff_element(child, parent_model=dst_model) return changed def sync_model( # pylint: disable=too-many-branches, unused-argument - self, src_model: Optional["DiffSyncModel"], dst_model: Optional["DiffSyncModel"], ids: Mapping, attrs: Mapping + self, src_model: Optional["DiffSyncModel"], dst_model: Optional["DiffSyncModel"], ids: Dict, attrs: Dict ) -> Tuple[bool, Optional["DiffSyncModel"]]: """Create/update/delete the current DiffSyncModel with current ids/attrs, and update self.status and self.message. Helper method to `sync_diff_element`. Returns: - tuple: (changed, model) where model may be None if an error occurred + (changed, model) where model may be None if an error occurred """ if self.action is None: status = DiffSyncStatus.SUCCESS @@ -443,7 +453,7 @@ def sync_model( # pylint: disable=too-many-branches, unused-argument return (True, dst_model) - def log_sync_status(self, action: Optional[str], status: DiffSyncStatus, message: str): + def log_sync_status(self, action: Optional[str], status: DiffSyncStatus, message: str) -> None: """Log the current sync status at the appropriate verbosity with appropriate context. Helper method to `sync_diff_element`/`sync_model`. diff --git a/diffsync/logging.py b/diffsync/logging.py index f96cef4c..904d196e 100644 --- a/diffsync/logging.py +++ b/diffsync/logging.py @@ -22,7 +22,7 @@ from packaging import version -def enable_console_logging(verbosity=0): +def enable_console_logging(verbosity: int = 0) -> None: """Enable formatted logging to console with the specified verbosity. See https://www.structlog.org/en/stable/development.html as a reference @@ -52,7 +52,7 @@ def enable_console_logging(verbosity=0): processors.append(structlog.dev.ConsoleRenderer()) structlog.configure( - processors=processors, + processors=processors, # type: ignore[arg-type] context_class=dict, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, @@ -60,7 +60,7 @@ def enable_console_logging(verbosity=0): ) -def _structlog_exception_formatter_required(): +def _structlog_exception_formatter_required() -> bool: """Determine if structlog exception formatter is needed. Return True if structlog exception formatter should be loaded diff --git a/diffsync/store/__init__.py b/diffsync/store/__init__.py index 0b750594..77f1a9f4 100644 --- a/diffsync/store/__init__.py +++ b/diffsync/store/__init__.py @@ -1,5 +1,5 @@ """BaseStore module.""" -from typing import Dict, List, Mapping, Text, Tuple, Type, Union, TYPE_CHECKING, Optional, Set +from typing import Dict, List, Tuple, Type, Union, TYPE_CHECKING, Optional, Set, Any import structlog # type: ignore from diffsync.exceptions import ObjectNotFound @@ -13,14 +13,18 @@ class BaseStore: """Reference store to be implemented in different backends.""" def __init__( - self, *args, diffsync: Optional["DiffSync"] = None, name: str = "", **kwargs # pylint: disable=unused-argument + self, # pylint: disable=unused-argument + *args: Any, # pylint: disable=unused-argument + diffsync: Optional["DiffSync"] = None, + name: str = "", + **kwargs: Any, # pylint: disable=unused-argument ) -> None: """Init method for BaseStore.""" self.diffsync = diffsync self.name = name or self.__class__.__name__ self._log = structlog.get_logger().new(store=self) - def __str__(self): + def __str__(self) -> str: """Render store name.""" return self.name @@ -28,12 +32,12 @@ def get_all_model_names(self) -> Set[str]: """Get all the model names stored. Return: - Set[str]: Set of all the model names. + Set of all the model names. """ raise NotImplementedError def get( - self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[Text, Mapping] + self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[str, Dict] ) -> "DiffSyncModel": """Get one object from the data store based on its unique id. @@ -47,19 +51,19 @@ def get( """ raise NotImplementedError - def get_all(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]: + def get_all(self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]: """Get all objects of a given type. Args: model: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve Returns: - List[DiffSyncModel]: List of Object + List of Object """ raise NotImplementedError def get_by_uids( - self, *, uids: List[Text], model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]] + self, *, uids: List[str], model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]] ) -> List["DiffSyncModel"]: """Get multiple objects from the store by their unique IDs/Keys and type. @@ -72,16 +76,16 @@ def get_by_uids( """ raise NotImplementedError - def remove_item(self, modelname: str, uid: str): + def remove_item(self, modelname: str, uid: str) -> None: """Remove one item from store.""" raise NotImplementedError - def remove(self, *, obj: "DiffSyncModel", remove_children: bool = False): + def remove(self, *, obj: "DiffSyncModel", remove_children: bool = False) -> None: """Remove a DiffSyncModel object from the store. Args: - obj (DiffSyncModel): object to remove - remove_children (bool): If True, also recursively remove any children of this object + obj: object to remove + remove_children: If True, also recursively remove any children of this object Raises: ObjectNotFound: if the object is not present @@ -110,26 +114,26 @@ def remove(self, *, obj: "DiffSyncModel", remove_children: bool = False): parent_id=uid, ) - def add(self, *, obj: "DiffSyncModel"): + def add(self, *, obj: "DiffSyncModel") -> None: """Add a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to store + obj: Object to store Raises: ObjectAlreadyExists: if a different object with the same uid is already present. """ raise NotImplementedError - def update(self, *, obj: "DiffSyncModel"): + def update(self, *, obj: "DiffSyncModel") -> None: """Update a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to update + obj: Object to update """ raise NotImplementedError - def count(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"], None] = None) -> int: + def count(self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"], None] = None) -> int: """Returns the number of elements of a specific model, or all elements in the store if not specified.""" raise NotImplementedError @@ -139,12 +143,12 @@ def get_or_instantiate( """Attempt to get the object with provided identifiers or instantiate it with provided identifiers and attrs. Args: - model (DiffSyncModel): The DiffSyncModel to get or create. - ids (Mapping): Identifiers for the DiffSyncModel to get or create with. - attrs (Mapping, optional): Attributes when creating an object if it doesn't exist. Defaults to None. + model: The DiffSyncModel to get or create. + ids: Identifiers for the DiffSyncModel to get or create with. + attrs: Attributes when creating an object if it doesn't exist. Defaults to None. Returns: - Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not. + Provides the existing or new object and whether it was created or not. """ created = False try: @@ -183,12 +187,12 @@ def update_or_instantiate( """Attempt to update an existing object with provided ids/attrs or instantiate it with provided identifiers and attrs. Args: - model (DiffSyncModel): The DiffSyncModel to get or create. - ids (Dict): Identifiers for the DiffSyncModel to get or create with. - attrs (Dict): Attributes when creating/updating an object if it doesn't exist. Pass in empty dict, if no specific attrs. + model: The DiffSyncModel to get or create. + ids: Identifiers for the DiffSyncModel to get or create with. + attrs: Attributes when creating/updating an object if it doesn't exist. Pass in empty dict, if no specific attrs. Returns: - Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not. + Provides the existing or new object and whether it was created or not. """ created = False try: @@ -210,7 +214,7 @@ def update_or_add_model_instance(self, obj: "DiffSyncModel") -> Tuple["DiffSyncM """Attempt to update an existing object with provided ids/attrs or instantiate obj. Args: - instance: An instance of the DiffSyncModel to update or create. + obj: An instance of the DiffSyncModel to update or create. Returns: Provides the existing or new object and whether it was added or not. @@ -234,7 +238,7 @@ def update_or_add_model_instance(self, obj: "DiffSyncModel") -> Tuple["DiffSyncM return obj, added def _get_object_class_and_model( - self, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]] + self, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]] ) -> Tuple[Union["DiffSyncModel", Type["DiffSyncModel"], None], str]: """Get object class and model name for a model.""" if isinstance(model, str): @@ -250,9 +254,9 @@ def _get_object_class_and_model( @staticmethod def _get_uid( - model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]], + model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]], object_class: Union["DiffSyncModel", Type["DiffSyncModel"], None], - identifier: Union[Text, Mapping], + identifier: Union[str, Dict], ) -> str: """Get the related uid for a model and an identifier.""" if isinstance(identifier, str): diff --git a/diffsync/store/local.py b/diffsync/store/local.py index eaf6fe5a..82bb69ba 100644 --- a/diffsync/store/local.py +++ b/diffsync/store/local.py @@ -1,7 +1,7 @@ """LocalStore module.""" from collections import defaultdict -from typing import List, Mapping, Text, Type, Union, TYPE_CHECKING, Dict, Set +from typing import List, Type, Union, TYPE_CHECKING, Dict, Set, Any from diffsync.exceptions import ObjectNotFound, ObjectAlreadyExists from diffsync.store import BaseStore @@ -14,7 +14,7 @@ class LocalStore(BaseStore): """LocalStore class.""" - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args: Any, **kwargs: Any) -> None: """Init method for LocalStore.""" super().__init__(*args, **kwargs) @@ -24,12 +24,12 @@ def get_all_model_names(self) -> Set[str]: """Get all the model names stored. Return: - Set[str]: Set of all the model names. + Set of all the model names. """ return set(self._data.keys()) def get( - self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[Text, Mapping] + self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[str, Dict] ) -> "DiffSyncModel": """Get one object from the data store based on its unique id. @@ -49,14 +49,14 @@ def get( raise ObjectNotFound(f"{modelname} {uid} not present in {str(self)}") return self._data[modelname][uid] - def get_all(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]: + def get_all(self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]: """Get all objects of a given type. Args: model: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve Returns: - List[DiffSyncModel]: List of Object + List of Object """ if isinstance(model, str): modelname = model @@ -66,7 +66,7 @@ def get_all(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]) return list(self._data[modelname].values()) def get_by_uids( - self, *, uids: List[Text], model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]] + self, *, uids: List[str], model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]] ) -> List["DiffSyncModel"]: """Get multiple objects from the store by their unique IDs/Keys and type. @@ -89,11 +89,11 @@ def get_by_uids( results.append(self._data[modelname][uid]) return results - def add(self, *, obj: "DiffSyncModel"): + def add(self, *, obj: "DiffSyncModel") -> None: """Add a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to store + obj: Object to store Raises: ObjectAlreadyExists: if a different object with the same uid is already present. @@ -113,11 +113,11 @@ def add(self, *, obj: "DiffSyncModel"): self._data[modelname][uid] = obj - def update(self, *, obj: "DiffSyncModel"): + def update(self, *, obj: "DiffSyncModel") -> None: """Update a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to update + obj: Object to update """ modelname = obj.get_type() uid = obj.get_unique_id() @@ -128,13 +128,13 @@ def update(self, *, obj: "DiffSyncModel"): self._data[modelname][uid] = obj - def remove_item(self, modelname: str, uid: str): + def remove_item(self, modelname: str, uid: str) -> None: """Remove one item from store.""" if uid not in self._data[modelname]: raise ObjectNotFound(f"{modelname} {uid} not present in {str(self)}") del self._data[modelname][uid] - def count(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"], None] = None) -> int: + def count(self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"], None] = None) -> int: """Returns the number of elements of a specific model, or all elements in the store if unspecified.""" if not model: return sum(len(entries) for entries in self._data.values()) diff --git a/diffsync/store/redis.py b/diffsync/store/redis.py index 83ca68cb..927dbf5e 100644 --- a/diffsync/store/redis.py +++ b/diffsync/store/redis.py @@ -2,7 +2,7 @@ import copy import uuid from pickle import loads, dumps # nosec -from typing import List, Mapping, Text, Type, Union, TYPE_CHECKING, Set +from typing import List, Type, Union, TYPE_CHECKING, Set, Any, Optional, Dict try: from redis import Redis @@ -23,7 +23,16 @@ class RedisStore(BaseStore): """RedisStore class.""" - def __init__(self, *args, store_id=None, host=None, port=6379, url=None, db=0, **kwargs): + def __init__( + self, + *args: Any, + store_id: Optional[str] = None, + host: Optional[str] = None, + port: int = 6379, + url: Optional[str] = None, + db: int = 0, + **kwargs: Any, + ): """Init method for RedisStore.""" super().__init__(*args, **kwargs) @@ -33,11 +42,13 @@ def __init__(self, *args, store_id=None, host=None, port=6379, url=None, db=0, * try: if url: self._store = Redis.from_url(url, db=db) - else: + elif host: self._store = Redis(host=host, port=port, db=db) + else: + raise RedisConnectionError("Neither 'host' nor 'url' were specified.") if not self._store.ping(): - raise RedisConnectionError + raise RedisConnectionError() except RedisConnectionError: raise ObjectStoreException("Redis store is unavailable.") from RedisConnectionError @@ -45,24 +56,24 @@ def __init__(self, *args, store_id=None, host=None, port=6379, url=None, db=0, * self._store_label = f"{REDIS_DIFFSYNC_ROOT_LABEL}:{self._store_id}" - def __str__(self): + def __str__(self) -> str: """Render store name.""" return f"{self.name} ({self._store_id})" - def _get_object_from_redis_key(self, key): + def _get_object_from_redis_key(self, key: str) -> "DiffSyncModel": """Get the object from Redis key.""" - try: - obj_result = loads(self._store.get(key)) # nosec + pickled_object = self._store.get(key) + if pickled_object: + obj_result = loads(pickled_object) # nosec obj_result.diffsync = self.diffsync return obj_result - except TypeError as exc: - raise ObjectNotFound(f"{key} not present in Cache") from exc + raise ObjectNotFound(f"{key} not present in Cache") def get_all_model_names(self) -> Set[str]: """Get all the model names stored. Return: - Set[str]: Set of all the model names. + Set of all the model names. """ # TODO: optimize it all_model_names = set() @@ -74,11 +85,11 @@ def get_all_model_names(self) -> Set[str]: return all_model_names - def _get_key_for_object(self, modelname, uid): + def _get_key_for_object(self, modelname: str, uid: str) -> str: return f"{self._store_label}:{modelname}:{uid}" def get( - self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[Text, Mapping] + self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[str, Dict] ) -> "DiffSyncModel": """Get one object from the data store based on its unique id. @@ -96,28 +107,28 @@ def get( return self._get_object_from_redis_key(self._get_key_for_object(modelname, uid)) - def get_all(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]: + def get_all(self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]: """Get all objects of a given type. Args: model: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve Returns: - List[DiffSyncModel]: List of Object + List of Object """ if isinstance(model, str): modelname = model else: modelname = model.get_type() - results = [] + results: List["DiffSyncModel"] = [] for key in self._store.scan_iter(f"{self._store_label}:{modelname}:*"): - results.append(self._get_object_from_redis_key(key)) + results.append(self._get_object_from_redis_key(key)) # type: ignore[arg-type] return results def get_by_uids( - self, *, uids: List[Text], model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]] + self, *, uids: List[str], model: Union[str, "DiffSyncModel", Type["DiffSyncModel"]] ) -> List["DiffSyncModel"]: """Get multiple objects from the store by their unique IDs/Keys and type. @@ -139,11 +150,11 @@ def get_by_uids( return results - def add(self, *, obj: "DiffSyncModel"): + def add(self, *, obj: "DiffSyncModel") -> None: """Add a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to store + obj: Object to store Raises: ObjectAlreadyExists: if a different object with the same uid is already present. @@ -171,11 +182,11 @@ def add(self, *, obj: "DiffSyncModel"): self._store.set(object_key, dumps(obj_copy)) - def update(self, *, obj: "DiffSyncModel"): + def update(self, *, obj: "DiffSyncModel") -> None: """Update a DiffSyncModel object to the store. Args: - obj (DiffSyncModel): Object to update + obj: Object to update """ modelname = obj.get_type() uid = obj.get_unique_id() @@ -186,7 +197,7 @@ def update(self, *, obj: "DiffSyncModel"): self._store.set(object_key, dumps(obj_copy)) - def remove_item(self, modelname: str, uid: str): + def remove_item(self, modelname: str, uid: str) -> None: """Remove one item from store.""" object_key = self._get_key_for_object(modelname, uid) @@ -195,7 +206,7 @@ def remove_item(self, modelname: str, uid: str): self._store.delete(object_key) - def count(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"], None] = None) -> int: + def count(self, *, model: Union[str, "DiffSyncModel", Type["DiffSyncModel"], None] = None) -> int: """Returns the number of elements of a specific model, or all elements in the store if unspecified.""" search_pattern = f"{self._store_label}:*" if model is not None: diff --git a/diffsync/utils.py b/diffsync/utils.py index 94b0aa94..07462ff7 100644 --- a/diffsync/utils.py +++ b/diffsync/utils.py @@ -14,36 +14,41 @@ See the License for the specific language governing permissions and limitations under the License. """ - from collections import OrderedDict -from typing import Iterator, List, Dict, Optional +from typing import Iterator, List, Dict, Optional, TypeVar, Callable, Generic SPACE = " " BRANCH = "│ " TEE = "├── " LAST = "└── " +T = TypeVar("T") +K = TypeVar("K") +V = TypeVar("V") + -def intersection(lst1, lst2) -> List: +def intersection(lst1: List[T], lst2: List[T]) -> List[T]: """Calculate the intersection of two lists, with ordering based on the first list.""" lst3 = [value for value in lst1 if value in lst2] return lst3 -def symmetric_difference(lst1, lst2) -> List: +def symmetric_difference(lst1: List[T], lst2: List[T]) -> List[T]: """Calculate the symmetric difference of two lists.""" - return sorted(set(lst1) ^ set(lst2)) + # Type hinting this correct is kind of hard to do. _Probably_ the solution is using typing.Protocol to ensure the + # set members support the ^ operator / set.symmetric_difference, but I decided this wasn't worth figuring out. + return sorted(set(lst1) ^ set(lst2)) # type: ignore[type-var] -class OrderedDefaultDict(OrderedDict): +class OrderedDefaultDict(OrderedDict, Generic[K, V]): """A combination of collections.OrderedDict and collections.DefaultDict behavior.""" - def __init__(self, dict_type): + def __init__(self, dict_type: Callable[[], V]) -> None: """Create a new OrderedDefaultDict.""" self.factory = dict_type super().__init__(self) - def __missing__(self, key): + def __missing__(self, key: K) -> V: """When trying to access a nonexistent key, initialize the key value based on the internal factory.""" self[key] = value = self.factory() return value @@ -71,7 +76,7 @@ def tree_string(data: Dict, root: str) -> str: return "\n".join([root, *_tree(data)]) -def set_key(data: Dict, keys: List): +def set_key(data: Dict, keys: List) -> None: """Set a nested dictionary key given a list of keys.""" current_level = data for key in keys: diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index 05561453..542703ee 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -53,13 +53,14 @@ class MyAdapter(DiffSync): ### Supported Model Flags -| Name | Description | Binary Value | -|---|---|---| +| Name | Description | Binary Value | +|---|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---| | IGNORE | Do not render diffs containing this model; do not make any changes to this model when synchronizing. Can be used to indicate a model instance that exists but should not be changed by DiffSync. | 0b1 | -| SKIP_CHILDREN_ON_DELETE | When deleting this model, do not recursively delete its children. Can be used for the case where deletion of a model results in the automatic deletion of all its children. | 0b10 | -| SKIP_UNMATCHED_SRC | Ignore the model if it only exists in the source/"from" DiffSync when determining diffs and syncing. If this flag is set, no new model will be created in the target/"to" DiffSync. | 0b100 | -| SKIP_UNMATCHED_DST | Ignore the model if it only exists in the target/"to" DiffSync when determining diffs and syncing. If this flag is set, the model will not be deleted from the target/"to" DiffSync. | 0b1000 | -| SKIP_UNMATCHED_BOTH | Convenience value combining both SKIP_UNMATCHED_SRC and SKIP_UNMATCHED_DST into a single flag | 0b1100 | +| SKIP_CHILDREN_ON_DELETE | When deleting this model, do not recursively delete its children. Can be used for the case where deletion of a model results in the automatic deletion of all its children. | 0b10 | +| SKIP_UNMATCHED_SRC | Ignore the model if it only exists in the source/"from" DiffSync when determining diffs and syncing. If this flag is set, no new model will be created in the target/"to" DiffSync. | 0b100 | +| SKIP_UNMATCHED_DST | Ignore the model if it only exists in the target/"to" DiffSync when determining diffs and syncing. If this flag is set, the model will not be deleted from the target/"to" DiffSync. | 0b1000 | +| SKIP_UNMATCHED_BOTH | Convenience value combining both SKIP_UNMATCHED_SRC and SKIP_UNMATCHED_DST into a single flag | 0b1100 | +| NATURAL_DELETION_ORDER | When deleting, delete children before instances of this model. | 0b10000 | ## Working with flags diff --git a/poetry.lock b/poetry.lock index 5128cb6b..dc0752c7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "alabaster" version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -16,7 +15,6 @@ files = [ name = "astroid" version = "2.11.7" description = "An abstract syntax tree for Python with inference support." -category = "dev" optional = false python-versions = ">=3.6.2" files = [ @@ -35,7 +33,6 @@ wrapt = ">=1.11,<2" name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -50,7 +47,6 @@ typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} name = "attrs" version = "22.2.0" description = "Classes Without Boilerplate" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -69,7 +65,6 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy name = "babel" version = "2.12.1" description = "Internationalization utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -84,7 +79,6 @@ pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} name = "bandit" version = "1.7.4" description = "Security oriented static analyser for python code." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -107,7 +101,6 @@ yaml = ["PyYAML"] name = "black" version = "23.1.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -158,7 +151,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -170,7 +162,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "dev" optional = false python-versions = "*" files = [ @@ -247,7 +238,6 @@ pycparser = "*" name = "charset-normalizer" version = "3.0.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "dev" optional = false python-versions = "*" files = [ @@ -345,7 +335,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -361,7 +350,6 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -373,7 +361,6 @@ files = [ name = "coverage" version = "7.2.1" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -440,7 +427,6 @@ toml = ["tomli"] name = "cryptography" version = "39.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -486,7 +472,6 @@ tox = ["tox"] name = "dill" version = "0.3.6" description = "serialize all of python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -501,7 +486,6 @@ graph = ["objgraph (>=1.7.2)"] name = "docutils" version = "0.19" description = "Docutils -- Python Documentation Utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -513,7 +497,6 @@ files = [ name = "exceptiongroup" version = "1.1.0" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -528,7 +511,6 @@ test = ["pytest (>=6)"] name = "flake8" version = "5.0.4" description = "the modular source code checker: pep8 pyflakes and co" -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -546,7 +528,6 @@ pyflakes = ">=2.5.0,<2.6.0" name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -561,7 +542,6 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -577,7 +557,6 @@ typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\"" name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -589,7 +568,6 @@ files = [ name = "imagesize" version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -601,7 +579,6 @@ files = [ name = "importlib-metadata" version = "4.2.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -621,7 +598,6 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pep517", name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -631,21 +607,19 @@ files = [ [[package]] name = "invoke" -version = "2.0.0" +version = "2.1.3" description = "Pythonic task execution" -category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "invoke-2.0.0-py3-none-any.whl", hash = "sha256:a860582bcf7a4b336fe18ef53937f0f28cec1c0053ffa767c2fcf7ba0b850f59"}, - {file = "invoke-2.0.0.tar.gz", hash = "sha256:7ab5dd9cd76b787d560a78b1a9810d252367ab595985c50612702be21d671dd7"}, + {file = "invoke-2.1.3-py3-none-any.whl", hash = "sha256:51e86a08d964160e01c44eccd22f50b25842bd96a9c63c11177032594cb86740"}, + {file = "invoke-2.1.3.tar.gz", hash = "sha256:a3b15d52d50bbabd851b8a39582c772180b614000fa1612b4d92484d54d38c6b"}, ] [[package]] name = "isort" version = "5.11.5" description = "A Python utility / library to sort Python imports." -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -663,7 +637,6 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -681,7 +654,6 @@ i18n = ["Babel (>=2.7)"] name = "lazy-object-proxy" version = "1.9.0" description = "A fast and thorough lazy object proxy." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -727,7 +699,6 @@ files = [ name = "m2r2" version = "0.3.3.post2" description = "Markdown and reStructuredText in a single file." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -743,7 +714,6 @@ mistune = "0.8.4" name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -803,7 +773,6 @@ files = [ name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -815,7 +784,6 @@ files = [ name = "mirakuru" version = "2.5.1" description = "Process executor (not only) for tests." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -830,7 +798,6 @@ psutil = {version = ">=4.0.0", markers = "sys_platform != \"cygwin\""} name = "mistune" version = "0.8.4" description = "The fastest markdown parser in pure Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -840,45 +807,44 @@ files = [ [[package]] name = "mypy" -version = "1.0.1" +version = "1.4.1" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "mypy-1.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:71a808334d3f41ef011faa5a5cd8153606df5fc0b56de5b2e89566c8093a0c9a"}, - {file = "mypy-1.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:920169f0184215eef19294fa86ea49ffd4635dedfdea2b57e45cb4ee85d5ccaf"}, - {file = "mypy-1.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27a0f74a298769d9fdc8498fcb4f2beb86f0564bcdb1a37b58cbbe78e55cf8c0"}, - {file = "mypy-1.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:65b122a993d9c81ea0bfde7689b3365318a88bde952e4dfa1b3a8b4ac05d168b"}, - {file = "mypy-1.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5deb252fd42a77add936b463033a59b8e48eb2eaec2976d76b6878d031933fe4"}, - {file = "mypy-1.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2013226d17f20468f34feddd6aae4635a55f79626549099354ce641bc7d40262"}, - {file = "mypy-1.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:48525aec92b47baed9b3380371ab8ab6e63a5aab317347dfe9e55e02aaad22e8"}, - {file = "mypy-1.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c96b8a0c019fe29040d520d9257d8c8f122a7343a8307bf8d6d4a43f5c5bfcc8"}, - {file = "mypy-1.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:448de661536d270ce04f2d7dddaa49b2fdba6e3bd8a83212164d4174ff43aa65"}, - {file = "mypy-1.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:d42a98e76070a365a1d1c220fcac8aa4ada12ae0db679cb4d910fabefc88b994"}, - {file = "mypy-1.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e64f48c6176e243ad015e995de05af7f22bbe370dbb5b32bd6988438ec873919"}, - {file = "mypy-1.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fdd63e4f50e3538617887e9aee91855368d9fc1dea30da743837b0df7373bc4"}, - {file = "mypy-1.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbeb24514c4acbc78d205f85dd0e800f34062efcc1f4a4857c57e4b4b8712bff"}, - {file = "mypy-1.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a2948c40a7dd46c1c33765718936669dc1f628f134013b02ff5ac6c7ef6942bf"}, - {file = "mypy-1.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5bc8d6bd3b274dd3846597855d96d38d947aedba18776aa998a8d46fabdaed76"}, - {file = "mypy-1.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:17455cda53eeee0a4adb6371a21dd3dbf465897de82843751cf822605d152c8c"}, - {file = "mypy-1.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e831662208055b006eef68392a768ff83596035ffd6d846786578ba1714ba8f6"}, - {file = "mypy-1.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e60d0b09f62ae97a94605c3f73fd952395286cf3e3b9e7b97f60b01ddfbbda88"}, - {file = "mypy-1.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:0af4f0e20706aadf4e6f8f8dc5ab739089146b83fd53cb4a7e0e850ef3de0bb6"}, - {file = "mypy-1.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:24189f23dc66f83b839bd1cce2dfc356020dfc9a8bae03978477b15be61b062e"}, - {file = "mypy-1.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93a85495fb13dc484251b4c1fd7a5ac370cd0d812bbfc3b39c1bafefe95275d5"}, - {file = "mypy-1.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f546ac34093c6ce33f6278f7c88f0f147a4849386d3bf3ae193702f4fe31407"}, - {file = "mypy-1.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c6c2ccb7af7154673c591189c3687b013122c5a891bb5651eca3db8e6c6c55bd"}, - {file = "mypy-1.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:15b5a824b58c7c822c51bc66308e759243c32631896743f030daf449fe3677f3"}, - {file = "mypy-1.0.1-py3-none-any.whl", hash = "sha256:eda5c8b9949ed411ff752b9a01adda31afe7eae1e53e946dbdf9db23865e66c4"}, - {file = "mypy-1.0.1.tar.gz", hash = "sha256:28cea5a6392bb43d266782983b5a4216c25544cd7d80be681a155ddcdafd152d"}, + {file = "mypy-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:566e72b0cd6598503e48ea610e0052d1b8168e60a46e0bfd34b3acf2d57f96a8"}, + {file = "mypy-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca637024ca67ab24a7fd6f65d280572c3794665eaf5edcc7e90a866544076878"}, + {file = "mypy-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dde1d180cd84f0624c5dcaaa89c89775550a675aff96b5848de78fb11adabcd"}, + {file = "mypy-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8c4d8e89aa7de683e2056a581ce63c46a0c41e31bd2b6d34144e2c80f5ea53dc"}, + {file = "mypy-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:bfdca17c36ae01a21274a3c387a63aa1aafe72bff976522886869ef131b937f1"}, + {file = "mypy-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7549fbf655e5825d787bbc9ecf6028731973f78088fbca3a1f4145c39ef09462"}, + {file = "mypy-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:98324ec3ecf12296e6422939e54763faedbfcc502ea4a4c38502082711867258"}, + {file = "mypy-1.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141dedfdbfe8a04142881ff30ce6e6653c9685b354876b12e4fe6c78598b45e2"}, + {file = "mypy-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8207b7105829eca6f3d774f64a904190bb2231de91b8b186d21ffd98005f14a7"}, + {file = "mypy-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:16f0db5b641ba159eff72cff08edc3875f2b62b2fa2bc24f68c1e7a4e8232d01"}, + {file = "mypy-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:470c969bb3f9a9efcedbadcd19a74ffb34a25f8e6b0e02dae7c0e71f8372f97b"}, + {file = "mypy-1.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5952d2d18b79f7dc25e62e014fe5a23eb1a3d2bc66318df8988a01b1a037c5b"}, + {file = "mypy-1.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:190b6bab0302cec4e9e6767d3eb66085aef2a1cc98fe04936d8a42ed2ba77bb7"}, + {file = "mypy-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9d40652cc4fe33871ad3338581dca3297ff5f2213d0df345bcfbde5162abf0c9"}, + {file = "mypy-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01fd2e9f85622d981fd9063bfaef1aed6e336eaacca00892cd2d82801ab7c042"}, + {file = "mypy-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2460a58faeea905aeb1b9b36f5065f2dc9a9c6e4c992a6499a2360c6c74ceca3"}, + {file = "mypy-1.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2746d69a8196698146a3dbe29104f9eb6a2a4d8a27878d92169a6c0b74435b6"}, + {file = "mypy-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ae704dcfaa180ff7c4cfbad23e74321a2b774f92ca77fd94ce1049175a21c97f"}, + {file = "mypy-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:43d24f6437925ce50139a310a64b2ab048cb2d3694c84c71c3f2a1626d8101dc"}, + {file = "mypy-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c482e1246726616088532b5e964e39765b6d1520791348e6c9dc3af25b233828"}, + {file = "mypy-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43b592511672017f5b1a483527fd2684347fdffc041c9ef53428c8dc530f79a3"}, + {file = "mypy-1.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34a9239d5b3502c17f07fd7c0b2ae6b7dd7d7f6af35fbb5072c6208e76295816"}, + {file = "mypy-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5703097c4936bbb9e9bce41478c8d08edd2865e177dc4c52be759f81ee4dd26c"}, + {file = "mypy-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e02d700ec8d9b1859790c0475df4e4092c7bf3272a4fd2c9f33d87fac4427b8f"}, + {file = "mypy-1.4.1-py3-none-any.whl", hash = "sha256:45d32cec14e7b97af848bddd97d85ea4f0db4d5a149ed9676caa4eb2f7402bb4"}, + {file = "mypy-1.4.1.tar.gz", hash = "sha256:9bbcd9ab8ea1f2e1c8031c21445b511442cc45c89951e49bbf852cbb70755b1b"}, ] [package.dependencies] -mypy-extensions = ">=0.4.3" +mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.10" +typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] @@ -890,7 +856,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -902,7 +867,6 @@ files = [ name = "packaging" version = "23.0" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -914,7 +878,6 @@ files = [ name = "pathspec" version = "0.11.0" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -926,7 +889,6 @@ files = [ name = "pbr" version = "5.11.1" description = "Python Build Reasonableness" -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -938,7 +900,6 @@ files = [ name = "platformdirs" version = "3.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -957,7 +918,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytes name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -976,7 +936,6 @@ testing = ["pytest", "pytest-benchmark"] name = "port-for" version = "0.6.3" description = "Utility that helps with local TCP ports management. It can find an unused TCP localhost port and remember the association." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -988,7 +947,6 @@ files = [ name = "psutil" version = "5.9.4" description = "Cross-platform lib for process and system monitoring in Python." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1015,7 +973,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "pycodestyle" version = "2.9.1" description = "Python style guide checker" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1027,7 +984,6 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1039,7 +995,6 @@ files = [ name = "pydantic" version = "1.10.5" description = "Data validation and settings management using python type hints" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1092,7 +1047,6 @@ email = ["email-validator (>=1.0.3)"] name = "pydocstyle" version = "6.3.0" description = "Python docstring style checker" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1111,7 +1065,6 @@ toml = ["tomli (>=1.2.3)"] name = "pyflakes" version = "2.5.0" description = "passive checker of Python programs" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1123,7 +1076,6 @@ files = [ name = "pygments" version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1138,7 +1090,6 @@ plugins = ["importlib-metadata"] name = "pylint" version = "2.13.9" description = "python code static checker" -category = "dev" optional = false python-versions = ">=3.6.2" files = [ @@ -1163,7 +1114,6 @@ testutil = ["gitpython (>3)"] name = "pytest" version = "7.2.1" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1188,7 +1138,6 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2. name = "pytest-cov" version = "4.0.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1207,7 +1156,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-redis" version = "2.4.0" description = "Redis fixtures and fixture factories for Pytest." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1228,7 +1176,6 @@ tests = ["mock", "pytest-cov", "pytest-xdist"] name = "pytest-structlog" version = "0.6" description = "Structured logging assertions" -category = "dev" optional = false python-versions = "*" files = [ @@ -1244,7 +1191,6 @@ structlog = "*" name = "pytz" version = "2022.7.1" description = "World timezone definitions, modern and historical" -category = "dev" optional = false python-versions = "*" files = [ @@ -1256,7 +1202,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1306,7 +1251,6 @@ files = [ name = "redis" version = "4.5.1" description = "Python client for Redis database and key-value store" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1327,7 +1271,6 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" name = "requests" version = "2.28.2" description = "Python HTTP for Humans." -category = "dev" optional = false python-versions = ">=3.7, <4" files = [ @@ -1349,7 +1292,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "setuptools" version = "67.4.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1366,7 +1308,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1378,7 +1319,6 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -category = "dev" optional = false python-versions = "*" files = [ @@ -1390,7 +1330,6 @@ files = [ name = "sphinx" version = "3.5.3" description = "Python documentation generator" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1426,7 +1365,6 @@ test = ["cython", "html5lib", "pytest", "pytest-cov", "typed-ast"] name = "sphinx-rtd-theme" version = "0.5.1" description = "Read the Docs theme for Sphinx" -category = "dev" optional = false python-versions = "*" files = [ @@ -1444,7 +1382,6 @@ dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client"] name = "sphinxcontrib-applehelp" version = "1.0.2" description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1460,7 +1397,6 @@ test = ["pytest"] name = "sphinxcontrib-devhelp" version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1476,7 +1412,6 @@ test = ["pytest"] name = "sphinxcontrib-htmlhelp" version = "2.0.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1492,7 +1427,6 @@ test = ["html5lib", "pytest"] name = "sphinxcontrib-jsmath" version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1507,7 +1441,6 @@ test = ["flake8", "mypy", "pytest"] name = "sphinxcontrib-qthelp" version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1523,7 +1456,6 @@ test = ["pytest"] name = "sphinxcontrib-serializinghtml" version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1539,7 +1471,6 @@ test = ["pytest"] name = "stevedore" version = "3.5.2" description = "Manage dynamic plugins for Python applications" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1555,7 +1486,6 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" name = "structlog" version = "22.3.0" description = "Structured Logging for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1577,7 +1507,6 @@ typing = ["mypy", "rich", "twisted"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1589,7 +1518,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1601,7 +1529,6 @@ files = [ name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1635,7 +1562,6 @@ files = [ name = "types-pyopenssl" version = "23.0.0.4" description = "Typing stubs for pyOpenSSL" -category = "dev" optional = false python-versions = "*" files = [ @@ -1646,11 +1572,21 @@ files = [ [package.dependencies] cryptography = ">=35.0.0" +[[package]] +name = "types-python-slugify" +version = "8.0.0.1" +description = "Typing stubs for python-slugify" +optional = false +python-versions = "*" +files = [ + {file = "types-python-slugify-8.0.0.1.tar.gz", hash = "sha256:f2177d2e65ecf2eca742d28fbf236a8d919a72e68272d1d748c3fe965e5ea3ab"}, + {file = "types_python_slugify-8.0.0.1-py3-none-any.whl", hash = "sha256:1de288ce45c85627ef632c2b5098dedccfc7ebedb241d181ba69402f03704449"}, +] + [[package]] name = "types-redis" version = "4.5.1.4" description = "Typing stubs for redis" -category = "dev" optional = false python-versions = "*" files = [ @@ -1662,11 +1598,24 @@ files = [ cryptography = ">=35.0.0" types-pyOpenSSL = "*" +[[package]] +name = "types-requests" +version = "2.28.11.15" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.28.11.15.tar.gz", hash = "sha256:fc8eaa09cc014699c6b63c60c2e3add0c8b09a410c818b5ac6e65f92a26dde09"}, + {file = "types_requests-2.28.11.15-py3-none-any.whl", hash = "sha256:a05e4c7bc967518fba5789c341ea8b0c942776ee474c7873129a61161978e586"}, +] + +[package.dependencies] +types-urllib3 = "<1.27" + [[package]] name = "types-toml" version = "0.10.8.5" description = "Typing stubs for toml" -category = "dev" optional = false python-versions = "*" files = [ @@ -1674,11 +1623,21 @@ files = [ {file = "types_toml-0.10.8.5-py3-none-any.whl", hash = "sha256:2432017febe43174af0f3c65f03116e3d3cf43e7e1406b8200e106da8cf98992"}, ] +[[package]] +name = "types-urllib3" +version = "1.26.25.8" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.8.tar.gz", hash = "sha256:ecf43c42d8ee439d732a1110b4901e9017a79a38daca26f08e42c8460069392c"}, + {file = "types_urllib3-1.26.25.8-py3-none-any.whl", hash = "sha256:95ea847fbf0bf675f50c8ae19a665baedcf07e6b4641662c4c3c72e7b2edf1a9"}, +] + [[package]] name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1690,7 +1649,6 @@ files = [ name = "urllib3" version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -1707,7 +1665,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -1792,7 +1749,6 @@ files = [ name = "yamllint" version = "1.29.0" description = "A linter for YAML files." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1809,7 +1765,6 @@ setuptools = "*" name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1827,4 +1782,4 @@ redis = ["redis"] [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "37892839cc3207d00936d4f2d47780725bc0c299e1b1372a0bb4aba8907fe379" +content-hash = "2213a75a00f27293bb23f2a74792c2f8e46b97148e9197ddf620c9239d88b634" diff --git a/pyproject.toml b/pyproject.toml index e7191dae..c215ca1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "diffsync" -version = "1.8.0" +version = "1.9.0" description = "Library to easily sync/diff/update 2 different data sources" authors = ["Network to Code, LLC "] license = "Apache-2.0" @@ -26,7 +26,7 @@ redis = {version = "^4.3", optional = true} [tool.poetry.extras] redis = ["redis"] -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "*" pyyaml = "*" black = "*" @@ -47,6 +47,8 @@ toml = "*" types-toml = "*" types-redis = "*" pytest-redis = "^2.4.0" +types-requests = "^2.28.11.15" +types-python-slugify = "^8.0.0.1" [tool.black] line-length = 120 @@ -101,6 +103,7 @@ testpaths = [ [tool.mypy] warn_unused_configs = true +disallow_untyped_defs = true ignore_missing_imports = true [build-system] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 07b8fc8d..e0dd85c7 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from typing import ClassVar, List, Mapping, Optional, Tuple +from typing import ClassVar, List, Optional, Tuple, Dict import pytest @@ -35,7 +35,7 @@ class ErrorProneModelMixin: _counter: ClassVar[int] = 0 @classmethod - def create(cls, diffsync: DiffSync, ids: Mapping, attrs: Mapping): + def create(cls, diffsync: DiffSync, ids: Dict, attrs: Dict): """As DiffSyncModel.create(), but periodically throw exceptions.""" cls._counter += 1 if not cls._counter % 5: @@ -44,7 +44,7 @@ def create(cls, diffsync: DiffSync, ids: Mapping, attrs: Mapping): return None # non-fatal error return super().create(diffsync, ids, attrs) # type: ignore - def update(self, attrs: Mapping): + def update(self, attrs: Dict): """As DiffSyncModel.update(), but periodically throw exceptions.""" # pylint: disable=protected-access self.__class__._counter += 1 @@ -69,11 +69,11 @@ class ExceptionModelMixin: """Test class that always throws exceptions when creating/updating/deleting instances.""" @classmethod - def create(cls, diffsync: DiffSync, ids: Mapping, attrs: Mapping): + def create(cls, diffsync: DiffSync, ids: Dict, attrs: Dict): """As DiffSyncModel.create(), but always throw exceptions.""" raise NotImplementedError - def update(self, attrs: Mapping): + def update(self, attrs: Dict): """As DiffSyncModel.update(), but always throw exceptions.""" raise NotImplementedError diff --git a/tests/unit/test_diffsync.py b/tests/unit/test_diffsync.py index ebf0e494..74727be8 100644 --- a/tests/unit/test_diffsync.py +++ b/tests/unit/test_diffsync.py @@ -973,6 +973,7 @@ class NoDeleteInterfaceDiffSync(BackendA): extra_models.load() extra_device = extra_models.device(name="nyc-spine3", site_name="nyc", role="spine") extra_device.model_flags |= DiffSyncModelFlags.SKIP_CHILDREN_ON_DELETE + extra_device.model_flags |= DiffSyncModelFlags.NATURAL_DELETION_ORDER extra_models.get(extra_models.site, "nyc").add_child(extra_device) extra_models.add(extra_device) extra_interface = extra_models.interface(name="eth0", device_name="nyc-spine3") diff --git a/tests/unit/test_diffsync_model_flags.py b/tests/unit/test_diffsync_model_flags.py index b7950aef..76457ccf 100644 --- a/tests/unit/test_diffsync_model_flags.py +++ b/tests/unit/test_diffsync_model_flags.py @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. """ +from typing import List import pytest +from diffsync import DiffSync, DiffSyncModel from diffsync.enum import DiffSyncModelFlags from diffsync.exceptions import ObjectNotFound @@ -111,3 +113,52 @@ def test_diffsync_diff_with_ignore_flag_on_target_models(backend_a, backend_a_mi diff = backend_a.diff_from(backend_a_minus_some_models) print(diff.str()) # for debugging of any failure assert not diff.has_diffs() + + +def test_diffsync_diff_with_natural_deletion_order(): + # This list will contain the order in which the delete methods were called + call_order = [] + + class TestModelChild(DiffSyncModel): # pylint: disable=missing-class-docstring + _modelname = "child" + _identifiers = ("name",) + + name: str + + def delete(self): + call_order.append(self.name) + return super().delete() + + class TestModelParent(DiffSyncModel): # pylint: disable=missing-class-docstring + _modelname = "parent" + _identifiers = ("name",) + _children = {"child": "children"} + + name: str + children: List[TestModelChild] = [] + + def delete(self): + call_order.append(self.name) + return super().delete() + + class TestBackend(DiffSync): # pylint: disable=missing-class-docstring + top_level = ["parent"] + + parent = TestModelParent + child = TestModelChild + + def load(self): + parent = self.parent(name="Test-Parent") + parent.model_flags |= DiffSyncModelFlags.NATURAL_DELETION_ORDER + self.add(parent) + child = self.child(name="Test-Child") + parent.add_child(child) + self.add(child) + + source = TestBackend() + source.load() + destination = TestBackend() + destination.load() + source.remove(source.get("parent", {"name": "Test-Parent"}), remove_children=True) + source.sync_to(destination) + assert call_order == ["Test-Child", "Test-Parent"]