Merge pull request #483 from seperman/dev

8.1.0

Author: seperman

.github/workflows/main.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
         architecture: ["x64"]
     steps:
     - uses: actions/checkout@v2
@@ -44,34 +44,34 @@ jobs:
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
     - name: Upgrade setuptools
-      if: matrix.python-version == 3.12
+      if: matrix.python-version => 3.12
       run: |
-        # workaround for 3.12, SEE: https://github.com/pypa/setuptools/issues/3661#issuecomment-1813845177
+        # workaround for 3.13, SEE: https://github.com/pypa/setuptools/issues/3661#issuecomment-1813845177
         pip install --upgrade setuptools
     - name: Install dependencies
-      if: matrix.python-version != 3.8
+      if: matrix.python-version > 3.9
       run: pip install -r requirements-dev.txt
     - name: Install dependencies
-      if: matrix.python-version == 3.8
+      if: matrix.python-version <= 3.9
       run: pip install -r requirements-dev3.8.txt
     - name: Lint with flake8
-      if: matrix.python-version == 3.12
+      if: matrix.python-version == 3.13
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 deepdiff --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 deepdiff --count --exit-zero --max-complexity=26 --max-line-lengt=250 --statistics
     - name: Test with pytest and get the coverage
-      if: matrix.python-version == 3.12
+      if: matrix.python-version == 3.13
       run: |
         pytest --benchmark-disable --cov-report=xml --cov=deepdiff tests/ --runslow
     - name: Test with pytest and no coverage report
-      if: matrix.python-version != 3.12
+      if: matrix.python-version != 3.13
       run: |
         pytest --benchmark-disable
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4
-      if: matrix.python-version == 3.12
+      if: matrix.python-version == 3.13
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       with:

AUTHORS.md

@@ -63,3 +63,12 @@ Authors in order of the timeline of their contributions:
 - [sf-tcalhoun](https://github.com/sf-tcalhoun) for fixing "Instantiating a Delta with a flat_dict_list unexpectedly mutates the flat_dict_list"
 - [dtorres-sf](https://github.com/dtorres-sf) for fixing iterable moved items when iterable_compare_func is used.
 - [Florian Finkernagel](https://github.com/TyberiusPrime) for pandas and polars support.
+- Mathis Chenuet [artemisart](https://github.com/artemisart) for fixing slots classes comparison and PR review.
+- Sherjeel Shabih  [sherjeelshabih](https://github.com/sherjeelshabih) for fixing the issue where the key deep_distance is not returned when both compared items are equal #510
+- [Aaron D. Marasco](https://github.com/AaronDMarasco) for adding `prefix` option to `pretty()`
+- [Juergen Skrotzky](https://github.com/Jorgen-VikingGod) for adding empty `py.typed`
+- [Mate Valko](https://github.com/vmatt) for fixing the issue so we lower only if clean_key is instance of str via #504 
+- [jlaba](https://github.com/jlaba) for fixing #493 include_paths, when only certain keys are included via #499 
+- [Doron Behar](https://github.com/doronbehar) for fixing DeepHash for numpy booleans via #496 
+- [Aaron D. Marasco](https://github.com/AaronDMarasco) for adding print() options which allows a user-defined string (or callback function) to prefix every output when using the pretty() call.
+- [David Hotham](https://github.com/dimbleby) for relaxing orderly-set dependency via #486 

CHANGELOG.md

@@ -1,6 +1,25 @@
 # DeepDiff Change log
 
 
+- v8-1-0
+    - Removing deprecated lines from setup.py
+    - Added `prefix` option to `pretty()`
+    - Fixes hashing of numpy boolean values.
+    - Fixes __slots__ comparison when the attribute doesn't exist.
+    - Relaxing orderly-set reqs
+    - Added Python 3.13 support
+    - Only lower if clean_key is instance of str
+    - Only lower if clean_key is instance of str #504
+    - Fixes issue where the key deep_distance is not returned when both compared items are equal
+    - Fixes issue where the key deep_distance is not returned when both compared items are equal #510
+    - Fixes exclude_paths fails to work in certain cases
+    - exclude_paths fails to work #509
+    - Fixes to_json() method chokes on standard json.dumps() kwargs such as sort_keys
+    - to_dict() method chokes on standard json.dumps() kwargs  #490
+    - Fixes accessing the affected_root_keys property on the diff object returned by DeepDiff fails when one of the dicts is empty
+    - In version 8.0.1, accessing the affected_root_keys property on the diff object returned by DeepDiff fails when one of the dicts is empty #508
+
+
 - v8-0-1
     - Bugfix. Numpy should be optional.
 

README.md

@@ -23,6 +23,25 @@ Tested on Python 3.8+ and PyPy3.
 
 Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.
 
+DeepDiff 8-1-0
+
+- Removing deprecated lines from setup.py
+- Added `prefix` option to `pretty()`
+- Fixes hashing of numpy boolean values.
+- Fixes __slots__ comparison when the attribute doesn't exist.
+- Relaxing orderly-set reqs
+- Added Python 3.13 support
+- Only lower if clean_key is instance of str
+- Only lower if clean_key is instance of str #504
+- Fixes issue where the key deep_distance is not returned when both compared items are equal
+- Fixes issue where the key deep_distance is not returned when both compared items are equal #510
+- Fixes exclude_paths fails to work in certain cases
+- exclude_paths fails to work #509
+- Fixes to_json() method chokes on standard json.dumps() kwargs such as sort_keys
+- to_dict() method chokes on standard json.dumps() kwargs  #490
+- Fixes accessing the affected_root_keys property on the diff object returned by DeepDiff fails when one of the dicts is empty
+- In version 8.0.1, accessing the affected_root_keys property on the diff object returned by DeepDiff fails when one of the dicts is empty #508
+
 DeepDiff 8-0-1
 
 - Bugfix. Numpy should be optional.

deepdiff/deephash.py

@@ -12,7 +12,7 @@
                              convert_item_or_items_into_compiled_regexes_else_none,
                              get_id, type_is_subclass_of_type_group, type_in_type_group,
                              number_to_string, datetime_normalize, KEY_TO_VAL_STR, short_repr,
-                             get_truncate_datetime, dict_, add_root_to_paths)
+                             get_truncate_datetime, dict_, add_root_to_paths, PydanticBaseModel)
 from deepdiff.base import Base
 
 try:
@@ -24,6 +24,11 @@
     import polars
 except ImportError:
     polars = False
+try:
+    import numpy as np
+    booleanTypes = (bool, np.bool_)
+except ImportError:
+    booleanTypes = bool
 
 logger = logging.getLogger(__name__)
 
@@ -326,13 +331,15 @@ def values(self):
     def items(self):
         return ((i, v[0]) for i, v in self.hashes.items())
 
-    def _prep_obj(self, obj, parent, parents_ids=EMPTY_FROZENSET, is_namedtuple=False):
+    def _prep_obj(self, obj, parent, parents_ids=EMPTY_FROZENSET, is_namedtuple=False, is_pydantic_object=False):
         """prepping objects"""
         original_type = type(obj) if not isinstance(obj, type) else obj
 
         obj_to_dict_strategies = []
         if is_namedtuple:
             obj_to_dict_strategies.append(lambda o: o._asdict())
+        elif is_pydantic_object:
+            obj_to_dict_strategies.append(lambda o: {k: v for (k, v) in o.__dict__.items() if v !="model_fields_set"})
         else:
             obj_to_dict_strategies.append(lambda o: o.__dict__)
 
@@ -492,7 +499,7 @@ def _hash(self, obj, parent, parents_ids=EMPTY_FROZENSET):
         """The main hash method"""
         counts = 1
 
-        if isinstance(obj, bool):
+        if isinstance(obj, booleanTypes):
             obj = self._prep_bool(obj)
             result = None
         elif self.use_enum_value and isinstance(obj, Enum):
@@ -557,6 +564,8 @@ def gen():
 
         elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
             result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'
+        elif isinstance(obj, PydanticBaseModel):
+            result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True)
         else:
             result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids)
 

deepdiff/diff.py

@@ -80,6 +80,9 @@ def _report_progress(_stats, progress_logger, duration):
 PURGE_LEVEL_RANGE_MSG = 'cache_purge_level should be 0, 1, or 2.'
 _ENABLE_CACHE_EVERY_X_DIFF = '_ENABLE_CACHE_EVERY_X_DIFF'
 
+model_fields_set = frozenset(["model_fields_set"])
+
+
 # What is the threshold to consider 2 items to be pairs. Only used when ignore_order = True.
 CUTOFF_DISTANCE_FOR_PAIRS_DEFAULT = 0.3
 
@@ -421,7 +424,7 @@ def unmangle(attribute):
                 else:
                     all_slots.extend(slots)
 
-        return {i: getattr(object, unmangle(i)) for i in all_slots}
+        return {i: getattr(object, key) for i in all_slots if hasattr(object, key := unmangle(i))}
 
     def _diff_enum(self, level, parents_ids=frozenset(), local_tree=None):
         t1 = detailed__dict__(level.t1, include_keys=ENUM_INCLUDE_KEYS)
@@ -437,13 +440,16 @@ def _diff_enum(self, level, parents_ids=frozenset(), local_tree=None):
             local_tree=local_tree,
         )
 
-    def _diff_obj(self, level, parents_ids=frozenset(), is_namedtuple=False, local_tree=None):
+    def _diff_obj(self, level, parents_ids=frozenset(), is_namedtuple=False, local_tree=None, is_pydantic_object=False):
         """Difference of 2 objects"""
         processing_error = False
         try:
             if is_namedtuple:
                 t1 = level.t1._asdict()
                 t2 = level.t2._asdict()
+            elif is_pydantic_object:
+                t1 = detailed__dict__(level.t1, ignore_private_variables=self.ignore_private_variables, ignore_keys=model_fields_set)
+                t2 = detailed__dict__(level.t2, ignore_private_variables=self.ignore_private_variables, ignore_keys=model_fields_set)
             elif all('__dict__' in dir(t) for t in level):
                 t1 = detailed__dict__(level.t1, ignore_private_variables=self.ignore_private_variables)
                 t2 = detailed__dict__(level.t2, ignore_private_variables=self.ignore_private_variables)
@@ -510,6 +516,32 @@ def _skip_this(self, level):
 
         return skip
 
+    def _skip_this_key(self, level, key):
+        # if include_paths is not set, than treet every path as included
+        if self.include_paths is None:
+            return False
+        if "{}['{}']".format(level.path(), key) in self.include_paths:
+            return False
+        if level.path() in self.include_paths:
+            # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']"]
+            return False
+        for prefix in self.include_paths:
+            if "{}['{}']".format(level.path(), key) in prefix:
+                # matches as long the prefix is longer than this object key
+                # eg.: level+key root['foo']['bar'] matches prefix root['foo']['bar'] from include paths
+                #      level+key root['foo'] matches prefix root['foo']['bar'] from include_paths
+                #      level+key root['foo']['bar'] DOES NOT match root['foo'] from include_paths This needs to be handled afterwards
+                return False
+        # check if a higher level is included as a whole (=without any sublevels specified)
+        # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']"]
+        # but does not match, if it is level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']['fruits']"]
+        up = level.up
+        while up is not None:
+            if up.path() in self.include_paths:
+                return False
+            up = up.up
+        return True
+
     def _get_clean_to_keys_mapping(self, keys, level):
         """
         Get a dictionary of cleaned value of keys to the keys themselves.
@@ -530,7 +562,7 @@ def _get_clean_to_keys_mapping(self, keys, level):
                 clean_key = KEY_TO_VAL_STR.format(type_, clean_key)
             else:
                 clean_key = key
-            if self.ignore_string_case:
+            if self.ignore_string_case and isinstance(clean_key, str):
                 clean_key = clean_key.lower()
             if clean_key in result:
                 logger.warning(('{} and {} in {} become the same key when ignore_numeric_type_changes'
@@ -570,11 +602,11 @@ def _diff_dict(
             rel_class = DictRelationship
 
         if self.ignore_private_variables:
-            t1_keys = SetOrdered([key for key in t1 if not(isinstance(key, str) and key.startswith('__'))])
-            t2_keys = SetOrdered([key for key in t2 if not(isinstance(key, str) and key.startswith('__'))])
+            t1_keys = SetOrdered([key for key in t1 if not(isinstance(key, str) and key.startswith('__')) and not self._skip_this_key(level, key)])
+            t2_keys = SetOrdered([key for key in t2 if not(isinstance(key, str) and key.startswith('__')) and not self._skip_this_key(level, key)])
         else:
-            t1_keys = SetOrdered(t1.keys())
-            t2_keys = SetOrdered(t2.keys())
+            t1_keys = SetOrdered([key for key in t1 if not self._skip_this_key(level, key)])
+            t2_keys = SetOrdered([key for key in t2 if not self._skip_this_key(level, key)])
         if self.ignore_string_type_changes or self.ignore_numeric_type_changes or self.ignore_string_case:
             t1_clean_to_keys = self._get_clean_to_keys_mapping(keys=t1_keys, level=level)
             t2_clean_to_keys = self._get_clean_to_keys_mapping(keys=t2_keys, level=level)
@@ -584,11 +616,17 @@ def _diff_dict(
             t1_clean_to_keys = t2_clean_to_keys = None
 
         t_keys_intersect = t2_keys & t1_keys
-        t_keys_union = t2_keys | t1_keys
         t_keys_added = t2_keys - t_keys_intersect
         t_keys_removed = t1_keys - t_keys_intersect
+
         if self.threshold_to_diff_deeper:
-            if len(t_keys_union) > 1 and len(t_keys_intersect) / len(t_keys_union) < self.threshold_to_diff_deeper:
+            if self.exclude_paths:
+                t_keys_union = {f"{level.path()}[{repr(key)}]" for key in (t2_keys | t1_keys)}
+                t_keys_union -= self.exclude_paths
+                t_keys_union_len = len(t_keys_union)
+            else:
+                t_keys_union_len = len(t2_keys | t1_keys)
+            if t_keys_union_len > 1 and len(t_keys_intersect) / t_keys_union_len < self.threshold_to_diff_deeper:
                 self._report_result('values_changed', level, local_tree=local_tree)
                 return
 
@@ -1652,7 +1690,7 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None, local_tree=
             self._diff_numpy_array(level, parents_ids, local_tree=local_tree)
 
         elif isinstance(level.t1, PydanticBaseModel):
-            self._diff_obj(level, parents_ids, local_tree=local_tree)
+            self._diff_obj(level, parents_ids, local_tree=local_tree, is_pydantic_object=True)
 
         elif isinstance(level.t1, Iterable):
             self._diff_iterable(level, parents_ids, _original_type=_original_type, local_tree=local_tree)
@@ -1808,9 +1846,13 @@ def affected_root_keys(self):
             value = self.tree.get(key)
             if value:
                 if isinstance(value, SetOrdered):
-                    result |= SetOrdered([i.get_root_key() for i in value])
+                    values_list = value
                 else:
-                    result |= SetOrdered([i.get_root_key() for i in value.keys()])
+                    values_list = value.keys()
+                for item in values_list:
+                    root_key = item.get_root_key()
+                    if root_key is not notpresent:
+                        result.add(root_key)
         return result
 
 

deepdiff/model.py

@@ -41,7 +41,7 @@ def remove_empty_keys(self):
         Remove empty keys from this object. Should always be called after the result is final.
         :return:
         """
-        empty_keys = [k for k, v in self.items() if not v]
+        empty_keys = [k for k, v in self.items() if not isinstance(v, (int)) and not v]
 
         for k in empty_keys:
             del self[k]
@@ -88,7 +88,13 @@ def __getitem__(self, item):
         return self.get(item)
 
     def __len__(self):
-        return sum([len(i) for i in self.values() if isinstance(i, SetOrdered)])
+        length = 0
+        for value in self.values():
+            if isinstance(value, SetOrdered):
+                length += len(value)
+            elif isinstance(value, int):
+                length += 1
+        return length
 
 
 class TextResult(ResultDict):
@@ -659,7 +665,9 @@ def get_root_key(self, use_t2=False):
         else:
             next_rel = root_level.t1_child_rel or root_level.t2_child_rel  # next relationship object to get a formatted param from
 
-        return next_rel.param
+        if next_rel:
+            return next_rel.param
+        return notpresent
 
     def path(self, root="root", force=None, get_parent_too=False, use_t2=False, output_format='str'):
         """