Merge pull request #432 from seperman/dev

6.7.1
seperman · Nov 14, 2023 · 89c5cc2 · 89c5cc2
2 parents 39c3a3d + db9f667
commit 89c5cc2
Show file tree

Hide file tree

Showing 16 changed files with 353 additions and 85 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # DeepDiff Change log
 
+- v6-7-1
+    - Support for subtracting delta objects when iterable_compare_func is used.
+    - Better handling of force adding a delta to an object. 
+    - Fix for [`Can't compare dicts with both single and double quotes in keys`](https://github.com/seperman/deepdiff/issues/430)
+    - Updated docs for Inconsistent Behavior with math_epsilon and ignore_order = True
 - v6-7-0
     - Delta can be subtracted from other objects now.
     - verify_symmetry is deprecated. Use bidirectional instead.

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# DeepDiff v 6.7.0
+# DeepDiff v 6.7.1
 
 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat)
 ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat)
@@ -17,13 +17,20 @@
 
 Tested on Python 3.7+ and PyPy3.
 
-- **[Documentation](https://zepworks.com/deepdiff/6.7.0/)**
+- **[Documentation](https://zepworks.com/deepdiff/6.7.1/)**
 
 ## What is new?
 
 Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.
 
-DeepDiff v6-7-0
+DeepDiff 6-7-1
+
+- Support for subtracting delta objects when iterable_compare_func is used.
+- Better handling of force adding a delta to an object. 
+- Fix for [`Can't compare dicts with both single and double quotes in keys`](https://github.com/seperman/deepdiff/issues/430)
+- Updated docs for Inconsistent Behavior with math_epsilon and ignore_order = True
+
+DeepDiff 6-7-0
 
 - Delta can be subtracted from other objects now.
 - verify_symmetry is deprecated. Use bidirectional instead.
@@ -98,11 +105,11 @@ Thank you!
 
 How to cite this library (APA style):
 
-    Dehpour, S. (2023). DeepDiff (Version 6.7.0) [Software]. Available from https://github.com/seperman/deepdiff.
+    Dehpour, S. (2023). DeepDiff (Version 6.7.1) [Software]. Available from https://github.com/seperman/deepdiff.
 
 How to cite this library (Chicago style):
 
-    Dehpour, Sep. 2023. DeepDiff (version 6.7.0).
+    Dehpour, Sep. 2023. DeepDiff (version 6.7.1).
 
 # Authors
 

diff --git a/deepdiff/__init__.py b/deepdiff/__init__.py
@@ -1,6 +1,6 @@
 """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes."""
 # flake8: noqa
-__version__ = '6.7.0'
+__version__ = '6.7.1'
 import logging
 
 if __name__ == '__main__':

diff --git a/deepdiff/delta.py b/deepdiff/delta.py
@@ -71,6 +71,7 @@ def __init__(
         diff=None,
         delta_path=None,
         delta_file=None,
+        delta_diff=None,
         flat_dict_list=None,
         deserializer=pickle_load,
         log_errors=True,
@@ -81,6 +82,7 @@ def __init__(
         verify_symmetry=None,
         bidirectional=False,
         always_include_values=False,
+        iterable_compare_func_was_used=None,
         force=False,
     ):
         if hasattr(deserializer, '__code__') and 'safe_to_import' in set(deserializer.__code__.co_varnames):
@@ -114,6 +116,8 @@ def _deserializer(obj, safe_to_import=None):
             with open(delta_path, 'rb') as the_file:
                 content = the_file.read()
             self.diff = _deserializer(content, safe_to_import=safe_to_import)
+        elif delta_diff:
+            self.diff = delta_diff
         elif delta_file:
             try:
                 content = delta_file.read()
@@ -128,7 +132,10 @@ def _deserializer(obj, safe_to_import=None):
         self.mutate = mutate
         self.raise_errors = raise_errors
         self.log_errors = log_errors
-        self._numpy_paths = self.diff.pop('_numpy_paths', False)
+        self._numpy_paths = self.diff.get('_numpy_paths', False)
+        # When we create the delta from a list of flat dictionaries, details such as iterable_compare_func_was_used get lost.
+        # That's why we allow iterable_compare_func_was_used to be explicitly set.
+        self._iterable_compare_func_was_used = self.diff.get('_iterable_compare_func_was_used', iterable_compare_func_was_used)
         self.serializer = serializer
         self.deserializer = deserializer
         self.force = force
@@ -198,7 +205,17 @@ def _do_verify_changes(self, path, expected_old_value, current_old_value):
             self._raise_or_log(VERIFICATION_MSG.format(
                 path_str, expected_old_value, current_old_value, VERIFY_BIDIRECTIONAL_MSG))
 
-    def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expected_old_value, elem=None, action=None, forced_old_value=None):
+    def _get_elem_and_compare_to_old_value(
+        self,
+        obj,
+        path_for_err_reporting,
+        expected_old_value,
+        elem=None,
+        action=None,
+        forced_old_value=None,
+        next_element=None,
+    ):
+        # if forced_old_value is not None:
         try:
             if action == GET:
                 current_old_value = obj[elem]
@@ -208,9 +225,21 @@ def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expect
                 raise DeltaError(INVALID_ACTION_WHEN_CALLING_GET_ELEM.format(action))
         except (KeyError, IndexError, AttributeError, TypeError) as e:
             if self.force:
-                _forced_old_value = {} if forced_old_value is None else forced_old_value
+                if forced_old_value is None:
+                    if next_element is None or isinstance(next_element, str):
+                        _forced_old_value = {}
+                    else:
+                        _forced_old_value = []    
+                else:
+                    _forced_old_value = forced_old_value
                 if action == GET:
-                    obj[elem] = _forced_old_value
+                    if isinstance(obj, list):
+                        if isinstance(elem, int) and elem < len(obj):
+                            obj[elem] = _forced_old_value
+                        else:
+                            obj.append(_forced_old_value)
+                    else:
+                        obj[elem] = _forced_old_value
                 elif action == GETATTR:
                     setattr(obj, elem, _forced_old_value)
                 return _forced_old_value
@@ -277,6 +306,11 @@ def _set_new_value(self, parent, parent_to_obj_elem, parent_to_obj_action,
                 parent, obj, path, parent_to_obj_elem,
                 parent_to_obj_action, elements,
                 to_type=list, from_type=tuple)
+        if elem != 0 and self.force and isinstance(obj, list) and len(obj) == 0:
+            # it must have been a dictionary    
+            obj = {}
+            self._simple_set_elem_value(obj=parent, path_for_err_reporting=path, elem=parent_to_obj_elem,
+                                        value=obj, action=parent_to_obj_action)
         self._simple_set_elem_value(obj=obj, path_for_err_reporting=path, elem=elem,
                                     value=new_value, action=action)
 
@@ -404,14 +438,21 @@ def _get_elements_and_details(self, path):
         try:
             elements = _path_to_elements(path)
             if len(elements) > 1:
-                parent = self.get_nested_obj(obj=self, elements=elements[:-2])
+                elements_subset = elements[:-2]
+                if len(elements_subset) != len(elements):
+                    next_element = elements[-2][0]
+                    next2_element = elements[-1][0]
+                else:
+                    next_element = None
+                parent = self.get_nested_obj(obj=self, elements=elements_subset, next_element=next_element)
                 parent_to_obj_elem, parent_to_obj_action = elements[-2]
                 obj = self._get_elem_and_compare_to_old_value(
                     obj=parent, path_for_err_reporting=path, expected_old_value=None,
-                    elem=parent_to_obj_elem, action=parent_to_obj_action)
+                    elem=parent_to_obj_elem, action=parent_to_obj_action, next_element=next2_element)
             else:
                 parent = parent_to_obj_elem = parent_to_obj_action = None
-                obj = self.get_nested_obj(obj=self, elements=elements[:-1])
+                obj = self
+                # obj = self.get_nested_obj(obj=self, elements=elements[:-1])
             elem, action = elements[-1]
         except Exception as e:
             self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path, e))
@@ -458,6 +499,55 @@ def _do_values_or_type_changed(self, changes, is_type_change=False, verify_chang
                 self._do_verify_changes(path, expected_old_value, current_old_value)
 
     def _do_item_removed(self, items):
+        """
+        Handle removing items.
+        """
+        # Sorting the iterable_item_removed in reverse order based on the paths.
+        # So that we delete a bigger index before a smaller index
+        for path, expected_old_value in sorted(items.items(), key=self._sort_key_for_item_added, reverse=True):
+            elem_and_details = self._get_elements_and_details(path)
+            if elem_and_details:
+                elements, parent, parent_to_obj_elem, parent_to_obj_action, obj, elem, action = elem_and_details
+            else:
+                continue  # pragma: no cover. Due to cPython peephole optimizer, this line doesn't get covered. https://github.com/nedbat/coveragepy/issues/198
+
+            look_for_expected_old_value = False
+            current_old_value = not_found
+            try:
+                if action == GET:
+                    current_old_value = obj[elem]
+                    look_for_expected_old_value = current_old_value != expected_old_value
+                elif action == GETATTR:
+                    current_old_value = getattr(obj, elem)
+                    look_for_expected_old_value = current_old_value != expected_old_value
+            except (KeyError, IndexError, AttributeError, TypeError):
+                look_for_expected_old_value = True
+
+            if look_for_expected_old_value and isinstance(obj, list) and not self._iterable_compare_func_was_used:
+                # It may return None if it doesn't find it
+                elem = self._find_closest_iterable_element_for_index(obj, elem, expected_old_value)
+                if elem is not None:
+                    current_old_value = expected_old_value
+            if current_old_value is not_found or elem is None:
+                continue
+
+            self._del_elem(parent, parent_to_obj_elem, parent_to_obj_action,
+                           obj, elements, path, elem, action)
+            self._do_verify_changes(path, expected_old_value, current_old_value)
+
+    def _find_closest_iterable_element_for_index(self, obj, elem, expected_old_value):
+        closest_elem = None
+        closest_distance = float('inf')
+        for index, value in enumerate(obj):
+            dist = abs(index - elem)
+            if dist > closest_distance:
+                break
+            if value == expected_old_value and dist < closest_distance:
+                closest_elem = index
+                closest_distance = dist
+        return closest_elem
+
+    def _do_item_removedOLD(self, items):
         """
         Handle removing items.
         """
@@ -695,10 +785,9 @@ def _from_flat_dicts(flat_dict_list):
         Create the delta's diff object from the flat_dict_list
         """
         result = {}
-
-        DEFLATTENING_NEW_ACTION_MAP = {
-            'iterable_item_added': 'iterable_items_added_at_indexes',
-            'iterable_item_removed': 'iterable_items_removed_at_indexes',
+        FLATTENING_NEW_ACTION_MAP = {
+            'unordered_iterable_item_added': 'iterable_items_added_at_indexes',
+            'unordered_iterable_item_removed': 'iterable_items_removed_at_indexes',
         }
         for flat_dict in flat_dict_list:
             index = None
@@ -710,8 +799,8 @@ def _from_flat_dicts(flat_dict_list):
                 raise ValueError("Flat dict need to include the 'action'.")
             if path is None:
                 raise ValueError("Flat dict need to include the 'path'.")
-            if action in DEFLATTENING_NEW_ACTION_MAP:
-                action = DEFLATTENING_NEW_ACTION_MAP[action]
+            if action in FLATTENING_NEW_ACTION_MAP:
+                action = FLATTENING_NEW_ACTION_MAP[action]
                 index = path.pop()
             if action in {'attribute_added', 'attribute_removed'}:
                 root_element = ('root', GETATTR)
@@ -729,8 +818,8 @@ def _from_flat_dicts(flat_dict_list):
                     result[action][path_str] = set()
                 result[action][path_str].add(value)
             elif action in {
-                'dictionary_item_added', 'dictionary_item_removed', 'iterable_item_added',
-                'iterable_item_removed', 'attribute_removed', 'attribute_added'
+                'dictionary_item_added', 'dictionary_item_removed',
+                'attribute_removed', 'attribute_added', 'iterable_item_added', 'iterable_item_removed',
             }:
                 result[action][path_str] = value
             elif action == 'values_changed':
@@ -843,10 +932,12 @@ def to_flat_dicts(self, include_action_in_path=False, report_type_changes=True):
             ]
 
         FLATTENING_NEW_ACTION_MAP = {
-            'iterable_items_added_at_indexes': 'iterable_item_added',
-            'iterable_items_removed_at_indexes': 'iterable_item_removed',
+            'iterable_items_added_at_indexes': 'unordered_iterable_item_added',
+            'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed',
         }
         for action, info in self.diff.items():
+            if action.startswith('_'):
+                continue
             if action in FLATTENING_NEW_ACTION_MAP:
                 new_action = FLATTENING_NEW_ACTION_MAP[action]
                 for path, index_to_value in info.items():

diff --git a/deepdiff/diff.py b/deepdiff/diff.py
@@ -231,7 +231,7 @@ def _group_by_sort_key(x):
             self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes)
             self.math_epsilon = math_epsilon
             if self.math_epsilon is not None and self.ignore_order:
-                logger.warning("math_epsilon will be ignored. It cannot be used when ignore_order is True.")
+                logger.warning("math_epsilon in conjunction with ignore_order=True is only used for flat object comparisons. Custom math_epsilon will not have an effect when comparing nested objects.")
             self.truncate_datetime = get_truncate_datetime(truncate_datetime)
             self.number_format_notation = number_format_notation
             if verbose_level in {0, 1, 2}:

diff --git a/deepdiff/path.py b/deepdiff/path.py
@@ -22,7 +22,7 @@ def _add_to_elements(elements, elem, inside):
         return
     if not elem.startswith('__'):
         remove_quotes = False
-        if '\\' in elem:
+        if '𝆺𝅥𝅯' in elem or '\\' in elem:
             remove_quotes = True
         else:
             try:
@@ -62,7 +62,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
     inside_quotes = False
     quote_used = ''
     for char in path:
-        if prev_char == '\\':
+        if prev_char == '𝆺𝅥𝅯':
             elem += char
         elif char in {'"', "'"}:
             elem += char
@@ -115,7 +115,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
     return tuple(elements)
 
 
-def _get_nested_obj(obj, elements):
+def _get_nested_obj(obj, elements, next_element=None):
     for (elem, action) in elements:
         if action == GET:
             obj = obj[elem]
@@ -124,21 +124,50 @@ def _get_nested_obj(obj, elements):
     return obj
 
 
-def _get_nested_obj_and_force(obj, elements):
-    for (elem, action) in elements:
+def _guess_type(elements, elem, index, next_element):
+    # If we are not at the last elements
+    if index < len(elements) - 1:
+        # We assume it is a nested dictionary not a nested list
+        return {}
+    if isinstance(next_element, int):
+        return []
+    return {}
+
+
+def _get_nested_obj_and_force(obj, elements, next_element=None):
+    prev_elem = None
+    prev_action = None
+    prev_obj = obj
+    for index, (elem, action) in enumerate(elements):
+        _prev_obj = obj
         if action == GET:
             try:
                 obj = obj[elem]
+                prev_obj = _prev_obj
             except KeyError:
-                obj[elem] = {}
+                obj[elem] = _guess_type(elements, elem, index, next_element)
                 obj = obj[elem]
+                prev_obj = _prev_obj
             except IndexError:
                 if isinstance(obj, list) and isinstance(elem, int) and elem >= len(obj):
                     obj.extend([None] * (elem - len(obj)))
-                    obj.append({})
+                    obj.append(_guess_type(elements, elem, index), next_element)
                     obj = obj[-1]
+                    prev_obj = _prev_obj
+                elif isinstance(obj, list) and len(obj) == 0 and prev_elem:
+                    # We ran into an empty list that should have been a dictionary
+                    # We need to change it from an empty list to a dictionary
+                    obj = {elem: _guess_type(elements, elem, index, next_element)}
+                    if prev_action == GET:
+                        prev_obj[prev_elem] = obj
+                    else:
+                        setattr(prev_obj, prev_elem, obj)
+                    obj = obj[elem]
         elif action == GETATTR:
             obj = getattr(obj, elem)
+            prev_obj = _prev_obj
+        prev_elem = elem
+        prev_action = action
     return obj
 
 
@@ -241,13 +270,13 @@ def parse_path(path, root_element=DEFAULT_FIRST_ELEMENT, include_actions=False):
 def stringify_element(param, quote_str=None):
     has_quote = "'" in param
     has_double_quote = '"' in param
-    if has_quote and has_double_quote:
+    if has_quote and has_double_quote and not quote_str:
         new_param = []
         for char in param:
             if char in {'"', "'"}:
-                new_param.append('\\')
+                new_param.append('𝆺𝅥𝅯')
             new_param.append(char)
-        param = ''.join(new_param)
+        result = '"' + ''.join(new_param) + '"'
     elif has_quote:
         result = f'"{param}"'
     elif has_double_quote: