From 2178878c58ff56a572c348e00967ec6727fb351b Mon Sep 17 00:00:00 2001
From: Seperman <sep@zepworks.com>
Date: Mon, 13 Nov 2023 22:07:57 -0800
Subject: [PATCH 1/7] subtract delta fixed when iterable_compare_func is used.
 Better handling of force adding a delta to an object. We change between an
 empty list. and an empty dictionary when needed. We find the closest list
 item when removing items from iterable and force=True.

---
 deepdiff/delta.py         | 130 ++++++++++++++++++++++++++----
 deepdiff/path.py          |  42 ++++++++--
 deepdiff/serialization.py |   3 +
 docs/delta.rst            |   4 +
 tests/test_delta.py       | 163 ++++++++++++++++++++++++++++----------
 tests/test_diff_text.py   |   8 ++
 6 files changed, 283 insertions(+), 67 deletions(-)

diff --git a/deepdiff/delta.py b/deepdiff/delta.py
index b2edd96..0976fb3 100644
--- a/deepdiff/delta.py
+++ b/deepdiff/delta.py
@@ -71,6 +71,7 @@ def __init__(
         diff=None,
         delta_path=None,
         delta_file=None,
+        delta_diff=None,
         flat_dict_list=None,
         deserializer=pickle_load,
         log_errors=True,
@@ -81,6 +82,7 @@ def __init__(
         verify_symmetry=None,
         bidirectional=False,
         always_include_values=False,
+        iterable_compare_func_was_used=None,
         force=False,
     ):
         if hasattr(deserializer, '__code__') and 'safe_to_import' in set(deserializer.__code__.co_varnames):
@@ -114,6 +116,8 @@ def _deserializer(obj, safe_to_import=None):
             with open(delta_path, 'rb') as the_file:
                 content = the_file.read()
             self.diff = _deserializer(content, safe_to_import=safe_to_import)
+        elif delta_diff:
+            self.diff = delta_diff
         elif delta_file:
             try:
                 content = delta_file.read()
@@ -128,7 +132,10 @@ def _deserializer(obj, safe_to_import=None):
         self.mutate = mutate
         self.raise_errors = raise_errors
         self.log_errors = log_errors
-        self._numpy_paths = self.diff.pop('_numpy_paths', False)
+        self._numpy_paths = self.diff.get('_numpy_paths', False)
+        # When we create the delta from a list of flat dictionaries, details such as iterable_compare_func_was_used get lost.
+        # That's why we allow iterable_compare_func_was_used to be explicitly set.
+        self._iterable_compare_func_was_used = self.diff.get('_iterable_compare_func_was_used', iterable_compare_func_was_used)
         self.serializer = serializer
         self.deserializer = deserializer
         self.force = force
@@ -198,7 +205,17 @@ def _do_verify_changes(self, path, expected_old_value, current_old_value):
             self._raise_or_log(VERIFICATION_MSG.format(
                 path_str, expected_old_value, current_old_value, VERIFY_BIDIRECTIONAL_MSG))
 
-    def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expected_old_value, elem=None, action=None, forced_old_value=None):
+    def _get_elem_and_compare_to_old_value(
+        self,
+        obj,
+        path_for_err_reporting,
+        expected_old_value,
+        elem=None,
+        action=None,
+        forced_old_value=None,
+        next_element=None,
+    ):
+        # if forced_old_value is not None:
         try:
             if action == GET:
                 current_old_value = obj[elem]
@@ -208,9 +225,21 @@ def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expect
                 raise DeltaError(INVALID_ACTION_WHEN_CALLING_GET_ELEM.format(action))
         except (KeyError, IndexError, AttributeError, TypeError) as e:
             if self.force:
-                _forced_old_value = {} if forced_old_value is None else forced_old_value
+                if forced_old_value is None:
+                    if next_element is None or isinstance(next_element, str):
+                        _forced_old_value = {}
+                    else:
+                        _forced_old_value = []    
+                else:
+                    _forced_old_value = forced_old_value
                 if action == GET:
-                    obj[elem] = _forced_old_value
+                    if isinstance(obj, list):
+                        if isinstance(elem, int) and elem < len(obj):
+                            obj[elem] = _forced_old_value
+                        else:
+                            obj.append(_forced_old_value)
+                    else:
+                        obj[elem] = _forced_old_value
                 elif action == GETATTR:
                     setattr(obj, elem, _forced_old_value)
                 return _forced_old_value
@@ -277,6 +306,11 @@ def _set_new_value(self, parent, parent_to_obj_elem, parent_to_obj_action,
                 parent, obj, path, parent_to_obj_elem,
                 parent_to_obj_action, elements,
                 to_type=list, from_type=tuple)
+        if elem != 0 and self.force and isinstance(obj, list) and len(obj) == 0:
+            # it must have been a dictionary    
+            obj = {}
+            self._simple_set_elem_value(obj=parent, path_for_err_reporting=path, elem=parent_to_obj_elem,
+                                        value=obj, action=parent_to_obj_action)
         self._simple_set_elem_value(obj=obj, path_for_err_reporting=path, elem=elem,
                                     value=new_value, action=action)
 
@@ -356,6 +390,9 @@ def _do_item_added(self, items, sort=True, insert=False):
         else:
             items = items.items()
 
+        # if getattr(self, 'DEBUG', None):
+        #     import pytest; pytest.set_trace()
+
         for path, new_value in items:
             elem_and_details = self._get_elements_and_details(path)
             if elem_and_details:
@@ -404,14 +441,21 @@ def _get_elements_and_details(self, path):
         try:
             elements = _path_to_elements(path)
             if len(elements) > 1:
-                parent = self.get_nested_obj(obj=self, elements=elements[:-2])
+                elements_subset = elements[:-2]
+                if len(elements_subset) != len(elements):
+                    next_element = elements[-2][0]
+                    next2_element = elements[-1][0]
+                else:
+                    next_element = None
+                parent = self.get_nested_obj(obj=self, elements=elements_subset, next_element=next_element)
                 parent_to_obj_elem, parent_to_obj_action = elements[-2]
                 obj = self._get_elem_and_compare_to_old_value(
                     obj=parent, path_for_err_reporting=path, expected_old_value=None,
-                    elem=parent_to_obj_elem, action=parent_to_obj_action)
+                    elem=parent_to_obj_elem, action=parent_to_obj_action, next_element=next2_element)
             else:
                 parent = parent_to_obj_elem = parent_to_obj_action = None
-                obj = self.get_nested_obj(obj=self, elements=elements[:-1])
+                obj = self
+                # obj = self.get_nested_obj(obj=self, elements=elements[:-1])
             elem, action = elements[-1]
         except Exception as e:
             self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path, e))
@@ -458,6 +502,57 @@ def _do_values_or_type_changed(self, changes, is_type_change=False, verify_chang
                 self._do_verify_changes(path, expected_old_value, current_old_value)
 
     def _do_item_removed(self, items):
+        """
+        Handle removing items.
+        """
+        # Sorting the iterable_item_removed in reverse order based on the paths.
+        # So that we delete a bigger index before a smaller index
+        # if hasattr(self, 'DEBUG'):
+        #     import pytest; pytest.set_trace()
+        for path, expected_old_value in sorted(items.items(), key=self._sort_key_for_item_added, reverse=True):
+            elem_and_details = self._get_elements_and_details(path)
+            if elem_and_details:
+                elements, parent, parent_to_obj_elem, parent_to_obj_action, obj, elem, action = elem_and_details
+            else:
+                continue  # pragma: no cover. Due to cPython peephole optimizer, this line doesn't get covered. https://github.com/nedbat/coveragepy/issues/198
+
+            look_for_expected_old_value = False
+            current_old_value = not_found
+            try:
+                if action == GET:
+                    current_old_value = obj[elem]
+                    look_for_expected_old_value = current_old_value != expected_old_value
+                elif action == GETATTR:
+                    current_old_value = getattr(obj, elem)
+                    look_for_expected_old_value = current_old_value != expected_old_value
+            except (KeyError, IndexError, AttributeError, TypeError):
+                look_for_expected_old_value = True
+
+            if look_for_expected_old_value and isinstance(obj, list) and not self._iterable_compare_func_was_used:
+                # It may return None if it doesn't find it
+                elem = self._find_closest_iterable_element_for_index(obj, elem, expected_old_value)
+                if elem is not None:
+                    current_old_value = expected_old_value
+            if current_old_value is not_found or elem is None:
+                continue
+
+            self._del_elem(parent, parent_to_obj_elem, parent_to_obj_action,
+                           obj, elements, path, elem, action)
+            self._do_verify_changes(path, expected_old_value, current_old_value)
+
+    def _find_closest_iterable_element_for_index(self, obj, elem, expected_old_value):
+        closest_elem = None
+        closest_distance = float('inf')
+        for index, value in enumerate(obj):
+            dist = abs(index - elem)
+            if dist > closest_distance:
+                break
+            if value == expected_old_value and dist < closest_distance:
+                closest_elem = index
+                closest_distance = dist
+        return closest_elem
+
+    def _do_item_removedOLD(self, items):
         """
         Handle removing items.
         """
@@ -695,10 +790,9 @@ def _from_flat_dicts(flat_dict_list):
         Create the delta's diff object from the flat_dict_list
         """
         result = {}
-
-        DEFLATTENING_NEW_ACTION_MAP = {
-            'iterable_item_added': 'iterable_items_added_at_indexes',
-            'iterable_item_removed': 'iterable_items_removed_at_indexes',
+        FLATTENING_NEW_ACTION_MAP = {
+            'unordered_iterable_item_added': 'iterable_items_added_at_indexes',
+            'unordered_iterable_item_removed': 'iterable_items_removed_at_indexes',
         }
         for flat_dict in flat_dict_list:
             index = None
@@ -710,8 +804,8 @@ def _from_flat_dicts(flat_dict_list):
                 raise ValueError("Flat dict need to include the 'action'.")
             if path is None:
                 raise ValueError("Flat dict need to include the 'path'.")
-            if action in DEFLATTENING_NEW_ACTION_MAP:
-                action = DEFLATTENING_NEW_ACTION_MAP[action]
+            if action in FLATTENING_NEW_ACTION_MAP:
+                action = FLATTENING_NEW_ACTION_MAP[action]
                 index = path.pop()
             if action in {'attribute_added', 'attribute_removed'}:
                 root_element = ('root', GETATTR)
@@ -729,8 +823,8 @@ def _from_flat_dicts(flat_dict_list):
                     result[action][path_str] = set()
                 result[action][path_str].add(value)
             elif action in {
-                'dictionary_item_added', 'dictionary_item_removed', 'iterable_item_added',
-                'iterable_item_removed', 'attribute_removed', 'attribute_added'
+                'dictionary_item_added', 'dictionary_item_removed',
+                'attribute_removed', 'attribute_added', 'iterable_item_added', 'iterable_item_removed',
             }:
                 result[action][path_str] = value
             elif action == 'values_changed':
@@ -843,10 +937,12 @@ def to_flat_dicts(self, include_action_in_path=False, report_type_changes=True):
             ]
 
         FLATTENING_NEW_ACTION_MAP = {
-            'iterable_items_added_at_indexes': 'iterable_item_added',
-            'iterable_items_removed_at_indexes': 'iterable_item_removed',
+            'iterable_items_added_at_indexes': 'unordered_iterable_item_added',
+            'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed',
         }
         for action, info in self.diff.items():
+            if action.startswith('_'):
+                continue
             if action in FLATTENING_NEW_ACTION_MAP:
                 new_action = FLATTENING_NEW_ACTION_MAP[action]
                 for path, index_to_value in info.items():
diff --git a/deepdiff/path.py b/deepdiff/path.py
index 0390a6d..641111e 100644
--- a/deepdiff/path.py
+++ b/deepdiff/path.py
@@ -115,7 +115,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
     return tuple(elements)
 
 
-def _get_nested_obj(obj, elements):
+def _get_nested_obj(obj, elements, next_element=None):
     for (elem, action) in elements:
         if action == GET:
             obj = obj[elem]
@@ -124,21 +124,50 @@ def _get_nested_obj(obj, elements):
     return obj
 
 
-def _get_nested_obj_and_force(obj, elements):
-    for (elem, action) in elements:
+def _guess_type(elements, elem, index, next_element):
+    # If we are not at the last elements
+    if index < len(elements) - 1:
+        # We assume it is a nested dictionary not a nested list
+        return {}
+    if isinstance(next_element, int):
+        return []
+    return {}
+
+
+def _get_nested_obj_and_force(obj, elements, next_element=None):
+    prev_elem = None
+    prev_action = None
+    prev_obj = obj
+    for index, (elem, action) in enumerate(elements):
+        _prev_obj = obj
         if action == GET:
             try:
                 obj = obj[elem]
+                prev_obj = _prev_obj
             except KeyError:
-                obj[elem] = {}
+                obj[elem] = _guess_type(elements, elem, index, next_element)
                 obj = obj[elem]
+                prev_obj = _prev_obj
             except IndexError:
                 if isinstance(obj, list) and isinstance(elem, int) and elem >= len(obj):
                     obj.extend([None] * (elem - len(obj)))
-                    obj.append({})
+                    obj.append(_guess_type(elements, elem, index), next_element)
                     obj = obj[-1]
+                    prev_obj = _prev_obj
+                elif isinstance(obj, list) and len(obj) == 0 and prev_elem:
+                    # We ran into an empty list that should have been a dictionary
+                    # We need to change it from an empty list to a dictionary
+                    obj = {elem: _guess_type(elements, elem, index, next_element)}
+                    if prev_action == GET:
+                        prev_obj[prev_elem] = obj
+                    else:
+                        setattr(prev_obj, prev_elem, obj)
+                    obj = obj[elem]
         elif action == GETATTR:
             obj = getattr(obj, elem)
+            prev_obj = _prev_obj
+        prev_elem = elem
+        prev_action = action
     return obj
 
 
@@ -245,9 +274,10 @@ def stringify_element(param, quote_str=None):
         new_param = []
         for char in param:
             if char in {'"', "'"}:
+                import pytest; pytest.set_trace()
                 new_param.append('\\')
             new_param.append(char)
-        param = ''.join(new_param)
+        result = '"' + ''.join(new_param) + '"'
     elif has_quote:
         result = f'"{param}"'
     elif has_double_quote:
diff --git a/deepdiff/serialization.py b/deepdiff/serialization.py
index 1ee2904..d2e8537 100644
--- a/deepdiff/serialization.py
+++ b/deepdiff/serialization.py
@@ -256,6 +256,9 @@ def _to_delta_dict(self, directed=True, report_repetition_required=True, always_
             # and will be omitted when counting distance. (Look inside the distance module.)
             result['_numpy_paths'] = self._numpy_paths
 
+        if self.iterable_compare_func:
+            result['_iterable_compare_func_was_used'] = True
+
         return deepcopy(dict(result))
 
     def pretty(self):
diff --git a/docs/delta.rst b/docs/delta.rst
index 418daa2..751dfba 100644
--- a/docs/delta.rst
+++ b/docs/delta.rst
@@ -21,6 +21,10 @@ delta_path : String, default=None.
 delta_file : File Object, default=None.
     :ref:`delta_file_label` is the file object containing the delta data.
 
+delta_diff : Delta diff, default=None.
+    This is a slightly different diff than the output of DeepDiff. When Delta object is initiated from the DeepDiff output, it transforms the diff into a slightly different structure that is more suitable for delta. You can find that object via delta.diff.
+    It is the same object that is serialized when you create a delta dump. If you already have the delta_diff object, you can pass it to Delta via the delta_diff parameter.
+
 flat_dict_list : List of flat dictionaries, default=None,
     :ref:`flat_dict_list_label` can be used to load the delta object from a list of flat dictionaries.
 
diff --git a/tests/test_delta.py b/tests/test_delta.py
index 08e23a8..13a7f40 100644
--- a/tests/test_delta.py
+++ b/tests/test_delta.py
@@ -381,18 +381,19 @@ def test_list_difference_delta_if_item_is_already_removed(self, mock_logger):
                 "root[3]": 'to_be_removed2'
             }
         }
-        expected_msg = VERIFICATION_MSG.format("root[3]", 'to_be_removed2', not_found, 'list index out of range')
         delta = Delta(diff, bidirectional=True, raise_errors=True)
-        with pytest.raises(DeltaError) as excinfo:
-            delta + t1
-        assert expected_msg == str(excinfo.value)
+        assert delta + t1 == t2, (
+            "We used to throw errors when the item to be removed was not found. "
+            "Instead, we try to look for the item to be removed even when the "
+            "index of it in delta is different than the index of it in the object."
+        )
 
         delta2 = Delta(diff, bidirectional=False, raise_errors=False)
         assert t1 + delta2 == t2
         expected_msg = UNABLE_TO_GET_PATH_MSG.format('root[3]')
-        mock_logger.assert_called_with(expected_msg)
+        assert 0 == mock_logger.call_count
 
-    def test_list_difference_delta_raises_error_if_prev_value_changed(self):
+    def test_list_difference_delta_does_not_raise_error_if_prev_value_changed(self):
         t1 = {
             1: 1,
             2: 2,
@@ -410,15 +411,17 @@ def test_list_difference_delta_raises_error_if_prev_value_changed(self):
                 "root[4]['b'][3]": 'to_be_removed2'
             }
         }
-        expected_msg = VERIFICATION_MSG.format("root[4]['b'][2]", 'to_be_removed', 'wrong', VERIFY_BIDIRECTIONAL_MSG)
 
+        # The previous behavior was to throw an error here because the original value for "root[4]['b'][2]" was not 'wrong' anymore.
+        # However, I decided to change that behavior to what makes more sense and is consistent with the bidirectional flag.
+        # No more verify_symmetry flag.
         delta = Delta(diff, bidirectional=True, raise_errors=True)
-        with pytest.raises(ValueError) as excinfo:
-            delta + t1
-        assert expected_msg == str(excinfo.value)
+        assert delta + t1 != t2
+        expected = {1: 1, 2: 2, 3: 3, 4: {'a': 'hello', 'b': [1, 2, 'wrong']}}
+        assert expected == delta + t1
 
         delta2 = Delta(diff, bidirectional=False, raise_errors=True)
-        assert t1 + delta2 == t2
+        assert expected == t1 + delta2
 
     def test_delta_dict_items_added_retain_order(self):
         t1 = {
@@ -1235,24 +1238,25 @@ def test_list_ignore_order_various_deltas2(self):
 
         flat_result1 = delta1.to_flat_dicts()
         flat_expected1 = [
-            {'path': [0], 'value': 7, 'action': 'iterable_item_added'},
-            {'path': [6], 'value': 8, 'action': 'iterable_item_added'},
-            {'path': [1], 'value': 4, 'action': 'iterable_item_added'},
-            {'path': [2], 'value': 4, 'action': 'iterable_item_added'},
-            {'path': [5], 'value': 4, 'action': 'iterable_item_added'},
-            {'path': [6], 'value': 6, 'action': 'iterable_item_removed'},
-            {'path': [0], 'value': 5, 'action': 'iterable_item_removed'},
+            {'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added'},
+            {'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added'},
+            {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'},
+            {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'},
+            {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'},
+            {'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed'},
+            {'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed'},
         ]
         assert flat_expected1 == flat_result1
 
         delta1_again = Delta(flat_dict_list=flat_expected1)
+        assert t1_plus_delta1 == t1 + delta1_again
         assert delta1.diff == delta1_again.diff
 
         flat_result2 = delta2.to_flat_dicts()
         flat_expected2 = [
-            {'path': [1], 'value': 4, 'action': 'iterable_item_added'},
-            {'path': [2], 'value': 4, 'action': 'iterable_item_added'},
-            {'path': [5], 'value': 4, 'action': 'iterable_item_added'},
+            {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'},
+            {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'},
+            {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'},
             {'path': [6], 'action': 'values_changed', 'value': 7},
             {'path': [0], 'action': 'values_changed', 'value': 8},
         ]
@@ -1304,6 +1308,7 @@ def test_delta_view_and_to_delta_dict_are_equal_when_parameteres_passed(self):
             'custom_operators': [],
             'encodings': None,
             'ignore_encoding_errors': False,
+            'iterable_compare_func': None,
         }
 
         expected = {'iterable_items_added_at_indexes': {'root': {1: 1, 2: 1, 3: 1}}, 'iterable_items_removed_at_indexes': {'root': {1: 2, 2: 2}}}
@@ -1507,7 +1512,7 @@ def test_delta_to_dict(self):
         assert expected == result
 
         flat_result = delta.to_flat_dicts()
-        flat_expected = [{'action': 'iterable_item_removed', 'path': [2], 'value': 'B'}]
+        flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B'}]
         assert flat_expected == flat_result
 
         delta_again = Delta(flat_dict_list=flat_expected)
@@ -1707,23 +1712,21 @@ def test_compare_func_with_duplicates_removed(self):
         ]
         assert flat_expected == flat_result
 
-        Delta.DEBUG = True
-        delta_again = Delta(flat_dict_list=flat_expected)
+        # Delta.DEBUG = True
+        delta_again = Delta(flat_dict_list=flat_expected, iterable_compare_func_was_used=True)
         expected_delta_dict = {
-            'iterable_items_removed_at_indexes': {
-                'root': {
-                    2: {
-                        'id': 1,
-                        'val': 3
-                    },
-                    0: {
-                        'id': 1,
-                        'val': 3
-                    },
-                    3: {
-                        'id': 3,
-                        'val': 3
-                    }
+            'iterable_item_removed': {
+                'root[2]': {
+                    'id': 1,
+                    'val': 3
+                },
+                'root[0]': {
+                    'id': 1,
+                    'val': 3
+                },
+                'root[3]': {
+                    'id': 3,
+                    'val': 3
                 }
             },
             'iterable_item_moved': {
@@ -1941,7 +1944,8 @@ def test_flatten_list_with_one_item_added(self):
         assert flat_expected == flat_result
 
         delta_again = Delta(flat_dict_list=flat_expected, force=True)
-        assert {'iterable_items_added_at_indexes': {"root['field2']": {0: 'James'}}} == delta_again.diff
+        assert {'iterable_item_added': {"root['field2'][0]": 'James'}} == delta_again.diff
+        # delta_again.DEBUG = True
         assert t2 == t1 + delta_again
 
         diff2 = DeepDiff(t2, t3)
@@ -1952,7 +1956,7 @@ def test_flatten_list_with_one_item_added(self):
 
         delta_again2 = Delta(flat_dict_list=flat_expected2, force=True)
 
-        assert {'iterable_items_added_at_indexes': {"root['field2']": {1: 'Jack'}}} == delta_again2.diff
+        assert {'iterable_item_added': {"root['field2'][1]": 'Jack'}} == delta_again2.diff
         assert t3 == t2 + delta_again2
 
     def test_flatten_set_with_one_item_added(self):
@@ -1986,21 +1990,25 @@ def test_flatten_tuple_with_one_item_added(self):
         t3 = {"field1": {"joe": "Joe"}, "field2": ("James", "Jack")}
         diff = DeepDiff(t1, t2)
         delta = Delta(diff=diff, always_include_values=True)
+        assert t2 == t1 + delta
         flat_expected = delta.to_flat_dicts(report_type_changes=False)
         expected_result = [{'path': ['field2', 0], 'value': 'James', 'action': 'iterable_item_added'}]
         assert expected_result == flat_expected
 
-        delta_again = Delta(flat_dict_list=flat_expected)
-        assert {'iterable_items_added_at_indexes': {"root['field2']": {0: 'James'}}} == delta_again.diff
+        delta_again = Delta(flat_dict_list=flat_expected, force=True)
+        assert {'iterable_item_added': {"root['field2'][0]": 'James'}} == delta_again.diff
+        assert {'field1': {'joe': 'Joe'}, 'field2': ['James']} == t1 + delta_again, "We lost the information about tuple when we convert to flat dict."
 
         diff = DeepDiff(t2, t3)
-        delta2 = Delta(diff=diff, always_include_values=True)
+        delta2 = Delta(diff=diff, always_include_values=True, force=True)
         flat_result2 = delta2.to_flat_dicts(report_type_changes=False)
         expected_result2 = [{'path': ['field2', 1], 'value': 'Jack', 'action': 'iterable_item_added'}]
         assert expected_result2 == flat_result2
+        assert t3 == t2 + delta2
 
         delta_again2 = Delta(flat_dict_list=flat_result2)
-        assert {'iterable_items_added_at_indexes': {"root['field2']": {1: 'Jack'}}} == delta_again2.diff
+        assert {'iterable_item_added': {"root['field2'][1]": 'Jack'}} == delta_again2.diff
+        assert t3 == t2 + delta_again2
 
     def test_flatten_list_with_multiple_item_added(self):
         t1 = {"field1": {"joe": "Joe"}}
@@ -2057,3 +2065,70 @@ def test_flatten_when_simple_type_change(self):
 
         delta_again = Delta(flat_dict_list=flat_result3)
         assert {'values_changed': {'root[2]': {'new_value': 3, 'old_value': '3'}}} == delta_again.diff
+
+    def test_subtract_delta1(self):
+        t1 = {'field_name1': ['yyy']}
+        t2 = {'field_name1': ['xxx', 'yyy']}
+        delta_diff = {'iterable_items_removed_at_indexes': {"root['field_name1']": {(0, 'GET'): 'xxx'}}}
+        expected_reverse_diff = {'iterable_items_added_at_indexes': {"root['field_name1']": {(0, 'GET'): 'xxx'}}}
+
+        delta = Delta(delta_diff=delta_diff, bidirectional=True)
+        reversed_diff = delta._get_reverse_diff()
+        assert expected_reverse_diff == reversed_diff
+        assert t2 != {'field_name1': ['yyy', 'xxx']} == t1 - delta, "Since iterable_items_added_at_indexes is used when ignore_order=True, the order is not necessarily the original order."
+
+    def test_subtract_delta_made_from_flat_dicts1(self):
+        t1 = {'field_name1': ['xxx', 'yyy']}
+        t2 = {'field_name1': []}
+        diff = DeepDiff(t1, t2)
+        delta = Delta(diff=diff, bidirectional=True)
+        flat_dict_list = delta.to_flat_dicts(include_action_in_path=False, report_type_changes=True)
+        expected_flat_dicts = [{
+            'path': ['field_name1', 0],
+            'value': 'xxx',
+            'action': 'iterable_item_removed'
+        }, {
+            'path': ['field_name1', 1],
+            'value': 'yyy',
+            'action': 'iterable_item_removed'
+        }]
+        assert expected_flat_dicts == flat_dict_list
+
+        delta1 = Delta(flat_dict_list=flat_dict_list, bidirectional=True, force=True)
+        assert t1 == t2 - delta1
+
+        delta2 = Delta(flat_dict_list=[flat_dict_list[0]], bidirectional=True, force=True)
+        middle_t = t2 - delta2
+        assert {'field_name1': ['xxx']} == middle_t
+
+        delta3 = Delta(flat_dict_list=[flat_dict_list[1]], bidirectional=True, force=True)
+        assert t1 == middle_t - delta3
+
+    def test_subtract_delta_made_from_flat_dicts2(self):
+        t1 = {'field_name1': []}
+        t2 = {'field_name1': ['xxx', 'yyy']}
+        diff = DeepDiff(t1, t2)
+        delta = Delta(diff=diff, bidirectional=True)
+        flat_dict_list = delta.to_flat_dicts(include_action_in_path=False, report_type_changes=True)
+        expected_flat_dicts = [{
+            'path': ['field_name1', 0],
+            'value': 'xxx',
+            'action': 'iterable_item_added'
+        }, {
+            'path': ['field_name1', 1],
+            'value': 'yyy',
+            'action': 'iterable_item_added'
+        }]
+        assert expected_flat_dicts == flat_dict_list
+
+        delta1 = Delta(flat_dict_list=flat_dict_list, bidirectional=True, force=True)
+        assert t1 == t2 - delta1
+
+        # We need to subtract the changes in the reverse order if we want to feed the flat dict rows individually to Delta
+        delta2 = Delta(flat_dict_list=[flat_dict_list[0]], bidirectional=True, force=True)
+        middle_t = t2 - delta2
+        assert {'field_name1': ['yyy']} == middle_t
+
+        delta3 = Delta(flat_dict_list=[flat_dict_list[1]], bidirectional=True, force=True)
+        delta3.DEBUG = True
+        assert t1 == middle_t - delta3
diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py
index d47b0f3..7cd5342 100755
--- a/tests/test_diff_text.py
+++ b/tests/test_diff_text.py
@@ -308,6 +308,13 @@ def test_diff_quote_in_string(self):
         expected = {'values_changed': {'''root["a']['b']['c"]''': {'new_value': 2, 'old_value': 1}}}
         assert expected == diff
 
+    def test_diff_quote_and_double_quote_in_string(self):
+        t1 = {'''a'"a''': 1}
+        t2 = {'''a'"a''': 2}
+        diff = DeepDiff(t1, t2)
+        expected = {'values_changed': {"root['a\\'\\\"a']": {'new_value': 2, 'old_value': 1}}}
+        assert expected == diff
+
     def test_bytes(self):
         t1 = {
             1: 1,
@@ -2001,3 +2008,4 @@ class Bar(PydanticBaseModel):
         diff = DeepDiff(t1, t2)
         expected = {'values_changed': {'root.stuff[0].thing': {'new_value': 2, 'old_value': 1}}}
         assert expected == diff
+

From 32182c8676e1b8cb30a2a2074a6984787dc92d5a Mon Sep 17 00:00:00 2001
From: Seperman <sep@zepworks.com>
Date: Mon, 13 Nov 2023 22:42:49 -0800
Subject: [PATCH 2/7] fixes #430

---
 deepdiff/path.py        | 9 ++++-----
 tests/test_diff_text.py | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/deepdiff/path.py b/deepdiff/path.py
index 641111e..dd74144 100644
--- a/deepdiff/path.py
+++ b/deepdiff/path.py
@@ -22,7 +22,7 @@ def _add_to_elements(elements, elem, inside):
         return
     if not elem.startswith('__'):
         remove_quotes = False
-        if '\\' in elem:
+        if '𝆺𝅥𝅯' in elem or '\\' in elem:
             remove_quotes = True
         else:
             try:
@@ -62,7 +62,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
     inside_quotes = False
     quote_used = ''
     for char in path:
-        if prev_char == '\\':
+        if prev_char == '𝆺𝅥𝅯':
             elem += char
         elif char in {'"', "'"}:
             elem += char
@@ -270,12 +270,11 @@ def parse_path(path, root_element=DEFAULT_FIRST_ELEMENT, include_actions=False):
 def stringify_element(param, quote_str=None):
     has_quote = "'" in param
     has_double_quote = '"' in param
-    if has_quote and has_double_quote:
+    if has_quote and has_double_quote and not quote_str:
         new_param = []
         for char in param:
             if char in {'"', "'"}:
-                import pytest; pytest.set_trace()
-                new_param.append('\\')
+                new_param.append('𝆺𝅥𝅯')
             new_param.append(char)
         result = '"' + ''.join(new_param) + '"'
     elif has_quote:
diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py
index 7cd5342..d952170 100755
--- a/tests/test_diff_text.py
+++ b/tests/test_diff_text.py
@@ -312,7 +312,7 @@ def test_diff_quote_and_double_quote_in_string(self):
         t1 = {'''a'"a''': 1}
         t2 = {'''a'"a''': 2}
         diff = DeepDiff(t1, t2)
-        expected = {'values_changed': {"root['a\\'\\\"a']": {'new_value': 2, 'old_value': 1}}}
+        expected = {'values_changed': {'root["a\'"a"]': {'new_value': 2, 'old_value': 1}}}
         assert expected == diff
 
     def test_bytes(self):

From d5b66b7f9c7240de3b58800d4705e700ebb5cf07 Mon Sep 17 00:00:00 2001
From: Seperman <sep@zepworks.com>
Date: Mon, 13 Nov 2023 22:43:47 -0800
Subject: [PATCH 3/7] removing trace

---
 deepdiff/delta.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/deepdiff/delta.py b/deepdiff/delta.py
index 0976fb3..d167bb5 100644
--- a/deepdiff/delta.py
+++ b/deepdiff/delta.py
@@ -390,9 +390,6 @@ def _do_item_added(self, items, sort=True, insert=False):
         else:
             items = items.items()
 
-        # if getattr(self, 'DEBUG', None):
-        #     import pytest; pytest.set_trace()
-
         for path, new_value in items:
             elem_and_details = self._get_elements_and_details(path)
             if elem_and_details:
@@ -507,8 +504,6 @@ def _do_item_removed(self, items):
         """
         # Sorting the iterable_item_removed in reverse order based on the paths.
         # So that we delete a bigger index before a smaller index
-        # if hasattr(self, 'DEBUG'):
-        #     import pytest; pytest.set_trace()
         for path, expected_old_value in sorted(items.items(), key=self._sort_key_for_item_added, reverse=True):
             elem_and_details = self._get_elements_and_details(path)
             if elem_and_details:

From b5d1484637a7d3cba7ee4c3ea403bf52e053f537 Mon Sep 17 00:00:00 2001
From: Seperman <sep@zepworks.com>
Date: Mon, 13 Nov 2023 22:45:09 -0800
Subject: [PATCH 4/7] fixes #418

---
 tests/test_delta.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_delta.py b/tests/test_delta.py
index 13a7f40..d3a614d 100644
--- a/tests/test_delta.py
+++ b/tests/test_delta.py
@@ -116,7 +116,7 @@ def test_delta_dump_and_read2(self, tmp_path):
         t2 = [1, 2, 3, 5]
         diff = DeepDiff(t1, t2)
         delta_content = Delta(diff).dumps()
-        path = os.path.join('tmp_path, delta_test2.delta')
+        path = os.path.join(tmp_path, 'delta_test2.delta')
         with open(path, 'wb') as the_file:
             the_file.write(delta_content)
         delta = Delta(delta_path=path)
@@ -128,7 +128,7 @@ def test_delta_dump_and_read3(self, tmp_path):
         t2 = [1, 2, 3, 5]
         diff = DeepDiff(t1, t2)
         delta_content = Delta(diff).dumps()
-        path = os.path.join('tmp_path, delta_test2.delta')
+        path = os.path.join(tmp_path, 'delta_test2.delta')
         with open(path, 'wb') as the_file:
             the_file.write(delta_content)
         with pytest.raises(ValueError) as excinfo:

From 01210c80dc2beacccdec105b08f835ef69a0d1a3 Mon Sep 17 00:00:00 2001
From: Seperman <sep@zepworks.com>
Date: Mon, 13 Nov 2023 23:03:27 -0800
Subject: [PATCH 5/7] updating docs for Inconsistent Behavior with math_epsilon
 and ignore_order. Fixes #431

---
 deepdiff/diff.py           |  2 +-
 tests/test_diff_text.py    |  1 -
 tests/test_ignore_order.py | 26 ++++++++++++++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/deepdiff/diff.py b/deepdiff/diff.py
index 8765cc3..d95b747 100755
--- a/deepdiff/diff.py
+++ b/deepdiff/diff.py
@@ -231,7 +231,7 @@ def _group_by_sort_key(x):
             self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes)
             self.math_epsilon = math_epsilon
             if self.math_epsilon is not None and self.ignore_order:
-                logger.warning("math_epsilon will be ignored. It cannot be used when ignore_order is True.")
+                logger.warning("math_epsilon in conjunction with ignore_order=True is only used for flat object comparisons. Custom math_epsilon will not have an effect when comparing nested objects.")
             self.truncate_datetime = get_truncate_datetime(truncate_datetime)
             self.number_format_notation = number_format_notation
             if verbose_level in {0, 1, 2}:
diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py
index d952170..d1e305a 100755
--- a/tests/test_diff_text.py
+++ b/tests/test_diff_text.py
@@ -2008,4 +2008,3 @@ class Bar(PydanticBaseModel):
         diff = DeepDiff(t1, t2)
         expected = {'values_changed': {'root.stuff[0].thing': {'new_value': 2, 'old_value': 1}}}
         assert expected == diff
-
diff --git a/tests/test_ignore_order.py b/tests/test_ignore_order.py
index 41e4166..3385293 100644
--- a/tests/test_ignore_order.py
+++ b/tests/test_ignore_order.py
@@ -897,6 +897,32 @@ def test_ignore_order_and_group_by4(self):
 
         assert expected == diff
 
+    def test_math_epsilon_when_ignore_order_in_dictionary(self):
+        a = {'x': 0.001}
+        b = {'x': 0.0011}
+        diff = DeepDiff(a, b, ignore_order=True)
+        assert {'values_changed': {"root['x']": {'new_value': 0.0011, 'old_value': 0.001}}} == diff
+
+        diff2 = DeepDiff(a, b, ignore_order=True, math_epsilon=0.01)
+        assert {} == diff2
+
+    def test_math_epsilon_when_ignore_order_in_list(self):
+        a = [0.001, 2]
+        b = [2, 0.0011]
+        diff = DeepDiff(a, b, ignore_order=True)
+        assert {'values_changed': {'root[0]': {'new_value': 0.0011, 'old_value': 0.001}}} == diff
+
+        diff2 = DeepDiff(a, b, ignore_order=True, math_epsilon=0.01)
+        assert {} == diff2
+
+    def test_math_epsilon_when_ignore_order_in_nested_list(self):
+        a = [{'x': 0.001}, {'y': 2.00002}]
+        b = [{'x': 0.0011}, {'y': 2}]
+
+        diff = DeepDiff(a, b, ignore_order=True, math_epsilon=0.01)
+        expected = {'values_changed': {'root[0]': {'new_value': {'x': 0.0011}, 'old_value': {'x': 0.001}}, 'root[1]': {'new_value': {'y': 2}, 'old_value': {'y': 2.00002}}}}
+        assert expected == diff
+
 
 class TestCompareFuncIgnoreOrder:
 

From 119c6ccbb64207728f2ca7bc321a60d10683cc18 Mon Sep 17 00:00:00 2001
From: Seperman <sep@zepworks.com>
Date: Mon, 13 Nov 2023 23:07:27 -0800
Subject: [PATCH 6/7] =?UTF-8?q?Bump=20version:=206.7.0=20=E2=86=92=206.7.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 README.md            | 8 ++++----
 deepdiff/__init__.py | 2 +-
 docs/conf.py         | 4 ++--
 docs/index.rst       | 2 +-
 setup.cfg            | 2 +-
 setup.py             | 2 +-
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 739e484..b2f65be 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# DeepDiff v 6.7.0
+# DeepDiff v 6.7.1
 
 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat)
 ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat)
@@ -17,7 +17,7 @@
 
 Tested on Python 3.7+ and PyPy3.
 
-- **[Documentation](https://zepworks.com/deepdiff/6.7.0/)**
+- **[Documentation](https://zepworks.com/deepdiff/6.7.1/)**
 
 ## What is new?
 
@@ -98,11 +98,11 @@ Thank you!
 
 How to cite this library (APA style):
 
-    Dehpour, S. (2023). DeepDiff (Version 6.7.0) [Software]. Available from https://github.com/seperman/deepdiff.
+    Dehpour, S. (2023). DeepDiff (Version 6.7.1) [Software]. Available from https://github.com/seperman/deepdiff.
 
 How to cite this library (Chicago style):
 
-    Dehpour, Sep. 2023. DeepDiff (version 6.7.0).
+    Dehpour, Sep. 2023. DeepDiff (version 6.7.1).
 
 # Authors
 
diff --git a/deepdiff/__init__.py b/deepdiff/__init__.py
index 3cea1ce..e15f347 100644
--- a/deepdiff/__init__.py
+++ b/deepdiff/__init__.py
@@ -1,6 +1,6 @@
 """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes."""
 # flake8: noqa
-__version__ = '6.7.0'
+__version__ = '6.7.1'
 import logging
 
 if __name__ == '__main__':
diff --git a/docs/conf.py b/docs/conf.py
index 5e7b70f..03fcdf5 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -61,9 +61,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '6.7.0'
+version = '6.7.1'
 # The full version, including alpha/beta/rc tags.
-release = '6.7.0'
+release = '6.7.1'
 
 load_dotenv(override=True)
 DOC_VERSION = os.environ.get('DOC_VERSION', version)
diff --git a/docs/index.rst b/docs/index.rst
index bea3614..4793556 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -4,7 +4,7 @@
    contain the root `toctree` directive.
 
 
-DeepDiff 6.7.0 documentation!
+DeepDiff 6.7.1 documentation!
 =============================
 
 *******
diff --git a/setup.cfg b/setup.cfg
index 96fffc3..5630d3a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 6.7.0
+current_version = 6.7.1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/setup.py b/setup.py
index bbbf3a2..2660a66 100755
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@
 if os.environ.get('USER', '') == 'vagrant':
     del os.link
 
-version = '6.7.0'
+version = '6.7.1'
 
 
 def get_reqs(filename):

From db9f6678ad88cff1068cdca5df3d7010ab443717 Mon Sep 17 00:00:00 2001
From: Seperman <sep@zepworks.com>
Date: Mon, 13 Nov 2023 23:12:41 -0800
Subject: [PATCH 7/7] updating docs

---
 CHANGELOG.md       |  5 +++++
 README.md          |  9 ++++++++-
 docs/changelog.rst | 10 ++++++++++
 docs/index.rst     | 11 +++++++++++
 4 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01dd971..24300d0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # DeepDiff Change log
 
+- v6-7-1
+    - Support for subtracting delta objects when iterable_compare_func is used.
+    - Better handling of force adding a delta to an object. 
+    - Fix for [`Can't compare dicts with both single and double quotes in keys`](https://github.com/seperman/deepdiff/issues/430)
+    - Updated docs for Inconsistent Behavior with math_epsilon and ignore_order = True
 - v6-7-0
     - Delta can be subtracted from other objects now.
     - verify_symmetry is deprecated. Use bidirectional instead.
diff --git a/README.md b/README.md
index b2f65be..23f4384 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,14 @@ Tested on Python 3.7+ and PyPy3.
 
 Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.
 
-DeepDiff v6-7-0
+DeepDiff 6-7-1
+
+- Support for subtracting delta objects when iterable_compare_func is used.
+- Better handling of force adding a delta to an object. 
+- Fix for [`Can't compare dicts with both single and double quotes in keys`](https://github.com/seperman/deepdiff/issues/430)
+- Updated docs for Inconsistent Behavior with math_epsilon and ignore_order = True
+
+DeepDiff 6-7-0
 
 - Delta can be subtracted from other objects now.
 - verify_symmetry is deprecated. Use bidirectional instead.
diff --git a/docs/changelog.rst b/docs/changelog.rst
index de09ee3..3e44fd7 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -5,6 +5,16 @@ Changelog
 
 DeepDiff Changelog
 
+-  v6-7-1
+
+   -  Support for subtracting delta objects when iterable_compare_func
+      is used.
+   -  Better handling of force adding a delta to an object.
+   -  Fix for
+      ```Can't compare dicts with both single and double quotes in keys`` <https://github.com/seperman/deepdiff/issues/430>`__
+   -  Updated docs for Inconsistent Behavior with math_epsilon and
+      ignore_order = True
+
 -  v6-7-0
 
    -  Delta can be subtracted from other objects now.
diff --git a/docs/index.rst b/docs/index.rst
index 4793556..b337d0c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -32,6 +32,17 @@ What Is New
 ***********
 
 
+DeepDiff 6-7-1
+--------------
+
+   -  Support for subtracting delta objects when iterable_compare_func
+      is used.
+   -  Better handling of force adding a delta to an object.
+   -  Fix for
+      ```Can't compare dicts with both single and double quotes in keys`` <https://github.com/seperman/deepdiff/issues/430>`__
+   -  Updated docs for Inconsistent Behavior with math_epsilon and
+      ignore_order = True
+
 DeepDiff 6-7-0
 --------------