Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

6.7.1 #432

Merged
merged 7 commits into from
Nov 14, 2023
Merged

6.7.1 #432

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# DeepDiff Change log

- v6-7-1
- Support for subtracting delta objects when iterable_compare_func is used.
- Better handling of force adding a delta to an object.
- Fix for [`Can't compare dicts with both single and double quotes in keys`](https://github.com/seperman/deepdiff/issues/430)
- Updated docs for Inconsistent Behavior with math_epsilon and ignore_order = True
- v6-7-0
- Delta can be subtracted from other objects now.
- verify_symmetry is deprecated. Use bidirectional instead.
Expand Down
17 changes: 12 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# DeepDiff v 6.7.0
# DeepDiff v 6.7.1

![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat)
![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat)
Expand All @@ -17,13 +17,20 @@

Tested on Python 3.7+ and PyPy3.

- **[Documentation](https://zepworks.com/deepdiff/6.7.0/)**
- **[Documentation](https://zepworks.com/deepdiff/6.7.1/)**

## What is new?

Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.

DeepDiff v6-7-0
DeepDiff 6-7-1

- Support for subtracting delta objects when iterable_compare_func is used.
- Better handling of force adding a delta to an object.
- Fix for [`Can't compare dicts with both single and double quotes in keys`](https://github.com/seperman/deepdiff/issues/430)
- Updated docs for Inconsistent Behavior with math_epsilon and ignore_order = True

DeepDiff 6-7-0

- Delta can be subtracted from other objects now.
- verify_symmetry is deprecated. Use bidirectional instead.
Expand Down Expand Up @@ -98,11 +105,11 @@ Thank you!

How to cite this library (APA style):

Dehpour, S. (2023). DeepDiff (Version 6.7.0) [Software]. Available from https://github.com/seperman/deepdiff.
Dehpour, S. (2023). DeepDiff (Version 6.7.1) [Software]. Available from https://github.com/seperman/deepdiff.

How to cite this library (Chicago style):

Dehpour, Sep. 2023. DeepDiff (version 6.7.0).
Dehpour, Sep. 2023. DeepDiff (version 6.7.1).

# Authors

Expand Down
2 changes: 1 addition & 1 deletion deepdiff/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes."""
# flake8: noqa
__version__ = '6.7.0'
__version__ = '6.7.1'
import logging

if __name__ == '__main__':
Expand Down
125 changes: 108 additions & 17 deletions deepdiff/delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def __init__(
diff=None,
delta_path=None,
delta_file=None,
delta_diff=None,
flat_dict_list=None,
deserializer=pickle_load,
log_errors=True,
Expand All @@ -81,6 +82,7 @@ def __init__(
verify_symmetry=None,
bidirectional=False,
always_include_values=False,
iterable_compare_func_was_used=None,
force=False,
):
if hasattr(deserializer, '__code__') and 'safe_to_import' in set(deserializer.__code__.co_varnames):
Expand Down Expand Up @@ -114,6 +116,8 @@ def _deserializer(obj, safe_to_import=None):
with open(delta_path, 'rb') as the_file:
content = the_file.read()
self.diff = _deserializer(content, safe_to_import=safe_to_import)
elif delta_diff:
self.diff = delta_diff
elif delta_file:
try:
content = delta_file.read()
Expand All @@ -128,7 +132,10 @@ def _deserializer(obj, safe_to_import=None):
self.mutate = mutate
self.raise_errors = raise_errors
self.log_errors = log_errors
self._numpy_paths = self.diff.pop('_numpy_paths', False)
self._numpy_paths = self.diff.get('_numpy_paths', False)
# When we create the delta from a list of flat dictionaries, details such as iterable_compare_func_was_used get lost.
# That's why we allow iterable_compare_func_was_used to be explicitly set.
self._iterable_compare_func_was_used = self.diff.get('_iterable_compare_func_was_used', iterable_compare_func_was_used)
self.serializer = serializer
self.deserializer = deserializer
self.force = force
Expand Down Expand Up @@ -198,7 +205,17 @@ def _do_verify_changes(self, path, expected_old_value, current_old_value):
self._raise_or_log(VERIFICATION_MSG.format(
path_str, expected_old_value, current_old_value, VERIFY_BIDIRECTIONAL_MSG))

def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expected_old_value, elem=None, action=None, forced_old_value=None):
def _get_elem_and_compare_to_old_value(
self,
obj,
path_for_err_reporting,
expected_old_value,
elem=None,
action=None,
forced_old_value=None,
next_element=None,
):
# if forced_old_value is not None:
try:
if action == GET:
current_old_value = obj[elem]
Expand All @@ -208,9 +225,21 @@ def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expect
raise DeltaError(INVALID_ACTION_WHEN_CALLING_GET_ELEM.format(action))
except (KeyError, IndexError, AttributeError, TypeError) as e:
if self.force:
_forced_old_value = {} if forced_old_value is None else forced_old_value
if forced_old_value is None:
if next_element is None or isinstance(next_element, str):
_forced_old_value = {}
else:
_forced_old_value = []
else:
_forced_old_value = forced_old_value
if action == GET:
obj[elem] = _forced_old_value
if isinstance(obj, list):
if isinstance(elem, int) and elem < len(obj):
obj[elem] = _forced_old_value
else:
obj.append(_forced_old_value)
else:
obj[elem] = _forced_old_value
elif action == GETATTR:
setattr(obj, elem, _forced_old_value)
return _forced_old_value
Expand Down Expand Up @@ -277,6 +306,11 @@ def _set_new_value(self, parent, parent_to_obj_elem, parent_to_obj_action,
parent, obj, path, parent_to_obj_elem,
parent_to_obj_action, elements,
to_type=list, from_type=tuple)
if elem != 0 and self.force and isinstance(obj, list) and len(obj) == 0:
# it must have been a dictionary
obj = {}
self._simple_set_elem_value(obj=parent, path_for_err_reporting=path, elem=parent_to_obj_elem,
value=obj, action=parent_to_obj_action)
self._simple_set_elem_value(obj=obj, path_for_err_reporting=path, elem=elem,
value=new_value, action=action)

Expand Down Expand Up @@ -404,14 +438,21 @@ def _get_elements_and_details(self, path):
try:
elements = _path_to_elements(path)
if len(elements) > 1:
parent = self.get_nested_obj(obj=self, elements=elements[:-2])
elements_subset = elements[:-2]
if len(elements_subset) != len(elements):
next_element = elements[-2][0]
next2_element = elements[-1][0]
else:
next_element = None
parent = self.get_nested_obj(obj=self, elements=elements_subset, next_element=next_element)
parent_to_obj_elem, parent_to_obj_action = elements[-2]
obj = self._get_elem_and_compare_to_old_value(
obj=parent, path_for_err_reporting=path, expected_old_value=None,
elem=parent_to_obj_elem, action=parent_to_obj_action)
elem=parent_to_obj_elem, action=parent_to_obj_action, next_element=next2_element)
else:
parent = parent_to_obj_elem = parent_to_obj_action = None
obj = self.get_nested_obj(obj=self, elements=elements[:-1])
obj = self
# obj = self.get_nested_obj(obj=self, elements=elements[:-1])
elem, action = elements[-1]
except Exception as e:
self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path, e))
Expand Down Expand Up @@ -458,6 +499,55 @@ def _do_values_or_type_changed(self, changes, is_type_change=False, verify_chang
self._do_verify_changes(path, expected_old_value, current_old_value)

def _do_item_removed(self, items):
"""
Handle removing items.
"""
# Sorting the iterable_item_removed in reverse order based on the paths.
# So that we delete a bigger index before a smaller index
for path, expected_old_value in sorted(items.items(), key=self._sort_key_for_item_added, reverse=True):
elem_and_details = self._get_elements_and_details(path)
if elem_and_details:
elements, parent, parent_to_obj_elem, parent_to_obj_action, obj, elem, action = elem_and_details
else:
continue # pragma: no cover. Due to cPython peephole optimizer, this line doesn't get covered. https://github.com/nedbat/coveragepy/issues/198

look_for_expected_old_value = False
current_old_value = not_found
try:
if action == GET:
current_old_value = obj[elem]
look_for_expected_old_value = current_old_value != expected_old_value
elif action == GETATTR:
current_old_value = getattr(obj, elem)
look_for_expected_old_value = current_old_value != expected_old_value
except (KeyError, IndexError, AttributeError, TypeError):
look_for_expected_old_value = True

if look_for_expected_old_value and isinstance(obj, list) and not self._iterable_compare_func_was_used:
# It may return None if it doesn't find it
elem = self._find_closest_iterable_element_for_index(obj, elem, expected_old_value)
if elem is not None:
current_old_value = expected_old_value
if current_old_value is not_found or elem is None:
continue

self._del_elem(parent, parent_to_obj_elem, parent_to_obj_action,
obj, elements, path, elem, action)
self._do_verify_changes(path, expected_old_value, current_old_value)

def _find_closest_iterable_element_for_index(self, obj, elem, expected_old_value):
closest_elem = None
closest_distance = float('inf')
for index, value in enumerate(obj):
dist = abs(index - elem)
if dist > closest_distance:
break
if value == expected_old_value and dist < closest_distance:
closest_elem = index
closest_distance = dist
return closest_elem

def _do_item_removedOLD(self, items):
"""
Handle removing items.
"""
Expand Down Expand Up @@ -695,10 +785,9 @@ def _from_flat_dicts(flat_dict_list):
Create the delta's diff object from the flat_dict_list
"""
result = {}

DEFLATTENING_NEW_ACTION_MAP = {
'iterable_item_added': 'iterable_items_added_at_indexes',
'iterable_item_removed': 'iterable_items_removed_at_indexes',
FLATTENING_NEW_ACTION_MAP = {
'unordered_iterable_item_added': 'iterable_items_added_at_indexes',
'unordered_iterable_item_removed': 'iterable_items_removed_at_indexes',
}
for flat_dict in flat_dict_list:
index = None
Expand All @@ -710,8 +799,8 @@ def _from_flat_dicts(flat_dict_list):
raise ValueError("Flat dict need to include the 'action'.")
if path is None:
raise ValueError("Flat dict need to include the 'path'.")
if action in DEFLATTENING_NEW_ACTION_MAP:
action = DEFLATTENING_NEW_ACTION_MAP[action]
if action in FLATTENING_NEW_ACTION_MAP:
action = FLATTENING_NEW_ACTION_MAP[action]
index = path.pop()
if action in {'attribute_added', 'attribute_removed'}:
root_element = ('root', GETATTR)
Expand All @@ -729,8 +818,8 @@ def _from_flat_dicts(flat_dict_list):
result[action][path_str] = set()
result[action][path_str].add(value)
elif action in {
'dictionary_item_added', 'dictionary_item_removed', 'iterable_item_added',
'iterable_item_removed', 'attribute_removed', 'attribute_added'
'dictionary_item_added', 'dictionary_item_removed',
'attribute_removed', 'attribute_added', 'iterable_item_added', 'iterable_item_removed',
}:
result[action][path_str] = value
elif action == 'values_changed':
Expand Down Expand Up @@ -843,10 +932,12 @@ def to_flat_dicts(self, include_action_in_path=False, report_type_changes=True):
]

FLATTENING_NEW_ACTION_MAP = {
'iterable_items_added_at_indexes': 'iterable_item_added',
'iterable_items_removed_at_indexes': 'iterable_item_removed',
'iterable_items_added_at_indexes': 'unordered_iterable_item_added',
'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed',
}
for action, info in self.diff.items():
if action.startswith('_'):
continue
if action in FLATTENING_NEW_ACTION_MAP:
new_action = FLATTENING_NEW_ACTION_MAP[action]
for path, index_to_value in info.items():
Expand Down
2 changes: 1 addition & 1 deletion deepdiff/diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def _group_by_sort_key(x):
self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes)
self.math_epsilon = math_epsilon
if self.math_epsilon is not None and self.ignore_order:
logger.warning("math_epsilon will be ignored. It cannot be used when ignore_order is True.")
logger.warning("math_epsilon in conjunction with ignore_order=True is only used for flat object comparisons. Custom math_epsilon will not have an effect when comparing nested objects.")
self.truncate_datetime = get_truncate_datetime(truncate_datetime)
self.number_format_notation = number_format_notation
if verbose_level in {0, 1, 2}:
Expand Down
49 changes: 39 additions & 10 deletions deepdiff/path.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def _add_to_elements(elements, elem, inside):
return
if not elem.startswith('__'):
remove_quotes = False
if '\\' in elem:
if '𝆺𝅥𝅯' in elem or '\\' in elem:
remove_quotes = True
else:
try:
Expand Down Expand Up @@ -62,7 +62,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
inside_quotes = False
quote_used = ''
for char in path:
if prev_char == '\\':
if prev_char == '𝆺𝅥𝅯':
elem += char
elif char in {'"', "'"}:
elem += char
Expand Down Expand Up @@ -115,7 +115,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
return tuple(elements)


def _get_nested_obj(obj, elements):
def _get_nested_obj(obj, elements, next_element=None):
for (elem, action) in elements:
if action == GET:
obj = obj[elem]
Expand All @@ -124,21 +124,50 @@ def _get_nested_obj(obj, elements):
return obj


def _get_nested_obj_and_force(obj, elements):
for (elem, action) in elements:
def _guess_type(elements, elem, index, next_element):
# If we are not at the last elements
if index < len(elements) - 1:
# We assume it is a nested dictionary not a nested list
return {}
if isinstance(next_element, int):
return []
return {}


def _get_nested_obj_and_force(obj, elements, next_element=None):
prev_elem = None
prev_action = None
prev_obj = obj
for index, (elem, action) in enumerate(elements):
_prev_obj = obj
if action == GET:
try:
obj = obj[elem]
prev_obj = _prev_obj
except KeyError:
obj[elem] = {}
obj[elem] = _guess_type(elements, elem, index, next_element)
obj = obj[elem]
prev_obj = _prev_obj
except IndexError:
if isinstance(obj, list) and isinstance(elem, int) and elem >= len(obj):
obj.extend([None] * (elem - len(obj)))
obj.append({})
obj.append(_guess_type(elements, elem, index), next_element)
obj = obj[-1]
prev_obj = _prev_obj
elif isinstance(obj, list) and len(obj) == 0 and prev_elem:
# We ran into an empty list that should have been a dictionary
# We need to change it from an empty list to a dictionary
obj = {elem: _guess_type(elements, elem, index, next_element)}
if prev_action == GET:
prev_obj[prev_elem] = obj
else:
setattr(prev_obj, prev_elem, obj)
obj = obj[elem]
elif action == GETATTR:
obj = getattr(obj, elem)
prev_obj = _prev_obj
prev_elem = elem
prev_action = action
return obj


Expand Down Expand Up @@ -241,13 +270,13 @@ def parse_path(path, root_element=DEFAULT_FIRST_ELEMENT, include_actions=False):
def stringify_element(param, quote_str=None):
has_quote = "'" in param
has_double_quote = '"' in param
if has_quote and has_double_quote:
if has_quote and has_double_quote and not quote_str:
new_param = []
for char in param:
if char in {'"', "'"}:
new_param.append('\\')
new_param.append('𝆺𝅥𝅯')
new_param.append(char)
param = ''.join(new_param)
result = '"' + ''.join(new_param) + '"'
elif has_quote:
result = f'"{param}"'
elif has_double_quote:
Expand Down
Loading
Loading