Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Implement apply_filter_policy and FilterPolicy.MERGE for the new filters #8042

Merged
merged 19 commits into from
Aug 9, 2024
Merged

feat: Implement apply_filter_policy and FilterPolicy.MERGE for the new filters #8042

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
6808107
Implement apply_filter_policy and FilterPolicy.MERGE for the new filters
vblagoje Jul 18, 2024
e82f9ac
Add reno note
vblagoje Jul 18, 2024
f04c805
Rewrite filters using exhisting internal version
vblagoje Jul 26, 2024
24fbc29
Merge branch 'main' into new_merge_filters
vblagoje Jul 26, 2024
57a1a1a
Fixes
vblagoje Jul 26, 2024
2b7bb67
More unit tests (new filters)
vblagoje Aug 6, 2024
9e9d59a
Merge branch 'main' into new_merge_filters
vblagoje Aug 6, 2024
f03be57
Revert test_filter tests
vblagoje Aug 6, 2024
c8a53f2
PR feedaback
vblagoje Aug 7, 2024
ce60022
Revert change in unrelated release note
vblagoje Aug 7, 2024
ad47b0a
Increase test coverage, add corner cases tests
vblagoje Aug 7, 2024
ad4b59c
Make Enums case insensitive
vblagoje Aug 7, 2024
b09826c
Handle string based logical operators for backward compatibility
vblagoje Aug 7, 2024
fe45233
Update release note with doc link
vblagoje Aug 7, 2024
3c3ff75
Assume logical filters are not nested
vblagoje Aug 7, 2024
519de1c
Output pydoc fix
vblagoje Aug 7, 2024
a2621ad
Convert all apply_filter_policy logical operators to str representations
vblagoje Aug 8, 2024
aeac10d
Alternative impl to ensure str operators in logical filters
vblagoje Aug 8, 2024
a1cc6f9
Revert back to Literal instead of LogicalOperator Enum
vblagoje Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 29 additions & 9 deletions haystack/document_stores/types/filter_policy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,19 @@ def convert_logical_operators(filter_dict: Dict[str, Any]) -> Dict[str, Any]:
return filter_dict


def logical_operators_to_str(filter_dict: Dict[str, Any]) -> Dict[str, Any]:
"""
Convert any logical operators found in a filter dictionary to string.

:param filter_dict: A dictionary representing a filter with potential LogicalOperator enums.
:return: A new dictionary with LogicalOperator strings in place of enums.
"""
# If the dictionary represents a logical filter, update the 'operator'
if is_logical_filter(filter_dict) and isinstance(filter_dict["operator"], LogicalOperator):
filter_dict["operator"] = filter_dict["operator"].value
return filter_dict
shadeMe marked this conversation as resolved.
Show resolved Hide resolved


def is_comparison_filter(filter_item: Dict[str, Any]) -> bool:
"""
Check if the given filter is a comparison filter.
Expand Down Expand Up @@ -133,7 +146,7 @@ def combine_two_logical_filters(
)
# Output:
{
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand Down Expand Up @@ -186,7 +199,7 @@ def combine_init_comparison_and_runtime_logical_filters(
)
# Output:
{
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand Down Expand Up @@ -243,7 +256,7 @@ def combine_runtime_comparison_and_init_logical_filters(
)
# Output:
{
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand Down Expand Up @@ -293,7 +306,7 @@ def combine_two_comparison_filters(
)
# Output:
{
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.date", "operator": ">=", "value": "2015-01-01"},
Expand Down Expand Up @@ -340,16 +353,23 @@ def apply_filter_policy(
runtime_filters = convert_logical_operators(runtime_filters)
# now we merge filters
if is_comparison_filter(init_filters) and is_comparison_filter(runtime_filters):
return combine_two_comparison_filters(init_filters, runtime_filters, default_logical_operator)
combined_filters = combine_two_comparison_filters(init_filters, runtime_filters, default_logical_operator)
return logical_operators_to_str(combined_filters)
elif is_comparison_filter(init_filters) and is_logical_filter(runtime_filters):
return combine_init_comparison_and_runtime_logical_filters(
combined_filters = combine_init_comparison_and_runtime_logical_filters(
init_filters, runtime_filters, default_logical_operator
)
return logical_operators_to_str(combined_filters)
elif is_logical_filter(init_filters) and is_comparison_filter(runtime_filters):
return combine_runtime_comparison_and_init_logical_filters(
combined_filters = combine_runtime_comparison_and_init_logical_filters(
runtime_filters, init_filters, default_logical_operator
)
return logical_operators_to_str(combined_filters)
elif is_logical_filter(init_filters) and is_logical_filter(runtime_filters):
return combine_two_logical_filters(init_filters, runtime_filters)
combined_filters = combine_two_logical_filters(init_filters, runtime_filters)
return logical_operators_to_str(combined_filters)

return runtime_filters or init_filters
resulting_filter = runtime_filters or init_filters
if resulting_filter:
resulting_filter = logical_operators_to_str(resulting_filter)
return resulting_filter
22 changes: 11 additions & 11 deletions test/document_stores/test_filter_policy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def test_merge_two_comparison_filters():
runtime_filters = {"field": "meta.type", "operator": "==", "value": "article"}
result = apply_filter_policy(FilterPolicy.MERGE, init_filters, runtime_filters)
assert result == {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.date", "operator": ">=", "value": "2015-01-01"},
{"field": "meta.type", "operator": "==", "value": "article"},
Expand All @@ -60,15 +60,15 @@ def test_merge_init_comparison_and_runtime_logical_filters():
"""
init_filters = {"field": "meta.date", "operator": ">=", "value": "2015-01-01"}
runtime_filters = {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
],
}
result = apply_filter_policy(FilterPolicy.MERGE, init_filters, runtime_filters)
assert result == {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand Down Expand Up @@ -99,7 +99,7 @@ def test_merge_runtime_comparison_and_init_logical_filters_with_string_operators
}
result = apply_filter_policy(FilterPolicy.MERGE, init_filters, runtime_filters)
assert result == {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand All @@ -115,7 +115,7 @@ def test_merge_runtime_comparison_and_init_logical_filters():
Result: AND operator with both filters
"""
init_filters = {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand All @@ -124,7 +124,7 @@ def test_merge_runtime_comparison_and_init_logical_filters():
runtime_filters = {"field": "meta.date", "operator": ">=", "value": "2015-01-01"}
result = apply_filter_policy(FilterPolicy.MERGE, init_filters, runtime_filters)
assert result == {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand All @@ -139,22 +139,22 @@ def test_merge_two_logical_filters():
Result: AND operator with both filters
"""
init_filters = {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
],
}
runtime_filters = {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.genre", "operator": "IN", "value": ["economy", "politics"]},
{"field": "meta.publisher", "operator": "==", "value": "nytimes"},
],
}
result = apply_filter_policy(FilterPolicy.MERGE, init_filters, runtime_filters)
assert result == {
"operator": LogicalOperator.AND,
"operator": "AND",
"conditions": [
{"field": "meta.type", "operator": "==", "value": "article"},
{"field": "meta.rating", "operator": ">=", "value": 3},
Expand All @@ -171,7 +171,7 @@ def test_merge_with_different_logical_operators():
"""
init_filters = {"operator": "AND", "conditions": [{"field": "meta.type", "operator": "==", "value": "article"}]}
runtime_filters = {
"operator": LogicalOperator.OR,
"operator": "OR",
"conditions": [{"field": "meta.genre", "operator": "IN", "value": ["economy", "politics"]}],
}
result = apply_filter_policy(FilterPolicy.MERGE, init_filters, runtime_filters)
Expand Down Expand Up @@ -204,7 +204,7 @@ def test_merge_with_custom_logical_operator(logical_operator):
default_logical_operator=LogicalOperator.from_str(logical_operator),
)
assert result == {
"operator": LogicalOperator.from_str(logical_operator),
"operator": logical_operator,
"conditions": [
{"field": "meta.date", "operator": ">=", "value": "2015-01-01"},
{"field": "meta.type", "operator": "==", "value": "article"},
Expand Down