WIP POC Expression filter

fixes pulp#2480
fixes pulp#3914

CHANGES/2480.feature

@@ -0,0 +1 @@
+Added a filter `q` that supports complex filter expressions.

CHANGES/3914.feature

@@ -0,0 +1 @@
+Added a filter `q` that supports complex filter expressions.

pulpcore/app/viewsets/task.py

@@ -60,9 +60,9 @@ class TaskFilter(BaseFilterSet):
     class Meta:
         model = Task
         fields = {
-            "state": ["exact", "in"],
-            "worker": ["exact", "in"],
-            "name": ["exact", "contains", "in"],
+            "state": ["exact", "in", "ne"],
+            "worker": ["exact", "in", "isnull"],
+            "name": ["exact", "contains", "in", "ne"],
             "logging_cid": ["exact", "contains"],
             "started_at": DATETIME_FILTER_OPTIONS,
             "finished_at": DATETIME_FILTER_OPTIONS,

pulpcore/filters.py

@@ -1,5 +1,7 @@
 from gettext import gettext as _
 
+import pyparsing as pp
+
 from functools import lru_cache
 from urllib.parse import urlparse
 from uuid import UUID
@@ -150,6 +152,57 @@ def pulp_type_choices(model):
         return choices
 
 
+class ExpressionFilter(filters.CharFilter):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        # TODO Proper error handling
+        if value not in EMPTY_VALUES:
+
+            def group_action(tokens):
+                key = tokens[0].key
+                value = tokens[0].value
+                filter = self.parent.filters[key]
+                if isinstance(filter, ExpressionFilter):
+                    raise serializers.ValidationError("Recursion: see Recursion!")
+                if isinstance(filter, filters.OrderingFilter):
+                    raise serializers.ValidationError(
+                        "order You the query filter. cannot the inside q"
+                    )
+                form = self.parent.form.__class__({key: value})
+                form.full_clean()
+                return filter.filter(qs, form.cleaned_data[key])
+
+            def not_action(tokens):
+                return qs.difference(tokens[0][1])
+
+            def and_action(tokens):
+                return tokens[0][0].intersection(*tokens[0][2::2])
+
+            def or_action(tokens):
+                return tokens[0][0].union(*tokens[0][2::2])
+
+            slug = pp.Word(pp.alphas, pp.alphanums + "_")
+            word = pp.Word(pp.alphanums + pp.alphas8bit + ",_-*'")
+            rhs = word | pp.quoted_string
+
+            group = pp.Group(
+                slug.set_results_name("key") + pp.Literal("=") + rhs.set_results_name("value")
+            ).set_parse_action(group_action)
+
+            expr = pp.infix_notation(
+                group,
+                [
+                    (pp.Keyword("NOT"), 1, pp.opAssoc.RIGHT, not_action),
+                    (pp.Keyword("AND"), 2, pp.opAssoc.LEFT, and_action),
+                    (pp.Keyword("OR"), 2, pp.opAssoc.LEFT, or_action),
+                ],
+            )
+            qs = expr.parse_string(value, parse_all=True)[0]
+        return qs
+
+
 class BaseFilterSet(filterset.FilterSet):
     """
     Class to override django_filter's FilterSet and provide a way to set help text
@@ -165,6 +218,7 @@ class BaseFilterSet(filterset.FilterSet):
     help_text = {}
     pulp_id__in = IdInFilter(field_name="pk", lookup_expr="in")
     pulp_href__in = HREFInFilter(field_name="pk", method="filter_pulp_href")
+    q = ExpressionFilter()
 
     FILTER_DEFAULTS = {
         **filterset.FilterSet.FILTER_DEFAULTS,

requirements.txt

@@ -1,7 +1,7 @@
-asyncio-throttle>=1.0,<=1.0.2
-aiohttp>=3.8.1,<3.8.6
 aiodns>=3.0,<=3.0.0
 aiofiles>=22.1,<23.3.0
+aiohttp>=3.8.1,<3.8.6
+asyncio-throttle>=1.0,<=1.0.2
 backoff>=2.1.2,<2.2.2
 click>=8.1.0,<=8.1.6
 cryptography>=38.0.1,<41.0.4
@@ -17,19 +17,20 @@ drf-nested-routers>=0.93.4,<=0.93.4
 drf-spectacular==0.26.4  # We monkeypatch this so we need a very narrow requirement string
 dynaconf>=3.1.12,<3.2.1
 gunicorn>=20.1,<=21.2.0
+importlib-metadata>=6.0.1,<=6.0.1  # Pinned to fix opentelemetry dependency solving issues with pip 
 jinja2>=3.1,<=3.1.2
 naya>=1.1.1,<=1.1.1
-importlib-metadata>=6.0.1,<=6.0.1  # Pinned to fix opentelemetry dependency solving issues with pip 
 opentelemetry-distro[otlp]>=0.38b0,<=0.40b0
 opentelemetry-exporter-otlp-proto-http>=1.17.0,<=1.19.0
 opentelemetry-instrumentation-django>=0.38b0,<=0.40b0
 opentelemetry-instrumentation-wsgi>=0.38b0,<=0.40b0
-pulp-glue>=0.18.0,<0.22
 protobuf>=4.21.1,<4.24.1
+pulp-glue>=0.18.0,<0.22
 pygtrie>=2.5,<=2.5.0
 psycopg[binary]>=3.1.8,<=3.1.10
-PyYAML>=5.1.1,<=6.0.1
+pyparsing>=3.1.0,<=3.1.1
 python-gnupg>=0.5,<=0.5.1
+PyYAML>=5.1.1,<=6.0.1
 redis>=4.3,<4.6.1
 setuptools>=39.2,<68.1.0
 url-normalize>=1.4.3,<=1.4.3