Expression filter

fixes pulp#2480
fixes pulp#3914

CHANGES/2480.feature

@@ -0,0 +1 @@
+Added a filter `q` that supports complex filter expressions.

CHANGES/3914.feature

@@ -0,0 +1 @@
+Added a filter `q` that supports complex filter expressions.

docs/workflows/advanced-filtering.rst

@@ -0,0 +1,33 @@
+Advanced Filtering
+==================
+
+In addition to the usual querystring filters, Pulp provides a special ``q`` filter, that allows you
+to combine other filters with `NOT`, `AND` and `OR` operations.
+
+For a given list endpoint, all the other existing (non ordering) filters can be used in these
+expressions.
+
+The grammar is basically::
+
+    EXPRESSION = SIMPLE_EXPR | "(" SIMPLE_EXPR | COMPOSIT_EXPR ")"
+    COMPOSIT_EXPR = NOT_EXPR | AND_EXPR | OR_EXPR
+    NOT_EXPR = "NOT" WS EXPRESSION
+    AND_EXPRESSION = EXPRESSION "AND" EXPRESSION
+    OR_EXPRESSION = EXPRESSION "OR" EXPRESSION
+    SIMPLE_EXPRESSION = FILTERNAME "=" STRING
+    STRING = SIMPLE_STRING | QUOTED_STRING
+
+Some example ``q`` expressions are::
+
+    pulp_type__in='core.rbac'
+    NOT pulp_type="core.rbac"
+    pulp_type__in=core.rbac,core.content_redirect
+    pulp_type="core.rbac" OR pulp_type="core.content_redirect"
+    pulp_type="core.rbac" AND name__contains=GGGG
+    pulp_type="core.rbac" AND name__iexact={prefix}-gGgG
+    pulp_type="core.rbac" AND name__icontains=gg AND NOT name__contains=HH
+    NOT (pulp_type="core.rbac" AND name__icontains=gGgG)
+    pulp_type="core.rbac" AND name__contains="naïve"
+    pulp_type="core.rbac" AND NOT name__contains="naïve"
+    pulp_type="core.rbac" AND(   name__icontains=gh OR name__contains="naïve")
+    pulp_type="core.rbac" OR name__icontains=gh OR name__contains="naïve"

docs/workflows/index.rst

@@ -25,3 +25,4 @@ assumes that the reader is familiar with the fundamentals discussed in the :doc:
    plugin-removal
    troubleshooting
    domains-multi-tenancy
+   advanced-filtering

pulpcore/app/viewsets/base.py

@@ -25,7 +25,15 @@
 from pulpcore.tasking.tasks import dispatch
 
 # These should be used to prevent duplication and keep things consistent
-NAME_FILTER_OPTIONS = ["exact", "in", "icontains", "contains", "startswith"]
+NAME_FILTER_OPTIONS = [
+    "exact",
+    "iexact",
+    "in",
+    "contains",
+    "icontains",
+    "startswith",
+    "istartswith",
+]
 # e.g.
 # /?name=foo
 # /?name__in=foo,bar

pulpcore/app/viewsets/task.py

@@ -60,9 +60,9 @@ class TaskFilter(BaseFilterSet):
     class Meta:
         model = Task
         fields = {
-            "state": ["exact", "in"],
-            "worker": ["exact", "in"],
-            "name": ["exact", "contains", "in"],
+            "state": ["exact", "in", "ne"],
+            "worker": ["exact", "in", "isnull"],
+            "name": ["exact", "contains", "in", "ne"],
             "logging_cid": ["exact", "contains"],
             "started_at": DATETIME_FILTER_OPTIONS,
             "finished_at": DATETIME_FILTER_OPTIONS,

pulpcore/filters.py

@@ -1,8 +1,11 @@
 from gettext import gettext as _
 
-from functools import lru_cache
+import pyparsing as pp
+
+from functools import lru_cache, partial
 from urllib.parse import urlparse
 from uuid import UUID
+from django import forms
 from django.db import models
 from django.forms.utils import ErrorList
 from django.urls import Resolver404, resolve
@@ -128,7 +131,7 @@ class HREFInFilter(BaseInFilter, filters.CharFilter):
     pass
 
 
-class PulpTypeInFilter(BaseInFilter, filters.ChoiceFilter):
+class PulpTypeFilter(filters.ChoiceFilter):
     """Special pulp_type filter only added to generic list endpoints."""
 
     def __init__(self, *args, **kwargs):
@@ -150,6 +153,104 @@ def pulp_type_choices(model):
         return choices
 
 
+class PulpTypeInFilter(BaseInFilter, PulpTypeFilter):
+    """Special pulp_type filter only added to generic list endpoints."""
+
+
+class ExpressionFilterField(forms.CharField):
+    class _FilterAction:
+        def __init__(self, filterset, tokens):
+            key = tokens[0].key
+            value = tokens[0].value
+            self.filter = filterset.filters.get(key)
+            if self.filter is None:
+                raise forms.ValidationError(_("Filter '{key}' does not exist.").format(key=key))
+            if isinstance(self.filter, ExpressionFilter):
+                raise forms.ValidationError(
+                    _("You cannot use '{key}' in complex filtering.").format(key=key)
+                )
+            if isinstance(self.filter, filters.OrderingFilter):
+                raise forms.ValidationError(
+                    _("An ordering filter cannot be used in complex filtering.")
+                )
+            form = filterset.form.__class__({key: value})
+            if not form.is_valid():
+                raise forms.ValidationError(form.errors.as_json())
+            self.value = form.cleaned_data[key]
+
+        def evaluate(self, qs):
+            return self.filter.filter(qs, self.value)
+
+    class _NotAction:
+        def __init__(self, tokens):
+            self.expr = tokens[0][0]
+
+        def evaluate(self, qs):
+            return qs.difference(self.expr.evaluate(qs))
+
+    class _AndAction:
+        def __init__(self, tokens):
+            self.exprs = tokens[0]
+
+        def evaluate(self, qs):
+            return (
+                self.exprs[0]
+                .evaluate(qs)
+                .intersection(*[expr.evaluate(qs) for expr in self.exprs[1:]])
+            )
+
+    class _OrAction:
+        def __init__(self, tokens):
+            self.exprs = tokens[0]
+
+        def evaluate(self, qs):
+            return self.exprs[0].evaluate(qs).union(*[expr.evaluate(qs) for expr in self.exprs[1:]])
+
+    def __init__(self, *args, **kwargs):
+        self.filterset = kwargs.pop("filter").parent
+        super().__init__(*args, **kwargs)
+
+    def clean(self, value):
+        value = super().clean(value)
+        if value not in EMPTY_VALUES:
+            slug = pp.Word(pp.alphas, pp.alphanums + "_")
+            word = pp.Word(pp.alphanums + pp.alphas8bit + ".,_-*")
+            rhs = word | pp.quoted_string.set_parse_action(pp.remove_quotes)
+            group = pp.Group(
+                slug.set_results_name("key")
+                + pp.Suppress(pp.Literal("="))
+                + rhs.set_results_name("value")
+            ).set_parse_action(partial(self._FilterAction, self.filterset))
+
+            expr = pp.infix_notation(
+                group,
+                [
+                    (pp.Suppress(pp.Keyword("NOT")), 1, pp.opAssoc.RIGHT, self._NotAction),
+                    (pp.Suppress(pp.Keyword("AND")), 2, pp.opAssoc.LEFT, self._AndAction),
+                    (pp.Suppress(pp.Keyword("OR")), 2, pp.opAssoc.LEFT, self._OrAction),
+                ],
+            )
+            try:
+                result = expr.parse_string(value, parse_all=True)[0]
+            except pp.ParseException:
+                raise forms.ValidationError(_("Syntax error in expression."))
+            return result
+        return None
+
+
+class ExpressionFilter(filters.CharFilter):
+    field_class = ExpressionFilterField
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.extra["filter"] = self
+
+    def filter(self, qs, value):
+        if value is not None:
+            qs = value.evaluate(qs)
+        return qs
+
+
 class BaseFilterSet(filterset.FilterSet):
     """
     Class to override django_filter's FilterSet and provide a way to set help text
@@ -165,6 +266,7 @@ class BaseFilterSet(filterset.FilterSet):
     help_text = {}
     pulp_id__in = IdInFilter(field_name="pk", lookup_expr="in")
     pulp_href__in = HREFInFilter(field_name="pk", method="filter_pulp_href")
+    q = ExpressionFilter()
 
     FILTER_DEFAULTS = {
         **filterset.FilterSet.FILTER_DEFAULTS,
@@ -296,6 +398,7 @@ def get_filterset_class(self, view, queryset=None):
             if hasattr(view, "is_master_viewset") and view.is_master_viewset():
 
                 class PulpTypeFilterSet(filterset_class):
+                    pulp_type = PulpTypeFilter(field_name="pulp_type", model=queryset.model)
                     pulp_type__in = PulpTypeInFilter(field_name="pulp_type", model=queryset.model)
 
                 return PulpTypeFilterSet