ES|QL query builder (elastic#2997)

* ES|QL query builder

* add missing esql api documentation

* add FORK command

* initial attempt at generating all functions

* unit tests

* more operators

* documentation

* integration tests

* add new COMPLETION command

* show ES|QL in all docs examples

* Docstring fixes

* add technical preview warning

Author: miguelgrinberg

docs/sphinx/esql.rst

@@ -0,0 +1,100 @@
+ES|QL Query Builder
+===================
+
+Commands
+--------
+
+.. autoclass:: elasticsearch.esql.ESQL
+   :inherited-members:
+   :members:
+
+.. autoclass:: elasticsearch.esql.esql.ESQLBase
+   :inherited-members:
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.From
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Row
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Show
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.ChangePoint
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Completion
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Dissect
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Drop
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Enrich
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Eval
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Fork
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Grok
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Keep
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Limit
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.LookupJoin
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.MvExpand
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Rename
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Sample
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Sort
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Stats
+   :members:
+   :exclude-members: __init__
+
+.. autoclass:: elasticsearch.esql.esql.Where
+   :members:
+   :exclude-members: __init__
+
+Functions
+---------
+
+.. automodule:: elasticsearch.esql.functions
+   :members:

elasticsearch/dsl/__init__.py

@@ -19,7 +19,7 @@
 from .aggs import A, Agg
 from .analysis import analyzer, char_filter, normalizer, token_filter, tokenizer
 from .document import AsyncDocument, Document
-from .document_base import InnerDoc, M, MetaField, mapped_field
+from .document_base import E, InnerDoc, M, MetaField, mapped_field
 from .exceptions import (
     ElasticsearchDslException,
     IllegalOperation,
@@ -135,6 +135,7 @@
     "Double",
     "DoubleRange",
     "DslBase",
+    "E",
     "ElasticsearchDslException",
     "EmptySearch",
     "Facet",

elasticsearch/dsl/document_base.py

@@ -15,6 +15,7 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
+import json
 from datetime import date, datetime
 from fnmatch import fnmatch
 from typing import (
@@ -56,7 +57,163 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.args, self.kwargs = args, kwargs
 
 
-class InstrumentedField:
+class InstrumentedExpression:
+    """Proxy object for a ES|QL expression."""
+
+    def __init__(self, expr: str):
+        self._expr = expr
+
+    def _render_value(self, value: Any) -> str:
+        if isinstance(value, InstrumentedExpression):
+            return str(value)
+        return json.dumps(value)
+
+    def __str__(self) -> str:
+        return self._expr
+
+    def __repr__(self) -> str:
+        return f"InstrumentedExpression[{self._expr}]"
+
+    def __pos__(self) -> "InstrumentedExpression":
+        return self
+
+    def __neg__(self) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"-({self._expr})")
+
+    def __eq__(self, value: Any) -> "InstrumentedExpression":  # type: ignore[override]
+        return InstrumentedExpression(f"{self._expr} == {self._render_value(value)}")
+
+    def __ne__(self, value: Any) -> "InstrumentedExpression":  # type: ignore[override]
+        return InstrumentedExpression(f"{self._expr} != {self._render_value(value)}")
+
+    def __lt__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} < {self._render_value(value)}")
+
+    def __gt__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} > {self._render_value(value)}")
+
+    def __le__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} <= {self._render_value(value)}")
+
+    def __ge__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} >= {self._render_value(value)}")
+
+    def __add__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} + {self._render_value(value)}")
+
+    def __radd__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._render_value(value)} + {self._expr}")
+
+    def __sub__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} - {self._render_value(value)}")
+
+    def __rsub__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._render_value(value)} - {self._expr}")
+
+    def __mul__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} * {self._render_value(value)}")
+
+    def __rmul__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._render_value(value)} * {self._expr}")
+
+    def __truediv__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} / {self._render_value(value)}")
+
+    def __rtruediv__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._render_value(value)} / {self._expr}")
+
+    def __mod__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._expr} % {self._render_value(value)}")
+
+    def __rmod__(self, value: Any) -> "InstrumentedExpression":
+        return InstrumentedExpression(f"{self._render_value(value)} % {self._expr}")
+
+    def is_null(self) -> "InstrumentedExpression":
+        """Compare the expression against NULL."""
+        return InstrumentedExpression(f"{self._expr} IS NULL")
+
+    def is_not_null(self) -> "InstrumentedExpression":
+        """Compare the expression against NOT NULL."""
+        return InstrumentedExpression(f"{self._expr} IS NOT NULL")
+
+    def in_(self, *values: Any) -> "InstrumentedExpression":
+        """Test if the expression equals one of the given values."""
+        rendered_values = ", ".join([f"{value}" for value in values])
+        return InstrumentedExpression(f"{self._expr} IN ({rendered_values})")
+
+    def like(self, *patterns: str) -> "InstrumentedExpression":
+        """Filter the expression using a string pattern."""
+        if len(patterns) == 1:
+            return InstrumentedExpression(
+                f"{self._expr} LIKE {self._render_value(patterns[0])}"
+            )
+        else:
+            return InstrumentedExpression(
+                f'{self._expr} LIKE ({", ".join([self._render_value(p) for p in patterns])})'
+            )
+
+    def rlike(self, *patterns: str) -> "InstrumentedExpression":
+        """Filter the expression using a regular expression."""
+        if len(patterns) == 1:
+            return InstrumentedExpression(
+                f"{self._expr} RLIKE {self._render_value(patterns[0])}"
+            )
+        else:
+            return InstrumentedExpression(
+                f'{self._expr} RLIKE ({", ".join([self._render_value(p) for p in patterns])})'
+            )
+
+    def match(self, query: str) -> "InstrumentedExpression":
+        """Perform a match query on the field."""
+        return InstrumentedExpression(f"{self._expr}:{self._render_value(query)}")
+
+    def asc(self) -> "InstrumentedExpression":
+        """Return the field name representation for ascending sort order.
+
+        For use in ES|QL queries only.
+        """
+        return InstrumentedExpression(f"{self._expr} ASC")
+
+    def desc(self) -> "InstrumentedExpression":
+        """Return the field name representation for descending sort order.
+
+        For use in ES|QL queries only.
+        """
+        return InstrumentedExpression(f"{self._expr} DESC")
+
+    def nulls_first(self) -> "InstrumentedExpression":
+        """Return the field name representation for nulls first sort order.
+
+        For use in ES|QL queries only.
+        """
+        return InstrumentedExpression(f"{self._expr} NULLS FIRST")
+
+    def nulls_last(self) -> "InstrumentedExpression":
+        """Return the field name representation for nulls last sort order.
+
+        For use in ES|QL queries only.
+        """
+        return InstrumentedExpression(f"{self._expr} NULLS LAST")
+
+    def where(
+        self, *expressions: Union[str, "InstrumentedExpression"]
+    ) -> "InstrumentedExpression":
+        """Add a condition to be met for the row to be included.
+
+        Use only in expressions given in the ``STATS`` command.
+        """
+        if len(expressions) == 1:
+            return InstrumentedExpression(f"{self._expr} WHERE {expressions[0]}")
+        else:
+            return InstrumentedExpression(
+                f'{self._expr} WHERE {" AND ".join([f"({expr})" for expr in expressions])}'
+            )
+
+
+E = InstrumentedExpression
+
+
+class InstrumentedField(InstrumentedExpression):
     """Proxy object for a mapped document field.
 
     An object of this instance is returned when a field is accessed as a class
@@ -71,8 +228,8 @@ class MyDocument(Document):
         s = s.sort(-MyDocument.name)  # sort by name in descending order
     """
 
-    def __init__(self, name: str, field: Field):
-        self._name = name
+    def __init__(self, name: str, field: Optional[Field]):
+        super().__init__(name)
         self._field = field
 
     # note that the return value type here assumes classes will only be used to
@@ -83,26 +240,29 @@ def __getattr__(self, attr: str) -> "InstrumentedField":
             # first let's see if this is an attribute of this object
             return super().__getattribute__(attr)  # type: ignore[no-any-return]
         except AttributeError:
-            try:
-                # next we see if we have a sub-field with this name
-                return InstrumentedField(f"{self._name}.{attr}", self._field[attr])
-            except KeyError:
-                # lastly we let the wrapped field resolve this attribute
-                return getattr(self._field, attr)  # type: ignore[no-any-return]
-
-    def __pos__(self) -> str:
+            if self._field:
+                try:
+                    # next we see if we have a sub-field with this name
+                    return InstrumentedField(f"{self._expr}.{attr}", self._field[attr])
+                except KeyError:
+                    # lastly we let the wrapped field resolve this attribute
+                    return getattr(self._field, attr)  # type: ignore[no-any-return]
+            else:
+                raise
+
+    def __pos__(self) -> str:  # type: ignore[override]
         """Return the field name representation for ascending sort order"""
-        return f"{self._name}"
+        return f"{self._expr}"
 
-    def __neg__(self) -> str:
+    def __neg__(self) -> str:  # type: ignore[override]
         """Return the field name representation for descending sort order"""
-        return f"-{self._name}"
+        return f"-{self._expr}"
 
     def __str__(self) -> str:
-        return self._name
+        return self._expr
 
     def __repr__(self) -> str:
-        return f"InstrumentedField[{self._name}]"
+        return f"InstrumentedField[{self._expr}]"
 
 
 class DocumentMeta(type):

elasticsearch/dsl/utils.py

@@ -333,7 +333,7 @@ def __init__(self, _expand__to_dot: Optional[bool] = None, **params: Any) -> Non
             _expand__to_dot = EXPAND__TO_DOT
         self._params: Dict[str, Any] = {}
         for pname, pvalue in params.items():
-            if pvalue == DEFAULT:
+            if pvalue is DEFAULT:
                 continue
             # expand "__" to dots
             if "__" in pname and _expand__to_dot:

elasticsearch/esql/__init__.py

@@ -0,0 +1,18 @@
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from .esql import ESQL, and_, not_, or_  # noqa: F401