Merge remote-tracking branch 'origin/main' into 8.x

.github/workflows/backport.yml

@@ -1,16 +1,26 @@
 name: Backport
 on:
- pull_request:
+ pull_request_target:
  types:
  - closed
  - labeled
 
 jobs:
  backport:
- runs-on: ubuntu-latest
  name: Backport
+ runs-on: ubuntu-latest
+ # Only react to merged PRs for security reasons.
+ # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
+ if: >
+ github.event.pull_request.merged
+ && (
+ github.event.action == 'closed'
+ || (
+ github.event.action == 'labeled'
+ && contains(github.event.label.name, 'backport')
+ )
+ )
  steps:
- - name: Backport
- uses: tibdex/backport@v2
+ - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # v2.0.4
  with:
  github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version: "3.11"
+ python-version: "3.12"
  - name: Install dependencies
  run: |
  python3 -m pip install setuptools wheel twine
@@ -32,7 +32,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version: "3.11"
+ python-version: "3.12"
  - name: Install dependencies
  run: |
  python3 -m pip install nox
@@ -47,7 +47,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version: "3.11"
+ python-version: "3.12"
  - name: Install dependencies
  run: |
  python3 -m pip install nox
@@ -66,8 +66,9 @@ jobs:
  "3.9",
  "3.10",
  "3.11",
+ "3.12",
  ]
- es-version: [8.0.0, 8.9.0]
+ es-version: [8.0.0, 8.11.0]
 
  steps:
  - name: Checkout Repository
@@ -82,7 +83,7 @@ jobs:
  with:
  python-version: ${{ matrix.python-version }}
  - name: Set up Python for Nox
- if: matrix.python-version != '3.11'
+ if: matrix.python-version != '3.12'
  uses: actions/setup-python@v4
  with:
  python-version: 3
@@ -93,4 +94,4 @@ jobs:
  run: |
  nox -rs test-${{ matrix.python-version }}
  env:
- WAIT_FOR_ES: "1"
+ WAIT_FOR_ES: "1"

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
  os: ubuntu-22.04
  tools:
- python: "3"
+ python: "3.12"
 
 python:
  install:

Changelog.rst

@@ -3,6 +3,16 @@
 Changelog
 =========
 
+8.11.0 (2023-11-13)
+-------------------
+
+* Added support for Python 3.12 (`#1680`_)
+* Added support for Search.collase() (`#1649`_, contributed by `@qcoumes`_)
+
+.. _@qcoumes: https://github.com/qcoumes
+.. _#1680: https://github.com/elastic/elasticsearch-dsl-py/pull/1680
+.. _#1649: https://github.com/elastic/elasticsearch-dsl-py/pull/1649
+
 8.9.0 (2023-09-07)
 ------------------
 

docs/search_dsl.rst

@@ -18,6 +18,12 @@ The ``Search`` object represents the entire search request:
 
  * pagination
 
+ * highlighting
+
+ * suggestions
+
+ * collapsing
+
  * additional parameters
 
  * associated client
@@ -433,7 +439,26 @@ keyword arguments will be added to the suggest's json as-is which means that it
 should be one of ``term``, ``phrase`` or ``completion`` to indicate which type
 of suggester should be used.
 
+Collapsing
+~~~~~~~~~~
+
+To collapse search results use the ``collapse`` method on your ``Search`` object:
+
+.. code:: python
+
+ s = Search().query("match", message="GET /search")
+ # collapse results by user_id
+ s = s.collapse("user_id")
+
+The top hits will only include one result per ``user_id``. You can also expand
+each collapsed top hit with the ``inner_hits`` parameter,
+``max_concurrent_group_searches`` being the number of concurrent requests
+allowed to retrieve the inner hits per group:
+
+.. code:: python
 
+ inner_hits = {"name": "recent_search", "size": 5, "sort": [{"@timestamp": "desc"}]}
+ s = s.collapse("user_id", inner_hits=inner_hits, max_concurrent_group_searches=4)
 
 More Like This Query
 ~~~~~~~~~~~~~~~~~~~~

elasticsearch_dsl/__init__.py

@@ -84,7 +84,7 @@
 from .utils import AttrDict, AttrList, DslBase
 from .wrappers import Range
 
-VERSION = (8, 9, 0)
+VERSION = (8, 11, 0)
 __version__ = VERSION
 __versionstr__ = ".".join(map(str, VERSION))
 __all__ = [

elasticsearch_dsl/search.py

@@ -120,6 +120,7 @@ def __init__(self, using="default", index=None, doc_type=None, extra=None):
 
  self._doc_type = []
  self._doc_type_map = {}
+ self._collapse = {}
  if isinstance(doc_type, (tuple, list)):
  self._doc_type.extend(doc_type)
  elif isinstance(doc_type, collections.abc.Mapping):
@@ -293,6 +294,7 @@ def _clone(self):
  s = self.__class__(
  using=self._using, index=self._index, doc_type=self._doc_type
  )
+ s._collapse = self._collapse.copy()
  s._doc_type_map = self._doc_type_map.copy()
  s._extra = self._extra.copy()
  s._params = self._params.copy()
@@ -318,6 +320,7 @@ def __init__(self, **kwargs):
 
  self.aggs = AggsProxy(self)
  self._sort = []
+ self._collapse = {}
  self._source = None
  self._highlight = {}
  self._highlight_opts = {}
@@ -568,6 +571,27 @@ def sort(self, *keys):
  s._sort.append(k)
  return s
 
+ def collapse(self, field=None, inner_hits=None, max_concurrent_group_searches=None):
+ """
+ Add collapsing information to the search request.
+ If called without providing ``field``, it will remove all collapse
+ requirements, otherwise it will replace them with the provided
+ arguments.
+ The API returns a copy of the Search object and can thus be chained.
+ """
+ s = self._clone()
+ s._collapse = {}
+
+ if field is None:
+ return s
+
+ s._collapse["field"] = field
+ if inner_hits:
+ s._collapse["inner_hits"] = inner_hits
+ if max_concurrent_group_searches:
+ s._collapse["max_concurrent_group_searches"] = max_concurrent_group_searches
+ return s
+
  def highlight_options(self, **kwargs):
  """
  Update the global highlighting options used for this request. For
@@ -663,6 +687,9 @@ def to_dict(self, count=False, **kwargs):
  if self._sort:
  d["sort"] = self._sort
 
+ if self._collapse:
+ d["collapse"] = self._collapse
+
  d.update(recursive_to_dict(self._extra))
 
  if self._source not in (None, {}):

noxfile.py

@@ -35,6 +35,7 @@
  "3.9",
  "3.10",
  "3.11",
+ "3.12",
  ]
 )
 def test(session):
@@ -52,7 +53,7 @@ def test(session):
  session.run("pytest", *argv)
 
 
-@nox.session()
+@nox.session(python="3.12")
 def format(session):
  session.install("black~=23.0", "isort")
  session.run("black", "--target-version=py37", *SOURCE_FILES)
@@ -62,7 +63,7 @@ def format(session):
  lint(session)
 
 
-@nox.session
+@nox.session(python="3.12")
 def lint(session):
  session.install("flake8", "black~=23.0", "isort")
  session.run("black", "--check", "--target-version=py37", *SOURCE_FILES)

setup.cfg

@@ -13,3 +13,4 @@ filterwarnings =
  # https://github.com/elastic/elasticsearch-py/issues/2181#issuecomment-1490932964
  ignore:The 'body' parameter is deprecated .*:DeprecationWarning
  ignore:Legacy index templates are deprecated in favor of composable templates.:elasticsearch.exceptions.ElasticsearchWarning
+ ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version..*:DeprecationWarning

setup.py

@@ -19,7 +19,7 @@
 
 from setuptools import find_packages, setup
 
-VERSION = (8, 9, 0)
+VERSION = (8, 11, 0)
 __version__ = VERSION
 __versionstr__ = ".".join(map(str, VERSION))
 
@@ -68,6 +68,7 @@
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
  "Programming Language :: Python :: Implementation :: CPython",
  "Programming Language :: Python :: Implementation :: PyPy",
  ],

tests/test_search.py

@@ -256,6 +256,38 @@ def test_sort_by_score():
  s.sort("-_score")
 
 
+def test_collapse():
+ s = search.Search()
+
+ inner_hits = {"name": "most_recent", "size": 5, "sort": [{"@timestamp": "desc"}]}
+ s = s.collapse("user.id", inner_hits=inner_hits, max_concurrent_group_searches=4)
+
+ assert {
+ "field": "user.id",
+ "inner_hits": {
+ "name": "most_recent",
+ "size": 5,
+ "sort": [{"@timestamp": "desc"}],
+ },
+ "max_concurrent_group_searches": 4,
+ } == s._collapse
+ assert {
+ "collapse": {
+ "field": "user.id",
+ "inner_hits": {
+ "name": "most_recent",
+ "size": 5,
+ "sort": [{"@timestamp": "desc"}],
+ },
+ "max_concurrent_group_searches": 4,
+ }
+ } == s.to_dict()
+
+ s = s.collapse()
+ assert {} == s._collapse
+ assert search.Search().to_dict() == s.to_dict()
+
+
 def test_slice():
  s = search.Search()
  assert {"from": 3, "size": 7} == s[3:10].to_dict()
@@ -305,6 +337,7 @@ def test_complex_example():
  s.query("match", title="python")
  .query(~Q("match", title="ruby"))
  .filter(Q("term", category="meetup") | Q("term", category="conference"))
+ .collapse("user_id")
  .post_filter("terms", tags=["prague", "czech"])
  .script_fields(more_attendees="doc['attendees'].value + 42")
  )
@@ -342,6 +375,7 @@ def test_complex_example():
  "aggs": {"avg_attendees": {"avg": {"field": "attendees"}}},
  }
  },
+ "collapse": {"field": "user_id"},
  "highlight": {
  "order": "score",
  "fields": {"title": {"fragment_size": 50}, "body": {"fragment_size": 50}},