From a228bb2a208a86b1c7d5a0698a3e1d8cc681e1ba Mon Sep 17 00:00:00 2001 From: Miguel Grinberg Date: Mon, 11 Nov 2024 17:08:22 +0000 Subject: [PATCH] support composable templates --- elasticsearch_dsl/__init__.py | 11 +++- elasticsearch_dsl/_async/index.py | 54 +++++++++++++++++++ elasticsearch_dsl/_sync/index.py | 50 +++++++++++++++++ elasticsearch_dsl/index.py | 8 ++- examples/alias_migration.py | 3 +- examples/async/alias_migration.py | 3 +- examples/async/parent_child.py | 2 +- examples/parent_child.py | 2 +- tests/conftest.py | 4 ++ tests/test_integration/_async/test_index.py | 25 ++++++++- tests/test_integration/_sync/test_index.py | 34 +++++++++++- .../_async/test_alias_migration.py | 2 +- .../test_examples/_async/test_parent_child.py | 2 +- .../_sync/test_alias_migration.py | 2 +- .../test_examples/_sync/test_parent_child.py | 2 +- utils/run-unasync.py | 1 + 16 files changed, 191 insertions(+), 14 deletions(-) diff --git a/elasticsearch_dsl/__init__.py b/elasticsearch_dsl/__init__.py index 656924bf..f9372829 100644 --- a/elasticsearch_dsl/__init__.py +++ b/elasticsearch_dsl/__init__.py @@ -79,7 +79,14 @@ construct_field, ) from .function import SF -from .index import AsyncIndex, AsyncIndexTemplate, Index, IndexTemplate +from .index import ( + AsyncIndex, + AsyncIndexTemplate, + AsyncNewIndexTemplate, + Index, + IndexTemplate, + NewIndexTemplate, +) from .mapping import AsyncMapping, Mapping from .query import Q, Query from .response import AggResponse, Response, UpdateByQueryResponse @@ -109,6 +116,7 @@ "AsyncIndexTemplate", "AsyncMapping", "AsyncMultiSearch", + "AsyncNewIndexTemplate", "AsyncSearch", "AsyncUpdateByQuery", "AttrDict", @@ -158,6 +166,7 @@ "Murmur3", "Nested", "NestedFacet", + "NewIndexTemplate", "Object", "Percolator", "Q", diff --git a/elasticsearch_dsl/_async/index.py b/elasticsearch_dsl/_async/index.py index 765e7438..35fc0030 100644 --- a/elasticsearch_dsl/_async/index.py +++ b/elasticsearch_dsl/_async/index.py @@ -73,6 +73,47 @@ async def save( ) +class AsyncNewIndexTemplate: + def __init__( + self, + name: str, + template: str, + index: Optional["AsyncIndex"] = None, + priority: Optional[int] = None, + **kwargs: Any, + ): + if index is None: + self._index = AsyncIndex(template, **kwargs) + else: + if kwargs: + raise ValueError( + "You cannot specify options for Index when" + " passing an Index instance." + ) + self._index = index.clone() + self._index._name = template + self._template_name = name + self.priority = priority + + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._index, attr_name) + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"template": self._index.to_dict()} + d["index_patterns"] = [self._index._name] + if self.priority is not None: + d["priority"] = self.priority + return d + + async def save( + self, using: Optional[AsyncUsingType] = None + ) -> "ObjectApiResponse[Any]": + es = get_connection(using or self._index._using) + return await es.indices.put_index_template( + name=self._template_name, **self.to_dict() + ) + + class AsyncIndex(IndexBase): _using: AsyncUsingType @@ -109,6 +150,19 @@ def as_template( template_name, pattern or self._name, index=self, order=order ) + def as_new_template( + self, + template_name: str, + pattern: Optional[str] = None, + priority: Optional[int] = None, + ) -> AsyncNewIndexTemplate: + # TODO: should we allow pattern to be a top-level arg? + # or maybe have an IndexPattern that allows for it and have + # Document._index be that? + return AsyncNewIndexTemplate( + template_name, pattern or self._name, index=self, priority=priority + ) + async def load_mappings(self, using: Optional[AsyncUsingType] = None) -> None: await self.get_or_create_mapping().update_from_es( self._name, using=using or self._using diff --git a/elasticsearch_dsl/_sync/index.py b/elasticsearch_dsl/_sync/index.py index 59508d51..a2193d04 100644 --- a/elasticsearch_dsl/_sync/index.py +++ b/elasticsearch_dsl/_sync/index.py @@ -69,6 +69,43 @@ def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]": return es.indices.put_template(name=self._template_name, body=self.to_dict()) +class NewIndexTemplate: + def __init__( + self, + name: str, + template: str, + index: Optional["Index"] = None, + priority: Optional[int] = None, + **kwargs: Any, + ): + if index is None: + self._index = Index(template, **kwargs) + else: + if kwargs: + raise ValueError( + "You cannot specify options for Index when" + " passing an Index instance." + ) + self._index = index.clone() + self._index._name = template + self._template_name = name + self.priority = priority + + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._index, attr_name) + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"template": self._index.to_dict()} + d["index_patterns"] = [self._index._name] + if self.priority is not None: + d["priority"] = self.priority + return d + + def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]": + es = get_connection(using or self._index._using) + return es.indices.put_index_template(name=self._template_name, **self.to_dict()) + + class Index(IndexBase): _using: UsingType @@ -103,6 +140,19 @@ def as_template( template_name, pattern or self._name, index=self, order=order ) + def as_new_template( + self, + template_name: str, + pattern: Optional[str] = None, + priority: Optional[int] = None, + ) -> NewIndexTemplate: + # TODO: should we allow pattern to be a top-level arg? + # or maybe have an IndexPattern that allows for it and have + # Document._index be that? + return NewIndexTemplate( + template_name, pattern or self._name, index=self, priority=priority + ) + def load_mappings(self, using: Optional[UsingType] = None) -> None: self.get_or_create_mapping().update_from_es( self._name, using=using or self._using diff --git a/elasticsearch_dsl/index.py b/elasticsearch_dsl/index.py index ef9f4b0b..f840e4b2 100644 --- a/elasticsearch_dsl/index.py +++ b/elasticsearch_dsl/index.py @@ -15,5 +15,9 @@ # specific language governing permissions and limitations # under the License. -from ._async.index import AsyncIndex, AsyncIndexTemplate # noqa: F401 -from ._sync.index import Index, IndexTemplate # noqa: F401 +from ._async.index import ( # noqa: F401 + AsyncIndex, + AsyncIndexTemplate, + AsyncNewIndexTemplate, +) +from ._sync.index import Index, IndexTemplate, NewIndexTemplate # noqa: F401 diff --git a/examples/alias_migration.py b/examples/alias_migration.py index c9fe4ede..b8cb906a 100644 --- a/examples/alias_migration.py +++ b/examples/alias_migration.py @@ -44,6 +44,7 @@ ALIAS = "test-blog" PATTERN = ALIAS + "-*" +PRIORITY = 100 class BlogPost(Document): @@ -81,7 +82,7 @@ def setup() -> None: deploy. """ # create an index template - index_template = BlogPost._index.as_template(ALIAS, PATTERN) + index_template = BlogPost._index.as_new_template(ALIAS, PATTERN, priority=PRIORITY) # upload the template into elasticsearch # potentially overriding the one already there index_template.save() diff --git a/examples/async/alias_migration.py b/examples/async/alias_migration.py index bede9098..9e01368b 100644 --- a/examples/async/alias_migration.py +++ b/examples/async/alias_migration.py @@ -45,6 +45,7 @@ ALIAS = "test-blog" PATTERN = ALIAS + "-*" +PRIORITY = 100 class BlogPost(AsyncDocument): @@ -82,7 +83,7 @@ async def setup() -> None: deploy. """ # create an index template - index_template = BlogPost._index.as_template(ALIAS, PATTERN) + index_template = BlogPost._index.as_new_template(ALIAS, PATTERN, priority=PRIORITY) # upload the template into elasticsearch # potentially overriding the one already there await index_template.save() diff --git a/examples/async/parent_child.py b/examples/async/parent_child.py index 6668a77c..70fc2325 100644 --- a/examples/async/parent_child.py +++ b/examples/async/parent_child.py @@ -226,7 +226,7 @@ async def save(self, **kwargs: Any) -> None: # type: ignore[override] async def setup() -> None: """Create an IndexTemplate and save it into elasticsearch.""" - index_template = Post._index.as_template("base") + index_template = Post._index.as_new_template("base", priority=100) await index_template.save() diff --git a/examples/parent_child.py b/examples/parent_child.py index 5acbbd72..335205a4 100644 --- a/examples/parent_child.py +++ b/examples/parent_child.py @@ -225,7 +225,7 @@ def save(self, **kwargs: Any) -> None: # type: ignore[override] def setup() -> None: """Create an IndexTemplate and save it into elasticsearch.""" - index_template = Post._index.as_template("base") + index_template = Post._index.as_new_template("base", priority=100) index_template.save() diff --git a/tests/conftest.py b/tests/conftest.py index 1a07670e..6a2589b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -122,6 +122,7 @@ def teardown_method(self, _: Any) -> None: ) self.client.indices.delete(index="*", expand_wildcards=expand_wildcards) self.client.indices.delete_template(name="*") + self.client.indices.delete_index_template(name="*") def es_version(self) -> Tuple[int, ...]: if not hasattr(self, "_es_version"): @@ -172,6 +173,9 @@ def write_client(client: Elasticsearch) -> Generator[Elasticsearch, None, None]: for index_name in client.indices.get(index="test-*", expand_wildcards="all"): client.indices.delete(index=index_name) client.options(ignore_status=404).indices.delete_template(name="test-template") + client.options(ignore_status=404).indices.delete_index_template( + name="test-template" + ) @pytest_asyncio.fixture diff --git a/tests/test_integration/_async/test_index.py b/tests/test_integration/_async/test_index.py index 21e4fa7c..e2f19252 100644 --- a/tests/test_integration/_async/test_index.py +++ b/tests/test_integration/_async/test_index.py @@ -22,6 +22,7 @@ AsyncDocument, AsyncIndex, AsyncIndexTemplate, + AsyncNewIndexTemplate, Date, Text, analysis, @@ -35,7 +36,29 @@ class Post(AsyncDocument): @pytest.mark.asyncio async def test_index_template_works(async_write_client: AsyncElasticsearch) -> None: - it = AsyncIndexTemplate("test-template", "test-*") + it = AsyncIndexTemplate("test-template", "test-legacy-*") + it.document(Post) + it.settings(number_of_replicas=0, number_of_shards=1) + await it.save() + + i = AsyncIndex("test-legacy-blog") + await i.create() + + assert { + "test-legacy-blog": { + "mappings": { + "properties": { + "title": {"type": "text", "analyzer": "my_analyzer"}, + "published_from": {"type": "date"}, + } + } + } + } == await async_write_client.indices.get_mapping(index="test-legacy-blog") + + +@pytest.mark.asyncio +async def test_new_index_template_works(async_write_client: AsyncElasticsearch) -> None: + it = AsyncNewIndexTemplate("test-template", "test-*") it.document(Post) it.settings(number_of_replicas=0, number_of_shards=1) await it.save() diff --git a/tests/test_integration/_sync/test_index.py b/tests/test_integration/_sync/test_index.py index ff435bdf..277faa2c 100644 --- a/tests/test_integration/_sync/test_index.py +++ b/tests/test_integration/_sync/test_index.py @@ -18,7 +18,15 @@ import pytest from elasticsearch import Elasticsearch -from elasticsearch_dsl import Date, Document, Index, IndexTemplate, Text, analysis +from elasticsearch_dsl import ( + Date, + Document, + Index, + IndexTemplate, + NewIndexTemplate, + Text, + analysis, +) class Post(Document): @@ -28,7 +36,29 @@ class Post(Document): @pytest.mark.sync def test_index_template_works(write_client: Elasticsearch) -> None: - it = IndexTemplate("test-template", "test-*") + it = IndexTemplate("test-template", "test-legacy-*") + it.document(Post) + it.settings(number_of_replicas=0, number_of_shards=1) + it.save() + + i = Index("test-legacy-blog") + i.create() + + assert { + "test-legacy-blog": { + "mappings": { + "properties": { + "title": {"type": "text", "analyzer": "my_analyzer"}, + "published_from": {"type": "date"}, + } + } + } + } == write_client.indices.get_mapping(index="test-legacy-blog") + + +@pytest.mark.sync +def test_new_index_template_works(write_client: Elasticsearch) -> None: + it = NewIndexTemplate("test-template", "test-*") it.document(Post) it.settings(number_of_replicas=0, number_of_shards=1) it.save() diff --git a/tests/test_integration/test_examples/_async/test_alias_migration.py b/tests/test_integration/test_examples/_async/test_alias_migration.py index 81202706..dae4c973 100644 --- a/tests/test_integration/test_examples/_async/test_alias_migration.py +++ b/tests/test_integration/test_examples/_async/test_alias_migration.py @@ -28,7 +28,7 @@ async def test_alias_migration(async_write_client: AsyncElasticsearch) -> None: await alias_migration.setup() # verify that template, index, and alias has been set up - assert await async_write_client.indices.exists_template(name=ALIAS) + assert await async_write_client.indices.exists_index_template(name=ALIAS) assert await async_write_client.indices.exists(index=PATTERN) assert await async_write_client.indices.exists_alias(name=ALIAS) diff --git a/tests/test_integration/test_examples/_async/test_parent_child.py b/tests/test_integration/test_examples/_async/test_parent_child.py index 9a1027f4..b12d445f 100644 --- a/tests/test_integration/test_examples/_async/test_parent_child.py +++ b/tests/test_integration/test_examples/_async/test_parent_child.py @@ -45,7 +45,7 @@ @pytest_asyncio.fixture async def question(async_write_client: AsyncElasticsearch) -> Question: await setup() - assert await async_write_client.indices.exists_template(name="base") + assert await async_write_client.indices.exists_index_template(name="base") # create a question object q = Question( diff --git a/tests/test_integration/test_examples/_sync/test_alias_migration.py b/tests/test_integration/test_examples/_sync/test_alias_migration.py index 59cdb372..9a74b699 100644 --- a/tests/test_integration/test_examples/_sync/test_alias_migration.py +++ b/tests/test_integration/test_examples/_sync/test_alias_migration.py @@ -28,7 +28,7 @@ def test_alias_migration(write_client: Elasticsearch) -> None: alias_migration.setup() # verify that template, index, and alias has been set up - assert write_client.indices.exists_template(name=ALIAS) + assert write_client.indices.exists_index_template(name=ALIAS) assert write_client.indices.exists(index=PATTERN) assert write_client.indices.exists_alias(name=ALIAS) diff --git a/tests/test_integration/test_examples/_sync/test_parent_child.py b/tests/test_integration/test_examples/_sync/test_parent_child.py index dcbbde86..12d93914 100644 --- a/tests/test_integration/test_examples/_sync/test_parent_child.py +++ b/tests/test_integration/test_examples/_sync/test_parent_child.py @@ -44,7 +44,7 @@ @pytest.fixture def question(write_client: Elasticsearch) -> Question: setup() - assert write_client.indices.exists_template(name="base") + assert write_client.indices.exists_index_template(name="base") # create a question object q = Question( diff --git a/utils/run-unasync.py b/utils/run-unasync.py index bae0c7a6..27d1f04b 100644 --- a/utils/run-unasync.py +++ b/utils/run-unasync.py @@ -57,6 +57,7 @@ def main(check=False): "AsyncIndexMeta": "IndexMeta", "AsyncIndexTemplate": "IndexTemplate", "AsyncIndex": "Index", + "AsyncNewIndexTemplate": "NewIndexTemplate", "AsyncUpdateByQuery": "UpdateByQuery", "AsyncMapping": "Mapping", "AsyncFacetedSearch": "FacetedSearch",