From 85a31872c36599b388586c17a4cb7d114eba6938 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 31 Jan 2023 16:40:29 +0100 Subject: [PATCH] Add `ignore_missing_component_templates` config option (#92436) This change introduces the configuration option `ignore_missing_component_templates` as discussed in https://github.com/elastic/elasticsearch/issues/92426 The implementation [option 6](https://github.com/elastic/elasticsearch/issues/92426#issuecomment-1372675683) was picked with a slight adjustment meaning no patterns are allowed. ## Implementation During the creation of an index template, the list of component templates is checked if all component templates exist. This check is extended to skip any component templates which are listed under `ignore_missing_component_templates`. An index template that skips the check for the component template `logs-foo@custom` looks as following: ``` PUT _index_template/logs-foo { "index_patterns": ["logs-foo-*"], "data_stream": { }, "composed_of": ["logs-foo@package", "logs-foo@custom"], "ignore_missing_component_templates": ["logs-foo@custom"], "priority": 500 } ``` The component template `logs-foo@package` has to exist before creation. It can be created with: ``` PUT _component_template/logs-foo@custom { "template": { "mappings": { "properties": { "host.ip": { "type": "ip" } } } } } ``` ## Testing For manual testing, different scenarios can be tested. To simplify testing, the commands from `.http` file are added. Before each test run, a clean cluster is expected. ### New behaviour, missing component template With the new config option, it must be possible to create an index template with a missing component templates without getting an error: ``` ### Add logs-foo@package component template PUT http://localhost:9200/ _component_template/logs-foo@package Authorization: Basic elastic password Content-Type: application/json { "template": { "mappings": { "properties": { "host.name": { "type": "keyword" } } } } } ### Add logs-foo index template PUT http://localhost:9200/ _index_template/logs-foo Authorization: Basic elastic password Content-Type: application/json { "index_patterns": ["logs-foo-*"], "data_stream": { }, "composed_of": ["logs-foo@package", "logs-foo@custom"], "ignore_missing_component_templates": ["logs-foo@custom"], "priority": 500 } ### Create data stream PUT http://localhost:9200/ _data_stream/logs-foo-bar Authorization: Basic elastic password Content-Type: application/json ### Check if mappings exist GET http://localhost:9200/ logs-foo-bar Authorization: Basic elastic password Content-Type: application/json ``` It is checked if all templates could be created and data stream mappings are correct. ### Old behaviour, with all component templates In the following, a component template is made optional but it already exists. It is checked, that it will show up in the mappings: ``` ### Add logs-foo@package component template PUT http://localhost:9200/ _component_template/logs-foo@package Authorization: Basic elastic password Content-Type: application/json { "template": { "mappings": { "properties": { "host.name": { "type": "keyword" } } } } } ### Add logs-foo@custom component template PUT http://localhost:9200/ _component_template/logs-foo@custom Authorization: Basic elastic password Content-Type: application/json { "template": { "mappings": { "properties": { "host.ip": { "type": "ip" } } } } } ### Add logs-foo index template PUT http://localhost:9200/ _index_template/logs-foo Authorization: Basic elastic password Content-Type: application/json { "index_patterns": ["logs-foo-*"], "data_stream": { }, "composed_of": ["logs-foo@package", "logs-foo@custom"], "ignore_missing_component_templates": ["logs-foo@custom"], "priority": 500 } ### Create data stream PUT http://localhost:9200/ _data_stream/logs-foo-bar Authorization: Basic elastic password Content-Type: application/json ### Check if mappings exist GET http://localhost:9200/ logs-foo-bar Authorization: Basic elastic password Content-Type: application/json ``` ### Check old behaviour Ensure, that the old behaviour still exists when a component template is used that is not part of `ignore_missing_component_templates`: ``` ### Add logs-foo index template PUT http://localhost:9200/ _index_template/logs-foo Authorization: Basic elastic password Content-Type: application/json { "index_patterns": ["logs-foo-*"], "data_stream": { }, "composed_of": ["logs-foo@package", "logs-foo@custom"], "ignore_missing_component_templates": ["logs-foo@custom"], "priority": 500 } ``` Co-authored-by: Lee Hinman --- docs/changelog/92436.yaml | 6 + ...gnore-missing-component-templates.asciidoc | 95 +++++++++++ .../indices/index-templates.asciidoc | 2 + .../15_composition.yml | 63 ++++++++ .../metadata/ComposableIndexTemplate.java | 58 ++++++- .../MetadataIndexTemplateService.java | 25 ++- .../ComposableIndexTemplateTests.java | 40 ++++- .../MetadataIndexTemplateServiceTests.java | 150 +++++++++++++++++- ...adataMigrateToDataTiersRoutingService.java | 3 + 9 files changed, 423 insertions(+), 19 deletions(-) create mode 100644 docs/changelog/92436.yaml create mode 100644 docs/reference/indices/ignore-missing-component-templates.asciidoc diff --git a/docs/changelog/92436.yaml b/docs/changelog/92436.yaml new file mode 100644 index 0000000000000..1f8b4a9bf1877 --- /dev/null +++ b/docs/changelog/92436.yaml @@ -0,0 +1,6 @@ +pr: 92436 +summary: Add `ignore_missing_component_templates` config option +area: Indices APIs +type: enhancement +issues: + - 92426 diff --git a/docs/reference/indices/ignore-missing-component-templates.asciidoc b/docs/reference/indices/ignore-missing-component-templates.asciidoc new file mode 100644 index 0000000000000..8337be779c709 --- /dev/null +++ b/docs/reference/indices/ignore-missing-component-templates.asciidoc @@ -0,0 +1,95 @@ +[[ignore_missing_component_templates]] +== Config ignore_missing_component_templates + +The configuration option `ignore_missing_component_templates` can be used when an index template references a component template that might not exist. Every time a data stream is created based on the index template, the existence of the component template will be checked. If it exists, it will used to form the index's composite settings. If it does not exist, it is ignored. + +=== Usage example + +In the following, one component template and an index template are created. The index template references two component templates, but only the `@package` one exists. + + +Create the component template `logs-foo_component1`. This has to be created before the index template as it is not optional: + +[source,console] +---- +PUT _component_template/logs-foo_component1 +{ + "template": { + "mappings": { + "properties": { + "host.name": { + "type": "keyword" + } + } + } + } +} +---- + +Next, the index template will be created and it references two component templates: + +[source,JSON] +---- + "composed_of": ["logs-foo_component1", "logs-foo_component2"] +---- + +Before, only the `logs-foo_component1` compontent template was created, meaning the `logs-foo_component2` is missing. Because of this the following entry was added to the config: + +[source,JSON] +---- + "ignore_missing_component_templates": ["logs-foo_component2"], +---- + +During creation of the template, it will not validate that `logs-foo_component2` exists: + + +[source,console] +---- +PUT _index_template/logs-foo +{ + "index_patterns": ["logs-foo-*"], + "data_stream": { }, + "composed_of": ["logs-foo_component1", "logs-foo_component2"], + "ignore_missing_component_templates": ["logs-foo_component2"], + "priority": 500 +} +---- +// TEST[continued] + +The index template `logs-foo` was successfully created. A data stream can be created based on this template: + +[source,console] +---- +PUT _data_stream/logs-foo-bar +---- +// TEST[continued] + +Looking at the mappings of the data stream, it will contain the `host.name` field. + +At a later stage, the missing component template might be added: + +[source,console] +---- +PUT _component_template/logs-foo_component2 +{ + "template": { + "mappings": { + "properties": { + "host.ip": { + "type": "ip" + } + } + } + } +} +---- +// TEST[continued] + +This will not have an immediate effect on the data stream. The mapping `host.ip` will only show up in the data stream mappings when the data stream is rolled over automatically next time or a manual rollover is triggered: + +[source,console] +---- +POST logs-foo-bar/_rollover +---- +// TEST[continued] +// TEST[teardown:data_stream_cleanup] diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 8a4c985970b26..6128ab48998f3 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -161,3 +161,5 @@ DELETE _component_template/component_template1 //// include::simulate-multi-component-templates.asciidoc[] + +include::ignore-missing-component-templates.asciidoc[] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml index 5eef78a8c63ba..2aaf492f0ff0d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml @@ -286,3 +286,66 @@ - is_false: purple-index.mappings.properties.nested.include_in_root - is_true: purple-index.mappings.properties.nested.include_in_parent + +--- +"Index template ignore_missing_component_template valid": + - skip: + version: " - 8.6.99" + reason: "index template v2 ignore_missing_component_template config not available before 8.7" + features: allowed_warnings + + - do: + cluster.put_component_template: + name: red + body: + template: + mappings: + properties: + foo: + type: keyword + + - do: + allowed_warnings: + - "index template [blue] has index patterns [purple-index] matching patterns from existing older templates [global] with patterns (global => [*]); this template [blue] will take precedence during new index creation" + indices.put_index_template: + name: blue + body: + index_patterns: ["purple-index"] + composed_of: ["red", "blue"] + ignore_missing_component_templates: ["blue"] + + - do: + indices.create: + index: purple-index + + - do: + indices.get: + index: purple-index + + - match: {purple-index.mappings.properties.foo: {type: keyword}} + +--- +"Index template ignore_missing_component_template invalid": + - skip: + version: " - 8.6.99" + reason: "index template v2 ignore_missing_component_template config not available before 8.7" + features: allowed_warnings + + - do: + cluster.put_component_template: + name: red + body: + template: + mappings: + properties: + foo: + type: keyword + + - do: + catch: /index_template \[blue\] invalid, cause \[index template \[blue\] specifies a missing component templates \[blue\] that does not exist/ + indices.put_index_template: + name: blue + body: + index_patterns: ["purple-index"] + composed_of: ["red", "blue"] + ignore_missing_component_templates: ["foo"] diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 367b0f9f6f00d..377d91d60a99e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.TransportVersion; +import org.elasticsearch.Version; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.common.Strings; @@ -46,6 +47,7 @@ public class ComposableIndexTemplate implements SimpleDiffable PARSER = new ConstructingObjectParser<>( @@ -59,7 +61,8 @@ public class ComposableIndexTemplate implements SimpleDiffable) a[5], (DataStreamTemplate) a[6], - (Boolean) a[7] + (Boolean) a[7], + (List) a[8] ) ); @@ -72,6 +75,7 @@ public class ComposableIndexTemplate implements SimpleDiffable p.map(), METADATA); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_AUTO_CREATE); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), IGNORE_MISSING_COMPONENT_TEMPLATES); } private final List indexPatterns; @@ -89,6 +93,8 @@ public class ComposableIndexTemplate implements SimpleDiffable ignoreMissingComponentTemplates; static Diff readITV2DiffFrom(StreamInput in) throws IOException { return SimpleDiffable.readDiffFrom(ComposableIndexTemplate::new, in); @@ -106,7 +112,7 @@ public ComposableIndexTemplate( @Nullable Long version, @Nullable Map metadata ) { - this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null); + this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null, null); } public ComposableIndexTemplate( @@ -118,7 +124,7 @@ public ComposableIndexTemplate( @Nullable Map metadata, @Nullable DataStreamTemplate dataStreamTemplate ) { - this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null); + this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null); } public ComposableIndexTemplate( @@ -130,6 +136,20 @@ public ComposableIndexTemplate( @Nullable Map metadata, @Nullable DataStreamTemplate dataStreamTemplate, @Nullable Boolean allowAutoCreate + ) { + this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null); + } + + public ComposableIndexTemplate( + List indexPatterns, + @Nullable Template template, + @Nullable List componentTemplates, + @Nullable Long priority, + @Nullable Long version, + @Nullable Map metadata, + @Nullable DataStreamTemplate dataStreamTemplate, + @Nullable Boolean allowAutoCreate, + @Nullable List ignoreMissingComponentTemplates ) { this.indexPatterns = indexPatterns; this.template = template; @@ -139,6 +159,7 @@ public ComposableIndexTemplate( this.metadata = metadata; this.dataStreamTemplate = dataStreamTemplate; this.allowAutoCreate = allowAutoCreate; + this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates; } public ComposableIndexTemplate(StreamInput in) throws IOException { @@ -154,6 +175,11 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { this.metadata = in.readMap(); this.dataStreamTemplate = in.readOptionalWriteable(DataStreamTemplate::new); this.allowAutoCreate = in.readOptionalBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_7_0)) { + this.ignoreMissingComponentTemplates = in.readOptionalStringList(); + } else { + this.ignoreMissingComponentTemplates = null; + } } public List indexPatterns() { @@ -204,6 +230,11 @@ public Boolean getAllowAutoCreate() { return this.allowAutoCreate; } + @Nullable + public List getIgnoreMissingComponentTemplates() { + return ignoreMissingComponentTemplates; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(this.indexPatterns); @@ -219,6 +250,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeGenericMap(this.metadata); out.writeOptionalWriteable(dataStreamTemplate); out.writeOptionalBoolean(allowAutoCreate); + if (out.getVersion().onOrAfter(Version.V_8_7_0)) { + out.writeOptionalStringCollection(ignoreMissingComponentTemplates); + } } @Override @@ -246,6 +280,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (this.allowAutoCreate != null) { builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate); } + if (this.ignoreMissingComponentTemplates != null) { + builder.stringListField(IGNORE_MISSING_COMPONENT_TEMPLATES.getPreferredName(), ignoreMissingComponentTemplates); + } builder.endObject(); return builder; } @@ -260,7 +297,8 @@ public int hashCode() { this.version, this.metadata, this.dataStreamTemplate, - this.allowAutoCreate + this.allowAutoCreate, + this.ignoreMissingComponentTemplates ); } @@ -280,7 +318,8 @@ && componentTemplatesEquals(this.componentTemplates, other.componentTemplates) && Objects.equals(this.version, other.version) && Objects.equals(this.metadata, other.metadata) && Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate) - && Objects.equals(this.allowAutoCreate, other.allowAutoCreate); + && Objects.equals(this.allowAutoCreate, other.allowAutoCreate) + && Objects.equals(this.ignoreMissingComponentTemplates, other.ignoreMissingComponentTemplates); } static boolean componentTemplatesEquals(List c1, List c2) { @@ -421,6 +460,7 @@ public static class Builder { private Map metadata; private DataStreamTemplate dataStreamTemplate; private Boolean allowAutoCreate; + private List ignoreMissingComponentTemplates; public Builder() {} @@ -464,6 +504,11 @@ public Builder allowAutoCreate(Boolean allowAutoCreate) { return this; } + public Builder ignoreMissingComponentTemplates(List ignoreMissingComponentTemplates) { + this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates; + return this; + } + public ComposableIndexTemplate build() { return new ComposableIndexTemplate( this.indexPatterns, @@ -473,7 +518,8 @@ public ComposableIndexTemplate build() { this.version, this.metadata, this.dataStreamTemplate, - this.allowAutoCreate + this.allowAutoCreate, + this.ignoreMissingComponentTemplates ); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 36a432b7625e9..3dca87dcbde41 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -505,17 +505,34 @@ public static void validateV2TemplateRequest(Metadata metadata, String name, Com } final Map componentTemplates = metadata.componentTemplates(); + final List ignoreMissingComponentTemplates = (template.getIgnoreMissingComponentTemplates() == null + ? List.of() + : template.getIgnoreMissingComponentTemplates()); final List missingComponentTemplates = template.composedOf() .stream() .filter(componentTemplate -> componentTemplates.containsKey(componentTemplate) == false) + .filter(componentTemplate -> ignoreMissingComponentTemplates.contains(componentTemplate) == false) .toList(); - if (missingComponentTemplates.size() > 0) { + if (missingComponentTemplates.size() > 0 && ignoreMissingComponentTemplates.size() == 0) { throw new InvalidIndexTemplateException( name, "index template [" + name + "] specifies component templates " + missingComponentTemplates + " that do not exist" ); } + + if (missingComponentTemplates.size() > 0 && ignoreMissingComponentTemplates.size() > 0) { + + throw new InvalidIndexTemplateException( + name, + "index template [" + + name + + "] specifies a missing component templates " + + missingComponentTemplates + + " " + + "that does not exist and is not part of 'ignore_missing_component_templates'" + ); + } } public ClusterState addIndexTemplateV2( @@ -579,7 +596,8 @@ public ClusterState addIndexTemplateV2( template.version(), template.metadata(), template.getDataStreamTemplate(), - template.getAllowAutoCreate() + template.getAllowAutoCreate(), + template.getIgnoreMissingComponentTemplates() ); } @@ -679,7 +697,8 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT indexTemplate.version(), indexTemplate.metadata(), indexTemplate.getDataStreamTemplate(), - indexTemplate.getAllowAutoCreate() + indexTemplate.getAllowAutoCreate(), + indexTemplate.getIgnoreMissingComponentTemplates() ); validate(name, templateToValidate); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index d4e7993ac0529..649355fb8b7f4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -75,6 +75,7 @@ public static ComposableIndexTemplate randomInstance() { } List indexPatterns = randomList(1, 4, () -> randomAlphaOfLength(4)); + List ignoreMissingComponentTemplates = randomList(0, 4, () -> randomAlphaOfLength(4)); return new ComposableIndexTemplate( indexPatterns, template, @@ -83,7 +84,8 @@ public static ComposableIndexTemplate randomInstance() { randomBoolean() ? null : randomNonNegativeLong(), meta, dataStreamTemplate, - randomBoolean() ? null : randomBoolean() + randomBoolean() ? null : randomBoolean(), + ignoreMissingComponentTemplates ); } @@ -149,7 +151,7 @@ protected ComposableIndexTemplate mutateInstance(ComposableIndexTemplate orig) { } public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate orig) { - switch (randomIntBetween(0, 6)) { + switch (randomIntBetween(0, 7)) { case 0: List newIndexPatterns = randomValueOtherThan( orig.indexPatterns(), @@ -177,7 +179,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.version(), orig.metadata(), orig.getDataStreamTemplate(), - orig.getAllowAutoCreate() + orig.getAllowAutoCreate(), + orig.getIgnoreMissingComponentTemplates() ); case 2: List newComposedOf = randomValueOtherThan(orig.composedOf(), () -> randomList(0, 10, () -> randomAlphaOfLength(5))); @@ -189,7 +192,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.version(), orig.metadata(), orig.getDataStreamTemplate(), - orig.getAllowAutoCreate() + orig.getAllowAutoCreate(), + orig.getIgnoreMissingComponentTemplates() ); case 3: return new ComposableIndexTemplate( @@ -200,7 +204,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.version(), orig.metadata(), orig.getDataStreamTemplate(), - orig.getAllowAutoCreate() + orig.getAllowAutoCreate(), + orig.getIgnoreMissingComponentTemplates() ); case 4: return new ComposableIndexTemplate( @@ -211,7 +216,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong), orig.metadata(), orig.getDataStreamTemplate(), - orig.getAllowAutoCreate() + orig.getAllowAutoCreate(), + orig.getIgnoreMissingComponentTemplates() ); case 5: return new ComposableIndexTemplate( @@ -222,7 +228,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.version(), randomValueOtherThan(orig.metadata(), ComposableIndexTemplateTests::randomMeta), orig.getDataStreamTemplate(), - orig.getAllowAutoCreate() + orig.getAllowAutoCreate(), + orig.getIgnoreMissingComponentTemplates() ); case 6: return new ComposableIndexTemplate( @@ -233,7 +240,24 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.version(), orig.metadata(), randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate), - orig.getAllowAutoCreate() + orig.getAllowAutoCreate(), + orig.getIgnoreMissingComponentTemplates() + ); + case 7: + List ignoreMissingComponentTemplates = randomValueOtherThan( + orig.getIgnoreMissingComponentTemplates(), + () -> randomList(1, 4, () -> randomAlphaOfLength(4)) + ); + return new ComposableIndexTemplate( + orig.indexPatterns(), + orig.template(), + orig.composedOf(), + orig.priority(), + orig.version(), + orig.metadata(), + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate(), + ignoreMissingComponentTemplates ); default: throw new IllegalStateException("illegal randomization branch"); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 9702810e3afc6..e8ad6d75736b2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -597,6 +597,7 @@ public void testRemoveIndexTemplateV2Wildcards() throws Exception { ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); assertThat(state.metadata().templatesV2().get("foo"), notNullValue()); + assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template); Exception e = expectThrows( @@ -1529,7 +1530,12 @@ public void testAddInvalidTemplate() throws Exception { }); assertThat(e.name(), equalTo("template")); - assertThat(e.getMessage(), containsString("index template [template] specifies " + "component templates [bad] that do not exist")); + assertThat( + e.getMessage(), + containsString( + "index_template [template] invalid, cause [index template [template] specifies component templates [bad] that do not exist]" + ) + ); } public void testRemoveComponentTemplate() throws Exception { @@ -2108,6 +2114,146 @@ public void testV2TemplateOverlaps() throws Exception { } } + /** + * Tests to add two component templates but ignores both with is valid + * + * @throws Exception + */ + public void testIgnoreMissingComponentTemplateValid() throws Exception { + + String indexTemplateName = "metric-test"; + + List componentTemplates = new ArrayList<>(); + componentTemplates.add("foo"); + componentTemplates.add("bar"); + + // Order of params is mixed up on purpose + List ignoreMissingComponentTemplates = new ArrayList<>(); + ignoreMissingComponentTemplates.add("bar"); + ignoreMissingComponentTemplates.add("foo"); + + ComposableIndexTemplate template = new ComposableIndexTemplate( + Arrays.asList("metrics-test-*"), + null, + componentTemplates, + 1L, + null, + null, + null, + null, + ignoreMissingComponentTemplates + ); + MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); + + ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, indexTemplateName, template); + MetadataIndexTemplateService.validateV2TemplateRequest(state.metadata(), indexTemplateName, template); + } + + public void testIgnoreMissingComponentTemplateInvalid() throws Exception { + + String indexTemplateName = "metric-test"; + + List componentTemplates = new ArrayList<>(); + componentTemplates.add("foo"); + componentTemplates.add("fail"); + + List ignoreMissingComponentTemplates = new ArrayList<>(); + ignoreMissingComponentTemplates.add("bar"); + ignoreMissingComponentTemplates.add("foo"); + + ComposableIndexTemplate template = new ComposableIndexTemplate( + Arrays.asList("metrics-foo-*"), + null, + componentTemplates, + 1L, + null, + null, + null, + null, + ignoreMissingComponentTemplates + ); + + MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); + ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, indexTemplateName, template); + + // try now the same thing with validation on + InvalidIndexTemplateException e = expectThrows( + InvalidIndexTemplateException.class, + () -> MetadataIndexTemplateService.validateV2TemplateRequest(state.metadata(), indexTemplateName, template) + + ); + assertThat(e.getMessage(), containsString("specifies a missing component templates [fail] that does not exist")); + } + + /** + * This is a similar test as above but with running the service + * @throws Exception + */ + public void testAddInvalidTemplateIgnoreService() throws Exception { + + String indexTemplateName = "metric-test"; + + List componentTemplates = new ArrayList<>(); + componentTemplates.add("foo"); + componentTemplates.add("fail"); + + List ignoreMissingComponentTemplates = new ArrayList<>(); + ignoreMissingComponentTemplates.add("bar"); + ignoreMissingComponentTemplates.add("foo"); + + ComposableIndexTemplate template = new ComposableIndexTemplate( + Arrays.asList("metrics-foo-*"), + null, + componentTemplates, + 1L, + null, + null, + null, + null, + ignoreMissingComponentTemplates + ); + + ComponentTemplate ct = new ComponentTemplate(new Template(Settings.EMPTY, null, null), null, null); + + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + CountDownLatch ctLatch = new CountDownLatch(1); + // Makes ure the foo template exists + service.putComponentTemplate( + "api", + randomBoolean(), + "foo", + TimeValue.timeValueSeconds(5), + ct, + ActionListener.wrap(r -> ctLatch.countDown(), e -> { + logger.error("unexpected error", e); + fail("unexpected error"); + }) + ); + ctLatch.await(5, TimeUnit.SECONDS); + InvalidIndexTemplateException e = expectThrows(InvalidIndexTemplateException.class, () -> { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference err = new AtomicReference<>(); + service.putIndexTemplateV2( + "api", + randomBoolean(), + "template", + TimeValue.timeValueSeconds(30), + template, + ActionListener.wrap(r -> fail("should have failed!"), exception -> { + err.set(exception); + latch.countDown(); + }) + ); + latch.await(5, TimeUnit.SECONDS); + if (err.get() != null) { + throw err.get(); + } + }); + + assertThat(e.name(), equalTo("template")); + assertThat(e.getMessage(), containsString("missing component templates [fail] that does not exist")); + } + private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { ThreadPool testThreadPool = mock(ThreadPool.class); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); @@ -2200,6 +2346,6 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { } public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) { - assertTrue(Objects.equals(actual, expected)); + assertEquals(actual, expected); } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index a0d64321041f7..ff9659235f2d3 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -693,6 +693,9 @@ static List migrateComposableTemplates(Metadata.Builder mb, ClusterState migratedComposableTemplateBuilder.metadata(composableTemplate.metadata()); migratedComposableTemplateBuilder.dataStreamTemplate(composableTemplate.getDataStreamTemplate()); migratedComposableTemplateBuilder.allowAutoCreate(composableTemplate.getAllowAutoCreate()); + migratedComposableTemplateBuilder.ignoreMissingComponentTemplates( + composableTemplate.getIgnoreMissingComponentTemplates() + ); mb.put(templateEntry.getKey(), migratedComposableTemplateBuilder.build()); migratedComposableTemplates.add(templateEntry.getKey());