From 5e70afd8751cc087f59f44a73d0a964e6ab977d7 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Thu, 25 Feb 2021 23:30:46 -0800 Subject: [PATCH 1/8] Revert "EXPER - fix ComposableTemplateIT by removing a test for checking data_stream not being used in composable template in oss distribution" This reverts commit 6046a417 --- .../template/ComposableTemplateIT.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index df8c85d9b51b2..00ca9e8f6e270 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -24,11 +24,19 @@ import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.collect.List; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; +import java.io.IOException; import java.util.Collections; +import static org.hamcrest.Matchers.equalTo; + public class ComposableTemplateIT extends ESIntegTestCase { // See: https://github.com/elastic/elasticsearch/issues/58643 @@ -83,4 +91,23 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit2)).get(); } + + public void testUsageOfDataStreamFails() throws IOException { + // Exception that would happen if a unknown field is provided in a composable template: + // The thrown exception will be used to compare against the exception that is thrown when providing + // a composable template with a data stream definition. + String content = "{\"index_patterns\":[\"logs-*-*\"],\"my_field\":\"bla\"}"; + XContentParser parser = + XContentHelper.createParser(xContentRegistry(), null, new BytesArray(content), XContentType.JSON); + Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); + + ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null, + null, new ComposableIndexTemplate.DataStreamTemplate()); + Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); + Exception actualException = (Exception) e.getCause(); + assertThat(actualException.getMessage(), + equalTo(expectedException.getMessage().replace("[1:32] ", "").replace("my_field", "data_stream"))); + assertThat(actualException.getMessage(), equalTo("[index_template] unknown field [data_stream]")); + } } From a5741bb3d4e27b41d6ff9049b213e6fdc00c69f6 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Thu, 25 Feb 2021 23:31:09 -0800 Subject: [PATCH 2/8] Revert "EXPER - fix MetadataIndexTemplateServiceTests by removing the dummy plugin for registering the timestamp filed" This reverts commit 87b1932a --- .../MetadataIndexTemplateServiceTests.java | 198 +++++++++++++----- 1 file changed, 147 insertions(+), 51 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index dab4cf6284524..fb270cc44a7d3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -38,17 +39,29 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -64,6 +77,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.common.settings.Settings.builder; +import static org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; @@ -77,6 +91,12 @@ import static org.hamcrest.Matchers.matchesRegex; public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return Collections.singletonList(DummyPlugin.class); + } + public void testIndexTemplateInvalidNumberOfShards() { PutRequest request = new PutRequest("test", "test_shards"); request.patterns(singletonList("test_shards*")); @@ -89,10 +109,10 @@ public void testIndexTemplateInvalidNumberOfShards() { assertEquals(throwables.size(), 1); assertThat(throwables.get(0), instanceOf(InvalidIndexTemplateException.class)); assertThat(throwables.get(0).getMessage(), - containsString("Failed to parse value [0] for setting [index.number_of_shards] must be >= 1")); + containsString("Failed to parse value [0] for setting [index.number_of_shards] must be >= 1")); assertThat(throwables.get(0).getMessage(), - containsString("unknown value for [index.shard.check_on_startup] " + - "must be one of [true, false, checksum] but was: blargh")); + containsString("unknown value for [index.shard.check_on_startup] " + + "must be one of [true, false, checksum] but was: blargh")); } public void testIndexTemplateValidationWithSpecifiedReplicas() throws Exception { @@ -100,9 +120,9 @@ public void testIndexTemplateValidationWithSpecifiedReplicas() throws Exception request.patterns(singletonList("test_shards_wait*")); Settings.Builder settingsBuilder = builder() - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") - .put(IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey(), "2"); + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey(), "2"); request.settings(settingsBuilder.build()); @@ -154,7 +174,7 @@ public void testIndexTemplateValidationAccumulatesValidationErrors() { assertThat(throwables.get(0).getMessage(), containsString("name must not contain a space")); assertThat(throwables.get(0).getMessage(), containsString("index_pattern [_test_shards*] must not start with '_'")); assertThat(throwables.get(0).getMessage(), - containsString("Failed to parse value [0] for setting [index.number_of_shards] must be >= 1")); + containsString("Failed to parse value [0] for setting [index.number_of_shards] must be >= 1")); } public void testIndexTemplateWithAliasNameEqualToTemplatePattern() { @@ -172,8 +192,8 @@ public void testIndexTemplateWithValidateMapping() throws Exception { PutRequest request = new PutRequest("api", "validate_template"); request.patterns(singletonList("te*")); request.putMapping("type1", Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("field2").field("type", "text").field("analyzer", "custom_1").endObject() - .endObject().endObject().endObject())); + .startObject("properties").startObject("field2").field("type", "text").field("analyzer", "custom_1").endObject() + .endObject().endObject().endObject())); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -268,7 +288,7 @@ public void testPutGlobalTemplateWithIndexHiddenSetting() throws Exception { assertThat(errors.get(0).getMessage(), containsString("global templates may not specify the setting index.hidden")); } - public void testAddComponentTemplate() throws Exception { + public void testAddComponentTemplate() throws Exception{ MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = ClusterState.EMPTY_STATE; Template template = new Template(Settings.builder().build(), null, ComponentTemplateTests.randomAliases()); @@ -314,18 +334,18 @@ public void testUpdateComponentTemplateWithIndexHiddenSetting() throws Exception assertNotNull(state.metadata().componentTemplates().get("foo")); ComposableIndexTemplate firstGlobalIndexTemplate = - new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), template, - org.elasticsearch.common.collect.List.of("foo"), 1L, null, null, null); + new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), template, + org.elasticsearch.common.collect.List.of("foo"), 1L, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate1", firstGlobalIndexTemplate); ComposableIndexTemplate secondGlobalIndexTemplate = - new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), template, - org.elasticsearch.common.collect.List.of("foo"), 2L, null, null, null); + new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), template, + org.elasticsearch.common.collect.List.of("foo"), 2L, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate2", secondGlobalIndexTemplate); ComposableIndexTemplate fooPatternIndexTemplate = - new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("foo-*"), template, - org.elasticsearch.common.collect.List.of("foo"), 3L, null, null, null); + new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("foo-*"), template, + org.elasticsearch.common.collect.List.of("foo"), 3L, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "foopatternindextemplate", fooPatternIndexTemplate); // update the component template to set the index.hidden setting @@ -629,10 +649,10 @@ public void testFindV2Templates() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); ComposableIndexTemplate it = new ComposableIndexTemplate(Collections.singletonList("i*"), - null, Collections.singletonList("ct"), null, 1L, null, null); + null, Collections.singletonList("ct"), null, 1L, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); ComposableIndexTemplate it2 = new ComposableIndexTemplate(Collections.singletonList("in*"), - null, Collections.singletonList("ct"), 10L, 2L, null, null); + null, Collections.singletonList("ct"), 10L, 2L, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean()); @@ -648,10 +668,10 @@ public void testFindV2TemplatesForHiddenIndex() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); ComposableIndexTemplate it = new ComposableIndexTemplate(Collections.singletonList("i*"), - null, Collections.singletonList("ct"), 0L, 1L, null, null); + null, Collections.singletonList("ct"), 0L, 1L, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); ComposableIndexTemplate it2 = new ComposableIndexTemplate(Collections.singletonList("*"), - null, Collections.singletonList("ct"), 10L, 2L, null, null); + null, Collections.singletonList("ct"), 10L, 2L, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true); @@ -666,8 +686,8 @@ public void testFindV2InvalidGlobalTemplate() { ComposableIndexTemplate invalidGlobalTemplate = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), templateWithHiddenSetting, org.elasticsearch.common.collect.List.of("ct"), 5L, 1L, null, null); Metadata invalidGlobalTemplateMetadata = Metadata.builder().putCustom(ComposableIndexTemplateMetadata.TYPE, - new ComposableIndexTemplateMetadata(org.elasticsearch.common.collect.Map.of("invalid_global_template", - invalidGlobalTemplate))).build(); + new ComposableIndexTemplateMetadata(org.elasticsearch.common.collect.Map.of("invalid_global_template", + invalidGlobalTemplate))).build(); MetadataIndexTemplateService.findV2Template(invalidGlobalTemplateMetadata, "index-name", false); fail("expecting an exception as the matching global template is invalid"); @@ -1177,7 +1197,7 @@ public void testRemoveComponentTemplateInUse() throws Exception { * Tests that we check that settings/mappings/etc are valid even after template composition, * when adding/updating a composable index template */ - public void testIndexTemplateFailsToOverrideComponentTemplateMappingField() throws Exception { + public void testIndexTemplateFailsToOverrideComponentTemplateMappingField() throws Exception { final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); ClusterState state = ClusterState.EMPTY_STATE; @@ -1400,22 +1420,22 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( - Settings.EMPTY, - null, - null, - null, - null, - createTestShardLimitService(randomIntBetween(1, 1000)), - new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null), - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - null, - xContentRegistry, - new SystemIndices(Collections.emptyMap()), - true + Settings.EMPTY, + null, + null, + null, + null, + createTestShardLimitService(randomIntBetween(1, 1000)), + new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null), + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + null, + xContentRegistry, + new SystemIndices(Collections.emptyMap()), + true ); MetadataIndexTemplateService service = new MetadataIndexTemplateService(null, createIndexService, - new AliasValidator(), null, - new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry); + new AliasValidator(), null, + new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry); final List throwables = new ArrayList<>(); service.putTemplate(request, new MetadataIndexTemplateService.PutListener() { @@ -1457,22 +1477,22 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { IndicesService indicesService = getInstanceFromNode(IndicesService.class); ClusterService clusterService = getInstanceFromNode(ClusterService.class); MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( - Settings.EMPTY, - clusterService, - indicesService, - null, - null, - createTestShardLimitService(randomIntBetween(1, 1000)), - new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null), - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - null, - xContentRegistry(), - new SystemIndices(Collections.emptyMap()), - true + Settings.EMPTY, + clusterService, + indicesService, + null, + null, + createTestShardLimitService(randomIntBetween(1, 1000)), + new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null), + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + null, + xContentRegistry(), + new SystemIndices(Collections.emptyMap()), + true ); return new MetadataIndexTemplateService( - clusterService, createIndexService, new AliasValidator(), indicesService, - new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry()); + clusterService, createIndexService, new AliasValidator(), indicesService, + new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry()); } @SuppressWarnings("unchecked") @@ -1525,4 +1545,80 @@ public static void assertTemplatesEqual(ComposableIndexTemplate actual, Composab } } } + + // Composable index template with data_stream definition need _timestamp meta field mapper, + // this is a dummy impl, so that tests don't fail with the fact that the _timestamp field can't be found. + // (tests using this dummy impl doesn't test the _timestamp validation, but need it to tests other functionality) + public static class DummyPlugin extends Plugin implements MapperPlugin { + + @Override + public Map getMetadataMappers() { + return Collections.singletonMap("_data_stream_timestamp", new MetadataFieldMapper.ConfigurableTypeParser( + c -> new MetadataTimestampFieldMapper(false), + c -> new MetadataTimestampFieldBuilder()) + ); + } + } + + private static MetadataTimestampFieldMapper toType(FieldMapper in) { + return (MetadataTimestampFieldMapper) in; + } + + public static class MetadataTimestampFieldBuilder extends MetadataFieldMapper.Builder { + + private final Parameter enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false); + + protected MetadataTimestampFieldBuilder() { + super("_data_stream_timestamp"); + } + + @Override + protected List> getParameters() { + return Collections.singletonList(enabled); + } + + @Override + public MetadataFieldMapper build(Mapper.BuilderContext context) { + return new MetadataTimestampFieldMapper(enabled.getValue()); + } + } + + public static class MetadataTimestampFieldMapper extends MetadataFieldMapper { + final boolean enabled; + + public MetadataTimestampFieldMapper(boolean enabled) { + super(new MappedFieldType("_data_stream_timestamp", false, false, false, TextSearchInfo.NONE, Collections.emptyMap()) { + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return "_data_stream_timestamp"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return null; + } + }); + this.enabled = enabled; + } + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new MetadataTimestampFieldBuilder().init(this); + } + + @Override + protected String contentType() { + return "_data_stream_timestamp"; + } + } } From eb87d5bb670ca018e3d27712d795dd5405ea632e Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Thu, 25 Feb 2021 23:31:24 -0800 Subject: [PATCH 3/8] Revert "EXPER - fix IndicesModuleTests by adding the timestamp field back to metadata fileds" This reverts commit b8630600 --- .../java/org/elasticsearch/indices/IndicesModuleTests.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 9fc6aa622a53e..81b6e91a48575 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.index.mapper.AllFieldMapper; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; @@ -78,11 +77,11 @@ public Map getMetadataMappers() { private static String[] EXPECTED_METADATA_FIELDS = new String[]{IgnoredFieldMapper.NAME, IdFieldMapper.NAME, RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, - VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, DataStreamTimestampFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; private static String[] EXPECTED_METADATA_FIELDS_6x = new String[]{AllFieldMapper.NAME, IgnoredFieldMapper.NAME, IdFieldMapper.NAME, RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, - VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, DataStreamTimestampFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); From e74a4ed9c42c292a519aeeeecf2e525b10500698 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Thu, 25 Feb 2021 23:31:34 -0800 Subject: [PATCH 4/8] Revert "EXPER - fix the MapperServiceTests by extending the max length of field name in the test" This reverts commit 0efae0a7 --- .../org/elasticsearch/index/mapper/MapperServiceTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 6c3513501faf7..a21a369904fbe 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -317,7 +317,7 @@ public void testDefaultMappingIsRejectedOn7() throws IOException { } public void testFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(25, 30); // extend the max length due to "_data_stream_timestamp" + int maxFieldNameLength = randomIntBetween(15, 20); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); @@ -376,7 +376,7 @@ public void testObjectNameLengthLimit() throws Throwable { } public void testAliasFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(25, 30); // extend the max length due to "_data_stream_timestamp" + int maxFieldNameLength = randomIntBetween(15, 20); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); From ca1e21386650aca9c8ca3dcaf0187959f03505de Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Thu, 25 Feb 2021 23:31:41 -0800 Subject: [PATCH 5/8] Revert "EXPER - Add DataStreamTimestampFieldMapper" This reverts commit 49ad8de2 --- .../DataStreamTimestampFieldMapper.java | 198 ------------------ .../elasticsearch/indices/IndicesModule.java | 2 - 2 files changed, 200 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java deleted file mode 100644 index c0eea6ec3894a..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.search.Query; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.search.lookup.SearchLookup; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - -public class DataStreamTimestampFieldMapper extends MetadataFieldMapper { - - public static final String NAME = "_data_stream_timestamp"; - private static final String DEFAULT_PATH = "@timestamp"; - - // For now the field shouldn't be useable in searches. - // In the future it should act as an alias to the actual data stream timestamp field. - public static final class TimestampFieldType extends MappedFieldType { - - public TimestampFieldType() { - super(NAME, false, false,false, TextSearchInfo.NONE, Collections.emptyMap()); - } - - @Override - public String typeName() { - return NAME; - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support term queries"); - } - - @Override - public Query existsQuery(QueryShardContext context) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exists queries"); - } - - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - } - - private static DataStreamTimestampFieldMapper toType(FieldMapper in) { - return (DataStreamTimestampFieldMapper) in; - } - - public static class Builder extends MetadataFieldMapper.Builder { - - private final Parameter enabled = Parameter.boolParam("enabled", false, m -> toType(m).enabled, false); - - public Builder() { - super(NAME); - } - - @Override - protected List> getParameters() { - return Collections.singletonList(enabled); - } - - @Override - public MetadataFieldMapper build(BuilderContext context) { - return new DataStreamTimestampFieldMapper(new TimestampFieldType(), enabled.getValue()); - } - } - - public static final TypeParser PARSER = new ConfigurableTypeParser( - c -> new DataStreamTimestampFieldMapper(new TimestampFieldType(), false), - c -> new Builder() - ); - - private final String path = DEFAULT_PATH; - private final boolean enabled; - - private DataStreamTimestampFieldMapper(MappedFieldType mappedFieldType, boolean enabled) { - super(mappedFieldType); - this.enabled = enabled; - } - - @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new Builder().init(this); - } - - public void doValidate(MappingLookup lookup) { - if (enabled == false) { - // not configured, so skip the validation - return; - } - - Mapper mapper = lookup.getMapper(path); - if (mapper == null) { - throw new IllegalArgumentException("the configured timestamp field [" + path + "] does not exist"); - } - - if (DateFieldMapper.CONTENT_TYPE.equals(mapper.typeName()) == false && - DateFieldMapper.DATE_NANOS_CONTENT_TYPE.equals(mapper.typeName()) == false) { - throw new IllegalArgumentException("the configured timestamp field [" + path + "] is of type [" + - mapper.typeName() + "], but [" + DateFieldMapper.CONTENT_TYPE + "," + DateFieldMapper.DATE_NANOS_CONTENT_TYPE + - "] is expected"); - } - - DateFieldMapper dateFieldMapper = (DateFieldMapper) mapper; - if (dateFieldMapper.fieldType().isSearchable() == false) { - throw new IllegalArgumentException("the configured timestamp field [" + path + "] is not indexed"); - } - if (dateFieldMapper.fieldType().hasDocValues() == false) { - throw new IllegalArgumentException("the configured timestamp field [" + path + "] doesn't have doc values"); - } - if (dateFieldMapper.getNullValue() != null) { - throw new IllegalArgumentException("the configured timestamp field [" + path + - "] has disallowed [null_value] attribute specified"); - } - if (dateFieldMapper.getIgnoreMalformed()) { - throw new IllegalArgumentException("the configured timestamp field [" + path + - "] has disallowed [ignore_malformed] attribute specified"); - } - - // Catch all validation that validates whether disallowed mapping attributes have been specified - // on the field this meta field refers to: - try (XContentBuilder builder = jsonBuilder()) { - builder.startObject(); - dateFieldMapper.doXContentBody(builder, false, EMPTY_PARAMS); - builder.endObject(); - Map configuredSettings = - XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2(); - - // Only type, meta and format attributes are allowed: - configuredSettings.remove("type"); - configuredSettings.remove("meta"); - configuredSettings.remove("format"); - // All other configured attributes are not allowed: - if (configuredSettings.isEmpty() == false) { - throw new IllegalArgumentException("the configured timestamp field [@timestamp] has disallowed attributes: " + - configuredSettings.keySet()); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - public void postParse(ParseContext context) throws IOException { - if (enabled == false) { - // not configured, so skip the validation - return; - } - - IndexableField[] fields = context.rootDoc().getFields(path); - if (fields.length == 0) { - throw new IllegalArgumentException("data stream timestamp field [" + path + "] is missing"); - } - - long numberOfValues = - Arrays.stream(fields) - .filter(indexableField -> indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) - .count(); - if (numberOfValues > 1) { - throw new IllegalArgumentException("data stream timestamp field [" + path + "] encountered multiple values"); - } - } - - @Override - protected String contentType() { - return NAME; - } -} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index b0d74ef981229..5c36edd8d4f14 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -50,7 +50,6 @@ import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; @@ -163,7 +162,6 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(TypeFieldMapper.NAME, TypeFieldMapper.PARSER); builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); - builtInMetadataMappers.put(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); //_field_names must be added last so that it has a chance to see all the other mappers builtInMetadataMappers.put(FieldNamesFieldMapper.NAME, FieldNamesFieldMapper.PARSER); return Collections.unmodifiableMap(builtInMetadataMappers); From 8b1ae28d372f906f2042de3ea84aa1f79bc14fd3 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Thu, 25 Feb 2021 23:56:10 -0800 Subject: [PATCH 6/8] Skip test in docs/reference/data-streams --- docs/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/build.gradle b/docs/build.gradle index 6df63264446cf..5b9d70b294f1e 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -22,9 +22,9 @@ import org.elasticsearch.gradle.info.BuildParams apply plugin: 'elasticsearch.docs-test' apply plugin: 'elasticsearch.rest-resources' -/* List of files that have snippets that will not work ... */ +/* List of files that have snippets that will not work until platinum tests can occur ... */ buildRestTests.expectedUnconvertedCandidates = [ -// File path from content root +// 'reference/ml/anomaly-detection/apis/update-job.asciidoc' ] restResources { @@ -80,7 +80,7 @@ buildRestTests.docs = fileTree(projectDir) { // Just syntax examples exclude 'README.asciidoc' // Broken code snippet tests - + exclude 'reference/data-streams' if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) { // We don't install/support this plugin in FIPS 140 exclude 'plugins/ingest-attachment.asciidoc' From 694c6c39e56a81586165ab4bc45ec15ace7a6c7c Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Fri, 26 Feb 2021 00:27:53 -0800 Subject: [PATCH 7/8] Skip other docs tests caused by DataStreamTimestampFieldMapper --- docs/build.gradle | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/build.gradle b/docs/build.gradle index 5b9d70b294f1e..0f3fd971490b5 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -81,6 +81,13 @@ buildRestTests.docs = fileTree(projectDir) { exclude 'README.asciidoc' // Broken code snippet tests exclude 'reference/data-streams' + exclude 'reference/indices/create-data-stream.asciidoc' + exclude 'reference/indices/data-stream-stats.asciidoc' + exclude 'reference/indices/delete-data-stream.asciidoc' + exclude 'reference/indices/get-data-stream.asciidoc' + exclude 'reference/indices/put-index-template.asciidoc' + exclude 'reference/indices/resolve.asciidoc' + exclude 'reference/indices/rollover-index.asciidoc' if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) { // We don't install/support this plugin in FIPS 140 exclude 'plugins/ingest-attachment.asciidoc' From 0341a7cd54746da1d55864685992f78bb009800f Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Fri, 26 Feb 2021 15:57:47 -0800 Subject: [PATCH 8/8] Resolve merge conflict in build.gradle --- docs/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build.gradle b/docs/build.gradle index 0f3fd971490b5..78f182852f3e0 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -22,9 +22,9 @@ import org.elasticsearch.gradle.info.BuildParams apply plugin: 'elasticsearch.docs-test' apply plugin: 'elasticsearch.rest-resources' -/* List of files that have snippets that will not work until platinum tests can occur ... */ +/* List of files that have snippets that will not work ... */ buildRestTests.expectedUnconvertedCandidates = [ -// 'reference/ml/anomaly-detection/apis/update-job.asciidoc' +// File path from content root ] restResources {