From 1780b792424e80577a1bec72759f25b978a10051 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 18 Aug 2020 15:11:55 -0500 Subject: [PATCH 01/25] migrate aliased indices to data stream --- .../cluster/metadata/DataStream.java | 5 +- .../cluster/metadata/IndexAbstraction.java | 2 - .../MetadataCreateDataStreamService.java | 115 ++++++++++--- .../MetadataMigrateToDataStreamService.java | 141 +++++++++++++++ .../MetadataCreateDataStreamServiceTests.java | 162 ++++++++++++++++++ ...tadataMigrateToDataStreamServiceTests.java | 104 +++++++++++ 6 files changed, 505 insertions(+), 24 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 2c3d446306b99..45ae3dba33a50 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -52,7 +52,6 @@ public DataStream(String name, TimestampField timeStampField, List indice this.indices = Collections.unmodifiableList(indices); this.generation = generation; assert indices.size() > 0; - assert indices.get(indices.size() - 1).getName().equals(getDefaultBackingIndexName(name, generation)); } public DataStream(String name, TimestampField timeStampField, List indices) { @@ -75,6 +74,10 @@ public long getGeneration() { return generation; } + public Index getWriteIndex() { + return indices.get(indices.size() - 1); + } + /** * Performs a rollover on a {@code DataStream} instance and returns a new instance containing * the updated list of backing indices and incremented generation. diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index ec2c6c1c33b81..1ab87c12b74bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -30,7 +30,6 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING; /** @@ -292,7 +291,6 @@ public DataStream(org.elasticsearch.cluster.metadata.DataStream dataStream, List this.dataStream = dataStream; this.dataStreamIndices = List.copyOf(dataStreamIndices); this.writeIndex = dataStreamIndices.get(dataStreamIndices.size() - 1); - assert writeIndex.getIndex().getName().equals(getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration())); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 66fe363272471..1bf0cb5ebf83c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -33,19 +33,24 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectPath; +import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; public class MetadataCreateDataStreamService { @@ -121,45 +126,113 @@ public CreateDataStreamClusterStateUpdateRequest(String name, static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, ClusterState currentState, - CreateDataStreamClusterStateUpdateRequest request) - throws Exception { + CreateDataStreamClusterStateUpdateRequest request) throws Exception { + return createDataStream(metadataCreateIndexService, currentState, request.name, List.of(), null); + } + + /** + * Creates a data stream with the specified properties. + * + * @param metadataCreateIndexService Used if a new write index must be created + * @param currentState Cluster state + * @param dataStreamName Name of the data stream + * @param backingIndices List of backing indices. May be empty + * @param writeIndex Write index for the data stream. If null, a new write index will be created. + * @return Cluster state containing the new data stream + */ + static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, + ClusterState currentState, + String dataStreamName, + List backingIndices, + IndexMetadata writeIndex) throws Exception + { + Objects.requireNonNull(metadataCreateIndexService); + Objects.requireNonNull(currentState); + Objects.requireNonNull(backingIndices); if (currentState.nodes().getMinNodeVersion().before(Version.V_7_9_0)) { throw new IllegalStateException("data streams require minimum node version of " + Version.V_7_9_0); } - if (currentState.metadata().dataStreams().containsKey(request.name)) { - throw new ResourceAlreadyExistsException("data_stream [" + request.name + "] already exists"); + if (currentState.metadata().dataStreams().containsKey(dataStreamName)) { + throw new ResourceAlreadyExistsException("data_stream [" + dataStreamName + "] already exists"); } - MetadataCreateIndexService.validateIndexOrAliasName(request.name, + MetadataCreateIndexService.validateIndexOrAliasName(dataStreamName, (s1, s2) -> new IllegalArgumentException("data_stream [" + s1 + "] " + s2)); - if (request.name.toLowerCase(Locale.ROOT).equals(request.name) == false) { - throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase"); + if (dataStreamName.toLowerCase(Locale.ROOT).equals(dataStreamName) == false) { + throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must be lowercase"); } - if (request.name.startsWith(".")) { - throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'"); + if (dataStreamName.startsWith(".")) { + throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '.'"); } - ComposableIndexTemplate template = lookupTemplateForDataStream(request.name, currentState.metadata()); + ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, currentState.metadata()); - String firstBackingIndexName = DataStream.getDefaultBackingIndexName(request.name, 1); - CreateIndexClusterStateUpdateRequest createIndexRequest = - new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName) - .dataStreamName(request.name) - .settings(Settings.builder().put("index.hidden", true).build()); - currentState = metadataCreateIndexService.applyCreateIndexRequest(currentState, createIndexRequest, false); - IndexMetadata firstBackingIndex = currentState.metadata().index(firstBackingIndexName); - assert firstBackingIndex != null; - assert firstBackingIndex.mapping() != null : "no mapping found for backing index [" + firstBackingIndexName + "]"; + if (backingIndices.size() > 0) { + validateBackingIndices(currentState, dataStreamName); + + // hide existing indices and remove aliases + for (IndexMetadata backingIndex : backingIndices) { + Metadata.Builder b = Metadata.builder(currentState.metadata()); + b.put(hideAndRemoveAlias(backingIndex, dataStreamName), false); + currentState = ClusterState.builder(currentState).metadata(b).build(); + } + } + + if (writeIndex == null) { + String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); + CreateIndexClusterStateUpdateRequest createIndexRequest = + new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName) + .dataStreamName(dataStreamName) + .settings(Settings.builder().put("index.hidden", true).build()); + currentState = metadataCreateIndexService.applyCreateIndexRequest(currentState, createIndexRequest, false); + writeIndex = currentState.metadata().index(firstBackingIndexName); + } else { + writeIndex = hideAndRemoveAlias(writeIndex, dataStreamName); + currentState = ClusterState.builder(currentState) + .metadata(Metadata.builder(currentState.metadata()).put(writeIndex, false).build()).build(); + } + assert writeIndex != null; + assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]"; String fieldName = template.getDataStreamTemplate().getTimestampField(); DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName); - DataStream newDataStream = new DataStream(request.name, timestampField, List.of(firstBackingIndex.getIndex())); + List dsBackingIndices = backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()); + dsBackingIndices.add(writeIndex.getIndex()); + DataStream newDataStream = new DataStream(dataStreamName, timestampField, dsBackingIndices, 1); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); - logger.info("adding data stream [{}]", request.name); + logger.info("adding data stream [{}]", dataStreamName); return ClusterState.builder(currentState).metadata(builder).build(); } + private static IndexMetadata hideAndRemoveAlias(IndexMetadata im, String dataStreamName) { + return IndexMetadata.builder(im) + .removeAlias(dataStreamName) + .settings(Settings.builder().put(im.getSettings()).put("index.hidden", "true").build()) + .build(); + } + + // package-visible for testing + static void validateBackingIndices(ClusterState currentState, String dataStreamName) { + IndexAbstraction ia = currentState.metadata().getIndicesLookup().get(dataStreamName); + if (ia == null || ia.getType() != IndexAbstraction.Type.ALIAS) { + throw new IllegalArgumentException("alias [" + dataStreamName + "] does not exist"); + } + IndexAbstraction.Alias alias = (IndexAbstraction.Alias) ia; + + // ensure that no other aliases reference indices + List indicesWithOtherAliases = new ArrayList<>(); + for (IndexMetadata im : alias.getIndices()) { + if (im.getAliases().size() > 1 || im.getAliases().containsKey(alias.getName()) == false) { + indicesWithOtherAliases.add(im.getIndex().getName()); + } + } + if (indicesWithOtherAliases.size() > 0) { + throw new IllegalArgumentException("other aliases referencing indices [" + + Strings.collectionToCommaDelimitedString(indicesWithOtherAliases) + "] must be removed before migrating to a data stream"); + } + } + public static ComposableIndexTemplate lookupTemplateForDataStream(String dataStreamName, Metadata metadata) { final String v2Template = MetadataIndexTemplateService.findV2Template(metadata, dataStreamName, false); if (v2Template == null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java new file mode 100644 index 0000000000000..94f45d75ecd2c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.ActiveShardsObserver; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; +import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +public class MetadataMigrateToDataStreamService { + + private static final Logger logger = LogManager.getLogger(MetadataMigrateToDataStreamService.class); + + private final ClusterService clusterService; + private final ActiveShardsObserver activeShardsObserver; + private final MetadataCreateIndexService metadataCreateIndexService; + + public MetadataMigrateToDataStreamService(ThreadPool threadPool, + ClusterService clusterService, + MetadataCreateIndexService metadataCreateIndexService) { + this.clusterService = clusterService; + this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); + this.metadataCreateIndexService = metadataCreateIndexService; + } + + public void migrateToDataStream(MigrateToDataStreamClusterStateUpdateRequest request, + ActionListener finalListener) { + AtomicReference writeIndexRef = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap( + response -> { + if (response.isAcknowledged()) { + String writeIndexName = writeIndexRef.get(); + assert writeIndexName != null; + activeShardsObserver.waitForActiveShards( + new String[]{writeIndexName}, + ActiveShardCount.DEFAULT, + request.masterNodeTimeout(), + shardsAcked -> { + finalListener.onResponse(new AcknowledgedResponse(true)); + }, + finalListener::onFailure); + } else { + finalListener.onResponse(new AcknowledgedResponse(false)); + } + }, + finalListener::onFailure + ); + clusterService.submitStateUpdateTask("migrate-to-data-stream [" + request.aliasName + "]", + new AckedClusterStateUpdateTask<>(Priority.HIGH, request, listener) { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ClusterState clusterState = migrateToDataStream(metadataCreateIndexService, currentState, request); + writeIndexRef.set(clusterState.metadata().dataStreams().get(request.aliasName).getWriteIndex().getName()); + return clusterState; + } + + @Override + protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { + return new ClusterStateUpdateResponse(acknowledged); + } + }); + } + + static ClusterState migrateToDataStream(MetadataCreateIndexService metadataCreateIndexService, + ClusterState currentState, + MigrateToDataStreamClusterStateUpdateRequest request) throws Exception { + validateRequest(currentState, request); + IndexAbstraction.Alias alias = (IndexAbstraction.Alias) currentState.metadata().getIndicesLookup().get(request.aliasName); + IndexMetadata writeIndex = alias.getWriteIndex(); + + List backingIndices = alias.getIndices() + .stream() + .filter(x -> writeIndex == null || x.getIndex().getName().equals(writeIndex.getIndex().getName()) == false) + .collect(Collectors.toList()); + + return MetadataCreateDataStreamService.createDataStream( + metadataCreateIndexService, currentState, request.aliasName, backingIndices, writeIndex); + } + + // package-visible for testing + static void validateRequest(ClusterState currentState, MigrateToDataStreamClusterStateUpdateRequest request) { + IndexAbstraction ia = currentState.metadata().getIndicesLookup().get(request.aliasName); + if (ia == null || ia.getType() != IndexAbstraction.Type.ALIAS) { + throw new IllegalArgumentException("alias [" + request.aliasName + "] does not exist"); + } + IndexAbstraction.Alias alias = (IndexAbstraction.Alias) ia; + + // check for "clean" alias without routing or filter query + AliasMetadata aliasMetadata = alias.getFirstAliasMetadata(); + assert aliasMetadata != null : "alias metadata may not be null"; + if (aliasMetadata.filteringRequired() || aliasMetadata.getIndexRouting() != null || aliasMetadata.getSearchRouting() != null) { + throw new IllegalArgumentException("alias [" + request.aliasName + "] may not have custom filtering or routing"); + } + } + + public static final class MigrateToDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { + + private final String aliasName; + + public MigrateToDataStreamClusterStateUpdateRequest(String aliasName, + TimeValue masterNodeTimeout, + TimeValue timeout) { + this.aliasName = aliasName; + masterNodeTimeout(masterNodeTimeout); + ackTimeout(timeout); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index d52f36afb1fb1..b1bb25743bba6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -24,16 +24,20 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -138,6 +142,164 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { equalTo("matching index template [template] for data stream [my-data-stream] has no data stream template")); } + public void testValidateRequestWithIndicesWithMultipleAliasReferences() { + String aliasName = "alias"; + AliasMetadata alias1 = AliasMetadata.builder(aliasName).build(); + AliasMetadata alias2 = AliasMetadata.builder(aliasName + "2").build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .putAlias(alias2) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo3") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .putAlias(alias2) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo4") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + ).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MetadataCreateDataStreamService.validateBackingIndices(cs, aliasName)); + String emsg = e.getMessage(); + assertThat(emsg, containsString("other aliases referencing indices [")); + assertThat(emsg, containsString("] must be removed before migrating to a data stream")); + String referencedIndices = emsg.substring(emsg.indexOf('[') + 1, emsg.indexOf(']')); + Set indices = Strings.commaDelimitedListToSet(referencedIndices); + assertThat(indices, containsInAnyOrder("foo2", "foo3")); + } + + public void testCreateDataStreamHidesBackingIndicesAndRemovesAlias() throws Exception { + String dataStreamName = "foo"; + AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); + IndexMetadata foo1 = IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + IndexMetadata foo2 = IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(foo1, false) + .put(foo2, false) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate()))) + .build(); + + ClusterState newState = MetadataCreateDataStreamService.createDataStream(getMetadataCreateIndexService(), cs, dataStreamName, + List.of(foo1, foo2), null); + IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); + assertThat(ds, notNullValue()); + assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + assertThat(ds.getIndices().size(), equalTo(3)); + List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); + assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2", DataStream.getDefaultBackingIndexName(dataStreamName, 1))); + for (IndexMetadata im : ds.getIndices()) { + assertThat(im.getSettings().get("index.hidden"), equalTo("true")); + assertThat(im.getAliases().size(), equalTo(0)); + } + } + + public void testCreateDataStreamWithSuppliedWriteIndex() throws Exception { + String dataStreamName = "foo"; + AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); + IndexMetadata foo1 = IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + IndexMetadata foo2 = IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(foo1, false) + .put(foo2, false) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate()))) + .build(); + + ClusterState newState = MetadataCreateDataStreamService.createDataStream(getMetadataCreateIndexService(), cs, dataStreamName, + List.of(foo1), foo2); + IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); + assertThat(ds, notNullValue()); + assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + assertThat(ds.getIndices().size(), equalTo(2)); + List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); + assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2")); + assertThat(ds.getWriteIndex().getIndex().getName(), equalTo("foo2")); + for (IndexMetadata im : ds.getIndices()) { + assertThat(im.getSettings().get("index.hidden"), equalTo("true")); + assertThat(im.getAliases().size(), equalTo(0)); + } + } + + public void testCreateDataStreamWithoutSuppliedWriteIndex() throws Exception { + String dataStreamName = "foo"; + AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); + IndexMetadata foo1 = IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + IndexMetadata foo2 = IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(foo1, false) + .put(foo2, false) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate()))) + .build(); + + ClusterState newState = MetadataCreateDataStreamService.createDataStream(getMetadataCreateIndexService(), cs, dataStreamName, + List.of(foo1, foo2), null); + IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); + assertThat(ds, notNullValue()); + assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + assertThat(ds.getIndices().size(), equalTo(3)); + List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); + assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2", DataStream.getDefaultBackingIndexName(dataStreamName, 1))); + assertThat(ds.getWriteIndex().getIndex().getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); + for (IndexMetadata im : ds.getIndices()) { + assertThat(im.getSettings().get("index.hidden"), equalTo("true")); + assertThat(im.getAliases().size(), equalTo(0)); + } + } + public static ClusterState createDataStream(final String dataStreamName) throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java new file mode 100644 index 0000000000000..a2c68a7b43eed --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.containsString; + +public class MetadataMigrateToDataStreamServiceTests extends ESTestCase { + + public void testValidateRequestWithNonexistentAlias() { + ClusterState cs = ClusterState.EMPTY_STATE; + String nonExistentAlias = "nonexistent_alias"; + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataMigrateToDataStreamService + .validateRequest(cs, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest( + nonExistentAlias, TimeValue.ZERO, TimeValue.ZERO))); + assertThat(e.getMessage(), containsString("alias [" + nonExistentAlias + "] does not exist")); + } + + public void testValidateRequestWithFilteredAlias() { + String filteredAliasName = "nonexistent_alias"; + AliasMetadata filteredAlias = AliasMetadata.builder(filteredAliasName).filter("{\"term\":{\"user.id\":\"kimchy\"}}").build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder().put(IndexMetadata.builder("foo") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(filteredAlias) + .numberOfShards(1) + .numberOfReplicas(0)) + ).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataMigrateToDataStreamService + .validateRequest(cs, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest( + filteredAliasName, TimeValue.ZERO, TimeValue.ZERO))); + assertThat(e.getMessage(), containsString("alias [" + filteredAliasName + "] may not have custom filtering or routing")); + } + + public void testValidateRequestWithAliasWithRouting() { + String routedAliasName = "nonexistent_alias"; + AliasMetadata aliasWithRouting = AliasMetadata.builder(routedAliasName).routing("foo").build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder().put(IndexMetadata.builder("foo") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(aliasWithRouting) + .numberOfShards(1) + .numberOfReplicas(0)) + ).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataMigrateToDataStreamService + .validateRequest(cs, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest( + routedAliasName, TimeValue.ZERO, TimeValue.ZERO))); + assertThat(e.getMessage(), containsString("alias [" + routedAliasName + "] may not have custom filtering or routing")); + } + + public void testValidateRequest() { + String aliasName = "alias"; + AliasMetadata alias1 = AliasMetadata.builder(aliasName).build(); + AliasMetadata alias2 = AliasMetadata.builder(aliasName + "2").build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo3") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo4") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + ).build(); + MetadataMigrateToDataStreamService.validateRequest(cs, + new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(aliasName, TimeValue.ZERO, TimeValue.ZERO)); + } +} From 36c4ace21b403e4ae95213661b9af4e0cd48493b Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 26 Aug 2020 08:09:28 -0500 Subject: [PATCH 02/25] update cluster state a single time for all backing indices --- .../MetadataCreateDataStreamService.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 1bf0cb5ebf83c..0411344d789e2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -172,11 +172,11 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn validateBackingIndices(currentState, dataStreamName); // hide existing indices and remove aliases + Metadata.Builder b = Metadata.builder(currentState.metadata()); for (IndexMetadata backingIndex : backingIndices) { - Metadata.Builder b = Metadata.builder(currentState.metadata()); - b.put(hideAndRemoveAlias(backingIndex, dataStreamName), false); - currentState = ClusterState.builder(currentState).metadata(b).build(); + hideAndRemoveAlias(b, backingIndex, dataStreamName); } + currentState = ClusterState.builder(currentState).metadata(b).build(); } if (writeIndex == null) { @@ -188,9 +188,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn currentState = metadataCreateIndexService.applyCreateIndexRequest(currentState, createIndexRequest, false); writeIndex = currentState.metadata().index(firstBackingIndexName); } else { - writeIndex = hideAndRemoveAlias(writeIndex, dataStreamName); - currentState = ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.metadata()).put(writeIndex, false).build()).build(); + Metadata.Builder b = Metadata.builder(currentState.metadata()); + hideAndRemoveAlias(b, writeIndex, dataStreamName); + currentState = ClusterState.builder(currentState).metadata(b).build(); } assert writeIndex != null; assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]"; @@ -205,11 +205,11 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn return ClusterState.builder(currentState).metadata(builder).build(); } - private static IndexMetadata hideAndRemoveAlias(IndexMetadata im, String dataStreamName) { - return IndexMetadata.builder(im) + private static void hideAndRemoveAlias(Metadata.Builder b, IndexMetadata im, String dataStreamName) { + b.put(IndexMetadata.builder(im) .removeAlias(dataStreamName) .settings(Settings.builder().put(im.getSettings()).put("index.hidden", "true").build()) - .build(); + .build(), false); } // package-visible for testing From 3bfdfda46c468168fc7f2fd07450f1d940ed8852 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 29 Sep 2020 17:20:23 -0500 Subject: [PATCH 03/25] add data stream timestamp field mapper --- .../metadata/MetadataCreateDataStreamService.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index c0b2c1e96ce1b..a2401afc25559 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -172,7 +172,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn // hide existing indices and remove aliases Metadata.Builder b = Metadata.builder(currentState.metadata()); for (IndexMetadata backingIndex : backingIndices) { - hideAndRemoveAlias(b, backingIndex, dataStreamName); + prepareBackingIndex(b, backingIndex, dataStreamName); } currentState = ClusterState.builder(currentState).metadata(b).build(); } @@ -195,7 +195,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn writeIndex = currentState.metadata().index(firstBackingIndexName); } else { Metadata.Builder b = Metadata.builder(currentState.metadata()); - hideAndRemoveAlias(b, writeIndex, dataStreamName); + prepareBackingIndex(b, writeIndex, dataStreamName); currentState = ClusterState.builder(currentState).metadata(b).build(); } assert writeIndex != null; @@ -211,10 +211,14 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn return ClusterState.builder(currentState).metadata(builder).build(); } - private static void hideAndRemoveAlias(Metadata.Builder b, IndexMetadata im, String dataStreamName) { + private static void prepareBackingIndex(Metadata.Builder b, IndexMetadata im, String dataStreamName) { + // hides the index, removes any aliases, and adds data stream timestamp field mapper + Map mapping = im.mapping().sourceAsMap(); + mapping.put("_data_stream_timestamp", Map.of("enabled", true)); b.put(IndexMetadata.builder(im) - .removeAlias(dataStreamName) - .settings(Settings.builder().put(im.getSettings()).put("index.hidden", "true").build()) + .removeAlias(dataStreamName) + .settings(Settings.builder().put(im.getSettings()).put("index.hidden", "true").build()) + .putMapping(new MappingMetadata("_doc", mapping)) .build(), false); } From 24eaac9dbd40e7cb493485b3f308f19dd089c9cc Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Thu, 1 Oct 2020 08:15:18 -0500 Subject: [PATCH 04/25] prohibit migration of aliases without a write index --- .../cluster/metadata/MetadataMigrateToDataStreamService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 94f45d75ecd2c..6ba04221e87fc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -117,6 +117,10 @@ static void validateRequest(ClusterState currentState, MigrateToDataStreamCluste } IndexAbstraction.Alias alias = (IndexAbstraction.Alias) ia; + if (alias.getWriteIndex() == null) { + throw new IllegalArgumentException("alias [" + request.aliasName + "] must specify a write index"); + } + // check for "clean" alias without routing or filter query AliasMetadata aliasMetadata = alias.getFirstAliasMetadata(); assert aliasMetadata != null : "alias metadata may not be null"; From d1d79ab5926ef0fbe8d7ab77020baed2941e3e65 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 13 Oct 2020 12:45:12 -0500 Subject: [PATCH 05/25] fix compilation and test --- .../MetadataMigrateToDataStreamService.java | 4 +- ...tadataMigrateToDataStreamServiceTests.java | 40 +++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 6ba04221e87fc..fe48c49d8ed95 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -67,11 +67,11 @@ public void migrateToDataStream(MigrateToDataStreamClusterStateUpdateRequest req ActiveShardCount.DEFAULT, request.masterNodeTimeout(), shardsAcked -> { - finalListener.onResponse(new AcknowledgedResponse(true)); + finalListener.onResponse(AcknowledgedResponse.TRUE); }, finalListener::onFailure); } else { - finalListener.onResponse(new AcknowledgedResponse(false)); + finalListener.onResponse(AcknowledgedResponse.FALSE); } }, finalListener::onFailure diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index a2c68a7b43eed..314d327bf7c73 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -40,7 +40,7 @@ public void testValidateRequestWithNonexistentAlias() { } public void testValidateRequestWithFilteredAlias() { - String filteredAliasName = "nonexistent_alias"; + String filteredAliasName = "filtered_alias"; AliasMetadata filteredAlias = AliasMetadata.builder(filteredAliasName).filter("{\"term\":{\"user.id\":\"kimchy\"}}").build(); ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( Metadata.builder().put(IndexMetadata.builder("foo") @@ -56,7 +56,7 @@ public void testValidateRequestWithFilteredAlias() { } public void testValidateRequestWithAliasWithRouting() { - String routedAliasName = "nonexistent_alias"; + String routedAliasName = "routed_alias"; AliasMetadata aliasWithRouting = AliasMetadata.builder(routedAliasName).routing("foo").build(); ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( Metadata.builder().put(IndexMetadata.builder("foo") @@ -71,10 +71,42 @@ public void testValidateRequestWithAliasWithRouting() { assertThat(e.getMessage(), containsString("alias [" + routedAliasName + "] may not have custom filtering or routing")); } + public void testValidateRequestWithAliasWithoutWriteIndex() { + String aliasWithoutWriteIndex = "alias"; + AliasMetadata alias1 = AliasMetadata.builder(aliasWithoutWriteIndex).build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo3") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo4") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + ).build(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataMigrateToDataStreamService + .validateRequest(cs, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest( + aliasWithoutWriteIndex, TimeValue.ZERO, TimeValue.ZERO))); + assertThat(e.getMessage(), containsString("alias [" + aliasWithoutWriteIndex + "] must specify a write index")); + } + public void testValidateRequest() { String aliasName = "alias"; AliasMetadata alias1 = AliasMetadata.builder(aliasName).build(); - AliasMetadata alias2 = AliasMetadata.builder(aliasName + "2").build(); ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( Metadata.builder() .put(IndexMetadata.builder("foo1") @@ -94,7 +126,7 @@ public void testValidateRequest() { .numberOfReplicas(0)) .put(IndexMetadata.builder("foo4") .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias1) + .putAlias(AliasMetadata.builder(aliasName).writeIndex(true)) .numberOfShards(1) .numberOfReplicas(0)) ).build(); From c1196d3cceee35cf97f120dd8906b4479492e220 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 13 Oct 2020 15:22:34 -0500 Subject: [PATCH 06/25] add transport actions --- .../xpack/core/MigrateToDataStreamAction.java | 88 +++++++++++++++++++ .../MigrateToDataStreamTransportAction.java | 71 +++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/MigrateToDataStreamAction.java create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/MigrateToDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/MigrateToDataStreamAction.java new file mode 100644 index 0000000000000..1adf02f72f9bd --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/MigrateToDataStreamAction.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +public class MigrateToDataStreamAction extends ActionType { + + public static final MigrateToDataStreamAction INSTANCE = new MigrateToDataStreamAction(); + public static final String NAME = "indices:admin/data_stream/migrate"; + + private MigrateToDataStreamAction() { + super(NAME, AcknowledgedResponse::readFrom); + } + + public static class Request extends AcknowledgedRequest implements IndicesRequest { + + private final String aliasName; + + public Request(String aliasName) { + this.aliasName = aliasName; + } + + public String getAliasName() { + return aliasName; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.hasText(aliasName) == false) { + validationException = ValidateActions.addValidationError("name is missing", validationException); + } + return validationException; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.aliasName = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(aliasName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MigrateToDataStreamAction.Request request = (MigrateToDataStreamAction.Request) o; + return aliasName.equals(request.aliasName); + } + + @Override + public int hashCode() { + return Objects.hash(aliasName); + } + + @Override + public String[] indices() { + return new String[]{aliasName}; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + } + +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java new file mode 100644 index 0000000000000..90d4e3ee908ec --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataMigrateToDataStreamService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.MigrateToDataStreamAction; + +public class MigrateToDataStreamTransportAction extends AcknowledgedTransportMasterNodeAction { + + private final MetadataMigrateToDataStreamService metadataMigrateToDataStreamService; + + @Inject + public MigrateToDataStreamTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataMigrateToDataStreamService metadataMigrateToDataStreamService + ) { + super( + MigrateToDataStreamAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + MigrateToDataStreamAction.Request::new, + indexNameExpressionResolver, + ThreadPool.Names.SAME + ); + this.metadataMigrateToDataStreamService = metadataMigrateToDataStreamService; + } + + @Override + protected void masterOperation( + Task task, + MigrateToDataStreamAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest updateRequest = + new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest( + request.getAliasName(), + request.masterNodeTimeout(), + request.timeout() + ); + metadataMigrateToDataStreamService.migrateToDataStream(updateRequest, listener); + } + + @Override + protected ClusterBlockException checkBlock(MigrateToDataStreamAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} From d583acd0e5ed517ede3af66d5c1ca3cb76570b83 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 13 Oct 2020 16:34:41 -0500 Subject: [PATCH 07/25] minor fixes --- .../metadata/MetadataMigrateToDataStreamService.java | 2 ++ .../xpack/security/authz/AuthorizationService.java | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index fe48c49d8ed95..1ed4960553ea2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.threadpool.ThreadPool; @@ -46,6 +47,7 @@ public class MetadataMigrateToDataStreamService { private final ActiveShardsObserver activeShardsObserver; private final MetadataCreateIndexService metadataCreateIndexService; + @Inject public MetadataMigrateToDataStreamService(ThreadPool threadPool, ClusterService clusterService, MetadataCreateIndexService metadataCreateIndexService) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 6d7d4d896c009..ecb9fd43cc7f1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -38,6 +38,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.MigrateToDataStreamAction; import org.elasticsearch.xpack.core.action.CreateDataStreamAction; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse; @@ -308,8 +309,10 @@ private void handleIndexActionAuthorizationResult(final IndexAuthorizationResult } //if we are creating an index we need to authorize potential aliases created at the same time if (IndexPrivilege.CREATE_INDEX_MATCHER.test(action)) { - assert (request instanceof CreateIndexRequest) || (request instanceof CreateDataStreamAction.Request); - if (request instanceof CreateDataStreamAction.Request || ((CreateIndexRequest) request).aliases().isEmpty()) { + assert (request instanceof CreateIndexRequest) || (request instanceof MigrateToDataStreamAction.Request) || + (request instanceof CreateDataStreamAction.Request); + if (request instanceof CreateDataStreamAction.Request || (request instanceof MigrateToDataStreamAction.Request) || + ((CreateIndexRequest) request).aliases().isEmpty()) { runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener); } else { Set aliases = ((CreateIndexRequest) request).aliases(); From 9fad1c395faaa4d4446a3429ed71285e37900ff0 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Oct 2020 16:01:46 -0500 Subject: [PATCH 08/25] wip on IT --- .../datastreams/DataStreamIT.java | 4 +- .../datastreams/DataStreamMigrationIT.java | 218 ++++++++++++++++++ .../xpack/datastreams/DataStreamsPlugin.java | 5 +- 3 files changed, 224 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 25cf50926de65..e33f1522b83a2 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1148,7 +1148,7 @@ private static void verifyResolvability( } } - private static void indexDocs(String dataStream, int numDocs) { + static void indexDocs(String dataStream, int numDocs) { BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); @@ -1169,7 +1169,7 @@ private static void indexDocs(String dataStream, int numDocs) { client().admin().indices().refresh(new RefreshRequest(dataStream)).actionGet(); } - private static void verifyDocs(String dataStream, long expectedNumHits, long minGeneration, long maxGeneration) { + static void verifyDocs(String dataStream, long expectedNumHits, long minGeneration, long maxGeneration) { SearchRequest searchRequest = new SearchRequest(dataStream); searchRequest.source().size((int) expectedNumHits); SearchResponse searchResponse = client().search(searchRequest).actionGet(); diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java new file mode 100644 index 0000000000000..3c22a34884108 --- /dev/null +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction; +import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.MigrateToDataStreamAction; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; +import org.junit.After; + +import java.util.Collection; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.datastreams.DataStreamIT.putComposableIndexTemplate; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; + +public class DataStreamMigrationIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class); + } + + @After + public void cleanup() { + AcknowledgedResponse response = client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(new String[] { "*" }) + ).actionGet(); + assertAcked(response); + + DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[] { "*" }); + client().execute(DeleteDataStreamAction.INSTANCE, deleteDSRequest).actionGet(); + DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); + client().execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteTemplateRequest).actionGet(); + } + + public void testBasicMigration() throws Exception { + putComposableIndexTemplate("id1", List.of("migrate*")); + + admin().indices().create(new CreateIndexRequest("index1")).get(); + admin().indices().create(new CreateIndexRequest("index2")).get(); + + int numDocs = randomIntBetween(2, 16); + indexDocs("index1", numDocs); + numDocs = randomIntBetween(2, 16); + indexDocs("index2", numDocs); + + String alias = "migrate-to-data-stream"; + IndicesAliasesRequest request = new IndicesAliasesRequest(); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index1").alias(alias).writeIndex(true)); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index2").alias(alias).writeIndex(false)); + assertAcked(admin().indices().aliases(request).get()); + + ResolveIndexAction.Response resolveResponse = + admin().indices().resolveIndex(new ResolveIndexAction.Request(new String[]{"*"})).get(); + assertThat(resolveResponse.getAliases().size(), equalTo(1)); + assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); + //assertThat(resolveResponse.getIndices().toArray(new ResolveIndexAction.ResolvedIndex[0]), arrayContaining()); + + + client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get(); + + + + /* + putComposableIndexTemplate("id2", List.of("metrics-bar*")); + createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); + + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); + GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); + getDataStreamResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName())); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(2)); + DataStream firstDataStream = getDataStreamResponse.getDataStreams().get(0).getDataStream(); + assertThat(firstDataStream.getName(), equalTo("metrics-bar")); + assertThat(firstDataStream.getTimeStampField().getName(), equalTo("@timestamp")); + assertThat(firstDataStream.getIndices().size(), equalTo(1)); + assertThat(firstDataStream.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 1))); + DataStream dataStream = getDataStreamResponse.getDataStreams().get(1).getDataStream(); + assertThat(dataStream.getName(), equalTo("metrics-foo")); + assertThat(dataStream.getTimeStampField().getName(), equalTo("@timestamp")); + assertThat(dataStream.getIndices().size(), equalTo(1)); + assertThat(dataStream.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 1))); + + String backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 1); + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); + assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); + assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + Map mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); + + backingIndex = DataStream.getDefaultBackingIndexName("metrics-foo", 1); + getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); + assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); + assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); + + int numDocsFoo = randomIntBetween(2, 16); + indexDocs("metrics-foo", numDocsFoo); + + verifyDocs("metrics-bar", numDocsBar, 1, 1); + verifyDocs("metrics-foo", numDocsFoo, 1, 1); + + RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-foo", null)).get(); + assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 2))); + assertTrue(rolloverResponse.isRolledOver()); + + rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-bar", null)).get(); + assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 2))); + assertTrue(rolloverResponse.isRolledOver()); + + backingIndex = DataStream.getDefaultBackingIndexName("metrics-foo", 2); + getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); + assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); + assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); + + backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 2); + getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); + assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); + assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); + + int numDocsBar2 = randomIntBetween(2, 16); + indexDocs("metrics-bar", numDocsBar2); + int numDocsFoo2 = randomIntBetween(2, 16); + indexDocs("metrics-foo", numDocsFoo2); + + verifyDocs("metrics-bar", numDocsBar + numDocsBar2, 1, 2); + verifyDocs("metrics-foo", numDocsFoo + numDocsFoo2, 1, 2); + + DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[] { "metrics-*" }); + client().execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest).actionGet(); + getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0)); + + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-bar", 1))) + .actionGet() + ); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-bar", 2))) + .actionGet() + ); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-foo", 1))) + .actionGet() + ); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-foo", 2))) + .actionGet() + ); + + */ + } + + static void indexDocs(String index, int numDocs) { + BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + bulkRequest.add( + new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat(bulkResponse.getItems().length, equalTo(numDocs)); + for (BulkItemResponse itemResponse : bulkResponse) { + assertThat(itemResponse.getFailureMessage(), nullValue()); + assertThat(itemResponse.status(), equalTo(RestStatus.CREATED)); + //assertThat(itemResponse.getIndex(), startsWith(index)); + } + client().admin().indices().refresh(new RefreshRequest(index)).actionGet(); + } + +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java index 84ab996507d7f..31a7b0b285fce 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java @@ -7,6 +7,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.xpack.core.MigrateToDataStreamAction; import org.elasticsearch.xpack.core.action.CreateDataStreamAction; import org.elasticsearch.xpack.core.action.DataStreamsStatsAction; import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; @@ -24,6 +25,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.xpack.datastreams.action.DataStreamsStatsTransportAction; +import org.elasticsearch.xpack.datastreams.action.MigrateToDataStreamTransportAction; import org.elasticsearch.xpack.datastreams.rest.RestCreateDataStreamAction; import org.elasticsearch.xpack.datastreams.rest.RestDataStreamsStatsAction; import org.elasticsearch.xpack.datastreams.rest.RestDeleteDataStreamAction; @@ -56,7 +58,8 @@ public Map getMetadataMappers() { var dsStatsAction = new ActionHandler<>(DataStreamsStatsAction.INSTANCE, DataStreamsStatsTransportAction.class); var dsUsageAction = new ActionHandler<>(XPackUsageFeatureAction.DATA_STREAMS, DataStreamUsageTransportAction.class); var dsInfoAction = new ActionHandler<>(XPackInfoFeatureAction.DATA_STREAMS, DataStreamInfoTransportAction.class); - return List.of(createDsAction, deleteDsInfoAction, getDsAction, dsStatsAction, dsUsageAction, dsInfoAction); + var migrateAction = new ActionHandler<>(MigrateToDataStreamAction.INSTANCE, MigrateToDataStreamTransportAction.class); + return List.of(createDsAction, deleteDsInfoAction, getDsAction, dsStatsAction, dsUsageAction, dsInfoAction, migrateAction); } @Override From ea390690d3a0c68d5c640485443b6b82309c66a3 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Oct 2020 17:05:36 -0500 Subject: [PATCH 09/25] basic IT --- .../MetadataCreateDataStreamService.java | 6 +- .../datastreams/DataStreamIT.java | 10 +- .../datastreams/DataStreamMigrationIT.java | 131 +++--------------- 3 files changed, 29 insertions(+), 118 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 1854b02249950..cdaf0d759b850 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -211,11 +212,14 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn private static void prepareBackingIndex(Metadata.Builder b, IndexMetadata im, String dataStreamName) { // hides the index, removes any aliases, and adds data stream timestamp field mapper - Map mapping = im.mapping().sourceAsMap(); + MappingMetadata mm = im.mapping(); + Map mapping = mm == null ? new HashMap<>() : mm.sourceAsMap(); mapping.put("_data_stream_timestamp", Map.of("enabled", true)); b.put(IndexMetadata.builder(im) .removeAlias(dataStreamName) .settings(Settings.builder().put(im.getSettings()).put("index.hidden", "true").build()) + .settingsVersion(im.getSettingsVersion() + 1) + .mappingVersion(im.getMappingVersion() + 1) .putMapping(new MappingMetadata("_doc", mapping)) .build(), false); } diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index e33f1522b83a2..777239155640f 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1169,17 +1169,23 @@ static void indexDocs(String dataStream, int numDocs) { client().admin().indices().refresh(new RefreshRequest(dataStream)).actionGet(); } - static void verifyDocs(String dataStream, long expectedNumHits, long minGeneration, long maxGeneration) { + static void verifyDocs(String dataStream, long expectedNumHits, List expectedIndices) { SearchRequest searchRequest = new SearchRequest(dataStream); searchRequest.source().size((int) expectedNumHits); SearchResponse searchResponse = client().search(searchRequest).actionGet(); assertThat(searchResponse.getHits().getTotalHits().value, equalTo(expectedNumHits)); + Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { + assertTrue(expectedIndices.contains(hit.getIndex())); + }); + } + + static void verifyDocs(String dataStream, long expectedNumHits, long minGeneration, long maxGeneration) { List expectedIndices = new ArrayList<>(); for (long k = minGeneration; k <= maxGeneration; k++) { expectedIndices.add(DataStream.getDefaultBackingIndexName(dataStream, k)); } - Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { assertTrue(expectedIndices.contains(hit.getIndex())); }); + verifyDocs(dataStream, expectedNumHits, expectedIndices); } public static void putComposableIndexTemplate(String id, List patterns) throws IOException { diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java index 3c22a34884108..9752186a6b53f 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -66,10 +67,10 @@ public void testBasicMigration() throws Exception { admin().indices().create(new CreateIndexRequest("index1")).get(); admin().indices().create(new CreateIndexRequest("index2")).get(); - int numDocs = randomIntBetween(2, 16); - indexDocs("index1", numDocs); - numDocs = randomIntBetween(2, 16); - indexDocs("index2", numDocs); + int numDocs1 = randomIntBetween(2, 16); + indexDocs("index1", numDocs1); + int numDocs2 = randomIntBetween(2, 16); + indexDocs("index2", numDocs2); String alias = "migrate-to-data-stream"; IndicesAliasesRequest request = new IndicesAliasesRequest(); @@ -77,123 +78,23 @@ public void testBasicMigration() throws Exception { request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index2").alias(alias).writeIndex(false)); assertAcked(admin().indices().aliases(request).get()); - ResolveIndexAction.Response resolveResponse = - admin().indices().resolveIndex(new ResolveIndexAction.Request(new String[]{"*"})).get(); + ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request(new String[]{"*"}, + IndicesOptions.fromOptions(true, true, true, true, true)); + ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); assertThat(resolveResponse.getAliases().size(), equalTo(1)); assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); - //assertThat(resolveResponse.getIndices().toArray(new ResolveIndexAction.ResolvedIndex[0]), arrayContaining()); - + assertThat(resolveResponse.getIndices().size(), equalTo(2)); client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get(); + resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); + assertThat(resolveResponse.getAliases().size(), equalTo(0)); + assertThat(resolveResponse.getDataStreams().size(), equalTo(1)); + assertThat(resolveResponse.getIndices().size(), equalTo(2)); - - /* - putComposableIndexTemplate("id2", List.of("metrics-bar*")); - createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); - client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); - - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - getDataStreamResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName())); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(2)); - DataStream firstDataStream = getDataStreamResponse.getDataStreams().get(0).getDataStream(); - assertThat(firstDataStream.getName(), equalTo("metrics-bar")); - assertThat(firstDataStream.getTimeStampField().getName(), equalTo("@timestamp")); - assertThat(firstDataStream.getIndices().size(), equalTo(1)); - assertThat(firstDataStream.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 1))); - DataStream dataStream = getDataStreamResponse.getDataStreams().get(1).getDataStream(); - assertThat(dataStream.getName(), equalTo("metrics-foo")); - assertThat(dataStream.getTimeStampField().getName(), equalTo("@timestamp")); - assertThat(dataStream.getIndices().size(), equalTo(1)); - assertThat(dataStream.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 1))); - - String backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 1); - GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); - assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); - assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); - Map mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); - assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); - - backingIndex = DataStream.getDefaultBackingIndexName("metrics-foo", 1); - getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); - assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); - assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); - mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); - assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); - - int numDocsFoo = randomIntBetween(2, 16); - indexDocs("metrics-foo", numDocsFoo); - - verifyDocs("metrics-bar", numDocsBar, 1, 1); - verifyDocs("metrics-foo", numDocsFoo, 1, 1); - - RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-foo", null)).get(); - assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 2))); - assertTrue(rolloverResponse.isRolledOver()); - - rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-bar", null)).get(); - assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 2))); - assertTrue(rolloverResponse.isRolledOver()); - - backingIndex = DataStream.getDefaultBackingIndexName("metrics-foo", 2); - getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); - assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); - assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); - mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); - assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); - - backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 2); - getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); - assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); - assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); - mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); - assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date")); - - int numDocsBar2 = randomIntBetween(2, 16); - indexDocs("metrics-bar", numDocsBar2); - int numDocsFoo2 = randomIntBetween(2, 16); - indexDocs("metrics-foo", numDocsFoo2); - - verifyDocs("metrics-bar", numDocsBar + numDocsBar2, 1, 2); - verifyDocs("metrics-foo", numDocsFoo + numDocsFoo2, 1, 2); - - DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[] { "metrics-*" }); - client().execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest).actionGet(); - getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0)); - - expectThrows( - IndexNotFoundException.class, - () -> client().admin() - .indices() - .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-bar", 1))) - .actionGet() - ); - expectThrows( - IndexNotFoundException.class, - () -> client().admin() - .indices() - .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-bar", 2))) - .actionGet() - ); - expectThrows( - IndexNotFoundException.class, - () -> client().admin() - .indices() - .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-foo", 1))) - .actionGet() - ); - expectThrows( - IndexNotFoundException.class, - () -> client().admin() - .indices() - .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-foo", 2))) - .actionGet() - ); - - */ + int numDocsDs = randomIntBetween(2, 16); + //DataStreamIT.indexDocs(alias, numDocsDs); + //DataStreamIT.verifyDocs(alias, numDocs1 + numDocs2 + numDocsDs, List.of("index1", "index2")); } static void indexDocs(String index, int numDocs) { From 9ad2cbf4d3f277d33ee4f7ca6be16193db97c9d5 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Oct 2020 17:18:35 -0500 Subject: [PATCH 10/25] spotless --- .../org/elasticsearch/datastreams/DataStreamIT.java | 4 +--- .../datastreams/DataStreamMigrationIT.java | 13 +++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 777239155640f..3ea62a6a40168 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1175,9 +1175,7 @@ static void verifyDocs(String dataStream, long expectedNumHits, List exp SearchResponse searchResponse = client().search(searchRequest).actionGet(); assertThat(searchResponse.getHits().getTotalHits().value, equalTo(expectedNumHits)); - Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { - assertTrue(expectedIndices.contains(hit.getIndex())); - }); + Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { assertTrue(expectedIndices.contains(hit.getIndex())); }); } static void verifyDocs(String dataStream, long expectedNumHits, long minGeneration, long maxGeneration) { diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java index 9752186a6b53f..8c7214da63f62 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -38,7 +38,6 @@ import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.startsWith; public class DataStreamMigrationIT extends ESIntegTestCase { @@ -78,8 +77,10 @@ public void testBasicMigration() throws Exception { request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index2").alias(alias).writeIndex(false)); assertAcked(admin().indices().aliases(request).get()); - ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request(new String[]{"*"}, - IndicesOptions.fromOptions(true, true, true, true, true)); + ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( + new String[] { "*" }, + IndicesOptions.fromOptions(true, true, true, true, true) + ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); assertThat(resolveResponse.getAliases().size(), equalTo(1)); assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); @@ -90,11 +91,12 @@ public void testBasicMigration() throws Exception { resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); assertThat(resolveResponse.getAliases().size(), equalTo(0)); assertThat(resolveResponse.getDataStreams().size(), equalTo(1)); + assertThat(resolveResponse.getDataStreams().get(0).getBackingIndices(), arrayContaining("index2", "index1")); assertThat(resolveResponse.getIndices().size(), equalTo(2)); int numDocsDs = randomIntBetween(2, 16); - //DataStreamIT.indexDocs(alias, numDocsDs); - //DataStreamIT.verifyDocs(alias, numDocs1 + numDocs2 + numDocsDs, List.of("index1", "index2")); + // DataStreamIT.indexDocs(alias, numDocsDs); + // DataStreamIT.verifyDocs(alias, numDocs1 + numDocs2 + numDocsDs, List.of("index1", "index2")); } static void indexDocs(String index, int numDocs) { @@ -111,7 +113,6 @@ static void indexDocs(String index, int numDocs) { for (BulkItemResponse itemResponse : bulkResponse) { assertThat(itemResponse.getFailureMessage(), nullValue()); assertThat(itemResponse.status(), equalTo(RestStatus.CREATED)); - //assertThat(itemResponse.getIndex(), startsWith(index)); } client().admin().indices().refresh(new RefreshRequest(index)).actionGet(); } From 847ff2bc19de3e0e3cc6b91e65a97627cfa5bbb9 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 20 Oct 2020 10:32:11 -0500 Subject: [PATCH 11/25] wip --- .../MetadataCreateDataStreamService.java | 2 +- .../datastreams/DataStreamMigrationIT.java | 25 +++++++++++++++++-- .../DataStreamTimestampFieldMapper.java | 2 +- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index cdaf0d759b850..3a94f8ec0372d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -211,7 +211,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn } private static void prepareBackingIndex(Metadata.Builder b, IndexMetadata im, String dataStreamName) { - // hides the index, removes any aliases, and adds data stream timestamp field mapper + // hides the index, removes the original alias, and adds data stream timestamp field mapper MappingMetadata mm = im.mapping(); Map mapping = mm == null ? new HashMap<>() : mm.sourceAsMap(); mapping.put("_data_stream_timestamp", Map.of("enabled", true)); diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java index 8c7214da63f62..dbcdc55859ed8 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -83,6 +83,7 @@ public void testBasicMigration() throws Exception { ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); assertThat(resolveResponse.getAliases().size(), equalTo(1)); + assertThat(resolveResponse.getAliases().get(0).getName(), equalTo(alias)); assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); assertThat(resolveResponse.getIndices().size(), equalTo(2)); @@ -91,14 +92,34 @@ public void testBasicMigration() throws Exception { resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); assertThat(resolveResponse.getAliases().size(), equalTo(0)); assertThat(resolveResponse.getDataStreams().size(), equalTo(1)); + assertThat(resolveResponse.getDataStreams().get(0).getName(), equalTo(alias)); assertThat(resolveResponse.getDataStreams().get(0).getBackingIndices(), arrayContaining("index2", "index1")); assertThat(resolveResponse.getIndices().size(), equalTo(2)); int numDocsDs = randomIntBetween(2, 16); - // DataStreamIT.indexDocs(alias, numDocsDs); - // DataStreamIT.verifyDocs(alias, numDocs1 + numDocs2 + numDocsDs, List.of("index1", "index2")); + indexDocs(alias, numDocsDs); + DataStreamIT.verifyDocs(alias, numDocs1 + numDocs2 + numDocsDs, List.of("index1", "index2")); } + /* + public void testMigrationWithoutTemplate() { + + } + + public void testMigrationWithoutIndexMappings() { + + } + + public void testMigrationWithoutWriteIndex() { + + } + + */ + + // missing alias + // filtered alias + // routed alias + static void indexDocs(String index, int numDocs) { BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numDocs; i++) { diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index 4b0c8a8bf4ea6..d97e5dee9bbb8 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -75,7 +75,7 @@ private static DataStreamTimestampFieldMapper toType(FieldMapper in) { public static class Builder extends MetadataFieldMapper.Builder { - private final Parameter enabled = Parameter.boolParam("enabled", false, m -> toType(m).enabled, false); + private final Parameter enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false); public Builder() { super(NAME); From 6017302aab2243c63aa6b11dad69c552aa6afdcb Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 21 Oct 2020 11:57:29 -0500 Subject: [PATCH 12/25] fix test --- .../cluster/metadata/MetadataCreateDataStreamService.java | 4 ++-- .../mapper/DataStreamTimestampFieldMapperTests.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 3a94f8ec0372d..0f3ae73509608 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -46,7 +46,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -213,7 +213,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn private static void prepareBackingIndex(Metadata.Builder b, IndexMetadata im, String dataStreamName) { // hides the index, removes the original alias, and adds data stream timestamp field mapper MappingMetadata mm = im.mapping(); - Map mapping = mm == null ? new HashMap<>() : mm.sourceAsMap(); + Map mapping = mm == null ? new LinkedHashMap<>() : mm.sourceAsMap(); mapping.put("_data_stream_timestamp", Map.of("enabled", true)); b.put(IndexMetadata.builder(im) .removeAlias(dataStreamName) diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java index 67d0c60ee5252..90185ce381330 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -277,17 +277,17 @@ public void testValidateNotDisallowedAttribute() throws IOException { assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] has disallowed attributes: [store]")); } - public void testCannotUpdateTimestampField() throws IOException { + public void testCanUpdateTimestampField() throws IOException { MapperService mapperService = createIndex("test").mapperService(); String mapping1 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":false}, \"properties\": {\"@timestamp\": {\"type\": \"date\"}}}}}"; String mapping2 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":true}, \"properties\": {\"@timestamp2\": " + "{\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; - assertConflicts(mapping1, mapping2, mapperService, "Mapper for [_data_stream_timestamp]", "[enabled] from [false] to [true]"); + assertConflicts(mapping1, mapping2, mapperService); mapping1 = "{\"type\":{\"properties\":{\"@timestamp\": {\"type\": \"date\"}}}}}"; mapping2 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":true}, \"properties\": " + "{\"@timestamp2\": {\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; - assertConflicts(mapping1, mapping2, mapperService, "Mapper for [_data_stream_timestamp]", "[enabled] from [false] to [true]"); + assertConflicts(mapping1, mapping2, mapperService); } } From 89348ed35f9e52c398d473568925a3dfcae86627 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 22 Oct 2020 11:37:47 +0200 Subject: [PATCH 13/25] Use MapperService to update the mapping. A side effect is moving logic that prepares indices to be backing indices to migrate service. * By using the mapper service directly in the migrate logic, mapping validation error occur before updating the cluster state. Whereas before the validation errors occurred when all mapper services in other nodes where updated in the cluster, which is too late, because the update already happened. * Updating the mapping using maps somehow changed the ordering, which caused assertions to trip. --- .../MetadataCreateDataStreamService.java | 53 --------------- .../MetadataMigrateToDataStreamService.java | 65 ++++++++++++++++++- .../MetadataCreateDataStreamServiceTests.java | 41 ------------ ...tadataMigrateToDataStreamServiceTests.java | 43 ++++++++++++ 4 files changed, 105 insertions(+), 97 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 0f3ae73509608..c1139e2a2eddc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -45,8 +44,6 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -165,17 +162,6 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, currentState.metadata()); - if (backingIndices.size() > 0) { - validateBackingIndices(currentState, dataStreamName); - - // hide existing indices and remove aliases - Metadata.Builder b = Metadata.builder(currentState.metadata()); - for (IndexMetadata backingIndex : backingIndices) { - prepareBackingIndex(b, backingIndex, dataStreamName); - } - currentState = ClusterState.builder(currentState).metadata(b).build(); - } - if (writeIndex == null) { String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); CreateIndexClusterStateUpdateRequest createIndexRequest = @@ -192,10 +178,6 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn RestStatus.BAD_REQUEST, e, firstBackingIndexName); } writeIndex = currentState.metadata().index(firstBackingIndexName); - } else { - Metadata.Builder b = Metadata.builder(currentState.metadata()); - prepareBackingIndex(b, writeIndex, dataStreamName); - currentState = ClusterState.builder(currentState).metadata(b).build(); } assert writeIndex != null; assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]"; @@ -210,41 +192,6 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn return ClusterState.builder(currentState).metadata(builder).build(); } - private static void prepareBackingIndex(Metadata.Builder b, IndexMetadata im, String dataStreamName) { - // hides the index, removes the original alias, and adds data stream timestamp field mapper - MappingMetadata mm = im.mapping(); - Map mapping = mm == null ? new LinkedHashMap<>() : mm.sourceAsMap(); - mapping.put("_data_stream_timestamp", Map.of("enabled", true)); - b.put(IndexMetadata.builder(im) - .removeAlias(dataStreamName) - .settings(Settings.builder().put(im.getSettings()).put("index.hidden", "true").build()) - .settingsVersion(im.getSettingsVersion() + 1) - .mappingVersion(im.getMappingVersion() + 1) - .putMapping(new MappingMetadata("_doc", mapping)) - .build(), false); - } - - // package-visible for testing - static void validateBackingIndices(ClusterState currentState, String dataStreamName) { - IndexAbstraction ia = currentState.metadata().getIndicesLookup().get(dataStreamName); - if (ia == null || ia.getType() != IndexAbstraction.Type.ALIAS) { - throw new IllegalArgumentException("alias [" + dataStreamName + "] does not exist"); - } - IndexAbstraction.Alias alias = (IndexAbstraction.Alias) ia; - - // ensure that no other aliases reference indices - List indicesWithOtherAliases = new ArrayList<>(); - for (IndexMetadata im : alias.getIndices()) { - if (im.getAliases().size() > 1 || im.getAliases().containsKey(alias.getName()) == false) { - indicesWithOtherAliases.add(im.getIndex().getName()); - } - } - if (indicesWithOtherAliases.size() > 0) { - throw new IllegalArgumentException("other aliases referencing indices [" + - Strings.collectionToCommaDelimitedString(indicesWithOtherAliases) + "] must be removed before migrating to a data stream"); - } - } - public static ComposableIndexTemplate lookupTemplateForDataStream(String dataStreamName, Metadata metadata) { final String v2Template = MetadataIndexTemplateService.findV2Template(metadata, dataStreamName, false); if (v2Template == null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 1ed4960553ea2..aed92e25b8003 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -31,11 +31,19 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -46,12 +54,15 @@ public class MetadataMigrateToDataStreamService { private final ClusterService clusterService; private final ActiveShardsObserver activeShardsObserver; private final MetadataCreateIndexService metadataCreateIndexService; + private final IndicesService indexServices; @Inject public MetadataMigrateToDataStreamService(ThreadPool threadPool, ClusterService clusterService, - MetadataCreateIndexService metadataCreateIndexService) { + MetadataCreateIndexService metadataCreateIndexService, + IndicesService indexServices) { this.clusterService = clusterService; + this.indexServices = indexServices; this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); this.metadataCreateIndexService = metadataCreateIndexService; } @@ -83,7 +94,7 @@ public void migrateToDataStream(MigrateToDataStreamClusterStateUpdateRequest req @Override public ClusterState execute(ClusterState currentState) throws Exception { - ClusterState clusterState = migrateToDataStream(metadataCreateIndexService, currentState, request); + ClusterState clusterState = migrateToDataStream(indexServices, metadataCreateIndexService, currentState, request); writeIndexRef.set(clusterState.metadata().dataStreams().get(request.aliasName).getWriteIndex().getName()); return clusterState; } @@ -95,11 +106,20 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { }); } - static ClusterState migrateToDataStream(MetadataCreateIndexService metadataCreateIndexService, + static ClusterState migrateToDataStream(IndicesService indicesServices, + MetadataCreateIndexService metadataCreateIndexService, ClusterState currentState, MigrateToDataStreamClusterStateUpdateRequest request) throws Exception { validateRequest(currentState, request); IndexAbstraction.Alias alias = (IndexAbstraction.Alias) currentState.metadata().getIndicesLookup().get(request.aliasName); + + validateBackingIndices(currentState, request.aliasName); + Metadata.Builder mb = Metadata.builder(currentState.metadata()); + for (IndexMetadata im : alias.getIndices()) { + prepareBackingIndex(mb, im, request.aliasName, indicesServices); + } + currentState = ClusterState.builder(currentState).metadata(mb).build(); + IndexMetadata writeIndex = alias.getWriteIndex(); List backingIndices = alias.getIndices() @@ -131,6 +151,45 @@ static void validateRequest(ClusterState currentState, MigrateToDataStreamCluste } } + private static void prepareBackingIndex(Metadata.Builder b, IndexMetadata im, String dataStreamName, IndicesService indicesService) throws IOException { + // hides the index, removes the original alias, and adds data stream timestamp field mapper + MappingMetadata mm = im.mapping(); + assert mm != null; + + MapperService mapperService = indicesService.createIndexMapperService(im); + mapperService.merge(im, MapperService.MergeReason.MAPPING_RECOVERY); + mapperService.merge("_doc", Map.of("_data_stream_timestamp", Map.of("enabled", true)), MapperService.MergeReason.MAPPING_UPDATE); + DocumentMapper mapper = mapperService.documentMapper(); + + b.put(IndexMetadata.builder(im) + .removeAlias(dataStreamName) + .settings(Settings.builder().put(im.getSettings()).put("index.hidden", "true").build()) + .settingsVersion(im.getSettingsVersion() + 1) + .mappingVersion(im.getMappingVersion() + 1) + .putMapping(new MappingMetadata(mapper))); + } + + // package-visible for testing + static void validateBackingIndices(ClusterState currentState, String dataStreamName) { + IndexAbstraction ia = currentState.metadata().getIndicesLookup().get(dataStreamName); + if (ia == null || ia.getType() != IndexAbstraction.Type.ALIAS) { + throw new IllegalArgumentException("alias [" + dataStreamName + "] does not exist"); + } + IndexAbstraction.Alias alias = (IndexAbstraction.Alias) ia; + + // ensure that no other aliases reference indices + List indicesWithOtherAliases = new ArrayList<>(); + for (IndexMetadata im : alias.getIndices()) { + if (im.getAliases().size() > 1 || im.getAliases().containsKey(alias.getName()) == false) { + indicesWithOtherAliases.add(im.getIndex().getName()); + } + } + if (indicesWithOtherAliases.size() > 0) { + throw new IllegalArgumentException("other aliases referencing indices [" + + Strings.collectionToCommaDelimitedString(indicesWithOtherAliases) + "] must be removed before migrating to a data stream"); + } + } + public static final class MigrateToDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { private final String aliasName; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index b1bb25743bba6..1e0915bf5722f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -24,14 +24,12 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; @@ -142,45 +140,6 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { equalTo("matching index template [template] for data stream [my-data-stream] has no data stream template")); } - public void testValidateRequestWithIndicesWithMultipleAliasReferences() { - String aliasName = "alias"; - AliasMetadata alias1 = AliasMetadata.builder(aliasName).build(); - AliasMetadata alias2 = AliasMetadata.builder(aliasName + "2").build(); - ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( - Metadata.builder() - .put(IndexMetadata.builder("foo1") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias1) - .numberOfShards(1) - .numberOfReplicas(0)) - .put(IndexMetadata.builder("foo2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias1) - .putAlias(alias2) - .numberOfShards(1) - .numberOfReplicas(0)) - .put(IndexMetadata.builder("foo3") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias1) - .putAlias(alias2) - .numberOfShards(1) - .numberOfReplicas(0)) - .put(IndexMetadata.builder("foo4") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias1) - .numberOfShards(1) - .numberOfReplicas(0)) - ).build(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.validateBackingIndices(cs, aliasName)); - String emsg = e.getMessage(); - assertThat(emsg, containsString("other aliases referencing indices [")); - assertThat(emsg, containsString("] must be removed before migrating to a data stream")); - String referencedIndices = emsg.substring(emsg.indexOf('[') + 1, emsg.indexOf(']')); - Set indices = Strings.commaDelimitedListToSet(referencedIndices); - assertThat(indices, containsInAnyOrder("foo2", "foo3")); - } - public void testCreateDataStreamHidesBackingIndicesAndRemovesAlias() throws Exception { String dataStreamName = "foo"; AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index 314d327bf7c73..7282a79016e9f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -22,10 +22,14 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; +import java.util.Set; + +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; public class MetadataMigrateToDataStreamServiceTests extends ESTestCase { @@ -133,4 +137,43 @@ public void testValidateRequest() { MetadataMigrateToDataStreamService.validateRequest(cs, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(aliasName, TimeValue.ZERO, TimeValue.ZERO)); } + + public void testValidateRequestWithIndicesWithMultipleAliasReferences() { + String aliasName = "alias"; + AliasMetadata alias1 = AliasMetadata.builder(aliasName).build(); + AliasMetadata alias2 = AliasMetadata.builder(aliasName + "2").build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .putAlias(alias2) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo3") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .putAlias(alias2) + .numberOfShards(1) + .numberOfReplicas(0)) + .put(IndexMetadata.builder("foo4") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias1) + .numberOfShards(1) + .numberOfReplicas(0)) + ).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MetadataMigrateToDataStreamService.validateBackingIndices(cs, aliasName)); + String emsg = e.getMessage(); + assertThat(emsg, containsString("other aliases referencing indices [")); + assertThat(emsg, containsString("] must be removed before migrating to a data stream")); + String referencedIndices = emsg.substring(emsg.indexOf('[') + 1, emsg.indexOf(']')); + Set indices = Strings.commaDelimitedListToSet(referencedIndices); + assertThat(indices, containsInAnyOrder("foo2", "foo3")); + } } From 3550fad78ee165baef932105566b138533bd5458 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Thu, 22 Oct 2020 12:52:43 -0500 Subject: [PATCH 14/25] add min node version check, fix spotless --- .../metadata/MetadataMigrateToDataStreamService.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index aed92e25b8003..99d05bae89215 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; @@ -110,6 +111,10 @@ static ClusterState migrateToDataStream(IndicesService indicesServices, MetadataCreateIndexService metadataCreateIndexService, ClusterState currentState, MigrateToDataStreamClusterStateUpdateRequest request) throws Exception { + if (currentState.nodes().getMinNodeVersion().before(Version.V_8_0_0)) { + throw new IllegalStateException("data stream migration requires minimum node version of " + Version.V_8_0_0); + } + validateRequest(currentState, request); IndexAbstraction.Alias alias = (IndexAbstraction.Alias) currentState.metadata().getIndicesLookup().get(request.aliasName); @@ -127,6 +132,7 @@ static ClusterState migrateToDataStream(IndicesService indicesServices, .filter(x -> writeIndex == null || x.getIndex().getName().equals(writeIndex.getIndex().getName()) == false) .collect(Collectors.toList()); + logger.info("submitting request to migrate alias [{}] to a data stream", request.aliasName); return MetadataCreateDataStreamService.createDataStream( metadataCreateIndexService, currentState, request.aliasName, backingIndices, writeIndex); } @@ -151,7 +157,11 @@ static void validateRequest(ClusterState currentState, MigrateToDataStreamCluste } } - private static void prepareBackingIndex(Metadata.Builder b, IndexMetadata im, String dataStreamName, IndicesService indicesService) throws IOException { + private static void prepareBackingIndex( + Metadata.Builder b, + IndexMetadata im, + String dataStreamName, + IndicesService indicesService) throws IOException { // hides the index, removes the original alias, and adds data stream timestamp field mapper MappingMetadata mm = im.mapping(); assert mm != null; From 4c806729b02d6d7e4cc4d7e6569084831fff3644 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Thu, 22 Oct 2020 13:46:59 -0500 Subject: [PATCH 15/25] add tests --- .../MetadataMigrateToDataStreamService.java | 4 +- .../datastreams/DataStreamMigrationIT.java | 162 ++++++++++++++++-- 2 files changed, 153 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 99d05bae89215..244217adc6413 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -164,7 +164,9 @@ private static void prepareBackingIndex( IndicesService indicesService) throws IOException { // hides the index, removes the original alias, and adds data stream timestamp field mapper MappingMetadata mm = im.mapping(); - assert mm != null; + if (mm == null) { + throw new IllegalArgumentException("backing index [" + im.getIndex().getName() + "] must have mappings for a timestamp field"); + } MapperService mapperService = indicesService.createIndexMapperService(im); mapperService.merge(im, MapperService.MergeReason.MAPPING_RECOVERY); diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java index dbcdc55859ed8..da5a3687d0f97 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -6,6 +6,7 @@ package org.elasticsearch.datastreams; +import joptsimple.internal.Strings; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -36,6 +37,7 @@ import static org.elasticsearch.datastreams.DataStreamIT.putComposableIndexTemplate; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -50,11 +52,11 @@ protected Collection> nodePlugins() { public void cleanup() { AcknowledgedResponse response = client().execute( DeleteDataStreamAction.INSTANCE, - new DeleteDataStreamAction.Request(new String[] { "*" }) + new DeleteDataStreamAction.Request(new String[]{"*"}) ).actionGet(); assertAcked(response); - DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[] { "*" }); + DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[]{"*"}); client().execute(DeleteDataStreamAction.INSTANCE, deleteDSRequest).actionGet(); DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); client().execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteTemplateRequest).actionGet(); @@ -78,7 +80,7 @@ public void testBasicMigration() throws Exception { assertAcked(admin().indices().aliases(request).get()); ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( - new String[] { "*" }, + new String[]{"*"}, IndicesOptions.fromOptions(true, true, true, true, true) ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); @@ -101,32 +103,168 @@ public void testBasicMigration() throws Exception { DataStreamIT.verifyDocs(alias, numDocs1 + numDocs2 + numDocsDs, List.of("index1", "index2")); } - /* - public void testMigrationWithoutTemplate() { + public void testMigrationWithoutTemplate() throws Exception { + admin().indices().create(new CreateIndexRequest("index1")).get(); + admin().indices().create(new CreateIndexRequest("index2")).get(); + + int numDocs1 = randomIntBetween(2, 16); + indexDocs("index1", numDocs1); + int numDocs2 = randomIntBetween(2, 16); + indexDocs("index2", numDocs2); + + String alias = "migrate-to-data-stream"; + IndicesAliasesRequest request = new IndicesAliasesRequest(); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index1").alias(alias).writeIndex(true)); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index2").alias(alias).writeIndex(false)); + assertAcked(admin().indices().aliases(request).get()); + + ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( + new String[]{"*"}, + IndicesOptions.fromOptions(true, true, true, true, true) + ); + ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); + assertThat(resolveResponse.getAliases().size(), equalTo(1)); + assertThat(resolveResponse.getAliases().get(0).getName(), equalTo(alias)); + assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); + assertThat(resolveResponse.getIndices().size(), equalTo(2)); + + Exception e = expectThrows( + Exception.class, + () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() + ); + + assertTrue(throwableOrItsCause( + e, IllegalArgumentException.class, "no matching index template found for data stream [" + alias + "]")); } - public void testMigrationWithoutIndexMappings() { + public void testMigrationWithoutIndexMappings() throws Exception { + putComposableIndexTemplate("id1", List.of("migrate*")); + + admin().indices().create(new CreateIndexRequest("index1")).get(); + admin().indices().create(new CreateIndexRequest("index2")).get(); + + String alias = "migrate-to-data-stream"; + IndicesAliasesRequest request = new IndicesAliasesRequest(); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index1").alias(alias).writeIndex(true)); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index2").alias(alias).writeIndex(false)); + assertAcked(admin().indices().aliases(request).get()); + + ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( + new String[]{"*"}, + IndicesOptions.fromOptions(true, true, true, true, true) + ); + ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); + assertThat(resolveResponse.getAliases().size(), equalTo(1)); + assertThat(resolveResponse.getAliases().get(0).getName(), equalTo(alias)); + assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); + assertThat(resolveResponse.getIndices().size(), equalTo(2)); + + Exception e = expectThrows( + Exception.class, + () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() + ); + + assertTrue(throwableOrItsCause( + e, IllegalArgumentException.class, "must have mappings for a timestamp field")); } - public void testMigrationWithoutWriteIndex() { + public void testMigrationWithoutTimestampMapping() throws Exception { + putComposableIndexTemplate("id1", List.of("migrate*")); + + admin().indices().create(new CreateIndexRequest("index1")).get(); + admin().indices().create(new CreateIndexRequest("index2")).get(); + + int numDocs1 = randomIntBetween(2, 16); + indexDocs("index1", numDocs1, "foo"); + int numDocs2 = randomIntBetween(2, 16); + indexDocs("index2", numDocs2, "foo"); + + String alias = "migrate-to-data-stream"; + IndicesAliasesRequest request = new IndicesAliasesRequest(); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index1").alias(alias).writeIndex(true)); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index2").alias(alias).writeIndex(false)); + assertAcked(admin().indices().aliases(request).get()); + + ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( + new String[]{"*"}, + IndicesOptions.fromOptions(true, true, true, true, true) + ); + ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); + assertThat(resolveResponse.getAliases().size(), equalTo(1)); + assertThat(resolveResponse.getAliases().get(0).getName(), equalTo(alias)); + assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); + assertThat(resolveResponse.getIndices().size(), equalTo(2)); + + Exception e = expectThrows( + Exception.class, + () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() + ); + + assertTrue(throwableOrItsCause( + e, IllegalArgumentException.class, "data stream timestamp field [@timestamp] does not exist")); } - */ - // missing alias - // filtered alias - // routed alias + public void testMigrationWithoutWriteIndex() throws Exception { + putComposableIndexTemplate("id1", List.of("migrate*")); + + admin().indices().create(new CreateIndexRequest("index1")).get(); + admin().indices().create(new CreateIndexRequest("index2")).get(); + + int numDocs1 = randomIntBetween(2, 16); + indexDocs("index1", numDocs1); + int numDocs2 = randomIntBetween(2, 16); + indexDocs("index2", numDocs2); + + String alias = "migrate-to-data-stream"; + IndicesAliasesRequest request = new IndicesAliasesRequest(); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index1").alias(alias).writeIndex(false)); + request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index2").alias(alias).writeIndex(false)); + assertAcked(admin().indices().aliases(request).get()); + + ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( + new String[]{"*"}, + IndicesOptions.fromOptions(true, true, true, true, true) + ); + ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); + assertThat(resolveResponse.getAliases().size(), equalTo(1)); + assertThat(resolveResponse.getAliases().get(0).getName(), equalTo(alias)); + assertThat(resolveResponse.getDataStreams().size(), equalTo(0)); + assertThat(resolveResponse.getIndices().size(), equalTo(2)); + + Exception e = expectThrows( + Exception.class, + () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() + ); + + assertTrue(throwableOrItsCause( + e, IllegalArgumentException.class, "alias [" + alias + "] must specify a write index")); + } + + static boolean throwableOrItsCause(Throwable t, Class clazz, String message) { + boolean found = false; + Throwable throwable = t; + while (throwable != null && found == false) { + found = throwable.getMessage().contains(message) && throwable.getClass().equals(clazz); + throwable = throwable.getCause(); + } + return found; + } static void indexDocs(String index, int numDocs) { + indexDocs(index, numDocs, Strings.EMPTY); + } + + static void indexDocs(String index, int numDocs, String fieldPrefix) { BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); bulkRequest.add( new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", fieldPrefix + DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) ); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); From 9cb64c85bc24185ba41db611d2322f32dcd203cf Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Thu, 22 Oct 2020 14:01:40 -0500 Subject: [PATCH 16/25] spotless --- .../datastreams/DataStreamMigrationIT.java | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java index da5a3687d0f97..0c05db90d0533 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -37,7 +37,6 @@ import static org.elasticsearch.datastreams.DataStreamIT.putComposableIndexTemplate; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.arrayContaining; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -52,11 +51,11 @@ protected Collection> nodePlugins() { public void cleanup() { AcknowledgedResponse response = client().execute( DeleteDataStreamAction.INSTANCE, - new DeleteDataStreamAction.Request(new String[]{"*"}) + new DeleteDataStreamAction.Request(new String[] { "*" }) ).actionGet(); assertAcked(response); - DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[]{"*"}); + DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[] { "*" }); client().execute(DeleteDataStreamAction.INSTANCE, deleteDSRequest).actionGet(); DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); client().execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteTemplateRequest).actionGet(); @@ -80,7 +79,7 @@ public void testBasicMigration() throws Exception { assertAcked(admin().indices().aliases(request).get()); ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( - new String[]{"*"}, + new String[] { "*" }, IndicesOptions.fromOptions(true, true, true, true, true) ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); @@ -103,7 +102,6 @@ public void testBasicMigration() throws Exception { DataStreamIT.verifyDocs(alias, numDocs1 + numDocs2 + numDocsDs, List.of("index1", "index2")); } - public void testMigrationWithoutTemplate() throws Exception { admin().indices().create(new CreateIndexRequest("index1")).get(); admin().indices().create(new CreateIndexRequest("index2")).get(); @@ -120,7 +118,7 @@ public void testMigrationWithoutTemplate() throws Exception { assertAcked(admin().indices().aliases(request).get()); ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( - new String[]{"*"}, + new String[] { "*" }, IndicesOptions.fromOptions(true, true, true, true, true) ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); @@ -134,11 +132,11 @@ public void testMigrationWithoutTemplate() throws Exception { () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() ); - assertTrue(throwableOrItsCause( - e, IllegalArgumentException.class, "no matching index template found for data stream [" + alias + "]")); + assertTrue( + throwableOrItsCause(e, IllegalArgumentException.class, "no matching index template found for data stream [" + alias + "]") + ); } - public void testMigrationWithoutIndexMappings() throws Exception { putComposableIndexTemplate("id1", List.of("migrate*")); @@ -152,7 +150,7 @@ public void testMigrationWithoutIndexMappings() throws Exception { assertAcked(admin().indices().aliases(request).get()); ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( - new String[]{"*"}, + new String[] { "*" }, IndicesOptions.fromOptions(true, true, true, true, true) ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); @@ -166,11 +164,9 @@ public void testMigrationWithoutIndexMappings() throws Exception { () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() ); - assertTrue(throwableOrItsCause( - e, IllegalArgumentException.class, "must have mappings for a timestamp field")); + assertTrue(throwableOrItsCause(e, IllegalArgumentException.class, "must have mappings for a timestamp field")); } - public void testMigrationWithoutTimestampMapping() throws Exception { putComposableIndexTemplate("id1", List.of("migrate*")); @@ -189,7 +185,7 @@ public void testMigrationWithoutTimestampMapping() throws Exception { assertAcked(admin().indices().aliases(request).get()); ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( - new String[]{"*"}, + new String[] { "*" }, IndicesOptions.fromOptions(true, true, true, true, true) ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); @@ -203,11 +199,9 @@ public void testMigrationWithoutTimestampMapping() throws Exception { () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() ); - assertTrue(throwableOrItsCause( - e, IllegalArgumentException.class, "data stream timestamp field [@timestamp] does not exist")); + assertTrue(throwableOrItsCause(e, IllegalArgumentException.class, "data stream timestamp field [@timestamp] does not exist")); } - public void testMigrationWithoutWriteIndex() throws Exception { putComposableIndexTemplate("id1", List.of("migrate*")); @@ -226,7 +220,7 @@ public void testMigrationWithoutWriteIndex() throws Exception { assertAcked(admin().indices().aliases(request).get()); ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request( - new String[]{"*"}, + new String[] { "*" }, IndicesOptions.fromOptions(true, true, true, true, true) ); ResolveIndexAction.Response resolveResponse = admin().indices().resolveIndex(resolveRequest).get(); @@ -240,8 +234,7 @@ public void testMigrationWithoutWriteIndex() throws Exception { () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get() ); - assertTrue(throwableOrItsCause( - e, IllegalArgumentException.class, "alias [" + alias + "] must specify a write index")); + assertTrue(throwableOrItsCause(e, IllegalArgumentException.class, "alias [" + alias + "] must specify a write index")); } static boolean throwableOrItsCause(Throwable t, Class clazz, String message) { From 9816cc384854054fea9f8fdaacb3ae151b2bea58 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 26 Oct 2020 13:49:06 -0500 Subject: [PATCH 17/25] fix merge error --- .../main/java/org/elasticsearch/cluster/metadata/DataStream.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 3189ff431361f..8437679a2453a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -81,6 +81,7 @@ public long getGeneration() { public Index getWriteIndex() { return indices.get(indices.size() - 1); + } @Nullable public Map getMetadata() { From 2d9ba916f9697b2442d699104ce52f8df84f296c Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 26 Oct 2020 14:11:44 -0500 Subject: [PATCH 18/25] moar compile errors --- .../cluster/metadata/ComposableIndexTemplate.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 28efd3e0593d4..9b0d7449c49c3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -111,6 +111,12 @@ public ComposableIndexTemplate(List indexPatterns, @Nullable Template te this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null); } + public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, + @Nullable Long priority, @Nullable Long version, @Nullable Map metadata, + @Nullable DataStreamTemplate dataStreamTemplate) { + this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null); + } + public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata, @Nullable DataStreamTemplate dataStreamTemplate, @Nullable Boolean allowAutoCreate) { From 497f64b998bee8c671f17c7e1a399196301563cd Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 27 Oct 2020 11:09:52 -0500 Subject: [PATCH 19/25] revise tests --- .../MetadataCreateDataStreamService.java | 4 +- .../MetadataMigrateToDataStreamService.java | 33 ++-- .../MetadataCreateDataStreamServiceTests.java | 122 -------------- ...tadataMigrateToDataStreamServiceTests.java | 151 +++++++++++++++++- 4 files changed, 173 insertions(+), 137 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 46c77b329d078..9a2d0cd9e8386 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -143,7 +143,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn List backingIndices, IndexMetadata writeIndex) throws Exception { - Objects.requireNonNull(metadataCreateIndexService); + if (writeIndex == null) { + Objects.requireNonNull(metadataCreateIndexService); + } Objects.requireNonNull(currentState); Objects.requireNonNull(backingIndices); if (currentState.metadata().dataStreams().containsKey(dataStreamName)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 244217adc6413..e0f956c408837 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.stream.Collectors; public class MetadataMigrateToDataStreamService { @@ -54,18 +55,15 @@ public class MetadataMigrateToDataStreamService { private final ClusterService clusterService; private final ActiveShardsObserver activeShardsObserver; - private final MetadataCreateIndexService metadataCreateIndexService; private final IndicesService indexServices; @Inject public MetadataMigrateToDataStreamService(ThreadPool threadPool, ClusterService clusterService, - MetadataCreateIndexService metadataCreateIndexService, IndicesService indexServices) { this.clusterService = clusterService; this.indexServices = indexServices; this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); - this.metadataCreateIndexService = metadataCreateIndexService; } public void migrateToDataStream(MigrateToDataStreamClusterStateUpdateRequest request, @@ -95,7 +93,16 @@ public void migrateToDataStream(MigrateToDataStreamClusterStateUpdateRequest req @Override public ClusterState execute(ClusterState currentState) throws Exception { - ClusterState clusterState = migrateToDataStream(indexServices, metadataCreateIndexService, currentState, request); + ClusterState clusterState = migrateToDataStream( + currentState, + indexMetadata -> { + try { + return indexServices.createIndexMapperService(indexMetadata); + } catch (IOException e) { + throw new IllegalStateException(e); + } + }, + request); writeIndexRef.set(clusterState.metadata().dataStreams().get(request.aliasName).getWriteIndex().getName()); return clusterState; } @@ -107,9 +114,8 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { }); } - static ClusterState migrateToDataStream(IndicesService indicesServices, - MetadataCreateIndexService metadataCreateIndexService, - ClusterState currentState, + static ClusterState migrateToDataStream(ClusterState currentState, + Function mapperSupplier, MigrateToDataStreamClusterStateUpdateRequest request) throws Exception { if (currentState.nodes().getMinNodeVersion().before(Version.V_8_0_0)) { throw new IllegalStateException("data stream migration requires minimum node version of " + Version.V_8_0_0); @@ -121,7 +127,7 @@ static ClusterState migrateToDataStream(IndicesService indicesServices, validateBackingIndices(currentState, request.aliasName); Metadata.Builder mb = Metadata.builder(currentState.metadata()); for (IndexMetadata im : alias.getIndices()) { - prepareBackingIndex(mb, im, request.aliasName, indicesServices); + prepareBackingIndex(mb, im, request.aliasName, mapperSupplier); } currentState = ClusterState.builder(currentState).metadata(mb).build(); @@ -133,8 +139,7 @@ static ClusterState migrateToDataStream(IndicesService indicesServices, .collect(Collectors.toList()); logger.info("submitting request to migrate alias [{}] to a data stream", request.aliasName); - return MetadataCreateDataStreamService.createDataStream( - metadataCreateIndexService, currentState, request.aliasName, backingIndices, writeIndex); + return MetadataCreateDataStreamService.createDataStream(null, currentState, request.aliasName, backingIndices, writeIndex); } // package-visible for testing @@ -161,14 +166,18 @@ private static void prepareBackingIndex( Metadata.Builder b, IndexMetadata im, String dataStreamName, - IndicesService indicesService) throws IOException { + Function mapperSupplier) throws IOException { // hides the index, removes the original alias, and adds data stream timestamp field mapper MappingMetadata mm = im.mapping(); if (mm == null) { throw new IllegalArgumentException("backing index [" + im.getIndex().getName() + "] must have mappings for a timestamp field"); } - MapperService mapperService = indicesService.createIndexMapperService(im); + //im.toXContent(null, Params) + //mm.source().uncompressed().toString(); + + + MapperService mapperService = mapperSupplier.apply(im); mapperService.merge(im, MapperService.MergeReason.MAPPING_RECOVERY); mapperService.merge("_doc", Map.of("_data_stream_timestamp", Map.of("enabled", true)), MapperService.MergeReason.MAPPING_UPDATE); DocumentMapper mapper = mapperService.documentMapper(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 019b848db273c..4c7159166cb8b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -30,12 +30,10 @@ import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -141,125 +139,6 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { equalTo("matching index template [template] for data stream [my-data-stream] has no data stream template")); } - public void testCreateDataStreamHidesBackingIndicesAndRemovesAlias() throws Exception { - String dataStreamName = "foo"; - AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); - IndexMetadata foo1 = IndexMetadata.builder("foo1") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping(generateMapping("@timestamp", "date")) - .build(); - IndexMetadata foo2 = IndexMetadata.builder("foo2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping(generateMapping("@timestamp", "date")) - .build(); - ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( - Metadata.builder() - .put(foo1, false) - .put(foo2, false) - .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()))) - .build(); - - ClusterState newState = MetadataCreateDataStreamService.createDataStream(getMetadataCreateIndexService(), cs, dataStreamName, - List.of(foo1, foo2), null); - IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); - assertThat(ds, notNullValue()); - assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - assertThat(ds.getIndices().size(), equalTo(3)); - List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); - assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2", DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - for (IndexMetadata im : ds.getIndices()) { - assertThat(im.getSettings().get("index.hidden"), equalTo("true")); - assertThat(im.getAliases().size(), equalTo(0)); - } - } - - public void testCreateDataStreamWithSuppliedWriteIndex() throws Exception { - String dataStreamName = "foo"; - AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); - IndexMetadata foo1 = IndexMetadata.builder("foo1") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping(generateMapping("@timestamp", "date")) - .build(); - IndexMetadata foo2 = IndexMetadata.builder("foo2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping(generateMapping("@timestamp", "date")) - .build(); - ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( - Metadata.builder() - .put(foo1, false) - .put(foo2, false) - .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()))) - .build(); - - ClusterState newState = MetadataCreateDataStreamService.createDataStream(getMetadataCreateIndexService(), cs, dataStreamName, - List.of(foo1), foo2); - IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); - assertThat(ds, notNullValue()); - assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - assertThat(ds.getIndices().size(), equalTo(2)); - List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); - assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2")); - assertThat(ds.getWriteIndex().getIndex().getName(), equalTo("foo2")); - for (IndexMetadata im : ds.getIndices()) { - assertThat(im.getSettings().get("index.hidden"), equalTo("true")); - assertThat(im.getAliases().size(), equalTo(0)); - } - } - - public void testCreateDataStreamWithoutSuppliedWriteIndex() throws Exception { - String dataStreamName = "foo"; - AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); - IndexMetadata foo1 = IndexMetadata.builder("foo1") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping(generateMapping("@timestamp", "date")) - .build(); - IndexMetadata foo2 = IndexMetadata.builder("foo2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .putAlias(alias) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping(generateMapping("@timestamp", "date")) - .build(); - ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( - Metadata.builder() - .put(foo1, false) - .put(foo2, false) - .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()))) - .build(); - - ClusterState newState = MetadataCreateDataStreamService.createDataStream(getMetadataCreateIndexService(), cs, dataStreamName, - List.of(foo1, foo2), null); - IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); - assertThat(ds, notNullValue()); - assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - assertThat(ds.getIndices().size(), equalTo(3)); - List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); - assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2", DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - assertThat(ds.getWriteIndex().getIndex().getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - for (IndexMetadata im : ds.getIndices()) { - assertThat(im.getSettings().get("index.hidden"), equalTo("true")); - assertThat(im.getAliases().size(), equalTo(0)); - } - } - public static ClusterState createDataStream(final String dataStreamName) throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, @@ -294,5 +173,4 @@ private static MetadataCreateIndexService getMetadataCreateIndexService() throws return s; } - } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index 7282a79016e9f..c3a236a8e1b56 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -25,14 +25,23 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.plugins.Plugin; +import java.io.IOException; +import java.util.Collection; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; -public class MetadataMigrateToDataStreamServiceTests extends ESTestCase { +public class MetadataMigrateToDataStreamServiceTests extends MapperServiceTestCase { public void testValidateRequestWithNonexistentAlias() { ClusterState cs = ClusterState.EMPTY_STATE; @@ -176,4 +185,142 @@ public void testValidateRequestWithIndicesWithMultipleAliasReferences() { Set indices = Strings.commaDelimitedListToSet(referencedIndices); assertThat(indices, containsInAnyOrder("foo2", "foo3")); } + + public void testCreateDataStreamWithSuppliedWriteIndex() throws Exception { + String dataStreamName = "foo"; + AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); + IndexMetadata + foo1 = + IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(AliasMetadata.builder(dataStreamName).writeIndex(true).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + IndexMetadata + foo2 = + IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")) + .metadata(Metadata.builder() + .put(foo1, false) + .put(foo2, false) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()))) + .build(); + + ClusterState newState = MetadataMigrateToDataStreamService.migrateToDataStream(cs, this::getMapperService, + new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(dataStreamName, + TimeValue.ZERO, + TimeValue.ZERO)); + IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); + assertThat(ds, notNullValue()); + assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + assertThat(ds.getIndices().size(), equalTo(2)); + List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); + assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2")); + assertThat(ds.getWriteIndex().getIndex().getName(), equalTo("foo1")); + for (IndexMetadata im : ds.getIndices()) { + assertThat(im.getSettings().get("index.hidden"), equalTo("true")); + assertThat(im.getAliases().size(), equalTo(0)); + } + } + + public void testCreateDataStreamHidesBackingIndicesAndRemovesAlias() throws Exception { + String dataStreamName = "foo"; + AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); + IndexMetadata foo1 = IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(AliasMetadata.builder(dataStreamName).writeIndex(true).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + IndexMetadata foo2 = IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")).metadata( + Metadata.builder() + .put(foo1, false) + .put(foo2, false) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate()))) + .build(); + + ClusterState newState = MetadataMigrateToDataStreamService.migrateToDataStream(cs, this::getMapperService, + new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(dataStreamName, + TimeValue.ZERO, + TimeValue.ZERO)); + IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); + assertThat(ds, notNullValue()); + assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + assertThat(ds.getIndices().size(), equalTo(2)); + List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); + assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2")); + assertThat(ds.getWriteIndex().getIndex().getName(), equalTo("foo1")); + for (IndexMetadata im : ds.getIndices()) { + assertThat(im.getSettings().get("index.hidden"), equalTo("true")); + assertThat(im.getAliases().size(), equalTo(0)); + } + } + + public void testCreateDataStreamWithoutSuppliedWriteIndex() throws Exception { + String dataStreamName = "foo"; + AliasMetadata alias = AliasMetadata.builder(dataStreamName).build(); + IndexMetadata + foo1 = + IndexMetadata.builder("foo1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + IndexMetadata + foo2 = + IndexMetadata.builder("foo2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .putAlias(alias) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp", "date")) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("dummy")) + .metadata(Metadata.builder() + .put(foo1, false) + .put(foo2, false) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()))) + .build(); + + IllegalArgumentException e = + expectThrows(IllegalArgumentException.class, + () -> MetadataMigrateToDataStreamService.migrateToDataStream(cs, + this::getMapperService, + new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(dataStreamName, + TimeValue.ZERO, + TimeValue.ZERO))); + assertThat(e.getMessage(), containsString("alias [" + dataStreamName + "] must specify a write index")); + } + + private MapperService getMapperService(IndexMetadata im) { + try { + return createMapperService("{\"_doc\": " + im.mapping().source().toString() + "}"); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + protected Collection getPlugins() { + return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin()); + } } From 51528252b1d6cf7ec82945c11f38fdf6a1017235 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 27 Oct 2020 11:28:13 -0500 Subject: [PATCH 20/25] spotless --- .../metadata/MetadataMigrateToDataStreamServiceTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index c3a236a8e1b56..9127f92b80fc0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -211,7 +211,8 @@ public void testCreateDataStreamWithSuppliedWriteIndex() throws Exception { .metadata(Metadata.builder() .put(foo1, false) .put(foo2, false) - .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()))) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate()))) .build(); ClusterState newState = MetadataMigrateToDataStreamService.migrateToDataStream(cs, this::getMapperService, @@ -298,7 +299,8 @@ public void testCreateDataStreamWithoutSuppliedWriteIndex() throws Exception { .metadata(Metadata.builder() .put(foo1, false) .put(foo2, false) - .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()))) + .put("template", new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, + new ComposableIndexTemplate.DataStreamTemplate()))) .build(); IllegalArgumentException e = From 768b163afa438ec8796fe7f3b8c5650f3f1db2a9 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Tue, 27 Oct 2020 14:07:34 -0500 Subject: [PATCH 21/25] remove test code --- .../cluster/metadata/MetadataMigrateToDataStreamService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index e0f956c408837..2d61873d25bd9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -173,10 +173,6 @@ private static void prepareBackingIndex( throw new IllegalArgumentException("backing index [" + im.getIndex().getName() + "] must have mappings for a timestamp field"); } - //im.toXContent(null, Params) - //mm.source().uncompressed().toString(); - - MapperService mapperService = mapperSupplier.apply(im); mapperService.merge(im, MapperService.MergeReason.MAPPING_RECOVERY); mapperService.merge("_doc", Map.of("_data_stream_timestamp", Map.of("enabled", true)), MapperService.MergeReason.MAPPING_UPDATE); From bbcec4f9f9965c59498e0ed74d833aa7f250bc2c Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 28 Oct 2020 07:27:48 -0500 Subject: [PATCH 22/25] revise log message --- .../cluster/metadata/MetadataCreateDataStreamService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 9a2d0cd9e8386..41e1192d78847 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -191,7 +192,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn DataStream newDataStream = new DataStream(dataStreamName, timestampField, dsBackingIndices, 1L, template.metadata() != null ? Map.copyOf(template.metadata()) : null); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); - logger.info("adding data stream [{}]", dataStreamName); + logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName, + writeIndex.getIndex().getName(), + Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray())); return ClusterState.builder(currentState).metadata(builder).build(); } From af7d4b489a44611b9577b7678d55eefd98fbe0aa Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 28 Oct 2020 07:38:46 -0500 Subject: [PATCH 23/25] validate that TSFM can be enabled but not disabled --- .../mapper/DataStreamTimestampFieldMapper.java | 3 ++- .../mapper/DataStreamTimestampFieldMapperTests.java | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index d97e5dee9bbb8..5e4dad6f5c68f 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -75,7 +75,8 @@ private static DataStreamTimestampFieldMapper toType(FieldMapper in) { public static class Builder extends MetadataFieldMapper.Builder { - private final Parameter enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false); + private final Parameter enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false) + .setMergeValidator((previous, current, conflicts) -> (previous == current) || (previous == false && current == true)); public Builder() { super(NAME); diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java index 90185ce381330..6e0d567f14b3c 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -277,7 +277,7 @@ public void testValidateNotDisallowedAttribute() throws IOException { assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] has disallowed attributes: [store]")); } - public void testCanUpdateTimestampField() throws IOException { + public void testCanUpdateTimestampFieldMapperFromDisabledToEnabled() throws IOException { MapperService mapperService = createIndex("test").mapperService(); String mapping1 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":false}, \"properties\": {\"@timestamp\": {\"type\": \"date\"}}}}}"; @@ -290,4 +290,13 @@ public void testCanUpdateTimestampField() throws IOException { + "{\"@timestamp2\": {\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; assertConflicts(mapping1, mapping2, mapperService); } + + public void testCannotUpdateTimestampFieldMapperFromEnabledToDisabled() throws IOException { + MapperService mapperService = createIndex("test").mapperService(); + String mapping1 = + "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":true}, \"properties\": {\"@timestamp\": {\"type\": \"date\"}}}}}"; + String mapping2 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":false}, \"properties\": {\"@timestamp2\": " + + "{\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; + assertConflicts(mapping1, mapping2, mapperService, "Cannot update parameter [enabled] from [true] to [false]"); + } } From b0c73d43c15d11733464abaf1b702c42f1355da4 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 28 Oct 2020 07:45:10 -0500 Subject: [PATCH 24/25] checkstyle --- .../datastreams/mapper/DataStreamTimestampFieldMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index 5e4dad6f5c68f..77184d7d92603 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -76,7 +76,8 @@ private static DataStreamTimestampFieldMapper toType(FieldMapper in) { public static class Builder extends MetadataFieldMapper.Builder { private final Parameter enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false) - .setMergeValidator((previous, current, conflicts) -> (previous == current) || (previous == false && current == true)); + // this field mapper may enabled but once enabled, may not be disabled + .setMergeValidator((previous, current, conflicts) -> (previous == current) || (previous == false && current)); public Builder() { super(NAME); From 4edaf1d73dffda4da4f116919762b2e8575997b5 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 28 Oct 2020 07:46:16 -0500 Subject: [PATCH 25/25] fix wording --- .../datastreams/mapper/DataStreamTimestampFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index 77184d7d92603..096ef9cb5b885 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -76,7 +76,7 @@ private static DataStreamTimestampFieldMapper toType(FieldMapper in) { public static class Builder extends MetadataFieldMapper.Builder { private final Parameter enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false) - // this field mapper may enabled but once enabled, may not be disabled + // this field mapper may be enabled but once enabled, may not be disabled .setMergeValidator((previous, current, conflicts) -> (previous == current) || (previous == false && current)); public Builder() {