diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index 69a45555e9401..4fb31e31e8554 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -79,7 +79,7 @@ indices. Indicates whether <> are pre-loaded for nested queries. Possible values are `true` (default) and `false`. -`index.hidden`:: +[[index-hidden]] `index.hidden`:: Indicates whether the index should be hidden by default. Hidden indices are not returned by default when using a wildcard expression. This behavior is controlled diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc index 6009189c4fe03..0692482d6a0e5 100644 --- a/docs/reference/indices/create-index.asciidoc +++ b/docs/reference/indices/create-index.asciidoc @@ -19,7 +19,7 @@ PUT /twitter [[indices-create-api-desc]] ==== {api-description-title} -You can use the create index API to add a new index to an {es} cluster. When +You can use the create index API to add a new index to an {es} cluster. When creating an index, you can specify the following: * Settings for the index @@ -44,6 +44,7 @@ Index names must meet the following criteria: - Cannot start with `-`, `_`, `+` - Cannot be `.` or `..` - Cannot be longer than 255 bytes (note it is bytes, so multi-byte characters will count towards the 255 limit faster) +- Names starting with `.` are deprecated, except for <> and internal indices managed by plugins // end::index-name-reqs[] -- diff --git a/modules/tasks/build.gradle b/modules/tasks/build.gradle new file mode 100644 index 0000000000000..b3ba90ccb6a11 --- /dev/null +++ b/modules/tasks/build.gradle @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Supports the Tasks API' + classname 'org.elasticsearch.tasksplugin.TasksPlugin' +} + +integTest.enabled = false diff --git a/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java b/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java new file mode 100644 index 0000000000000..b7d63991877db --- /dev/null +++ b/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tasksplugin; + +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; + +import java.util.Collection; +import java.util.Collections; + +import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; + +/** + * This plugin currently only exists to register `.tasks` as a system index. + */ +public class TasksPlugin extends Plugin implements SystemIndexPlugin { + + @Override + public Collection getSystemIndexDescriptors() { + return Collections.singletonList(new SystemIndexDescriptor(TASK_INDEX, this.getClass().getSimpleName())); + } +} diff --git a/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java b/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java new file mode 100644 index 0000000000000..48ec1e06098f3 --- /dev/null +++ b/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tasksplugin; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +public class TasksPluginTests extends ESTestCase { + + public void testDummy() { + // This is a dummy test case to satisfy the conventions + TasksPlugin plugin = new TasksPlugin(); + assertThat(plugin.getSystemIndexDescriptors(), Matchers.hasSize(1)); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 5f6ee87e87fd8..903bdae31c122 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -122,9 +122,9 @@ protected void masterOperation(Task task, final RolloverRequest rolloverRequest, ? rolloverRequest.getNewIndexName() : generateRolloverIndexName(sourceProvidedName, indexNameExpressionResolver); final String rolloverIndexName = indexNameExpressionResolver.resolveDateMathExpression(unresolvedName); - MetaDataCreateIndexService.validateIndexName(rolloverIndexName, state); // will fail if the index already exists final Boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.exists(rolloverRequest.getCreateIndexRequest().settings()) ? IndexMetaData.INDEX_HIDDEN_SETTING.get(rolloverRequest.getCreateIndexRequest().settings()) : null; + createIndexService.validateIndexName(rolloverIndexName, state, isHidden); // fails if the index already exists checkNoDuplicatedAliasInIndexTemplate(metaData, rolloverIndexName, rolloverRequest.getAlias(), isHidden); IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getAlias()) .clear() diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 51796ecc89e41..9ad1676c359f0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -53,6 +53,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -70,6 +71,7 @@ import org.elasticsearch.indices.IndexCreationException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexNameException; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import org.elasticsearch.threadpool.ThreadPool; @@ -78,6 +80,7 @@ import java.nio.file.Path; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -89,6 +92,7 @@ import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.util.stream.Collectors.toList; @@ -103,6 +107,7 @@ */ public class MetaDataCreateIndexService { private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger); public static final int MAX_INDEX_NAME_BYTES = 255; @@ -115,19 +120,21 @@ public class MetaDataCreateIndexService { private final IndexScopedSettings indexScopedSettings; private final ActiveShardsObserver activeShardsObserver; private final NamedXContentRegistry xContentRegistry; + private final Collection systemIndexDescriptors; private final boolean forbidPrivateIndexSettings; public MetaDataCreateIndexService( - final Settings settings, - final ClusterService clusterService, - final IndicesService indicesService, - final AllocationService allocationService, - final AliasValidator aliasValidator, - final Environment env, - final IndexScopedSettings indexScopedSettings, - final ThreadPool threadPool, - final NamedXContentRegistry xContentRegistry, - final boolean forbidPrivateIndexSettings) { + final Settings settings, + final ClusterService clusterService, + final IndicesService indicesService, + final AllocationService allocationService, + final AliasValidator aliasValidator, + final Environment env, + final IndexScopedSettings indexScopedSettings, + final ThreadPool threadPool, + final NamedXContentRegistry xContentRegistry, + final Collection systemIndexDescriptors, + final boolean forbidPrivateIndexSettings) { this.settings = settings; this.clusterService = clusterService; this.indicesService = indicesService; @@ -137,17 +144,40 @@ public MetaDataCreateIndexService( this.indexScopedSettings = indexScopedSettings; this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); this.xContentRegistry = xContentRegistry; + this.systemIndexDescriptors = systemIndexDescriptors; this.forbidPrivateIndexSettings = forbidPrivateIndexSettings; } /** * Validate the name for an index against some static rules and a cluster state. */ - public static void validateIndexName(String index, ClusterState state) { + public void validateIndexName(String index, ClusterState state, @Nullable Boolean isHidden) { validateIndexOrAliasName(index, InvalidIndexNameException::new); if (!index.toLowerCase(Locale.ROOT).equals(index)) { throw new InvalidIndexNameException(index, "must be lowercase"); } + + if (index.charAt(0) == '.') { + List matchingDescriptors = systemIndexDescriptors.stream() + .filter(descriptor -> descriptor.matchesIndexPattern(index)) + .collect(toList()); + if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) { + deprecationLogger.deprecated("index name [{}] starts with a dot '.', in the next major version, index names " + + "starting with a dot are reserved for hidden indices and system indices", index); + } else if (matchingDescriptors.size() > 1) { + // This should be prevented by erroring on overlapping patterns at startup time, but is here just in case. + StringBuilder errorMessage = new StringBuilder() + .append("index name [") + .append(index) + .append("] is claimed as a system index by multiple system index patterns: [") + .append(matchingDescriptors.stream() + .map(descriptor -> "pattern: [" + descriptor.getIndexPattern() + + "], description: [" + descriptor.getDescription() + "]").collect(Collectors.joining("; "))); + // Throw AssertionError if assertions are enabled, or a regular exception otherwise: + assert false : errorMessage.toString(); + throw new IllegalStateException(errorMessage.toString()); + } + } if (state.routingTable().hasIndex(index)) { throw new ResourceAlreadyExistsException(state.routingTable().index(index).getIndex()); } @@ -653,7 +683,8 @@ private static IndexService validateActiveShardCountAndCreateIndexService(String } private void validate(CreateIndexClusterStateUpdateRequest request, ClusterState state) { - validateIndexName(request.index(), state); + boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.get(request.settings()); + validateIndexName(request.index(), state, isHidden); validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings); } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java new file mode 100644 index 0000000000000..12b2d8356d5ca --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices; + +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.regex.Regex; + +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Describes a system index. Provides the information required to create and maintain the system index. + */ +public class SystemIndexDescriptor { + private final String indexPattern; + private final String description; + private final CharacterRunAutomaton indexPatternAutomaton; + + /** + * + * @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character. + * @param description The name of the plugin responsible for this system index. + */ + public SystemIndexDescriptor(String indexPattern, String description) { + Objects.requireNonNull(indexPattern, "system index pattern must not be null"); + if (indexPattern.length() < 2) { + throw new IllegalArgumentException("system index pattern provided as [" + indexPattern + + "] but must at least 2 characters in length"); + } + if (indexPattern.charAt(0) != '.') { + throw new IllegalArgumentException("system index pattern provided as [" + indexPattern + + "] but must start with the character [.]"); + } + if (indexPattern.charAt(1) == '*') { + throw new IllegalArgumentException("system index pattern provided as [" + indexPattern + + "] but must not start with the character sequence [.*] to prevent conflicts"); + } + this.indexPattern = indexPattern; + this.indexPatternAutomaton = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(indexPattern)); + this.description = description; + } + + /** + * @return The pattern of index names that this descriptor will be used for. + */ + public String getIndexPattern() { + return indexPattern; + } + + /** + * Checks whether an index name matches the system index name pattern for this descriptor. + * @param index The index name to be checked against the index pattern given at construction time. + * @return True if the name matches the pattern, false otherwise. + */ + public boolean matchesIndexPattern(String index) { + return indexPatternAutomaton.run(index); + } + + /** + * @return A short description of the purpose of this system index. + */ + public String getDescription() { + return description; + } + + @Override + public String toString() { + return "SystemIndexDescriptor[pattern=[" + indexPattern + "], description=[" + description + "]]"; + } + + /** + * Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed + * descriptors overlap with any of the other patterns. If any do, throws an exception. + * + * @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide. + * @throws IllegalStateException Thrown if any of the index patterns overlaps with another. + */ + public static void checkForOverlappingPatterns(Map> sourceToDescriptors) { + List> sourceDescriptorPair = sourceToDescriptors.entrySet().stream() + .flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) + .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message + .collect(Collectors.toList()); + + // This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the + // automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states + // per pattern should be low as well. If these assumptions change, this might need to be reworked. + sourceDescriptorPair.forEach(descriptorToCheck -> { + List> descriptorsMatchingThisPattern = sourceDescriptorPair.stream() + + .filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked + .filter(d -> overlaps(descriptorToCheck.v2(), d.v2())) + .collect(Collectors.toList()); + if (descriptorsMatchingThisPattern.isEmpty() == false) { + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("a system index descriptor [") + .append(descriptorToCheck.v2()) + .append("] from plugin [") + .append(descriptorToCheck.v1()) + .append("] overlaps with other system index descriptors: [") + .append(descriptorsMatchingThisPattern.stream() + .map(descriptor -> descriptor.v2() + " from plugin [" + descriptor.v1() + "]") + .collect(Collectors.joining(", "))); + throw new IllegalStateException(errorMessage.toString()); + } + }); + } + + private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) { + Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern()); + Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern()); + return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false; + } + + // TODO: Index settings and mapping + // TODO: getThreadpool() + // TODO: Upgrade handling (reindex script?) +} diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 5c4890ad90ca1..ff24a7236cd30 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -96,14 +96,15 @@ import org.elasticsearch.gateway.GatewayMetaState; import org.elasticsearch.gateway.GatewayModule; import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.gateway.MetaStateService; +import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; @@ -136,6 +137,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; @@ -429,6 +431,18 @@ protected Node( .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + final Map> systemIndexDescriptorMap = pluginsService + .filterPlugins(SystemIndexPlugin.class) + .stream() + .collect(Collectors.toUnmodifiableMap( + plugin -> plugin.getClass().getSimpleName(), + plugin -> plugin.getSystemIndexDescriptors())); + SystemIndexDescriptor.checkForOverlappingPatterns(systemIndexDescriptorMap); + + final List systemIndexDescriptors = systemIndexDescriptorMap.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + final IndicesService indicesService = new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry, analysisModule.getAnalysisRegistry(), clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry, @@ -447,6 +461,7 @@ protected Node( settingsModule.getIndexScopedSettings(), threadPool, xContentRegistry, + systemIndexDescriptors, forbidPrivateIndexSettings); Collection pluginComponents = pluginsService.filterPlugins(Plugin.class).stream() diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java new file mode 100644 index 0000000000000..a4d7a36f38670 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import org.elasticsearch.indices.SystemIndexDescriptor; + +import java.util.Collection; +import java.util.Collections; + +/** + * Plugin for defining system indices. Extends {@link ActionPlugin} because system indices must be accessed via APIs + * added by the plugin that owns the system index, rather than standard APIs. + */ +public interface SystemIndexPlugin extends ActionPlugin { + + /** + * Returns a {@link Collection} of {@link SystemIndexDescriptor}s that describe this plugin's system indices, including + * name, mapping, and settings. + * @return Descriptions of the system indices managed by this plugin. + */ + default Collection getSystemIndexDescriptors() { + return Collections.emptyList(); + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 84c08e02bf179..ae26e958b7aee 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -269,7 +269,8 @@ public ClusterState execute(ClusterState currentState) { if (currentIndexMetaData == null) { // Index doesn't exist - create it and start recovery // Make sure that the index we are about to create has a validate name - MetaDataCreateIndexService.validateIndexName(renamedIndexName, currentState); + boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.get(snapshotIndexMetaData.getSettings()); + createIndexService.validateIndexName(renamedIndexName, currentState, isHidden); createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetaData.getSettings(), false); IndexMetaData.Builder indexMdBuilder = IndexMetaData.builder(snapshotIndexMetaData) .state(IndexMetaData.State.OPEN) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java index b65809c1c268f..788339d1feb69 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java @@ -213,6 +213,7 @@ private static List putTemplate(NamedXContentRegistry xContentRegistr IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, null, xContentRegistry, + Collections.emptyList(), true); MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(null, createIndexService, new AliasValidator(), null, @@ -246,6 +247,7 @@ private List putTemplateDetail(PutRequest request) throws Exception { IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, null, xContentRegistry(), + Collections.emptyList(), true); MetaDataIndexTemplateService service = new MetaDataIndexTemplateService( clusterService, createIndexService, new AliasValidator(), indicesService, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 416491ce31ba5..f6ba70e917f58 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -56,9 +56,13 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.indices.InvalidIndexNameException; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.gateway.TestGatewayAllocator; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; import org.hamcrest.Matchers; import org.junit.Before; @@ -457,26 +461,43 @@ private DiscoveryNode newNode(String nodeId) { } public void testValidateIndexName() throws Exception { + ThreadPool testThreadPool = new TestThreadPool(getTestName()); + try { + MetaDataCreateIndexService checkerService = new MetaDataCreateIndexService( + Settings.EMPTY, + ClusterServiceUtils.createClusterService(testThreadPool), + null, + null, + null, + null, + null, + testThreadPool, + null, + Collections.emptyList(), + false + ); + validateIndexName(checkerService, "index?name", "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS); - validateIndexName("index?name", "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS); - - validateIndexName("index#name", "must not contain '#'"); + validateIndexName(checkerService, "index#name", "must not contain '#'"); - validateIndexName("_indexname", "must not start with '_', '-', or '+'"); - validateIndexName("-indexname", "must not start with '_', '-', or '+'"); - validateIndexName("+indexname", "must not start with '_', '-', or '+'"); + validateIndexName(checkerService, "_indexname", "must not start with '_', '-', or '+'"); + validateIndexName(checkerService, "-indexname", "must not start with '_', '-', or '+'"); + validateIndexName(checkerService, "+indexname", "must not start with '_', '-', or '+'"); - validateIndexName("INDEXNAME", "must be lowercase"); + validateIndexName(checkerService, "INDEXNAME", "must be lowercase"); - validateIndexName("..", "must not be '.' or '..'"); + validateIndexName(checkerService, "..", "must not be '.' or '..'"); - validateIndexName("foo:bar", "must not contain ':'"); + validateIndexName(checkerService, "foo:bar", "must not contain ':'"); + } finally { + testThreadPool.shutdown(); + } } - private void validateIndexName(String indexName, String errorMessage) { + private void validateIndexName(MetaDataCreateIndexService metaDataCreateIndexService, String indexName, String errorMessage) { InvalidIndexNameException e = expectThrows(InvalidIndexNameException.class, - () -> MetaDataCreateIndexService.validateIndexName(indexName, ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING - .getDefault(Settings.EMPTY)).build())); + () -> metaDataCreateIndexService.validateIndexName(indexName, ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING + .getDefault(Settings.EMPTY)).build(), false)); assertThat(e.getMessage(), endsWith(errorMessage)); } @@ -539,6 +560,58 @@ public void testShardLimit() { assertThat(e, hasToString(containsString(expectedMessage))); } + public void testValidateIndexNameChecksSystemIndexNames() { + List systemIndexDescriptors = new ArrayList<>(); + systemIndexDescriptors.add(new SystemIndexDescriptor(".test", "test")); + systemIndexDescriptors.add(new SystemIndexDescriptor(".test3", "test")); + systemIndexDescriptors.add(new SystemIndexDescriptor(".pattern-test*", "test-1")); + systemIndexDescriptors.add(new SystemIndexDescriptor(".pattern-test-overlapping", "test-2")); + + ThreadPool testThreadPool = new TestThreadPool(getTestName()); + try { + MetaDataCreateIndexService checkerService = new MetaDataCreateIndexService( + Settings.EMPTY, + ClusterServiceUtils.createClusterService(testThreadPool), + null, + null, + null, + null, + null, + testThreadPool, + null, + systemIndexDescriptors, + false + ); + // Check deprecations + checkerService.validateIndexName(".test2", ClusterState.EMPTY_STATE, false); + assertWarnings("index name [.test2] starts with a dot '.', in the next major version, index " + + "names starting with a dot are reserved for hidden indices and system indices"); + + // Check non-system hidden indices don't trigger a warning + checkerService.validateIndexName(".test2", ClusterState.EMPTY_STATE, true); + + // Check NO deprecation warnings if we give the index name + checkerService.validateIndexName(".test", ClusterState.EMPTY_STATE, false); + checkerService.validateIndexName(".test3", ClusterState.EMPTY_STATE, false); + + // Check that patterns with wildcards work + checkerService.validateIndexName(".pattern-test", ClusterState.EMPTY_STATE, false); + checkerService.validateIndexName(".pattern-test-with-suffix", ClusterState.EMPTY_STATE, false); + checkerService.validateIndexName(".pattern-test-other-suffix", ClusterState.EMPTY_STATE, false); + + // Check that an exception is thrown if more than one descriptor matches the index name + AssertionError exception = expectThrows(AssertionError.class, + () -> checkerService.validateIndexName(".pattern-test-overlapping", ClusterState.EMPTY_STATE, false)); + assertThat(exception.getMessage(), + containsString("index name [.pattern-test-overlapping] is claimed as a system index by multiple system index patterns:")); + assertThat(exception.getMessage(), containsString("pattern: [.pattern-test*], description: [test-1]")); + assertThat(exception.getMessage(), containsString("pattern: [.pattern-test-overlapping], description: [test-2]")); + + } finally { + testThreadPool.shutdown(); + } + } + public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exception { IndexTemplateMetaData templateMetaData = addMatchingTemplate(templateBuilder -> { templateBuilder.putAlias(AliasMetaData.builder("alias1")); diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java new file mode 100644 index 0000000000000..ed3d6515616a9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + +public class SystemIndexDescriptorTests extends ESTestCase { + + public void testValidation() { + { + Exception ex = expectThrows(NullPointerException.class, + () -> new SystemIndexDescriptor(null, randomAlphaOfLength(5))); + assertThat(ex.getMessage(), containsString("must not be null")); + } + + { + Exception ex = expectThrows(IllegalArgumentException.class, + () -> new SystemIndexDescriptor("", randomAlphaOfLength(5))); + assertThat(ex.getMessage(), containsString("must at least 2 characters in length")); + } + + { + Exception ex = expectThrows(IllegalArgumentException.class, + () -> new SystemIndexDescriptor(".", randomAlphaOfLength(5))); + assertThat(ex.getMessage(), containsString("must at least 2 characters in length")); + } + + { + Exception ex = expectThrows(IllegalArgumentException.class, + () -> new SystemIndexDescriptor(randomAlphaOfLength(10), randomAlphaOfLength(5))); + assertThat(ex.getMessage(), containsString("must start with the character [.]")); + } + + { + Exception ex = expectThrows(IllegalArgumentException.class, + () -> new SystemIndexDescriptor(".*", randomAlphaOfLength(5))); + assertThat(ex.getMessage(), containsString("must not start with the character sequence [.*] to prevent conflicts")); + } + { + Exception ex = expectThrows(IllegalArgumentException.class, + () -> new SystemIndexDescriptor(".*" + randomAlphaOfLength(10), randomAlphaOfLength(5))); + assertThat(ex.getMessage(), containsString("must not start with the character sequence [.*] to prevent conflicts")); + } + } + + public void testBasicOverlappingPatterns() { + SystemIndexDescriptor broadPattern = new SystemIndexDescriptor(".a*c*", "test"); + SystemIndexDescriptor notOverlapping = new SystemIndexDescriptor(".bbbddd*", "test"); + SystemIndexDescriptor overlapping1 = new SystemIndexDescriptor(".ac*", "test"); + SystemIndexDescriptor overlapping2 = new SystemIndexDescriptor(".aaaabbbccc", "test"); + SystemIndexDescriptor overlapping3 = new SystemIndexDescriptor(".aaabb*cccddd*", "test"); + + // These sources have fixed prefixes to make sure they sort in the same order, so that the error message is consistent + // across tests + String broadPatternSource = "AAA" + randomAlphaOfLength(5); + String otherSource = "ZZZ" + randomAlphaOfLength(6); + Map> descriptors = new HashMap<>(); + descriptors.put(broadPatternSource, Arrays.asList(broadPattern)); + descriptors.put(otherSource, Arrays.asList(notOverlapping, overlapping1, overlapping2, overlapping3)); + + IllegalStateException exception = expectThrows(IllegalStateException.class, + () -> SystemIndexDescriptor.checkForOverlappingPatterns(descriptors)); + assertThat(exception.getMessage(), containsString("a system index descriptor [" + broadPattern + + "] from plugin [" + broadPatternSource + "] overlaps with other system index descriptors:")); + String fromPluginString = " from plugin [" + otherSource + "]"; + assertThat(exception.getMessage(), containsString(overlapping1.toString() + fromPluginString)); + assertThat(exception.getMessage(), containsString(overlapping2.toString() + fromPluginString)); + assertThat(exception.getMessage(), containsString(overlapping3.toString() + fromPluginString)); + assertThat(exception.getMessage(), not(containsString(notOverlapping.toString()))); + } + + public void testComplexOverlappingPatterns() { + // These patterns are slightly more complex to detect because pattern1 does not match pattern2 and vice versa + SystemIndexDescriptor pattern1 = new SystemIndexDescriptor(".a*c", "test"); + SystemIndexDescriptor pattern2 = new SystemIndexDescriptor(".ab*", "test"); + + // These sources have fixed prefixes to make sure they sort in the same order, so that the error message is consistent + // across tests + String source1 = "AAA" + randomAlphaOfLength(5); + String source2 = "ZZZ" + randomAlphaOfLength(6); + Map> descriptors = new HashMap<>(); + descriptors.put(source1, Arrays.asList(pattern1)); + descriptors.put(source2, Arrays.asList(pattern2)); + + IllegalStateException exception = expectThrows(IllegalStateException.class, + () -> SystemIndexDescriptor.checkForOverlappingPatterns(descriptors)); + assertThat(exception.getMessage(), containsString("a system index descriptor [" + pattern1 + + "] from plugin [" + source1 + "] overlaps with other system index descriptors:")); + assertThat(exception.getMessage(), containsString(pattern2.toString() + " from plugin [" + source2 + "]")); + + } +} diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 901409739c1d6..2c757727791d4 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -205,7 +205,7 @@ public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version m allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, threadPool); MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(SETTINGS, clusterService, indicesService, allocationService, new AliasValidator(), environment, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry, true); + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry, Collections.emptyList(), true); transportCloseIndexAction = new TransportCloseIndexAction(SETTINGS, transportService, clusterService, threadPool, indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 0f6189e9db1a7..2278ed75a7f84 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1230,7 +1230,7 @@ public void onFailure(final Exception e) { final MetaDataCreateIndexService metaDataCreateIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService, allocationService, new AliasValidator(), environment, indexScopedSettings, - threadPool, namedXContentRegistry, false); + threadPool, namedXContentRegistry, Collections.emptyList(), false); actions.put(CreateIndexAction.INSTANCE, new TransportCreateIndexAction( transportService, clusterService, threadPool, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index 4b56c5b59a554..a6a8e90f8fe97 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -33,6 +33,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { public static final String ENRICH_INDEX_NAME_BASE = ".enrich-"; + public static final String ENRICH_INDEX_PATTERN = ENRICH_INDEX_NAME_BASE + "*"; public static final String MATCH_TYPE = "match"; public static final String GEO_MATCH_TYPE = "geo_match"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 4e2e735f9496a..ccb86f2a7b4ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -38,6 +38,7 @@ public class AnnotationIndex { public static final String WRITE_ALIAS_NAME = ".ml-annotations-write"; // Exposed for testing, but always use the aliases in non-test code public static final String INDEX_NAME = ".ml-annotations-6"; + public static final String INDEX_PATTERN = ".ml-annotations*"; /** * Create the .ml-annotations index with correct mappings if it does not already diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index 96f21876223ce..d62e2de207d9a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -8,8 +8,13 @@ public final class AnomalyDetectorsIndexFields { public static final String CONFIG_INDEX = ".ml-config"; + public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; + public static final String RESULTS_INDEX_PATTERN = RESULTS_INDEX_PREFIX + "*"; + public static final String STATE_INDEX_PREFIX = ".ml-state"; + public static final String STATE_INDEX_PATTERN = STATE_INDEX_PREFIX + "*"; + public static final String RESULTS_INDEX_DEFAULT = "shared"; private AnomalyDetectorsIndexFields() {} diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index 327c4f240f438..54dd0b219a7b7 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -24,11 +24,12 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.ingest.Processor; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; @@ -57,14 +58,16 @@ import org.elasticsearch.xpack.enrich.rest.RestPutEnrichPolicyAction; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import static org.elasticsearch.xpack.core.XPackSettings.ENRICH_ENABLED_SETTING; +import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.ENRICH_INDEX_PATTERN; -public class EnrichPlugin extends Plugin implements ActionPlugin, IngestPlugin { +public class EnrichPlugin extends Plugin implements SystemIndexPlugin, IngestPlugin { static final Setting ENRICH_FETCH_SIZE_SETTING = Setting.intSetting( "enrich.fetch_size", @@ -240,4 +243,11 @@ public List> getSettings() { ENRICH_MAX_FORCE_MERGE_ATTEMPTS ); } + + @Override + public Collection getSystemIndexDescriptors() { + return Collections.singletonList( + new SystemIndexDescriptor(ENRICH_INDEX_PATTERN, "Contains data to support enrich ingest processors.") + ); + } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 1ebb56480b62a..0ade749a26d7b 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -28,8 +28,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; @@ -50,9 +51,9 @@ import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; -import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; import org.elasticsearch.xpack.core.ilm.action.ExplainLifecycleAction; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction; @@ -133,7 +134,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; -public class IndexLifecycle extends Plugin implements ActionPlugin { +public class IndexLifecycle extends Plugin implements SystemIndexPlugin { private final SetOnce indexLifecycleInitialisationService = new SetOnce<>(); private final SetOnce ilmHistoryStore = new SetOnce<>(); private final SetOnce snapshotLifecycleService = new SetOnce<>(); @@ -327,4 +328,12 @@ public void close() { throw new ElasticsearchException("unable to close index lifecycle services", e); } } + + @Override + public Collection getSystemIndexDescriptors() { + //TODO: The SLM history store should be non-dot-prefixed hidden indices, but need to be here for now + // to prevent warnings + return Collections.singletonList(new SystemIndexDescriptor(".slm-history-*", + "Contains a history of Snapshot Lifecycle Management operations")); + } } diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index 7ea6e6d2e4daa..b9180fb2afe9f 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -12,14 +12,17 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.UnaryOperator; @@ -28,8 +31,9 @@ /** * This class supplies the logstash featureset and templates */ -public class Logstash extends Plugin implements ActionPlugin { +public class Logstash extends Plugin implements SystemIndexPlugin { + private static final String LOGSTASH_CONCRETE_INDEX_NAME = ".logstash"; private static final String LOGSTASH_TEMPLATE_FILE_NAME = "logstash-management"; private static final String LOGSTASH_INDEX_TEMPLATE_NAME = ".logstash-management"; private static final String OLD_LOGSTASH_INDEX_NAME = "logstash-index-template"; @@ -59,4 +63,10 @@ public UnaryOperator> getIndexTemplateMetaDat return templates; }; } + + @Override + public Collection getSystemIndexDescriptors() { + return Collections.singletonList(new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME, + "Contains data for Logstash Central Management")); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 98f1aba57090d..e742cefe8e6fb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -42,17 +42,18 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.ingest.Processor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.monitor.os.OsProbe; import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.persistent.PersistentTasksExecutor; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; @@ -68,7 +69,6 @@ import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; -import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction; import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction; import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction; @@ -80,6 +80,7 @@ import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction; import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAction; import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction; +import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.action.FindFileStructureAction; import org.elasticsearch.xpack.core.ml.action.FlushJobAction; @@ -130,6 +131,7 @@ import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction; import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction; import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction; +import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; @@ -140,7 +142,6 @@ import org.elasticsearch.xpack.core.ml.notifications.AuditorField; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.ml.action.TransportCloseJobAction; -import org.elasticsearch.xpack.ml.action.TransportExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarAction; import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarEventAction; import org.elasticsearch.xpack.ml.action.TransportDeleteDataFrameAnalyticsAction; @@ -152,6 +153,7 @@ import org.elasticsearch.xpack.ml.action.TransportDeleteModelSnapshotAction; import org.elasticsearch.xpack.ml.action.TransportDeleteTrainedModelAction; import org.elasticsearch.xpack.ml.action.TransportEvaluateDataFrameAction; +import org.elasticsearch.xpack.ml.action.TransportExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.action.TransportFinalizeJobExecutionAction; import org.elasticsearch.xpack.ml.action.TransportFindFileStructureAction; import org.elasticsearch.xpack.ml.action.TransportFlushJobAction; @@ -266,9 +268,9 @@ import org.elasticsearch.xpack.ml.rest.datafeeds.RestStartDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestStopDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestUpdateDatafeedAction; -import org.elasticsearch.xpack.ml.rest.dataframe.RestExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestDeleteDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestEvaluateDataFrameAction; +import org.elasticsearch.xpack.ml.rest.dataframe.RestExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsStatsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestPutDataFrameAnalyticsAction; @@ -323,7 +325,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; -public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlugin, IngestPlugin, PersistentTaskPlugin { +public class MachineLearning extends Plugin implements SystemIndexPlugin, AnalysisPlugin, IngestPlugin, PersistentTaskPlugin { public static final String NAME = "ml"; public static final String BASE_PATH = "/_ml/"; public static final String PRE_V7_BASE_PATH = "/_xpack/ml/"; @@ -1044,4 +1046,18 @@ public List getNamedXContent() { return namedXContent; } + @Override + public Collection getSystemIndexDescriptors() { + //TODO: Some of the below should be converted to non-dot-prefixed hidden indices. I am not sure which ones. + // Either way, they should be here until then to prevent deprecation warnings + return Collections.unmodifiableList(Arrays.asList( + new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, this.getClass().getSimpleName()), + new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, this.getClass().getSimpleName()), + new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, this.getClass().getSimpleName()), + new SystemIndexDescriptor(AnomalyDetectorsIndexFields.RESULTS_INDEX_PATTERN, this.getClass().getSimpleName()), + new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, this.getClass().getSimpleName()), + new SystemIndexDescriptor(".ml-notifications-*", this.getClass().getSimpleName()), + new SystemIndexDescriptor(AnnotationIndex.INDEX_PATTERN, this.getClass().getSimpleName()) + )); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index e295dfd696bc3..7c4ac4f8226a5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -42,12 +42,12 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.ingest.Processor; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.DiscoveryPlugin; import org.elasticsearch.plugins.ExtensiblePlugin; @@ -55,6 +55,7 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestHeaderDefinition; @@ -128,6 +129,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.security.support.Automatons; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.ssl.SSLConfiguration; @@ -272,7 +274,7 @@ import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; -public class Security extends Plugin implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, +public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin, MapperPlugin, ExtensiblePlugin { private static final Logger logger = LogManager.getLogger(Security.class); @@ -1064,4 +1066,16 @@ private synchronized NioGroupFactory getNioGroupFactory(Settings settings) { return groupFactory.get(); } } + + @Override + public Collection getSystemIndexDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, this.getClass().getSimpleName()), + new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, this.getClass().getSimpleName()), + new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, this.getClass().getSimpleName()), + + new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, this.getClass().getSimpleName()), + new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, this.getClass().getSimpleName()) + )); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml index e59b157a9f0ba..2ae4fe0527016 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml @@ -272,7 +272,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.create: - index: ".ml-special-index-to-avoid-picking-up-template" + index: "ml-special-index-to-avoid-picking-up-template" body: mappings: properties: @@ -283,28 +283,28 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.put_alias: - index: ".ml-special-index-to-avoid-picking-up-template" + index: "ml-special-index-to-avoid-picking-up-template" name: ".ml-anomalies-get-datafeed-stats-1" - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.put_alias: - index: ".ml-special-index-to-avoid-picking-up-template" + index: "ml-special-index-to-avoid-picking-up-template" name: ".ml-anomalies-.write-get-datafeed-stats-1" - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.put_alias: - index: ".ml-special-index-to-avoid-picking-up-template" + index: "ml-special-index-to-avoid-picking-up-template" name: ".ml-anomalies-get-datafeed-stats-2" - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.put_alias: - index: ".ml-special-index-to-avoid-picking-up-template" + index: "ml-special-index-to-avoid-picking-up-template" name: ".ml-anomalies-.write-get-datafeed-stats-2" - do: diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 29cee3c3af8aa..128218e3b3e01 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -27,11 +27,12 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.persistent.PersistentTasksExecutor; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; @@ -113,7 +114,7 @@ import static java.util.Collections.emptyList; -public class Transform extends Plugin implements ActionPlugin, PersistentTaskPlugin { +public class Transform extends Plugin implements SystemIndexPlugin, PersistentTaskPlugin { public static final String NAME = "transform"; public static final String TASK_THREAD_POOL_NAME = "transform_indexing"; @@ -313,4 +314,14 @@ public void close() { public List getNamedXContent() { return new TransformNamedXContentProvider().getNamedXContentParsers(); } + + @Override + public Collection getSystemIndexDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + // TODO: Verify that these should be system indices, rather than hidden indices + new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, this.getClass().getSimpleName()), + new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN, this.getClass().getSimpleName()), + new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED, this.getClass().getSimpleName()) + )); + } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index eb5d5eeb8b9cf..3abc363886ef7 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -39,12 +39,13 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.node.Node; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ReloadablePlugin; import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptContext; @@ -197,7 +198,7 @@ import static org.elasticsearch.common.settings.Setting.Property.NodeScope; import static org.elasticsearch.xpack.core.ClientHelper.WATCHER_ORIGIN; -public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin, ReloadablePlugin { +public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin, ReloadablePlugin { // This setting is only here for backward compatibility reasons as 6.x indices made use of it. It can be removed in 8.x. @Deprecated @@ -682,4 +683,16 @@ public void reload(Settings settings) { } reloadableServices.forEach(s -> s.reload(settings)); } + + @Override + public Collection getSystemIndexDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + new SystemIndexDescriptor(Watch.INDEX, this.getClass().getSimpleName()), + new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, this.getClass().getSimpleName()), + + //TODO: The below should be converted to hidden indices, but need to be included here until then to prevent + // deprecation warnings + new SystemIndexDescriptor(".watch-history-*", this.getClass().getSimpleName()) + )); + } }