diff --git a/docs/changelog/79512.yaml b/docs/changelog/79512.yaml new file mode 100644 index 0000000000000..b4d257e9f19f5 --- /dev/null +++ b/docs/changelog/79512.yaml @@ -0,0 +1,5 @@ +pr: 79512 +summary: All system indices are hidden indices +area: Infra/Core +type: enhancement +issues: [] diff --git a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java index f82461f70b785..50b72b905fd6d 100644 --- a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java +++ b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java @@ -14,6 +14,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.json.JsonXContent; @@ -218,6 +219,27 @@ public void testUpdateIndexSettings() throws IOException { assertThat(response.getStatusLine().getStatusCode(), is(200)); } + public void testCannotCreateVisibleSystemIndex() { + Request request = request("PUT", "/" + indexName); + request.setJsonEntity("{\"settings\": {\"index.hidden\":\"false\"}}"); + ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertThat( + exception.getMessage(), + containsString("Cannot create system index [" + indexName + "] with [index.hidden] set to 'false'") + ); + } + + public void testCannotSetVisible() throws IOException { + Request putIndexRequest = request("PUT", "/" + indexName); + Response response = client().performRequest(putIndexRequest); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + + Request putSettingsRequest = request("PUT", "/" + indexName + "/_settings"); + putSettingsRequest.setJsonEntity("{ \"index.hidden\" : false }"); + ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(putSettingsRequest)); + assertThat(exception.getMessage(), containsString("Cannot set [index.hidden] to 'false' on system indices: [" + indexName + "]")); + } + public void testGetIndex() throws IOException { Request request = request("PUT", "/" + indexName); Response response = client().performRequest(request); diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 6eeba181cb8b9..ec77c42477063 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -159,6 +159,18 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("reindex/20_validation/reindex without source gives useful error message", "exception with a type. Not much benefit adding _doc there.") task.skipTest("update_by_query/20_validation/update_by_query without source gives useful error message", "exception with a type. Not much benefit adding _doc there.") + + // these tests are all relying on a call to refresh all indices, when they could easily be changed + // in 7.x to call the specific index they want to refresh. + // See https://github.com/elastic/elasticsearch/issues/81188 + task.skipTest("delete_by_query/70_throttle/Rethrottle to -1 which turns off throttling", "test relies on system index being non-hidden") + task.skipTest("delete_by_query/80_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden") + task.skipTest("delete_by_query/80_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden") + task.skipTest("reindex/80_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden") + task.skipTest("reindex/80_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden") + task.skipTest("update_by_query/70_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden") + task.skipTest("update_by_query/70_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden") + task.addAllowedWarningRegex("\\[types removal\\].*") } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index daa88c8284620..20215c999611f 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -299,20 +299,22 @@ public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) thr String indexName = Optional.ofNullable(descriptor.getPrimaryIndex()).orElse(descriptor.getIndexPattern().replace("*", "old")); CreateIndexRequestBuilder createRequest = prepareCreate(indexName); createRequest.setWaitForActiveShards(ActiveShardCount.ALL); - if (descriptor.getSettings() != null) { - createRequest.setSettings( - Settings.builder() - .put("index.version.created", Version.CURRENT) - .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .build() - ); - } else { + if (SystemIndexDescriptor.DEFAULT_SETTINGS.equals(descriptor.getSettings())) { + // unmanaged createRequest.setSettings( createSimpleSettings( NEEDS_UPGRADE_VERSION, descriptor.isInternal() ? INTERNAL_UNMANAGED_FLAG_VALUE : EXTERNAL_UNMANAGED_FLAG_VALUE ) ); + } else { + // managed + createRequest.setSettings( + Settings.builder() + .put("index.version.created", Version.CURRENT) + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) + .build() + ); } if (descriptor.getMappings() == null) { createRequest.setMapping(createSimpleMapping(false, descriptor.isInternal())); diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml index 4e877765c7cf2..dc7f221ec3687 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml @@ -124,8 +124,6 @@ task_id: $task - do: - warnings: - - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" indices.refresh: {} - do: diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml index 884e19c363be9..efc669173328c 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml @@ -155,7 +155,8 @@ - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" - indices.refresh: {} + indices.refresh: + expand_wildcards: "open,hidden" - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" @@ -270,7 +271,8 @@ - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" - indices.refresh: {} + indices.refresh: + expand_wildcards: "open,hidden" - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml index 52d40aa9469ff..d5beb06449a97 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml @@ -164,7 +164,8 @@ - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" - indices.refresh: {} + indices.refresh: + expand_wildcards: "open,hidden" - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" @@ -282,7 +283,8 @@ - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" - indices.refresh: {} + indices.refresh: + expand_wildcards: "open,hidden" - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml index 234d2d712efb3..29330f5b32f58 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml @@ -147,7 +147,8 @@ - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" - indices.refresh: {} + indices.refresh: + expand_wildcards: "open,hidden" - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" @@ -256,7 +257,8 @@ - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" - indices.refresh: {} + indices.refresh: + expand_wildcards: "open,hidden" - do: warnings: - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java index f2245b5830aba..3c0f19ec76980 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -45,6 +46,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.test.rest.ESRestTestCase.entityAsMap; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; @@ -92,6 +94,17 @@ public void testSystemIndexAccessBlockedByDefault() throws Exception { // And with a total wildcard assertDeprecationWarningOnAccess(randomFrom("*", "_all"), SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + // If we're not expanding wildcards, we don't get anything + { + Request searchRequest = new Request("GET", "/" + randomFrom("*", "_all") + randomFrom("/_count", "/_search")); + searchRequest.setJsonEntity("{\"query\": {\"match\": {\"some_field\": \"some_value\"}}}"); + searchRequest.addParameter("allow_no_indices", "false"); + + ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(searchRequest)); + assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(exception.getMessage(), containsString("no such index")); + } + // Try to index a doc directly { String expectedWarning = "this request accesses system indices: [" @@ -115,6 +128,7 @@ private void assertDeprecationWarningOnAccess(String queryPattern, String warnin searchRequest.setJsonEntity("{\"query\": {\"match\": {\"some_field\": \"some_value\"}}}"); // Disallow no indices to cause an exception if this resolves to zero indices, so that we're sure it resolved the index searchRequest.addParameter("allow_no_indices", "false"); + searchRequest.addParameter("expand_wildcards", "open,hidden"); searchRequest.setOptions(expectWarnings(expectedWarning)); Response response = getRestClient().performRequest(searchRequest); diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java new file mode 100644 index 0000000000000..78826934d0037 --- /dev/null +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.system.indices; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.After; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class SystemAliasIT extends ESRestTestCase { + static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("rest_user", new SecureString("rest-user-password".toCharArray())); + + @After + public void resetFeatures() throws Exception { + client().performRequest(new Request("POST", "/_features/_reset")); + } + + @Override + protected Settings restClientSettings() { + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + + public void testCreatingSystemIndexWithAlias() throws Exception { + { + Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setJsonEntity("{\"aliases\": {\".internal-unmanaged-alias\": {}}}"); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + assertAliasIsHiddenInIndexResponse(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + assertAliasIsHiddenInAliasesEndpoint(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + } + + public void testCreatingSystemIndexWithLegacyAlias() throws Exception { + { + Request request = new Request("PUT", "/_template/system_template"); + request.setJsonEntity( + "{" + + " \"index_patterns\": [\".internal-unmanaged-*\"]," + + " \"aliases\": {" + + " \".internal-unmanaged-alias\": {}" + + " }" + + "}" + ); + request.setOptions(expectWarnings("Legacy index templates are deprecated in favor of composable templates.")); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + { + Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + assertAliasIsHiddenInIndexResponse(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + assertAliasIsHiddenInAliasesEndpoint(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + } + + public void testCreatingSystemIndexWithIndexAliasEndpoint() throws Exception { + { + Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + { + Request request = new Request("PUT", "/.internal-unmanaged-index-8/_alias/.internal-unmanaged-alias"); + request.setOptions( + expectWarnings( + "this request accesses system indices: [.internal-unmanaged-index-8], " + + "but in a future major version, direct access to system indices will be prevented by default" + ) + ); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + assertAliasIsHiddenInIndexResponse(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + assertAliasIsHiddenInAliasesEndpoint(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + } + + public void testCreatingSystemIndexWithAliasEndpoint() throws Exception { + { + Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + { + Request request = new Request("PUT", "/_alias/.internal-unmanaged-alias"); + request.setJsonEntity("{\"index\": \".internal-unmanaged-index-8\"}"); + request.setOptions( + expectWarnings( + "this request accesses system indices: [.internal-unmanaged-index-8], " + + "but in a future major version, direct access to system indices will be prevented by default" + ) + ); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + assertAliasIsHiddenInIndexResponse(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + assertAliasIsHiddenInAliasesEndpoint(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + } + + public void testCreatingSystemIndexWithAliasesEndpoint() throws Exception { + { + Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + { + Request request = new Request("POST", "/_aliases"); + request.setJsonEntity( + "{" + + " \"actions\": [" + + " {" + + " \"add\": {" + + " \"index\": \".internal-unmanaged-index-8\"," + + " \"alias\": \".internal-unmanaged-alias\"" + + " }" + + " }" + + " ]" + + "}" + ); + + request.setOptions( + expectWarnings( + "this request accesses system indices: [.internal-unmanaged-index-8], " + + "but in a future major version, direct access to system indices will be prevented by default" + ) + ); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + assertAliasIsHiddenInIndexResponse(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + assertAliasIsHiddenInAliasesEndpoint(".internal-unmanaged-index-8", ".internal-unmanaged-alias"); + } + + @SuppressWarnings("unchecked") + private void assertAliasIsHiddenInIndexResponse(String indexName, String aliasName) throws IOException { + Request request = new Request("GET", "/" + indexName); + request.setOptions( + expectWarnings( + "this request accesses system indices: [" + + indexName + + "], " + + "but in a future major version, direct access to system indices will be prevented by default" + ) + ); + Response response = client().performRequest(request); + Map responseMap = responseAsMap(response); + Map indexMap = (Map) responseMap.get(indexName); + Map settingsMap = (Map) indexMap.get("settings"); + Map indexSettingsMap = (Map) settingsMap.get("index"); + assertThat(indexSettingsMap.get("hidden"), equalTo("true")); + + Map aliasesMap = (Map) indexMap.get("aliases"); + assertThat(aliasesMap.keySet(), equalTo(Set.of(aliasName))); + Map aliasMap = (Map) aliasesMap.get(aliasName); + assertThat(aliasMap.get("is_hidden"), notNullValue()); + assertThat(aliasMap.get("is_hidden"), equalTo(true)); + } + + @SuppressWarnings("unchecked") + private void assertAliasIsHiddenInAliasesEndpoint(String indexName, String aliasName) throws IOException { + Request request = new Request("GET", "/_aliases"); + request.setOptions( + expectWarnings( + "this request accesses system indices: [" + + indexName + + "], " + + "but in a future major version, direct access to system indices will be prevented by default", + "this request accesses aliases with names reserved for system indices: [.internal-unmanaged-alias], " + + "but in a future major version, direct access to system indices and their aliases will not be allowed" + ) + ); + Response response = client().performRequest(request); + Map responseMap = responseAsMap(response); + Map indexAliasMap = (Map) responseMap.get(indexName); + Map aliasesMap = (Map) indexAliasMap.get("aliases"); + Map aliasMap = (Map) aliasesMap.get(aliasName); + assertThat(aliasMap.get("is_hidden"), notNullValue()); + assertThat(aliasMap.get("is_hidden"), equalTo(true)); + } +} diff --git a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java index fcf3177f2c94a..07df8541646bd 100644 --- a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java +++ b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java @@ -45,6 +45,9 @@ public class SystemIndicesQA extends Plugin implements SystemIndexPlugin, ActionPlugin { + private static final String INTERNAL_UNMANAGED_INDEX_NAME = ".internal-unmanaged-index*"; + private static final String INTERNAL_MANAGED_INDEX_NAME = ".internal-managed-index*"; + @Override public String getFeatureName() { return "system indices qa"; @@ -73,6 +76,32 @@ public Collection getSystemIndexDescriptors(Settings sett .setOrigin("net-new") .setVersionMetaKey("version") .setPrimaryIndex(".net-new-system-index-" + Version.CURRENT.major) + .build(), + SystemIndexDescriptor.builder() + .setIndexPattern(INTERNAL_UNMANAGED_INDEX_NAME) + .setDescription("internal unmanaged system index") + .setType(SystemIndexDescriptor.Type.INTERNAL_UNMANAGED) + .setOrigin("qa") + .setVersionMetaKey("version") + .setPrimaryIndex(".internal-unmanaged-index-" + Version.CURRENT.major) + .setAliasName(".internal-unmanaged-alias") + .build(), + SystemIndexDescriptor.builder() + .setIndexPattern(INTERNAL_MANAGED_INDEX_NAME) + .setDescription("internal managed system index") + .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) + .setMappings(mappings()) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .build() + ) + .setOrigin("qa") + .setVersionMetaKey("version") + .setPrimaryIndex(".internal-managed-index-" + Version.CURRENT.major) + .setAliasName(".internal-managed-alias") .build() ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java index 967e20f9be2ba..bda0af2445c4c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java @@ -8,24 +8,58 @@ package org.elasticsearch.action.admin.indices.create; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.TestSystemIndexPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.snapshots.SystemIndicesSnapshotIT; import org.elasticsearch.test.ESIntegTestCase; +import org.junit.After; import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME; import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class AutoCreateSystemIndexIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class); + return CollectionUtils.appendToCopy( + CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class), + UnmanagedSystemIndexTestPlugin.class + ); + } + + @After + public void afterEach() { + assertAcked(client().admin().indices().prepareDeleteTemplate("*").get()); + client().admin().indices().prepareDelete(PRIMARY_INDEX_NAME); } public void testAutoCreatePrimaryIndex() throws Exception { @@ -51,4 +85,132 @@ public void testAutoCreateNonPrimaryIndex() throws Exception { GetIndexResponse response = client().admin().indices().prepareGetIndex().addIndices(INDEX_NAME + "-2").get(); assertThat(response.indices().length, is(1)); } + + public void testSystemIndicesAutoCreatedAsHidden() throws Exception { + CreateIndexRequest request = new CreateIndexRequest(UnmanagedSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + client().execute(AutoCreateAction.INSTANCE, request).get(); + + GetIndexResponse response = client().admin() + .indices() + .prepareGetIndex() + .addIndices(UnmanagedSystemIndexTestPlugin.SYSTEM_INDEX_NAME) + .get(); + assertThat(response.indices().length, is(1)); + Settings settings = response.settings().get(UnmanagedSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + assertThat(settings, notNullValue()); + assertThat(settings.getAsBoolean(SETTING_INDEX_HIDDEN, false), is(true)); + } + + public void testSystemIndicesAutoCreateRejectedWhenNotHidden() { + CreateIndexRequest request = new CreateIndexRequest(UnmanagedSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + request.settings(Settings.builder().put(SETTING_INDEX_HIDDEN, false).build()); + ExecutionException exception = expectThrows( + ExecutionException.class, + () -> client().execute(AutoCreateAction.INSTANCE, request).get() + ); + + assertThat( + exception.getCause().getMessage(), + containsString("Cannot auto-create system index [.unmanaged-system-idx] with [index.hidden] set to 'false'") + ); + } + + /** + * Check that a template applying a system alias creates a hidden alias. + */ + public void testAutoCreateSystemAliasViaV1Template() throws Exception { + assertAcked( + client().admin() + .indices() + .preparePutTemplate("test-template") + .setPatterns(List.of(INDEX_NAME + "*")) + .addAlias(new Alias(INDEX_NAME + "-legacy-alias")) + .get() + ); + + String nonPrimaryIndex = INDEX_NAME + "-2"; + CreateIndexRequest request = new CreateIndexRequest(nonPrimaryIndex); + assertAcked(client().execute(AutoCreateAction.INSTANCE, request).get()); + + assertTrue(indexExists(nonPrimaryIndex)); + + assertAliasesHidden(nonPrimaryIndex, Set.of(".test-index", ".test-index-legacy-alias")); + + assertAcked(client().admin().indices().prepareDeleteTemplate("*").get()); + } + + /** + * Check that a composable template applying a system alias creates a hidden alias. + */ + public void testAutoCreateSystemAliasViaComposableTemplate() throws Exception { + ComposableIndexTemplate cit = new ComposableIndexTemplate( + Collections.singletonList(INDEX_NAME + "*"), + new Template( + null, + null, + Map.of(INDEX_NAME + "-composable-alias", AliasMetadata.builder(INDEX_NAME + "-composable-alias").build()) + ), + Collections.emptyList(), + 4L, + 5L, + Collections.emptyMap() + ); + assertAcked( + client().execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("test-composable-template").indexTemplate(cit) + ).get() + ); + + String nonPrimaryIndex = INDEX_NAME + "-2"; + CreateIndexRequest request = new CreateIndexRequest(nonPrimaryIndex); + assertAcked(client().execute(AutoCreateAction.INSTANCE, request).get()); + + assertTrue(indexExists(nonPrimaryIndex)); + + assertAliasesHidden(nonPrimaryIndex, Set.of(".test-index", ".test-index-composable-alias")); + + assertAcked( + client().execute( + DeleteComposableIndexTemplateAction.INSTANCE, + new DeleteComposableIndexTemplateAction.Request("test-composable-template") + ).get() + ); + } + + private void assertAliasesHidden(String nonPrimaryIndex, Set aliasNames) throws InterruptedException, ExecutionException { + final GetAliasesResponse getAliasesResponse = client().admin() + .indices() + .getAliases(new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())) + .get(); + + assertThat(getAliasesResponse.getAliases().size(), equalTo(1)); + assertThat(getAliasesResponse.getAliases().get(nonPrimaryIndex).size(), equalTo(2)); + assertThat( + getAliasesResponse.getAliases().get(nonPrimaryIndex).stream().map(AliasMetadata::alias).collect(Collectors.toSet()), + equalTo(aliasNames) + ); + getAliasesResponse.getAliases().get(nonPrimaryIndex).forEach(alias -> assertThat(alias.isHidden(), is(true))); + } + + public static class UnmanagedSystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_INDEX_NAME = ".unmanaged-system-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME + "*", "System indices for tests")); + } + + @Override + public String getFeatureName() { + return SystemIndicesSnapshotIT.SystemIndexTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "A simple test plugin"; + } + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 41a346696417e..b200e438adb7e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -8,11 +8,20 @@ package org.elasticsearch.action.admin.indices.create; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -21,12 +30,17 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentType; +import org.junit.After; import org.junit.Before; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME; import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME; @@ -42,6 +56,14 @@ public void beforeEach() { TestSystemIndexDescriptor.useNewMappings.set(false); } + @After + public void afterEach() throws Exception { + assertAcked(client().admin().indices().prepareDeleteTemplate("*").get()); + assertAcked( + client().execute(DeleteComposableIndexTemplateAction.INSTANCE, new DeleteComposableIndexTemplateAction.Request("*")).get() + ); + } + @Override protected Collection> nodePlugins() { return CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class); @@ -91,6 +113,70 @@ public void testCreateSystemIndexViaConcreteName() { doCreateTest(() -> assertAcked(prepareCreate(PRIMARY_INDEX_NAME)), PRIMARY_INDEX_NAME); } + /** + * Check that a legacy template applying a system alias creates a hidden alias. + */ + public void testCreateSystemAliasViaV1Template() throws Exception { + assertAcked( + client().admin() + .indices() + .preparePutTemplate("test-template") + .setPatterns(List.of(INDEX_NAME + "*")) + .addAlias(new Alias(INDEX_NAME + "-legacy-alias")) + .get() + ); + + assertAcked(prepareCreate(INDEX_NAME + "-2")); + ensureGreen(PRIMARY_INDEX_NAME); + + assertTrue(indexExists(PRIMARY_INDEX_NAME)); + assertFalse(indexExists(INDEX_NAME + "-2")); + + assertHasAliases(Set.of(".test-index", ".test-index-legacy-alias")); + + assertAcked(client().admin().indices().prepareDeleteTemplate("*").get()); + } + + /** + * Check that a composable template applying a system alias creates a hidden alias. + */ + public void testCreateSystemAliasViaComposableTemplate() throws Exception { + ComposableIndexTemplate cit = new ComposableIndexTemplate( + Collections.singletonList(INDEX_NAME + "*"), + new Template( + null, + null, + Map.of(INDEX_NAME + "-composable-alias", AliasMetadata.builder(INDEX_NAME + "-composable-alias").build()) + ), + Collections.emptyList(), + 4L, + 5L, + Collections.emptyMap() + ); + assertAcked( + client().execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("test-composable-template").indexTemplate(cit) + ).get() + ); + + assertAcked(prepareCreate(INDEX_NAME + "-2")); + ensureGreen(PRIMARY_INDEX_NAME); + + // Attempting to directly create a non-primary system index only creates the primary index + assertTrue(indexExists(PRIMARY_INDEX_NAME)); + assertFalse(indexExists(INDEX_NAME + "-2")); + + assertHasAliases(Set.of(".test-index", ".test-index-composable-alias")); + + assertAcked( + client().execute( + DeleteComposableIndexTemplateAction.INSTANCE, + new DeleteComposableIndexTemplateAction.Request("test-composable-template") + ).get() + ); + } + private void doCreateTest(Runnable runnable, String concreteIndex) { internalCluster().startNodes(1); @@ -112,6 +198,38 @@ private void doCreateTest(Runnable runnable, String concreteIndex) { ensureGreen(INDEX_NAME); assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings(), concreteIndex); + assertAliases(concreteIndex); + } + + /** + * Make sure that aliases are created hidden + */ + private void assertAliases(String concreteIndex) { + final GetAliasesResponse getAliasesResponse = client().admin() + .indices() + .getAliases(new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())) + .actionGet(); + + assertThat(getAliasesResponse.getAliases().size(), equalTo(1)); + assertThat(getAliasesResponse.getAliases().get(concreteIndex).size(), equalTo(1)); + assertThat(getAliasesResponse.getAliases().get(concreteIndex).get(0).isHidden(), equalTo(true)); + } + + private void assertHasAliases(Set aliasNames) throws InterruptedException, java.util.concurrent.ExecutionException { + final GetAliasesResponse getAliasesResponse = client().admin() + .indices() + .getAliases(new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())) + .get(); + + // Attempting to directly create a non-primary system index only creates the primary index + assertThat(getAliasesResponse.getAliases().size(), equalTo(1)); + assertThat(getAliasesResponse.getAliases().get(PRIMARY_INDEX_NAME).size(), equalTo(2)); + assertThat( + getAliasesResponse.getAliases().get(PRIMARY_INDEX_NAME).stream().map(AliasMetadata::alias).collect(Collectors.toSet()), + equalTo(aliasNames) + ); + assertThat(getAliasesResponse.getAliases().get(PRIMARY_INDEX_NAME).get(0).isHidden(), equalTo(true)); + assertThat(getAliasesResponse.getAliases().get(PRIMARY_INDEX_NAME).get(1).isHidden(), equalTo(true)); } /** diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java new file mode 100644 index 0000000000000..55b217be8943f --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.aliases; + +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.indices.TestSystemIndexPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Collection; +import java.util.stream.Collectors; + +import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME; +import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class SystemIndexAliasIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class); + } + + public void testCreateAliasForSystemIndex() throws Exception { + createIndex(PRIMARY_INDEX_NAME); + ensureGreen(); + assertAcked(admin().indices().prepareAliases().addAlias(PRIMARY_INDEX_NAME, INDEX_NAME + "-system-alias")); + + final GetAliasesResponse getAliasesResponse = client().admin() + .indices() + .getAliases(new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())) + .get(); + + assertThat(getAliasesResponse.getAliases().size(), equalTo(1)); + assertThat(getAliasesResponse.getAliases().get(PRIMARY_INDEX_NAME).size(), equalTo(2)); + assertThat( + getAliasesResponse.getAliases().get(PRIMARY_INDEX_NAME).stream().map(AliasMetadata::alias).collect(Collectors.toSet()), + containsInAnyOrder(".test-index", ".test-index-system-alias") + ); + getAliasesResponse.getAliases().get(PRIMARY_INDEX_NAME).forEach(alias -> assertThat(alias.isHidden(), is(true))); + + assertAcked(client().admin().indices().prepareDeleteTemplate("*").get()); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index 491e9c1f125d4..8378d90d1f307 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -143,7 +143,7 @@ public void testClusterInfoServiceCollectsInformation() { Settings.builder() .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0) .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, randomBoolean()) + .put(IndexMetadata.SETTING_INDEX_HIDDEN, indexName.equals(TEST_SYSTEM_INDEX_NAME) || randomBoolean()) .build() ) ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 616540e8a1bce..b01aa48df6a4c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -60,6 +61,7 @@ public class TransportIndicesAliasesAction extends AcknowledgedTransportMasterNo private final MetadataIndexAliasesService indexAliasesService; private final RequestValidators requestValidators; + private final SystemIndices systemIndices; @Inject public TransportIndicesAliasesAction( @@ -69,7 +71,8 @@ public TransportIndicesAliasesAction( final MetadataIndexAliasesService indexAliasesService, final ActionFilters actionFilters, final IndexNameExpressionResolver indexNameExpressionResolver, - final RequestValidators requestValidators + final RequestValidators requestValidators, + final SystemIndices systemIndices ) { super( IndicesAliasesAction.NAME, @@ -83,6 +86,7 @@ public TransportIndicesAliasesAction( ); this.indexAliasesService = indexAliasesService; this.requestValidators = Objects.requireNonNull(requestValidators); + this.systemIndices = systemIndices; } @Override @@ -209,7 +213,7 @@ protected void masterOperation( action.indexRouting(), action.searchRouting(), action.writeIndex(), - action.isHidden() + systemIndices.isSystemName(resolvedName) ? Boolean.TRUE : action.isHidden() ) ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 6e9c77e99b8c7..d8112d418b4bc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -41,9 +41,12 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; + /** * Api that auto creates an index or data stream that originate from requests that write into an index that doesn't yet exist. */ @@ -187,6 +190,22 @@ public ClusterState execute(ClusterState currentState) throws Exception { } updateRequest = buildSystemIndexUpdateRequest(indexName, descriptor); + } else if (isSystemIndex) { + updateRequest = buildUpdateRequest(indexName); + + if (Objects.isNull(request.settings())) { + updateRequest.settings(SystemIndexDescriptor.DEFAULT_SETTINGS); + } else if (false == request.settings().hasValue(SETTING_INDEX_HIDDEN)) { + updateRequest.settings( + Settings.builder().put(request.settings()).put(SETTING_INDEX_HIDDEN, true).build() + ); + } else if ("false".equals(request.settings().get(SETTING_INDEX_HIDDEN))) { + final String message = "Cannot auto-create system index [" + + indexName + + "] with [index.hidden] set to 'false'"; + logger.warn(message); + throw new IllegalStateException(message); + } } else { updateRequest = buildUpdateRequest(indexName); } @@ -231,7 +250,7 @@ private CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest( updateRequest.settings(settings); } if (aliasName != null) { - updateRequest.aliases(Set.of(new Alias(aliasName))); + updateRequest.aliases(Set.of(new Alias(aliasName).isHidden(true))); } if (logger.isDebugEnabled()) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 627c4943bafa0..62fa64c537b83 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -34,6 +34,9 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; /** * Create index action. @@ -90,7 +93,8 @@ protected void masterOperation( final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index(), resolvedAt); final SystemIndexDescriptor mainDescriptor = systemIndices.findMatchingDescriptor(indexName); - final boolean isSystemIndex = mainDescriptor != null && mainDescriptor.isAutomaticallyManaged(); + final boolean isSystemIndex = mainDescriptor != null; + final boolean isManagedSystemIndex = isSystemIndex && mainDescriptor.isAutomaticallyManaged(); if (mainDescriptor != null && mainDescriptor.isNetNew()) { final SystemIndexAccessLevel systemIndexAccessLevel = systemIndices.getSystemIndexAccessLevel(threadPool.getThreadContext()); if (systemIndexAccessLevel != SystemIndexAccessLevel.ALL) { @@ -107,13 +111,26 @@ protected void masterOperation( } } + if (isSystemIndex) { + if (Objects.isNull(request.settings())) { + request.settings(SystemIndexDescriptor.DEFAULT_SETTINGS); + } else if (false == request.settings().hasValue(SETTING_INDEX_HIDDEN)) { + request.settings(Settings.builder().put(request.settings()).put(SETTING_INDEX_HIDDEN, true).build()); + } else if (Boolean.FALSE.toString().equalsIgnoreCase(request.settings().get(SETTING_INDEX_HIDDEN))) { + final String message = "Cannot create system index [" + indexName + "] with [index.hidden] set to 'false'"; + logger.warn(message); + listener.onFailure(new IllegalStateException(message)); + return; + } + } + final CreateIndexClusterStateUpdateRequest updateRequest; // Requests that a cluster generates itself are permitted to create a system index with // different mappings, settings etc. This is so that rolling upgrade scenarios still work. // We check this via the request's origin. Eventually, `SystemIndexManager` will reconfigure // the index to the latest settings. - if (isSystemIndex && Strings.isNullOrEmpty(request.origin())) { + if (isManagedSystemIndex && Strings.isNullOrEmpty(request.origin())) { final SystemIndexDescriptor descriptor = mainDescriptor.getDescriptorCompatibleWith( state.nodes().getSmallestNonClientNodeVersion() ); @@ -140,11 +157,16 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest( String indexName, long nameResolvedAt ) { + Set aliases = request.aliases().stream().peek(alias -> { + if (systemIndices.isSystemName(alias.name())) { + alias.isHidden(true); + } + }).collect(Collectors.toSet()); return new CreateIndexClusterStateUpdateRequest(cause, indexName, request.index()).ackTimeout(request.timeout()) .masterNodeTimeout(request.masterNodeTimeout()) .settings(request.settings()) .mappings(request.mappings()) - .aliases(request.aliases()) + .aliases(aliases) .nameResolvedInstant(nameResolvedAt) .waitForActiveShards(request.waitForActiveShards()); } @@ -160,9 +182,10 @@ private CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest( if (descriptor.getAliasName() == null) { aliases = Set.of(); } else { - aliases = Set.of(new Alias(descriptor.getAliasName())); + aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true)); } + // Here, we override the user's requested index with the descriptor's primary index final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest( cause, descriptor.getPrimaryIndex(), diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index f2e174c092dc0..e320fb07eff26 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -109,10 +109,10 @@ protected void masterOperation( return; } - final List hiddenSystemIndexViolations = checkForHidingSystemIndex(concreteIndices, request); - if (hiddenSystemIndexViolations.isEmpty() == false) { - final String message = "Cannot set [index.hidden] to 'true' on system indices: " - + hiddenSystemIndexViolations.stream().map(entry -> "[" + entry + "]").collect(Collectors.joining(", ")); + final List unhiddenSystemIndexViolations = checkForUnhidingSystemIndex(concreteIndices, request); + if (unhiddenSystemIndexViolations.isEmpty() == false) { + final String message = "Cannot set [index.hidden] to 'false' on system indices: " + + unhiddenSystemIndexViolations.stream().map(entry -> "[" + entry + "]").collect(Collectors.joining(", ")); logger.warn(message); listener.onFailure(new IllegalStateException(message)); return; @@ -174,17 +174,17 @@ private Map> checkForSystemIndexViolations(Index[] concrete } /** - * Checks that the request isn't trying to add the "hidden" setting to a system + * Checks that the request isn't trying to remove the "hidden" setting on a system * index * * @param concreteIndices the indices being updated * @param request the update request - * @return a list of system indexes that this request would set to hidden + * @return a list of system indexes that this request would make visible */ - private List checkForHidingSystemIndex(Index[] concreteIndices, UpdateSettingsRequest request) { + private List checkForUnhidingSystemIndex(Index[] concreteIndices, UpdateSettingsRequest request) { // Requests that a cluster generates itself are permitted to have a difference in settings // so that rolling upgrade scenarios still work. We check this via the request's origin. - if (request.settings().getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false) == false) { + if (request.settings().getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, true)) { return List.of(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 157e51cc5cc57..65ed6ad1bcb9e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -59,6 +60,7 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea private final NamedXContentRegistry xContentRegistry; private final IndicesService indicesService; private final AliasValidator aliasValidator; + private final SystemIndices systemIndices; @Inject public TransportSimulateIndexTemplateAction( @@ -69,7 +71,8 @@ public TransportSimulateIndexTemplateAction( ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, - IndicesService indicesService + IndicesService indicesService, + SystemIndices systemIndices ) { super( SimulateIndexTemplateAction.NAME, @@ -86,6 +89,7 @@ public TransportSimulateIndexTemplateAction( this.xContentRegistry = xContentRegistry; this.indicesService = indicesService; this.aliasValidator = new AliasValidator(); + this.systemIndices = systemIndices; } @Override @@ -131,7 +135,8 @@ protected void masterOperation( stateWithTemplate, xContentRegistry, indicesService, - aliasValidator + aliasValidator, + systemIndices ); final Map> overlapping = new HashMap<>(); @@ -182,7 +187,8 @@ public static Template resolveTemplate( final ClusterState simulatedState, final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, - final AliasValidator aliasValidator + final AliasValidator aliasValidator, + final SystemIndices systemIndices ) throws Exception { Settings settings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -217,7 +223,8 @@ public static Template resolveTemplate( // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp tempIndexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - tempIndexService.dateMathExpressionResolverAt() + tempIndexService.dateMathExpressionResolverAt(), + systemIndices::isSystemName ) ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index f02a5757ccc95..6e726805fd549 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -48,6 +49,7 @@ public class TransportSimulateTemplateAction extends TransportMasterNodeReadActi private final NamedXContentRegistry xContentRegistry; private final IndicesService indicesService; private AliasValidator aliasValidator; + private final SystemIndices systemIndices; @Inject public TransportSimulateTemplateAction( @@ -58,7 +60,8 @@ public TransportSimulateTemplateAction( ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, - IndicesService indicesService + IndicesService indicesService, + SystemIndices systemIndices ) { super( SimulateTemplateAction.NAME, @@ -75,6 +78,7 @@ public TransportSimulateTemplateAction( this.xContentRegistry = xContentRegistry; this.indicesService = indicesService; this.aliasValidator = new AliasValidator(); + this.systemIndices = systemIndices; } @Override @@ -152,7 +156,8 @@ protected void masterOperation( stateWithTemplate, xContentRegistry, indicesService, - aliasValidator + aliasValidator, + systemIndices ); listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index a028a144ff6a1..dd56212398926 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -138,6 +138,7 @@ public DataStream( this.indices = Collections.unmodifiableList(indices); this.generation = generation; this.metadata = metadata; + assert system == false || hidden; // system indices must be hidden this.hidden = hidden; this.replicated = replicated; this.timeProvider = timeProvider; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 40c33011feae8..b8266b5e1bc72 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; @@ -202,7 +203,9 @@ static ClusterState createDataStream( firstBackingIndexName ).dataStreamName(dataStreamName).systemDataStreamDescriptor(systemDataStreamDescriptor); - if (isSystem == false) { + if (isSystem) { + createIndexRequest.settings(SystemIndexDescriptor.DEFAULT_SETTINGS); + } else { createIndexRequest.settings(MetadataRolloverService.HIDDEN_INDEX_SETTINGS); } @@ -228,7 +231,7 @@ static ClusterState createDataStream( DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName); List dsBackingIndices = backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()); dsBackingIndices.add(writeIndex.getIndex()); - boolean hidden = isSystem ? false : template.getDataStreamTemplate().isHidden(); + boolean hidden = isSystem || template.getDataStreamTemplate().isHidden(); DataStream newDataStream = new DataStream( dataStreamName, timestampField, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index ab0893aeaf5d1..bb65a61831a6b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -566,7 +566,8 @@ private ClusterState applyCreateIndexRequestWithV1Templates( // shard id and the current timestamp xContentRegistry, indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()) + indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()), + systemIndices::isSystemName ), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer @@ -632,7 +633,8 @@ private ClusterState applyCreateIndexRequestWithV2Template( xContentRegistry, // the context is used ony for validation so it's fine to pass fake values for the shard id and the current timestamp indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()) + indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()), + systemIndices::isSystemName ), Collections.singletonList(templateName), metadataTransformer @@ -688,7 +690,8 @@ private ClusterState applyCreateIndexRequestForSystemDataStream( aliasValidator, xContentRegistry, indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()) + indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()), + systemIndices::isSystemName ), List.of(), metadataTransformer @@ -783,7 +786,8 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()) + indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()), + systemIndices::isSystemName ), List.of(), metadataTransformer @@ -1027,7 +1031,8 @@ public static List resolveAndValidateAliases( AliasValidator aliasValidator, NamedXContentRegistry xContentRegistry, SearchExecutionContext searchExecutionContext, - Function indexNameExpressionResolver + Function indexNameExpressionResolver, + Predicate systemNamePredicate ) { // Keep a separate set to facilitate searches when processing aliases from the template @@ -1040,12 +1045,13 @@ public static List resolveAndValidateAliases( if (Strings.hasLength(alias.filter())) { aliasValidator.validateAliasFilter(alias.name(), alias.filter(), searchExecutionContext, xContentRegistry); } + AliasMetadata aliasMetadata = AliasMetadata.builder(alias.name()) .filter(alias.filter()) .indexRouting(alias.indexRouting()) .searchRouting(alias.searchRouting()) .writeIndex(alias.writeIndex()) - .isHidden(alias.isHidden()) + .isHidden(systemNamePredicate.test(alias.name()) ? Boolean.TRUE : alias.isHidden()) .build(); resolvedAliases.add(aliasMetadata); resolvedExpressions.add(new Alias(resolvedExpression)); @@ -1075,6 +1081,17 @@ public static List resolveAndValidateAliases( aliasMetadata = AliasMetadata.newAliasMetadata(aliasMetadata, templatedAlias); } + // set system aliases from templates to hidden + if (systemNamePredicate.test(aliasMetadata.alias())) { + aliasMetadata = AliasMetadata.builder(aliasMetadata.alias()) + .filter(aliasMetadata.filter()) + .indexRouting(aliasMetadata.indexRouting()) + .searchRouting(aliasMetadata.searchRouting()) + .writeIndex(aliasMetadata.writeIndex()) + .isHidden(true) + .build(); + } + aliasValidator.validateAliasMetadata(aliasMetadata, index, metadata); if (aliasMetadata.filter() != null) { aliasValidator.validateAliasFilter( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index a4507e90d59e7..c5848880b0ab2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -44,6 +44,7 @@ import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; @@ -113,6 +114,7 @@ public class MetadataIndexTemplateService { private final MetadataCreateIndexService metadataCreateIndexService; private final IndexScopedSettings indexScopedSettings; private final NamedXContentRegistry xContentRegistry; + private final SystemIndices systemIndices; @Inject public MetadataIndexTemplateService( @@ -121,7 +123,8 @@ public MetadataIndexTemplateService( AliasValidator aliasValidator, IndicesService indicesService, IndexScopedSettings indexScopedSettings, - NamedXContentRegistry xContentRegistry + NamedXContentRegistry xContentRegistry, + SystemIndices systemIndices ) { this.clusterService = clusterService; this.aliasValidator = aliasValidator; @@ -129,6 +132,7 @@ public MetadataIndexTemplateService( this.metadataCreateIndexService = metadataCreateIndexService; this.indexScopedSettings = indexScopedSettings; this.xContentRegistry = xContentRegistry; + this.systemIndices = systemIndices; } public void removeTemplates(final RemoveRequest request, final RemoveListener listener) { @@ -291,7 +295,8 @@ ClusterState addComponentTemplate( composableTemplateName, composableTemplate, indicesService, - xContentRegistry + xContentRegistry, + systemIndices ); } catch (Exception e) { if (validationFailure == null) { @@ -605,7 +610,7 @@ public ClusterState addIndexTemplateV2( // Finally, right before adding the template, we need to ensure that the composite settings, // mappings, and aliases are valid after it's been composed with the component templates try { - validateCompositeTemplate(currentState, name, finalIndexTemplate, indicesService, xContentRegistry); + validateCompositeTemplate(currentState, name, finalIndexTemplate, indicesService, xContentRegistry, systemIndices); } catch (Exception e) { throw new IllegalArgumentException( "composable template [" @@ -1309,7 +1314,8 @@ private static void validateCompositeTemplate( final String templateName, final ComposableIndexTemplate template, final IndicesService indicesService, - final NamedXContentRegistry xContentRegistry + final NamedXContentRegistry xContentRegistry, + final SystemIndices systemIndices ) throws Exception { final ClusterState stateWithTemplate = ClusterState.builder(state) .metadata(Metadata.builder(state.metadata()).put(templateName, template)) @@ -1356,7 +1362,8 @@ private static void validateCompositeTemplate( // shard id and the current timestamp xContentRegistry, tempIndexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - tempIndexService.dateMathExpressionResolverAt(System.currentTimeMillis()) + tempIndexService.dateMathExpressionResolverAt(System.currentTimeMillis()), + systemIndices::isSystemName ); // triggers inclusion of _timestamp field and its validation: diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java index eb86a2c9c2c8e..dae71851a294f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java @@ -93,12 +93,26 @@ public ClusterState execute(ClusterState currentState) throws Exception { builder.system(indexMetadata.isSystem() == false); updated = true; } - if (isSystem && indexMetadata.getSettings().getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false)) { - builder.settings( - Settings.builder().put(indexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_HIDDEN, false) - ); + boolean isHidden = indexMetadata.getSettings().getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false); + if (isSystem && isHidden == false) { + builder.settings(Settings.builder().put(indexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_HIDDEN, true)); updated = true; } + if (isSystem && indexMetadata.getAliases().values().stream().anyMatch(a -> a.isHidden() == false)) { + for (AliasMetadata aliasMetadata : indexMetadata.getAliases().values()) { + if (aliasMetadata.isHidden() == false) { + builder.removeAlias(aliasMetadata.alias()); + builder.putAlias( + AliasMetadata.builder(aliasMetadata.alias()) + .filter(aliasMetadata.filter()) + .indexRouting(aliasMetadata.indexRouting()) + .isHidden(true) + .searchRouting(aliasMetadata.searchRouting()) + .writeIndex(aliasMetadata.writeIndex()) + ); + } + } + } if (updated) { updatedMetadata.add(builder.build()); } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 0940c220947a5..4d9c4589f86cc 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -40,6 +40,9 @@ * creating the system index, upgrading its mappings, and creating an alias. */ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable { + + public static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(); + /** * A pattern, either with a wildcard or simple regex. Indices that match one of these patterns are considered system indices. * Note that this pattern must not overlap with any other {@link SystemIndexDescriptor}s and must allow an alphanumeric suffix @@ -326,10 +329,17 @@ public SystemIndexDescriptor(String indexPattern, String description, Type type, this.description = description; this.mappings = mappings; - if (Objects.nonNull(settings) && settings.getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false)) { - throw new IllegalArgumentException("System indices cannot have " + IndexMetadata.SETTING_INDEX_HIDDEN + " set to true."); + settings = Objects.isNull(settings) ? Settings.EMPTY : settings; + + if (settings.hasValue(IndexMetadata.SETTING_INDEX_HIDDEN) == false) { + settings = Settings.builder().put(settings).put(DEFAULT_SETTINGS).build(); + } + + if (settings.getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false)) { + this.settings = settings; + } else { + throw new IllegalArgumentException("System indices must have " + IndexMetadata.SETTING_INDEX_HIDDEN + " set to true."); } - this.settings = settings; this.indexFormat = indexFormat; this.versionMetaKey = versionMetaKey; this.origin = origin; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java new file mode 100644 index 0000000000000..a9c67bbd82f19 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.TransportService; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class TransportCreateIndexActionTests extends ESTestCase { + + private static final ClusterState CLUSTER_STATE = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().build()) + .build(); + + private static final String SYSTEM_INDEX_NAME = ".my-system"; + private static final String SYSTEM_ALIAS_NAME = ".my-alias"; + private static final SystemIndices SYSTEM_INDICES = new SystemIndices( + Map.of( + "test-feature", + new SystemIndices.Feature( + "test-feature", + "a test feature", + List.of( + SystemIndexDescriptor.builder() + .setIndexPattern(SYSTEM_INDEX_NAME + "*") + .setType(SystemIndexDescriptor.Type.INTERNAL_UNMANAGED) + .setAliasName(SYSTEM_ALIAS_NAME) + .build() + ) + ) + ) + ); + + private MetadataCreateIndexService metadataCreateIndexService; + private TransportCreateIndexAction action; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); + this.metadataCreateIndexService = mock(MetadataCreateIndexService.class); + this.action = new TransportCreateIndexAction( + mock(TransportService.class), + mock(ClusterService.class), + null, + metadataCreateIndexService, + mock(ActionFilters.class), + indexNameExpressionResolver, + SYSTEM_INDICES + ); + } + + public void testSystemIndicesCannotBeCreatedUnhidden() { + CreateIndexRequest request = new CreateIndexRequest(); + request.settings(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, false).build()); + request.index(SYSTEM_INDEX_NAME); + + @SuppressWarnings("unchecked") + ActionListener mockListener = mock(ActionListener.class); + + action.masterOperation(mock(Task.class), request, CLUSTER_STATE, mockListener); + + ArgumentCaptor exceptionArgumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(mockListener, times(0)).onResponse(any()); + verify(metadataCreateIndexService, times(0)).createIndex(any(), any()); + verify(mockListener, times(1)).onFailure(exceptionArgumentCaptor.capture()); + + Exception e = exceptionArgumentCaptor.getValue(); + assertThat(e.getMessage(), equalTo("Cannot create system index [.my-system] with [index.hidden] set to 'false'")); + } + + public void testSystemIndicesCreatedHiddenByDefault() { + CreateIndexRequest request = new CreateIndexRequest(); + request.index(SYSTEM_INDEX_NAME); + + @SuppressWarnings("unchecked") + ActionListener mockListener = mock(ActionListener.class); + + action.masterOperation(mock(Task.class), request, CLUSTER_STATE, mockListener); + + ArgumentCaptor createRequestArgumentCaptor = ArgumentCaptor.forClass( + CreateIndexClusterStateUpdateRequest.class + ); + verify(mockListener, times(0)).onFailure(any()); + verify(metadataCreateIndexService, times(1)).createIndex(createRequestArgumentCaptor.capture(), any()); + + CreateIndexClusterStateUpdateRequest processedRequest = createRequestArgumentCaptor.getValue(); + assertTrue(processedRequest.settings().getAsBoolean(SETTING_INDEX_HIDDEN, false)); + } + + public void testSystemAliasCreatedHiddenByDefault() { + CreateIndexRequest request = new CreateIndexRequest(); + request.index(SYSTEM_INDEX_NAME); + request.alias(new Alias(SYSTEM_ALIAS_NAME)); + + @SuppressWarnings("unchecked") + ActionListener mockListener = mock(ActionListener.class); + + action.masterOperation(mock(Task.class), request, CLUSTER_STATE, mockListener); + + ArgumentCaptor createRequestArgumentCaptor = ArgumentCaptor.forClass( + CreateIndexClusterStateUpdateRequest.class + ); + verify(mockListener, times(0)).onFailure(any()); + verify(metadataCreateIndexService, times(1)).createIndex(createRequestArgumentCaptor.capture(), any()); + + CreateIndexClusterStateUpdateRequest processedRequest = createRequestArgumentCaptor.getValue(); + assertTrue(processedRequest.aliases().contains(new Alias(SYSTEM_ALIAS_NAME).isHidden(true))); + } + +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java index ca804b1d38d96..26f036ae335a8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java @@ -88,8 +88,10 @@ public void setUp() throws Exception { ); } - public void testSystemIndicesCannotBeSetToHidden() { - UpdateSettingsRequest request = new UpdateSettingsRequest(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build()); + public void testSystemIndicesCannotBeSetToVisible() { + UpdateSettingsRequest request = new UpdateSettingsRequest( + Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, false).build() + ); request.indices(SYSTEM_INDEX_NAME); @SuppressWarnings("unchecked") @@ -102,6 +104,6 @@ public void testSystemIndicesCannotBeSetToHidden() { verify(mockListener, times(1)).onFailure(exceptionArgumentCaptor.capture()); Exception e = exceptionArgumentCaptor.getValue(); - assertThat(e.getMessage(), equalTo("Cannot set [index.hidden] to 'true' on system indices: [.my-system]")); + assertThat(e.getMessage(), equalTo("Cannot set [index.hidden] to 'false' on system indices: [.my-system]")); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 12669e36168bf..540982fa631d5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -2267,9 +2267,10 @@ public void testIgnoreThrottled() { } } - public void testFullWildcardSystemIndexResolutionAllowed() { + public void testFullWildcardSystemIndexResolutionWithExpandHiddenAllowed() { ClusterState state = systemIndexTestClusterState(); SearchRequest request = new SearchRequest(randomFrom("*", "_all")); + request.indicesOptions(IndicesOptions.strictExpandHidden()); List indexNames = resolveConcreteIndexNameList(state, request); assertThat(indexNames, containsInAnyOrder("some-other-index", ".ml-stuff", ".ml-meta", ".watches")); @@ -2299,10 +2300,19 @@ public void testSingleSystemIndexResolutionAllowed() { assertThat(indexNames, containsInAnyOrder(".ml-meta")); } + public void testFullWildcardSystemIndicesAreHidden() { + ClusterState state = systemIndexTestClusterState(); + SearchRequest request = new SearchRequest(randomFrom("*", "_all")); + + List indexNames = resolveConcreteIndexNameList(state, request); + assertThat(indexNames, containsInAnyOrder("some-other-index")); + } + public void testFullWildcardSystemIndexResolutionDeprecated() { threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.FALSE.toString()); ClusterState state = systemIndexTestClusterState(); SearchRequest request = new SearchRequest(randomFrom("*", "_all")); + request.indicesOptions(IndicesOptions.strictExpandHidden()); List indexNames = resolveConcreteIndexNameList(state, request); assertThat(indexNames, containsInAnyOrder("some-other-index", ".ml-stuff", ".ml-meta", ".watches")); @@ -2334,7 +2344,7 @@ public void testSingleSystemIndexResolutionDeprecated() { ); } - public void testWildcardSystemIndexReslutionSingleMatchDeprecated() { + public void testWildcardSystemIndexResolutionSingleMatchDeprecated() { threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.FALSE.toString()); ClusterState state = systemIndexTestClusterState(); SearchRequest request = new SearchRequest(".w*"); @@ -2952,11 +2962,10 @@ public void testRemoteIndex() { } private ClusterState systemIndexTestClusterState() { - Settings settings = Settings.builder().build(); Metadata.Builder mdBuilder = Metadata.builder() - .put(indexBuilder(".ml-meta", settings).state(State.OPEN).system(true)) - .put(indexBuilder(".watches", settings).state(State.OPEN).system(true)) - .put(indexBuilder(".ml-stuff", settings).state(State.OPEN).system(true)) + .put(indexBuilder(".ml-meta", SystemIndexDescriptor.DEFAULT_SETTINGS).state(State.OPEN).system(true)) + .put(indexBuilder(".watches", SystemIndexDescriptor.DEFAULT_SETTINGS).state(State.OPEN).system(true)) + .put(indexBuilder(".ml-stuff", SystemIndexDescriptor.DEFAULT_SETTINGS).state(State.OPEN).system(true)) .put(indexBuilder("some-other-index").state(State.OPEN)); SystemIndices systemIndices = new SystemIndices( Map.of( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 2cf5a921a767e..57cbfb4d07503 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -37,7 +37,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; @@ -206,12 +205,12 @@ public void testCreateSystemDataStream() throws Exception { assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(true)); - assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(true)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue()); assertThat( newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"), - nullValue() + equalTo("true") ); assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).isSystem(), is(true)); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 2d8688d730215..9f709a231b811 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -743,7 +743,8 @@ public void testInvalidAliasName() { aliasValidator, xContentRegistry(), searchExecutionContext, - indexNameExpressionResolver::resolveDateMathExpression + indexNameExpressionResolver::resolveDateMathExpression, + m -> false ) ); } @@ -761,7 +762,8 @@ public void testAliasNameWithMathExpression() { aliasValidator, xContentRegistry(), searchExecutionContext, - indexNameExpressionResolver::resolveDateMathExpression + indexNameExpressionResolver::resolveDateMathExpression, + m -> false ); assertEquals("date-math-based-2021-01-01", aliasMetadata.get(0).alias()); @@ -795,7 +797,8 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { aliasValidator, xContentRegistry(), searchExecutionContext, - indexNameExpressionResolver::resolveDateMathExpression + indexNameExpressionResolver::resolveDateMathExpression, + m -> false ); Settings aggregatedIndexSettings = aggregateIndexSettings( @@ -891,7 +894,8 @@ public void testTemplateOrder() throws Exception { aliasValidator, xContentRegistry(), searchExecutionContext, - indexNameExpressionResolver::resolveDateMathExpression + indexNameExpressionResolver::resolveDateMathExpression, + m -> false ); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("12")); @@ -932,7 +936,8 @@ public void testResolvedAliasInTemplate() { aliasValidator, xContentRegistry(), searchExecutionContext, - indexNameExpressionResolver::resolveDateMathExpression + indexNameExpressionResolver::resolveDateMathExpression, + m -> false ); assertThat(resolvedAliases.get(0).alias(), equalTo("jan-2021-01-01")); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 3e945efbbd947..ae5b340138cce 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -2110,7 +2110,8 @@ private static List putTemplate(NamedXContentRegistry xContentRegistr new AliasValidator(), null, new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), - xContentRegistry + xContentRegistry, + EmptySystemIndices.INSTANCE ); final List throwables = new ArrayList<>(); @@ -2172,7 +2173,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { new AliasValidator(), indicesService, new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), - xContentRegistry() + xContentRegistry(), + EmptySystemIndices.INSTANCE ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java index c46c8d9f2d57f..7fe4ceb634585 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java @@ -29,9 +29,11 @@ public class SystemIndexMetadataUpgradeServiceTests extends ESTestCase { private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; private static final String SYSTEM_INDEX_NAME = ".myindex-1"; + private static final String SYSTEM_ALIAS_NAME = ".myindex-alias"; private static final SystemIndexDescriptor DESCRIPTOR = SystemIndexDescriptor.builder() .setIndexPattern(".myindex-*") .setPrimaryIndex(SYSTEM_INDEX_NAME) + .setAliasName(SYSTEM_ALIAS_NAME) .setSettings(getSettingsBuilder().build()) .setMappings(MAPPINGS) .setVersionMetaKey("version") @@ -51,31 +53,49 @@ public void setUpTest() { /** * When we upgrade Elasticsearch versions, existing indices may be newly - * defined as system indices. If such indices are set to "hidden," we need - * to remove that setting. + * defined as system indices. If such indices are set without "hidden," we need + * to update that setting. */ - public void testUpgradeHiddenIndexToSystemIndex() throws Exception { + public void testUpgradeVisibleIndexToSystemIndex() throws Exception { // create an initial cluster state with a hidden index that matches the system index descriptor IndexMetadata.Builder hiddenIndexMetadata = IndexMetadata.builder(SYSTEM_INDEX_NAME) .system(false) - .settings(getSettingsBuilder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true)); + .settings(getSettingsBuilder().put(IndexMetadata.SETTING_INDEX_HIDDEN, false)); - assertSystemUpgradeRemovesHiddenSetting(hiddenIndexMetadata); + assertSystemUpgradeAppliesHiddenSetting(hiddenIndexMetadata); } /** - * If a system index erroneously is set to hidden, we should remedy that situation. + * If a system index erroneously is set to visible, we should remedy that situation. */ public void testHiddenSettingRemovedFromSystemIndices() throws Exception { // create an initial cluster state with a hidden index that matches the system index descriptor IndexMetadata.Builder hiddenIndexMetadata = IndexMetadata.builder(SYSTEM_INDEX_NAME) .system(true) - .settings(getSettingsBuilder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true)); + .settings(getSettingsBuilder().put(IndexMetadata.SETTING_INDEX_HIDDEN, false)); - assertSystemUpgradeRemovesHiddenSetting(hiddenIndexMetadata); + assertSystemUpgradeAppliesHiddenSetting(hiddenIndexMetadata); } - private void assertSystemUpgradeRemovesHiddenSetting(IndexMetadata.Builder hiddenIndexMetadata) throws Exception { + public void testUpgradeIndexWithVisibleAlias() throws Exception { + IndexMetadata.Builder visibleAliasMetadata = IndexMetadata.builder(SYSTEM_INDEX_NAME) + .system(false) + .settings(getSettingsBuilder()) + .putAlias(AliasMetadata.builder(SYSTEM_ALIAS_NAME).isHidden(false)); + + assertSystemUpgradeHidesAlias(visibleAliasMetadata); + } + + public void testSystemAliasesBecomeHidden() throws Exception { + IndexMetadata.Builder visibleAliasMetadata = IndexMetadata.builder(SYSTEM_INDEX_NAME) + .system(true) + .settings(getSettingsBuilder()) + .putAlias(AliasMetadata.builder(SYSTEM_ALIAS_NAME).isHidden(false)); + + assertSystemUpgradeHidesAlias(visibleAliasMetadata); + } + + private void assertSystemUpgradeAppliesHiddenSetting(IndexMetadata.Builder hiddenIndexMetadata) throws Exception { Metadata.Builder clusterMetadata = new Metadata.Builder(); clusterMetadata.put(hiddenIndexMetadata); @@ -89,7 +109,24 @@ private void assertSystemUpgradeRemovesHiddenSetting(IndexMetadata.Builder hidde IndexMetadata result = newState.metadata().index(SYSTEM_INDEX_NAME); assertThat(result.isSystem(), equalTo(true)); - assertThat(result.getSettings().get(IndexMetadata.SETTING_INDEX_HIDDEN), equalTo("false")); + assertThat(result.isHidden(), equalTo(true)); + } + + private void assertSystemUpgradeHidesAlias(IndexMetadata.Builder visibleAliasMetadata) throws Exception { + Metadata.Builder clusterMetadata = new Metadata.Builder(); + clusterMetadata.put(visibleAliasMetadata); + + ClusterState clusterState = ClusterState.builder(new ClusterName("system-index-metadata-upgrade-service-tests")) + .metadata(clusterMetadata.build()) + .customs(ImmutableOpenMap.of()) + .build(); + + // Get a metadata upgrade task and execute it on the initial cluster state + ClusterState newState = service.getTask().execute(clusterState); + + IndexMetadata result = newState.metadata().index(SYSTEM_INDEX_NAME); + assertThat(result.isSystem(), equalTo(true)); + assertThat(result.getAliases().values().stream().allMatch(AliasMetadata::isHidden), equalTo(true)); } private static Settings.Builder getSettingsBuilder() { diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 467a405f9442c..7720304196dce 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -249,7 +249,7 @@ public void testGetDescriptorCompatibleWith() { assertSame(prior, compat); } - public void testSystemIndicesCannotAlsoBeHidden() { + public void testSystemIndicesMustBeHidden() { SystemIndexDescriptor.Builder builder = SystemIndexDescriptor.builder() .setIndexPattern(".system*") .setDescription("system stuff") @@ -260,11 +260,11 @@ public void testSystemIndicesCannotAlsoBeHidden() { .setVersionMetaKey("version") .setOrigin("system"); - builder.setSettings(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build()); + builder.setSettings(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, false).build()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); - assertThat(e.getMessage(), equalTo("System indices cannot have index.hidden set to true.")); + assertThat(e.getMessage(), equalTo("System indices must have index.hidden set to true.")); } public void testSpecialCharactersAreReplacedWhenConvertingToAutomaton() { diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java index 38fc0f9982f0d..61c54db7a7337 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java @@ -334,10 +334,10 @@ private static IndexMetadata.Builder getIndexMetadata( ); final Settings.Builder settingsBuilder = Settings.builder(); - if (descriptor.getSettings() != null) { - settingsBuilder.put(descriptor.getSettings()); - } else { + if (SystemIndexDescriptor.DEFAULT_SETTINGS.equals(descriptor.getSettings())) { settingsBuilder.put(getSettings()); + } else { + settingsBuilder.put(descriptor.getSettings()); } settingsBuilder.put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), format); indexMetadata.settings(settingsBuilder.build()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index cdcca062638f8..bcdedca931115 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -1045,7 +1045,7 @@ public DatafeedConfig build() { setDefaultQueryDelay(); if (indicesOptions == null) { - indicesOptions = SearchRequest.DEFAULT_INDICES_OPTIONS; + indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED; } return new DatafeedConfig( id, diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java index 0bdd822733329..caa724c035b83 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java @@ -112,8 +112,21 @@ public void testSystemDataStreamCRUD() throws Exception { Response putResponse = restClient.performRequest(putRequest); assertThat(putResponse.getStatusLine().getStatusCode(), is(200)); + // system data streams are hidden + Request listAllVisibleRequest = new Request("GET", "/_data_stream"); + Response listAllVisibleResponse = restClient.performRequest(listAllVisibleRequest); + assertThat(listAllVisibleResponse.getStatusLine().getStatusCode(), is(200)); + Map visibleResponseMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(listAllVisibleResponse.getEntity()), + false + ); + List visibleDataStreams = (List) visibleResponseMap.get("data_streams"); + assertThat(visibleDataStreams.size(), is(0)); + // list - no header needed Request listAllRequest = new Request("GET", "/_data_stream"); + listAllRequest.addParameter("expand_wildcards", "open,hidden"); Response listAllResponse = restClient.performRequest(listAllRequest); assertThat(listAllResponse.getStatusLine().getStatusCode(), is(200)); Map responseMap = XContentHelper.convertToMap( @@ -158,6 +171,7 @@ public void testDataStreamStats() throws Exception { assertThat(putResponse.getStatusLine().getStatusCode(), is(200)); Request statsRequest = new Request("GET", "/_data_stream/_stats"); + statsRequest.addParameter("expand_wildcards", "open,hidden"); Response response = restClient.performRequest(statsRequest); assertThat(response.getStatusLine().getStatusCode(), is(200)); @@ -224,6 +238,7 @@ public void testSystemDataStreamReadWrite() throws Exception { // search all Request search = new Request("GET", "/_search"); + search.addParameter("expand_wildcards", "open,hidden"); search.setJsonEntity("{ \"query\": { \"match_all\": {} } }"); // no header diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java index eb46884f16395..c99c4f8344566 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java @@ -207,12 +207,7 @@ public void testCountWithSystemDataStream() throws Exception { // Create a system index - this one has an alias Request sysIdxRequest = new Request("PUT", ".fleet-artifacts"); assertOK(adminClient().performRequest(sysIdxRequest)); - assertThatAPIWildcardResolutionWorks( - singletonList( - "this request accesses system indices: [.fleet-artifacts-7], but in a future major version, direct access to system" - + " indices will be prevented by default" - ) - ); + assertThatAPIWildcardResolutionWorks(); assertThatAPIWildcardResolutionWorks( singletonList( "this request accesses system indices: [.fleet-artifacts-7], but in a future major version, direct access to system" @@ -227,12 +222,7 @@ public void testCountWithSystemDataStream() throws Exception { Request regularIdxRequest = new Request("PUT", regularIndex); regularIdxRequest.setJsonEntity("{\"aliases\": {\"" + regularAlias + "\": {}}}"); assertOK(client().performRequest(regularIdxRequest)); - assertThatAPIWildcardResolutionWorks( - singletonList( - "this request accesses system indices: [.fleet-artifacts-7], but in a future major version, direct access to system" - + " indices will be prevented by default" - ) - ); + assertThatAPIWildcardResolutionWorks(); assertThatAPIWildcardResolutionWorks(emptyList(), "r*"); } @@ -240,10 +230,6 @@ private void assertThatAPIWildcardResolutionWorks() throws Exception { assertThatAPIWildcardResolutionWorks(emptyList(), null); } - private void assertThatAPIWildcardResolutionWorks(List warningsExpected) throws Exception { - assertThatAPIWildcardResolutionWorks(warningsExpected, null); - } - private void assertThatAPIWildcardResolutionWorks(List warningsExpected, String indexPattern) throws Exception { String path = indexPattern == null || indexPattern.isEmpty() ? "/_count" : "/" + indexPattern + "/_count"; Request countRequest = new Request("GET", path); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java index 28b7751e99489..346f1a1790dc2 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java @@ -51,8 +51,7 @@ public void testCannotMountSnapshotBlobCacheIndex() throws Exception { } private void executeTest(final String indexName, final String featureName, final Client client) throws Exception { - final boolean isHidden = randomBoolean(); - createAndPopulateIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, isHidden)); + createAndPopulateIndex(indexName, Settings.builder()); final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createRepository(repositoryName, "fs"); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java index c41ec68cbb7db..921b2ad6fae8b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.security.DeleteRoleRequest; @@ -89,13 +90,19 @@ public void testSnapshotUserRoleCanSnapshotAndSeeAllIndices() { assertThat(getRepositoriesResponse.repositories().size(), is(1)); assertThat(getRepositoriesResponse.repositories().get(0).name(), is("repo")); // view all indices, including restricted ones - final GetIndexResponse getIndexResponse = client.admin().indices().prepareGetIndex().setIndices(randomFrom("_all", "*")).get(); + final GetIndexResponse getIndexResponse = client.admin() + .indices() + .prepareGetIndex() + .setIndices(randomFrom("_all", "*")) + .setIndicesOptions(IndicesOptions.strictExpandHidden()) + .get(); assertThat(Arrays.asList(getIndexResponse.indices()), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex)); // create snapshot that includes restricted indices final CreateSnapshotResponse snapshotResponse = client.admin() .cluster() .prepareCreateSnapshot("repo", "snap") .setIndices(randomFrom("_all", "*")) + .setIndicesOptions(IndicesOptions.strictExpandHidden()) .setWaitForCompletion(true) .get(); assertThat(snapshotResponse.getSnapshotInfo().state(), is(SnapshotState.SUCCESS)); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java index 95551a73339be..c4d1a6604cf36 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse; import org.elasticsearch.xpack.core.watcher.execution.ExecutionState; @@ -47,7 +48,11 @@ public void testTimeFields() throws Exception { assertWatchWithMinimumActionsCount("_id", ExecutionState.EXECUTED, 1); assertBusy(() -> { - GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings().get(); + GetMappingsResponse mappingsResponse = client().admin() + .indices() + .prepareGetMappings() + .setIndicesOptions(IndicesOptions.strictExpandHidden()) + .get(); assertThat(mappingsResponse, notNullValue()); assertThat(mappingsResponse.getMappings().isEmpty(), is(false)); for (ObjectObjectCursor metadatas : mappingsResponse.getMappings()) { diff --git a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml index 22441992a4f8b..36ea0b65f2aa5 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml @@ -22,7 +22,7 @@ - match: {indices.2.attributes.1: open} - match: {indices.2.data_stream: simple-data-stream2} - match: {indices.3.name: my_remote_cluster:.security-7} - - match: {indices.3.attributes.0: open} + - match: {indices.3.attributes.0: hidden} - match: {indices.4.name: my_remote_cluster:closed_index} - match: {indices.4.aliases.0: aliased_closed_index} - match: {indices.4.attributes.0: closed}