Migrate .tasks to be auto-managed (#66640)

Backport of #65959.

Part of #61656. Change the `.tasks` system index descriptor so that
the index can be automatically managed by Elasticsearch e.g. created
on-demand, mapping kept up-to-date, etc.

Also add an integration test to exercise the `SystemIndexManager`
end-to-end, and cherry-pick #66605 to add more system index tests.

server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java

@@ -0,0 +1,178 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.indices.TestSystemIndexDescriptor;
+import org.elasticsearch.indices.TestSystemIndexPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME;
+import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME;
+import static org.elasticsearch.test.XContentTestUtils.convertToXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * These tests exercise various scenarios where a system index is created, and ensure that when
+ * this happens then the correct mappings and settings are applied.
+ */
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class CreateSystemIndicesIT extends ESIntegTestCase {
+
+    @Before
+    public void beforeEach() {
+        TestSystemIndexDescriptor.useNewMappings.set(false);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class);
+    }
+
+    /**
+     * Check that a system index is auto-created with the expected mappings and
+     * settings when it is first used, when it is referenced via its alias.
+     */
+    public void testSystemIndexIsAutoCreatedViaAlias() {
+        doCreateTest(() -> indexDoc(INDEX_NAME));
+    }
+
+    /**
+     * Check that a system index is auto-created with the expected mappings and
+     * settings when it is first used, when it is referenced via its concrete
+     * index name.
+     */
+    public void testSystemIndexIsAutoCreatedViaConcreteName() {
+        doCreateTest(() -> indexDoc(PRIMARY_INDEX_NAME));
+    }
+
+    /**
+     * Check that a system index is created with the expected mappings and
+     * settings when it is explicitly created, when it is referenced via its alias.
+     */
+    public void testCreateSystemIndexViaAlias() {
+        doCreateTest(() -> assertAcked(prepareCreate(INDEX_NAME)));
+    }
+
+    /**
+     * Check that a system index is created with the expected mappings and
+     * settings when it is explicitly created, when it is referenced via its
+     * concrete index name.
+     */
+    public void testCreateSystemIndexViaConcreteName() {
+        doCreateTest(() -> assertAcked(prepareCreate(PRIMARY_INDEX_NAME)));
+    }
+
+    /**
+     * Check that when a system index is created, that an attempt to override the settings or mappings is ignored.
+     */
+    public void testCreateSystemIndexIgnoresExplicitSettingsAndMappings() {
+        doCreateTest(
+            () -> assertAcked(
+                prepareCreate(PRIMARY_INDEX_NAME).addMapping(
+                    "foo",
+                    Collections.singletonMap("foo", TestSystemIndexDescriptor.getNewMappings())
+                ).setSettings(Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 999).build())
+            )
+        );
+    }
+
+    private void doCreateTest(Runnable runnable) {
+        internalCluster().startNodes(1);
+
+        // Trigger the creation of the system index
+        runnable.run();
+        ensureGreen(INDEX_NAME);
+
+        assertMappingsAndSettings(TestSystemIndexDescriptor.getOldMappings());
+
+        // Remove the index and alias...
+        assertAcked(client().admin().indices().prepareAliases().removeAlias(PRIMARY_INDEX_NAME, INDEX_NAME).get());
+        assertAcked(client().admin().indices().prepareDelete(PRIMARY_INDEX_NAME));
+
+        // ...so that we can check that the they will still be auto-created again,
+        // but this time with updated settings
+        TestSystemIndexDescriptor.useNewMappings.set(true);
+
+        runnable.run();
+        ensureGreen(INDEX_NAME);
+
+        assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings());
+    }
+
+    /**
+     * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
+     * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters.
+     */
+    private void assertMappingsAndSettings(String expectedMappings) {
+        final GetMappingsResponse getMappingsResponse = client().admin()
+            .indices()
+            .getMappings(new GetMappingsRequest().indices(INDEX_NAME))
+            .actionGet();
+
+        final ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = getMappingsResponse.getMappings();
+        assertThat(
+            "Expected mappings to contain a key for [" + PRIMARY_INDEX_NAME + "], but found: " + mappings.toString(),
+            mappings.containsKey(PRIMARY_INDEX_NAME),
+            equalTo(true)
+        );
+        final Map<String, Object> sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).get(MapperService.SINGLE_MAPPING_NAME).getSourceAsMap();
+
+        try {
+            assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        final GetSettingsResponse getSettingsResponse = client().admin()
+            .indices()
+            .getSettings(new GetSettingsRequest().indices(INDEX_NAME))
+            .actionGet();
+
+        final Settings actual = getSettingsResponse.getIndexToSettings().get(PRIMARY_INDEX_NAME);
+
+        for (String settingName : TestSystemIndexDescriptor.SETTINGS.keySet()) {
+            assertThat(actual.get(settingName), equalTo(TestSystemIndexDescriptor.SETTINGS.get(settingName)));
+        }
+    }
+
+    private void indexDoc(String index) {
+        client().prepareIndex(index, MapperService.SINGLE_MAPPING_NAME).setId("1").setSource("foo", "bar").execute().actionGet();
+    }
+}

server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexManagerIT.java

@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices;
+
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME;
+import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME;
+import static org.elasticsearch.test.XContentTestUtils.convertToXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class SystemIndexManagerIT extends ESIntegTestCase {
+
+    @Before
+    public void beforeEach() {
+        TestSystemIndexDescriptor.useNewMappings.set(false);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class);
+    }
+
+    /**
+     * Check that if the the SystemIndexManager finds a managed index with out-of-date mappings, then
+     * the manager updates those mappings.
+     */
+    public void testSystemIndexManagerUpgradesMappings() throws Exception {
+        internalCluster().startNodes(1);
+
+        // Trigger the creation of the system index
+        assertAcked(prepareCreate(INDEX_NAME));
+        ensureGreen(INDEX_NAME);
+
+        assertMappingsAndSettings(TestSystemIndexDescriptor.getOldMappings());
+
+        // Poke the test descriptor so that the mappings are now "updated"
+        TestSystemIndexDescriptor.useNewMappings.set(true);
+
+        // Cause a cluster state update, so that the SystemIndexManager will update the mappings in our index
+        triggerClusterStateUpdates();
+
+        assertBusy(() -> assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings()));
+    }
+
+    /**
+     * Check that if the the SystemIndexManager finds a managed index with mappings that claim to be newer than
+     * what it expects, then those mappings are left alone.
+     */
+    public void testSystemIndexManagerLeavesNewerMappingsAlone() throws Exception {
+        TestSystemIndexDescriptor.useNewMappings.set(true);
+
+        internalCluster().startNodes(1);
+        // Trigger the creation of the system index
+        assertAcked(prepareCreate(INDEX_NAME));
+        ensureGreen(INDEX_NAME);
+
+        assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings());
+
+        // Poke the test descriptor so that the mappings are now out-dated.
+        TestSystemIndexDescriptor.useNewMappings.set(false);
+
+        // Cause a cluster state update, so that the SystemIndexManager's listener will execute
+        triggerClusterStateUpdates();
+
+        // Mappings should be unchanged.
+        assertBusy(() -> assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings()));
+    }
+
+    /**
+     * Performs a cluster state update in order to trigger any cluster state listeners - specifically, SystemIndexManager.
+     */
+    private void triggerClusterStateUpdates() {
+        final String name = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
+        client().admin().indices().putTemplate(new PutIndexTemplateRequest(name).patterns(singletonList(name))).actionGet();
+    }
+
+    /**
+     * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
+     * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters.
+     */
+    private void assertMappingsAndSettings(String expectedMappings) {
+        final GetMappingsResponse getMappingsResponse = client().admin()
+            .indices()
+            .getMappings(new GetMappingsRequest().indices(INDEX_NAME))
+            .actionGet();
+
+        final ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = getMappingsResponse.getMappings();
+        assertThat(
+            "Expected mappings to contain a key for [" + PRIMARY_INDEX_NAME + "], but found: " + mappings.toString(),
+            mappings.containsKey(PRIMARY_INDEX_NAME),
+            equalTo(true)
+        );
+        final Map<String, Object> sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).get(MapperService.SINGLE_MAPPING_NAME).getSourceAsMap();
+        try {
+            assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        final GetSettingsResponse getSettingsResponse = client().admin()
+            .indices()
+            .getSettings(new GetSettingsRequest().indices(INDEX_NAME))
+            .actionGet();
+        final Settings actual = getSettingsResponse.getIndexToSettings().get(PRIMARY_INDEX_NAME);
+        for (String settingName : TestSystemIndexDescriptor.SETTINGS.keySet()) {
+            assertThat(actual.get(settingName), equalTo(TestSystemIndexDescriptor.SETTINGS.get(settingName)));
+        }
+    }
+}