Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add more tests for creating system indices #66605

Merged
merged 2 commits into from
Dec 18, 2020
Merged

Add more tests for creating system indices #66605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
582c30b
Fix waiting for put template
pugnascotia Dec 18, 2020
9854f5d
Refactor SystemIndexManagerIT, add CreateSystemIndicesIT
pugnascotia Dec 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 154 additions & 0 deletions ...ClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.action.admin.indices.create;

import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.indices.TestSystemIndexDescriptor;
import org.elasticsearch.indices.TestSystemIndexPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Collection;
import java.util.Map;

import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME;
import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME;
import static org.elasticsearch.test.XContentTestUtils.convertToXContent;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;

@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
public class CreateSystemIndicesIT extends ESIntegTestCase {

@Before
public void beforeEach() {
TestSystemIndexDescriptor.useNewMappings.set(false);
}

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class);
}

/**
* Check that a system index is auto-created with the expected mappings and
* settings when it is first used, when it is referenced via its alias.
*/
public void testSystemIndexIsAutoCreatedViaAlias() {
doCreateTest(() -> indexDoc(INDEX_NAME, "1", "foo", "bar"));
}

/**
* Check that a system index is auto-created with the expected mappings and
* settings when it is first used, when it is referenced via its concrete
* index name.
*/
public void testSystemIndexIsAutoCreatedViaConcreteName() {
doCreateTest(() -> indexDoc(PRIMARY_INDEX_NAME, "1", "foo", "bar"));
}

/**
* Check that a system index is created with the expected mappings and
* settings when it is explicitly created, when it is referenced via its alias.
*/
public void testCreateSystemIndexViaAlias() {
doCreateTest(() -> assertAcked(prepareCreate(INDEX_NAME)));
}

/**
* Check that a system index is created with the expected mappings and
* settings when it is explicitly created, when it is referenced via its
* concrete index name.
*/
public void testCreateSystemIndexViaConcreteName() {
doCreateTest(() -> assertAcked(prepareCreate(PRIMARY_INDEX_NAME)));
}

private void doCreateTest(Runnable runnable) {
internalCluster().startNodes(1);

// Trigger the creation of the system index
runnable.run();
ensureGreen(INDEX_NAME);

assertMappingsAndSettings(TestSystemIndexDescriptor.getOldMappings());

// Remove the index and alias...
assertAcked(client().admin().indices().prepareAliases().removeAlias(PRIMARY_INDEX_NAME, INDEX_NAME).get());
assertAcked(client().admin().indices().prepareDelete(PRIMARY_INDEX_NAME));

// ...so that we can check that the they will still be auto-created again,
// but this time with updated settings
TestSystemIndexDescriptor.useNewMappings.set(true);

runnable.run();
ensureGreen(INDEX_NAME);

assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings());
}

/**
* Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
* Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters.
*/
private void assertMappingsAndSettings(String expectedMappings) {
final GetMappingsResponse getMappingsResponse = client().admin()
.indices()
.getMappings(new GetMappingsRequest().indices(INDEX_NAME))
.actionGet();

final ImmutableOpenMap<String, MappingMetadata> mappings = getMappingsResponse.getMappings();
assertThat(
"Expected mappings to contain a key for [" + PRIMARY_INDEX_NAME + "], but found: " + mappings.toString(),
mappings.containsKey(PRIMARY_INDEX_NAME),
equalTo(true)
);
final Map<String, Object> sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).getSourceAsMap();

try {
assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings));
} catch (IOException e) {
throw new UncheckedIOException(e);
}

final GetSettingsResponse getSettingsResponse = client().admin()
.indices()
.getSettings(new GetSettingsRequest().indices(INDEX_NAME))
.actionGet();

final Settings actual = getSettingsResponse.getIndexToSettings().get(PRIMARY_INDEX_NAME);

for (String settingName : TestSystemIndexDescriptor.SETTINGS.keySet()) {
assertThat(actual.get(settingName), equalTo(TestSystemIndexDescriptor.SETTINGS.get(settingName)));
}
}

}
161 changes: 36 additions & 125 deletions server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexManagerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,17 @@

package org.elasticsearch.indices;

import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.SystemIndexPlugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;

Expand All @@ -43,27 +39,24 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME;
import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME;
import static org.elasticsearch.test.XContentTestUtils.convertToXContent;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;

@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
public class SystemIndexManagerIT extends ESIntegTestCase {

private static final String INDEX_NAME = ".test-index";
private static final String PRIMARY_INDEX_NAME = INDEX_NAME + "-1";

@Before
public void beforeEach() {
TestSystemIndexDescriptor.useNewMappings.set(false);
}

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return CollectionUtils.appendToCopy(super.nodePlugins(), TestPlugin.class);
return CollectionUtils.appendToCopy(super.nodePlugins(), TestSystemIndexPlugin.class);
}

/**
Expand All @@ -77,15 +70,15 @@ public void testSystemIndexManagerUpgradesMappings() throws Exception {
assertAcked(prepareCreate(INDEX_NAME));
ensureGreen(INDEX_NAME);

assertMappings(TestSystemIndexDescriptor.getOldMappings());
assertMappingsAndSettings(TestSystemIndexDescriptor.getOldMappings());

// Poke the test descriptor so that the mappings are now "updated"
TestSystemIndexDescriptor.useNewMappings.set(true);

// Cause a cluster state update, so that the SystemIndexManager will update the mappings in our index
triggerClusterStateUpdates();

assertBusy(() -> assertMappings(TestSystemIndexDescriptor.getNewMappings()));
assertBusy(() -> assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings()));
}

/**
Expand All @@ -100,7 +93,7 @@ public void testSystemIndexManagerLeavesNewerMappingsAlone() throws Exception {
assertAcked(prepareCreate(INDEX_NAME));
ensureGreen(INDEX_NAME);

assertMappings(TestSystemIndexDescriptor.getNewMappings());
assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings());

// Poke the test descriptor so that the mappings are now out-dated.
TestSystemIndexDescriptor.useNewMappings.set(false);
Expand All @@ -109,7 +102,7 @@ public void testSystemIndexManagerLeavesNewerMappingsAlone() throws Exception {
triggerClusterStateUpdates();

// Mappings should be unchanged.
assertBusy(() -> assertMappings(TestSystemIndexDescriptor.getNewMappings()));
assertBusy(() -> assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings()));
}

/**
Expand All @@ -121,121 +114,39 @@ private void triggerClusterStateUpdates() {
}

/**
* Fetch the mappings for {@link #INDEX_NAME} and verify that they match the expected mappings. Note that this is just
* a dumb string comparison, so order of keys matters.
* Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
* Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters.
*/
private void assertMappings(String expectedMappings) {
client().admin().indices().getMappings(new GetMappingsRequest().indices(INDEX_NAME), new ActionListener<>() {
@Override
public void onResponse(GetMappingsResponse getMappingsResponse) {
final ImmutableOpenMap<String, MappingMetadata> mappings = getMappingsResponse.getMappings();
assertThat(
"Expected mappings to contain a key for [" + PRIMARY_INDEX_NAME + "], but found: " + mappings.toString(),
mappings.containsKey(PRIMARY_INDEX_NAME),
equalTo(true)
);
final Map<String, Object> sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).getSourceAsMap();

try {
assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

@Override
public void onFailure(Exception e) {
throw new AssertionError("Couldn't fetch mappings for " + INDEX_NAME, e);
}
});
}

/** A special kind of {@link SystemIndexDescriptor} that can toggle what kind of mappings it
* expects. A real descriptor is immutable. */
public static class TestSystemIndexDescriptor extends SystemIndexDescriptor {

public static final AtomicBoolean useNewMappings = new AtomicBoolean(false);
private static final Settings settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-1")
.put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE)
.build();

TestSystemIndexDescriptor() {
super(INDEX_NAME + "*", PRIMARY_INDEX_NAME, "Test system index", null, settings, INDEX_NAME, 0, "version", "stack");
private void assertMappingsAndSettings(String expectedMappings) {
final GetMappingsResponse getMappingsResponse = client().admin()
.indices()
.getMappings(new GetMappingsRequest().indices(INDEX_NAME))
.actionGet();

final ImmutableOpenMap<String, MappingMetadata> mappings = getMappingsResponse.getMappings();
assertThat(
"Expected mappings to contain a key for [" + PRIMARY_INDEX_NAME + "], but found: " + mappings.toString(),
mappings.containsKey(PRIMARY_INDEX_NAME),
equalTo(true)
);
final Map<String, Object> sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).getSourceAsMap();

try {
assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings));
} catch (IOException e) {
throw new UncheckedIOException(e);
}

@Override
public boolean isAutomaticallyManaged() {
return true;
}
final GetSettingsResponse getSettingsResponse = client().admin()
.indices()
.getSettings(new GetSettingsRequest().indices(INDEX_NAME))
.actionGet();

@Override
public String getMappings() {
return useNewMappings.get() ? getNewMappings() : getOldMappings();
}

public static String getOldMappings() {
try {
final XContentBuilder builder = jsonBuilder();

builder.startObject();
{
builder.startObject("_meta");
builder.field("version", Version.CURRENT.previousMajor().toString());
builder.endObject();

builder.startObject("properties");
{
builder.startObject("foo");
builder.field("type", "text");
builder.endObject();
}
builder.endObject();
}

builder.endObject();
return Strings.toString(builder);
} catch (IOException e) {
throw new UncheckedIOException("Failed to build .test-index-1 index mappings", e);
}
}
final Settings actual = getSettingsResponse.getIndexToSettings().get(PRIMARY_INDEX_NAME);

public static String getNewMappings() {
try {
final XContentBuilder builder = jsonBuilder();

builder.startObject();
{
builder.startObject("_meta");
builder.field("version", Version.CURRENT.toString());
builder.endObject();

builder.startObject("properties");
{
builder.startObject("bar");
builder.field("type", "text");
builder.endObject();
builder.startObject("foo");
builder.field("type", "text");
builder.endObject();
}
builder.endObject();
}

builder.endObject();
return Strings.toString(builder);
} catch (IOException e) {
throw new UncheckedIOException("Failed to build .test-index-1 index mappings", e);
}
for (String settingName : TestSystemIndexDescriptor.SETTINGS.keySet()) {
assertThat(actual.get(settingName), equalTo(TestSystemIndexDescriptor.SETTINGS.get(settingName)));
}
}

/** Just a test plugin to allow the test descriptor to be installed in the cluster. */
public static class TestPlugin extends Plugin implements SystemIndexPlugin {
@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
return List.of(new TestSystemIndexDescriptor());
}
}
}
Loading