From 5991ae6074035fde325a9f1b396e982f67455fb8 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 19 Jun 2017 15:57:45 +0200 Subject: [PATCH 1/3] IndexMetaData: Introduce internal format index setting This setting is supposed to ease index upgrades as it allows you to check for a new setting called `index.internal.version` which in turn can be configured in the index settings and has to be set to '6' in order to be valid. --- .../cluster/metadata/IndexMetaData.java | 31 ++++++++++-- .../common/settings/IndexScopedSettings.java | 1 + .../cluster/metadata/IndexMetaDataTests.java | 47 +++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 591b83c0eff7c..7bfcb53be0190 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -23,7 +23,6 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.Version; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.Diff; @@ -259,6 +258,14 @@ static Setting buildNumberOfShardsSetting() { Setting.Property.Dynamic, Setting.Property.IndexScope); + /** + * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading + */ + private static final String INDEX_INTERNAL_FORMAT = "index.internal.format"; + public static final Setting INDEX_INTERNAL_FORMAT_SETTING = + Setting.intSetting(INDEX_INTERNAL_FORMAT, 0, Setting.Property.IndexScope); + static final int INTERNAL_INDEX_FORMAT_CURRENT = 6; + public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations"; static final String KEY_VERSION = "version"; static final String KEY_ROUTING_NUM_SHARDS = "routing_num_shards"; @@ -279,6 +286,7 @@ static Setting buildNumberOfShardsSetting() { private final Index index; private final long version; private final long[] primaryTerms; + private final int internalIndexFormat; private final State state; @@ -309,7 +317,7 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat ImmutableOpenMap customs, ImmutableOpenIntMap> inSyncAllocationIds, DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters, Version indexCreatedVersion, Version indexUpgradedVersion, - int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards) { + int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards, int internalIndexFormat) { this.index = index; this.version = version; @@ -335,6 +343,7 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat this.routingPartitionSize = routingPartitionSize; this.waitForActiveShards = waitForActiveShards; assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards; + this.internalIndexFormat = internalIndexFormat; } public Index getIndex() { @@ -506,6 +515,16 @@ public DiscoveryNodeFilters excludeFilters() { return excludeFilters; } + /** + * Check that the 'index.internal.format' setting is set correctly for this index. + * This setting eases for upgrades + * + * @return true if the index format is configured to be {@link #INTERNAL_INDEX_FORMAT_CURRENT} + */ + public boolean isIndexInternalFormat() { + return internalIndexFormat == INTERNAL_INDEX_FORMAT_CURRENT; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -551,6 +570,9 @@ public boolean equals(Object o) { if (!inSyncAllocationIds.equals(that.inSyncAllocationIds)) { return false; } + if (internalIndexFormat != that.internalIndexFormat) { + return false; + } return true; } @@ -567,6 +589,7 @@ public int hashCode() { result = 31 * result + Long.hashCode(routingNumShards); result = 31 * result + Arrays.hashCode(primaryTerms); result = 31 * result + inSyncAllocationIds.hashCode(); + result = 31 * result + Integer.hashCode(internalIndexFormat); return result; } @@ -1051,9 +1074,11 @@ public IndexMetaData build() { } final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); + + int internalIndexFormat = INDEX_INTERNAL_FORMAT_SETTING.get(settings); return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(), tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters, - indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards); + indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards, internalIndexFormat); } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index ae4cf6cd41a36..a6de595f09071 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -77,6 +77,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING, IndexMetaData.INDEX_PRIORITY_SETTING, IndexMetaData.INDEX_DATA_PATH_SETTING, + IndexMetaData.INDEX_INTERNAL_FORMAT_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING, diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index 7b11f96ac4d70..946312f5feb11 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -32,6 +32,9 @@ import java.io.IOException; import java.util.Set; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INTERNAL_INDEX_FORMAT_CURRENT; +import static org.hamcrest.Matchers.is; + public class IndexMetaDataTests extends ESTestCase { public void testIndexMetaDataSerialization() throws IOException { @@ -121,4 +124,48 @@ public void testSelectShrinkShards() { assertEquals("the number of target shards (8) must be greater than the shard id: 8", expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectShrinkShards(8, metaData, 8)).getMessage()); } + + public void testInternalIndexFormat() { + Settings defaultSettings = Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 1) + .build(); + + // matching version + { + IndexMetaData metaData = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put(defaultSettings) + // intentionally not using the constant, so upgrading requires you to look at this test + // where you have to update this part and the next one + .put("index.internal.format", 6) + .build()) + .build(); + + assertThat(metaData.isIndexInternalFormat(), is(true)); + } + + // not matching version + { + IndexMetaData metaData = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put(defaultSettings) + .put("index.internal.format", randomFrom(randomIntBetween(0, 5), + randomIntBetween(7, Integer.MAX_VALUE))) + .build()) + .build(); + assertThat(metaData.isIndexInternalFormat(), is(false)); + } + + // no setting configured + { + IndexMetaData metaData = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put(defaultSettings) + .build()) + .build(); + assertThat(metaData.isIndexInternalFormat(), is(false)); + } + } } From ef1fd6583e058999229d455183772b5f53abfb51 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Tue, 20 Jun 2017 16:33:15 +0200 Subject: [PATCH 2/3] Incorporating review comments - Make the setting final - Do not expose an own getter, just extract the setting via IndexMetaData.getSettings() - also no need to put this into hashCode()/equals() - Make more generic and rename to `index.format` - Allow the user to choose the correct value and just return that value --- .../cluster/metadata/IndexMetaData.java | 25 +++---------------- .../common/settings/IndexScopedSettings.java | 2 +- .../cluster/metadata/IndexMetaDataTests.java | 22 ++++------------ 3 files changed, 10 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 7bfcb53be0190..c510ce60851e8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -261,10 +261,9 @@ static Setting buildNumberOfShardsSetting() { /** * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading */ - private static final String INDEX_INTERNAL_FORMAT = "index.internal.format"; - public static final Setting INDEX_INTERNAL_FORMAT_SETTING = - Setting.intSetting(INDEX_INTERNAL_FORMAT, 0, Setting.Property.IndexScope); - static final int INTERNAL_INDEX_FORMAT_CURRENT = 6; + private static final String INDEX_FORMAT = "index.format"; + public static final Setting INDEX_FORMAT_SETTING = + Setting.intSetting(INDEX_FORMAT, 0, Setting.Property.IndexScope, Setting.Property.Final); public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations"; static final String KEY_VERSION = "version"; @@ -286,7 +285,6 @@ static Setting buildNumberOfShardsSetting() { private final Index index; private final long version; private final long[] primaryTerms; - private final int internalIndexFormat; private final State state; @@ -343,7 +341,6 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat this.routingPartitionSize = routingPartitionSize; this.waitForActiveShards = waitForActiveShards; assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards; - this.internalIndexFormat = internalIndexFormat; } public Index getIndex() { @@ -515,16 +512,6 @@ public DiscoveryNodeFilters excludeFilters() { return excludeFilters; } - /** - * Check that the 'index.internal.format' setting is set correctly for this index. - * This setting eases for upgrades - * - * @return true if the index format is configured to be {@link #INTERNAL_INDEX_FORMAT_CURRENT} - */ - public boolean isIndexInternalFormat() { - return internalIndexFormat == INTERNAL_INDEX_FORMAT_CURRENT; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -570,9 +557,6 @@ public boolean equals(Object o) { if (!inSyncAllocationIds.equals(that.inSyncAllocationIds)) { return false; } - if (internalIndexFormat != that.internalIndexFormat) { - return false; - } return true; } @@ -589,7 +573,6 @@ public int hashCode() { result = 31 * result + Long.hashCode(routingNumShards); result = 31 * result + Arrays.hashCode(primaryTerms); result = 31 * result + inSyncAllocationIds.hashCode(); - result = 31 * result + Integer.hashCode(internalIndexFormat); return result; } @@ -1075,7 +1058,7 @@ public IndexMetaData build() { final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); - int internalIndexFormat = INDEX_INTERNAL_FORMAT_SETTING.get(settings); + int internalIndexFormat = INDEX_FORMAT_SETTING.get(settings); return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(), tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters, indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards, internalIndexFormat); diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index a6de595f09071..890a43107c53a 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -77,7 +77,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING, IndexMetaData.INDEX_PRIORITY_SETTING, IndexMetaData.INDEX_DATA_PATH_SETTING, - IndexMetaData.INDEX_INTERNAL_FORMAT_SETTING, + IndexMetaData.INDEX_FORMAT_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING, diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index 946312f5feb11..fa56c756fcc35 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -32,8 +32,8 @@ import java.io.IOException; import java.util.Set; -import static org.elasticsearch.cluster.metadata.IndexMetaData.INTERNAL_INDEX_FORMAT_CURRENT; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; public class IndexMetaDataTests extends ESTestCase { @@ -125,7 +125,7 @@ public void testSelectShrinkShards() { expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectShrinkShards(8, metaData, 8)).getMessage()); } - public void testInternalIndexFormat() { + public void testIndexFormat() { Settings defaultSettings = Settings.builder() .put("index.version.created", 1) .put("index.number_of_shards", 1) @@ -139,23 +139,11 @@ public void testInternalIndexFormat() { .put(defaultSettings) // intentionally not using the constant, so upgrading requires you to look at this test // where you have to update this part and the next one - .put("index.internal.format", 6) + .put("index.format", 6) .build()) .build(); - assertThat(metaData.isIndexInternalFormat(), is(true)); - } - - // not matching version - { - IndexMetaData metaData = IndexMetaData.builder("foo") - .settings(Settings.builder() - .put(defaultSettings) - .put("index.internal.format", randomFrom(randomIntBetween(0, 5), - randomIntBetween(7, Integer.MAX_VALUE))) - .build()) - .build(); - assertThat(metaData.isIndexInternalFormat(), is(false)); + assertThat(metaData.getSettings().getAsInt(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), 0), is(6)); } // no setting configured @@ -165,7 +153,7 @@ public void testInternalIndexFormat() { .put(defaultSettings) .build()) .build(); - assertThat(metaData.isIndexInternalFormat(), is(false)); + assertThat(metaData.getSettings().getAsInt(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), 0), is(0)); } } } From bbffd948c710aae8cd0ce5d5be543dd4245bde4a Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Tue, 20 Jun 2017 20:24:47 +0200 Subject: [PATCH 3/3] removed unused code --- .../org/elasticsearch/cluster/metadata/IndexMetaData.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index c510ce60851e8..47fc2526c4eeb 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -315,7 +315,7 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat ImmutableOpenMap customs, ImmutableOpenIntMap> inSyncAllocationIds, DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters, Version indexCreatedVersion, Version indexUpgradedVersion, - int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards, int internalIndexFormat) { + int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards) { this.index = index; this.version = version; @@ -1058,10 +1058,9 @@ public IndexMetaData build() { final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); - int internalIndexFormat = INDEX_FORMAT_SETTING.get(settings); return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(), tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters, - indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards, internalIndexFormat); + indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards); } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {