IndexMetaData: Introduce internal format index setting

core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -23,7 +23,6 @@
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.Diff;
@@ -259,6 +258,14 @@ static Setting<Integer> buildNumberOfShardsSetting() {
                       Setting.Property.Dynamic,
                       Setting.Property.IndexScope);
 
+    /**
+     * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading
+     */
+    private static final String INDEX_INTERNAL_FORMAT = "index.internal.format";
+    public static final Setting<Integer> INDEX_INTERNAL_FORMAT_SETTING =
+            Setting.intSetting(INDEX_INTERNAL_FORMAT, 0, Setting.Property.IndexScope);
+    static final int INTERNAL_INDEX_FORMAT_CURRENT = 6;
+
     public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations";
     static final String KEY_VERSION = "version";
     static final String KEY_ROUTING_NUM_SHARDS = "routing_num_shards";
@@ -279,6 +286,7 @@ static Setting<Integer> buildNumberOfShardsSetting() {
     private final Index index;
     private final long version;
     private final long[] primaryTerms;
+    private final int internalIndexFormat;
 
     private final State state;
 
@@ -309,7 +317,7 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat
                           ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> inSyncAllocationIds,
                           DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
                           Version indexCreatedVersion, Version indexUpgradedVersion,
-                          int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards) {
+                          int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards, int internalIndexFormat) {
 
         this.index = index;
         this.version = version;
@@ -335,6 +343,7 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat
         this.routingPartitionSize = routingPartitionSize;
         this.waitForActiveShards = waitForActiveShards;
         assert numberOfShards * routingFactor == routingNumShards :  routingNumShards + " must be a multiple of " + numberOfShards;
+        this.internalIndexFormat = internalIndexFormat;
     }
 
     public Index getIndex() {
@@ -506,6 +515,16 @@ public DiscoveryNodeFilters excludeFilters() {
         return excludeFilters;
     }
 
+    /**
+     * Check that the 'index.internal.format' setting is set correctly for this index.
+     * This setting eases for upgrades
+     *
+     * @return true if the index format is configured to be {@link #INTERNAL_INDEX_FORMAT_CURRENT}
+     */
+    public boolean isIndexInternalFormat() {
+        return internalIndexFormat == INTERNAL_INDEX_FORMAT_CURRENT;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -551,6 +570,9 @@ public boolean equals(Object o) {
         if (!inSyncAllocationIds.equals(that.inSyncAllocationIds)) {
             return false;
         }
+        if (internalIndexFormat != that.internalIndexFormat) {
+            return false;
+        }
         return true;
     }
 
@@ -567,6 +589,7 @@ public int hashCode() {
         result = 31 * result + Long.hashCode(routingNumShards);
         result = 31 * result + Arrays.hashCode(primaryTerms);
         result = 31 * result + inSyncAllocationIds.hashCode();
+        result = 31 * result + Integer.hashCode(internalIndexFormat);
         return result;
     }
 
@@ -1051,9 +1074,11 @@ public IndexMetaData build() {
             }
 
             final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
+
+            int internalIndexFormat = INDEX_INTERNAL_FORMAT_SETTING.get(settings);
             return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
                 tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters,
-                indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards);
+                indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards, internalIndexFormat);
         }
 
         public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {

core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -77,6 +77,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING,
         IndexMetaData.INDEX_PRIORITY_SETTING,
         IndexMetaData.INDEX_DATA_PATH_SETTING,
+        IndexMetaData.INDEX_INTERNAL_FORMAT_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING,

core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java

@@ -32,6 +32,9 @@
 import java.io.IOException;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.metadata.IndexMetaData.INTERNAL_INDEX_FORMAT_CURRENT;
+import static org.hamcrest.Matchers.is;
+
 public class IndexMetaDataTests extends ESTestCase {
 
     public void testIndexMetaDataSerialization() throws IOException {
@@ -121,4 +124,48 @@ public void testSelectShrinkShards() {
         assertEquals("the number of target shards (8) must be greater than the shard id: 8",
             expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectShrinkShards(8, metaData, 8)).getMessage());
     }
+
+    public void testInternalIndexFormat() {
+        Settings defaultSettings = Settings.builder()
+                .put("index.version.created", 1)
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 1)
+                .build();
+
+        // matching version
+        {
+            IndexMetaData metaData = IndexMetaData.builder("foo")
+                    .settings(Settings.builder()
+                            .put(defaultSettings)
+                            // intentionally not using the constant, so upgrading requires you to look at this test
+                            // where you have to update this part and the next one
+                            .put("index.internal.format", 6)
+                            .build())
+                    .build();
+
+            assertThat(metaData.isIndexInternalFormat(), is(true));
+        }
+
+        // not matching version
+        {
+            IndexMetaData metaData = IndexMetaData.builder("foo")
+                    .settings(Settings.builder()
+                            .put(defaultSettings)
+                            .put("index.internal.format", randomFrom(randomIntBetween(0, 5),
+                                    randomIntBetween(7, Integer.MAX_VALUE)))
+                            .build())
+                    .build();
+            assertThat(metaData.isIndexInternalFormat(), is(false));
+        }
+
+        // no setting configured
+        {
+            IndexMetaData metaData = IndexMetaData.builder("foo")
+                    .settings(Settings.builder()
+                            .put(defaultSettings)
+                            .build())
+                    .build();
+            assertThat(metaData.isIndexInternalFormat(), is(false));
+        }
+    }
 }