TSDB: add index timestamp range check

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml

@@ -117,3 +117,50 @@ no partitioning:
                 mode: time_series
                 shards: 5
                 routing_partition_size: 2
+
+---
+set start_time and end_time:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              time_series:
+                start_time: 1632625782000
+                end_time: 1632625792000
+
+  - do:
+      indices.put_settings:
+        index: test_index
+        body:
+          index:
+            time_series:
+              start_time: 1632625781000
+              end_time: 1632625793000
+
+  - do:
+      catch: /index.time_series.start_time must be smaller than pre value \[1632625781000\]/
+      indices.put_settings:
+        index: test_index
+        body:
+          index:
+            time_series:
+              start_time: 1632625783000
+
+  - do:
+      catch: /index.time_series.end_time must be larger than pre value \[1632625793000\]/
+      indices.put_settings:
+        index: test_index
+        body:
+          index:
+            time_series:
+              end_time: 1632625792000
+
+  - do:
+      indices.delete:
+        index: test_index

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -185,6 +185,8 @@ private static Set<Setting<?>> builtInIndexSettings() {
         }
         Set<Setting<?>> result = new HashSet<>(ALWAYS_ENABLED_BUILT_IN_INDEX_SETTINGS);
         result.add(IndexSettings.MODE);
+        result.add(IndexSettings.TIME_SERIES_START_TIME);
+        result.add(IndexSettings.TIME_SERIES_END_TIME);
         return Set.copyOf(result);
     }
 

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -8,16 +8,22 @@
 
 package org.elasticsearch.index;
 
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.DocumentParserContext;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Stream;
 
 import static java.util.stream.Collectors.toSet;
+import static org.elasticsearch.core.TimeValue.NSEC_PER_MSEC;
 
 /**
  * "Mode" that controls which behaviors and settings an index supports.
@@ -26,8 +32,13 @@ public enum IndexMode {
     STANDARD {
         @Override
         void validateWithOtherSettings(Map<Setting<?>, Object> settings) {}
+
+        @Override
+        public void validateWithSource(DocumentParserContext context) {}
     },
     TIME_SERIES {
+        public static final String TIMESTAMP_FIELD = "@timestamp";
+
         @Override
         void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
             if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) {
@@ -40,6 +51,45 @@ void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
             }
         }
 
+        @Override
+        public void validateWithSource(DocumentParserContext context) {
+            validateTimestamp(context);
+        }
+
+        private void validateTimestamp(DocumentParserContext context) {
+            IndexableField[] fields = context.rootDoc().getFields(TIMESTAMP_FIELD);
+            if (fields.length == 0) {
+                throw new IllegalArgumentException("time series index @timestamp field is missing");
+            }
+
+            long numberOfValues = Arrays.stream(fields)
+                .filter(indexableField -> indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC)
+                .count();
+            if (numberOfValues > 1) {
+                throw new IllegalArgumentException("time series index @timestamp field encountered multiple values");
+            }
+
+            long timestamp = fields[0].numericValue().longValue();
+            if (context.mappingLookup().getMapper(TIMESTAMP_FIELD).typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) {
+                timestamp /= NSEC_PER_MSEC;
+            }
+
+            long startTime = context.indexSettings().getTimeSeriesStartTime();
+            long endTime = context.indexSettings().getTimeSeriesEndTime();
+
+            if (timestamp < startTime) {
+                throw new IllegalArgumentException(
+                    "time series index @timestamp value [" + timestamp + "] must be larger than " + startTime
+                );
+            }
+
+            if (timestamp > endTime) {
+                throw new IllegalArgumentException(
+                    "time series index @timestamp value [" + timestamp + "] must be smaller than " + endTime
+                );
+            }
+        }
+
         private String error(Setting<?> unsupported) {
             return "[" + IndexSettings.MODE.getKey() + "=time_series] is incompatible with [" + unsupported.getKey() + "]";
         }
@@ -57,4 +107,6 @@ private String error(Setting<?> unsupported) {
     );
 
     abstract void validateWithOtherSettings(Map<Setting<?>, Object> settings);
+
+    public abstract void validateWithSource(DocumentParserContext context);
 }

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -363,6 +363,28 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
+    /**
+     * in time series mode, the start time of the index, timestamp must larger than start_time
+     */
+    public static final Setting<Long> TIME_SERIES_START_TIME = Setting.longSetting(
+        "index.time_series.start_time",
+        -1L,
+        -1L,
+        Property.IndexScope,
+        Property.Dynamic
+    );
+
+    /**
+     * in time series mode, the end time of the index, timestamp must smaller than start_time
+     */
+    public static final Setting<Long> TIME_SERIES_END_TIME = Setting.longSetting(
+        "index.time_series.end_time",
+        -1L,
+        -1L,
+        Property.IndexScope,
+        Property.Dynamic
+    );
+
     private final Index index;
     private final Version version;
     private final Logger logger;
@@ -373,6 +395,8 @@ public Iterator<Setting<?>> settings() {
      * The {@link IndexMode "mode"} of the index.
      */
     private final IndexMode mode;
+    private volatile long timeSeriesStartTime;
+    private volatile long timeSeriesEndTime;
 
     // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock
     private volatile Settings settings;
@@ -515,6 +539,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         this.indexMetadata = indexMetadata;
         numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null);
         mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD;
+        timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings);
+        timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings);
 
         this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
         this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
@@ -614,6 +640,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit);
+        scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_START_TIME, this::updateTimeSeriesStartTime);
+        scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_END_TIME, this::updateTimeSeriesEndTime);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) { this.searchIdleAfter = searchIdleAfter; }
@@ -1092,4 +1120,37 @@ public long getMappingDimensionFieldsLimit() {
     private void setMappingDimensionFieldsLimit(long value) {
         this.mappingDimensionFieldsLimit = value;
     }
+
+    public long getTimeSeriesStartTime() {
+        return timeSeriesStartTime;
+    }
+
+    public void updateTimeSeriesStartTime(long startTime) {
+        if (timeSeriesStartTime < 0) {
+            throw new IllegalArgumentException("index.time_series.start_time not set before, can not update value");
+        }
+
+        if (this.timeSeriesStartTime < startTime) {
+            throw new IllegalArgumentException(
+                "index.time_series.start_time must be smaller than pre value [" + this.timeSeriesStartTime + "]"
+            );
+        }
+        this.timeSeriesStartTime = startTime;
+    }
+
+    public long getTimeSeriesEndTime() {
+        return timeSeriesEndTime;
+    }
+
+    public void updateTimeSeriesEndTime(long endTime) {
+        if (timeSeriesEndTime < 0) {
+            throw new IllegalArgumentException("index.time_series.end_time not set before, can not update value");
+        }
+
+        if (this.timeSeriesEndTime > endTime) {
+            throw new IllegalArgumentException("index.time_series.end_time must be larger than pre value [" + this.timeSeriesEndTime + "]");
+        }
+
+        this.timeSeriesEndTime = endTime;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -139,6 +139,8 @@ private static void internalParseDocument(RootObjectMapper root, MetadataFieldMa
 
         executeIndexTimeScripts(context);
 
+        context.indexSettings().getMode().validateWithSource(context);
+
         for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) {
             metadataMapper.postParse(context);
         }

server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java

@@ -15,11 +15,13 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
+import org.hamcrest.Matchers;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -545,4 +547,66 @@ public void testCustomDataPathDeprecated() {
         assertThat(indexSettings.hasCustomDataPath(), is(true));
         assertSettingDeprecationsAndWarnings(new Setting<?>[] { IndexMetadata.INDEX_DATA_PATH_SETTING });
     }
+
+    public void testUpdateTimeSeriesTimeRange() {
+        long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000);
+        long endTime = startTime + randomLongBetween(1000, 86400000);
+        final Settings settings = Settings.builder()
+            .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime)
+            .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime)
+            .build();
+        IndexMetadata metadata = newIndexMeta("test", settings);
+        IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY);
+
+        // test update start_time
+        {
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> indexSettings.updateTimeSeriesStartTime(startTime + randomLongBetween(10, 1000))
+            );
+            assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time must be smaller than pre value"));
+
+            // success
+            long newStartTime = Math.max(startTime - randomLongBetween(10, 1000), 0);
+            indexSettings.updateTimeSeriesStartTime(newStartTime);
+            assertEquals(newStartTime, indexSettings.getTimeSeriesStartTime());
+        }
+
+        // test update end_time
+        {
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> indexSettings.updateTimeSeriesEndTime(endTime - randomLongBetween(10, 1000))
+            );
+            assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than pre value"));
+
+            // success
+            long newEndTime = endTime + randomLongBetween(10, 1000);
+            indexSettings.updateTimeSeriesEndTime(newEndTime);
+            assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime());
+        }
+    }
+
+    public void testUpdateTimeSeriesTimeRangeNotSet() {
+        final Settings settings = Settings.builder().build();
+        IndexMetadata metadata = newIndexMeta("test", settings);
+        IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY);
+
+        {
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> indexSettings.updateTimeSeriesStartTime(randomLongBetween(10, 1000))
+            );
+            assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time not set before, can not update value"));
+        }
+
+        {
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> indexSettings.updateTimeSeriesEndTime(randomLongBetween(10, 1000))
+            );
+
+            assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time not set before, can not update value"));
+        }
+    }
 }