Increase max number of dimensions from 16 to 21

server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java

@@ -0,0 +1,182 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.timeseries.support;
+
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.DocumentParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class TimeSeriesDimensionsLimitIT extends ESIntegTestCase {
+
+    public void testDimensionFieldNameLimit() throws IOException {
+        int dimensionFieldLimit = 21;
+        final String dimensionFieldName = randomAlphaOfLength(randomIntBetween(513, 1024));
+        createTimeSeriesIndex(mapping -> {
+            mapping.startObject("routing_field").field("type", "keyword").field("time_series_dimension", true).endObject();
+            mapping.startObject(dimensionFieldName).field("type", "keyword").field("time_series_dimension", true).endObject();
+        },
+            mapping -> mapping.startObject("gauge").field("type", "integer").field("time_series_metric", "gauge").endObject(),
+            () -> List.of("routing_field"),
+            dimensionFieldLimit
+        );
+        final Exception ex = expectThrows(
+            DocumentParsingException.class,
+            () -> client().prepareIndex("test")
+                .setSource(
+                    "routing_field",
+                    randomAlphaOfLength(10),
+                    dimensionFieldName,
+                    randomAlphaOfLength(1536),
+                    "gauge",
+                    randomIntBetween(10, 20),
+                    "@timestamp",
+                    Instant.now().toEpochMilli()
+                )
+                .get()
+        );
+        assertThat(
+            ex.getCause().getMessage(),
+            equalTo(
+                "Dimension name must be less than [512] bytes but [" + dimensionFieldName + "] was [" + dimensionFieldName.length() + "]."
+            )
+        );
+    }
+
+    public void testDimensionFieldValueLimit() throws IOException {
+        int dimensionFieldLimit = 21;
+        createTimeSeriesIndex(
+            mapping -> mapping.startObject("field").field("type", "keyword").field("time_series_dimension", true).endObject(),
+            mapping -> mapping.startObject("gauge").field("type", "integer").field("time_series_metric", "gauge").endObject(),
+            () -> List.of("field"),
+            dimensionFieldLimit
+        );
+        long startTime = Instant.now().toEpochMilli();
+        client().prepareIndex("test")
+            .setSource("field", randomAlphaOfLength(1536), "gauge", randomIntBetween(10, 20), "@timestamp", startTime)
+            .get();
+        final Exception ex = expectThrows(
+            DocumentParsingException.class,
+            () -> client().prepareIndex("test")
+                .setSource("field", randomAlphaOfLength(1537), "gauge", randomIntBetween(10, 20), "@timestamp", startTime + 1)
+                .get()
+        );
+        assertThat(ex.getCause().getMessage(), equalTo("Dimension fields must be less than [1536] bytes but was [1537]."));
+    }
+
+    public void testTotalNumberOfDimensionFieldsLimit() {
+        int dimensionFieldLimit = 21;
+        final Exception ex = expectThrows(IllegalArgumentException.class, () -> createTimeSeriesIndex(mapping -> {
+            mapping.startObject("routing_field").field("type", "keyword").field("time_series_dimension", true).endObject();
+            for (int i = 0; i < dimensionFieldLimit; i++) {
+                mapping.startObject(randomAlphaOfLength(10)).field("type", "keyword").field("time_series_dimension", true).endObject();
+            }
+        },
+            mapping -> mapping.startObject("gauge").field("type", "integer").field("time_series_metric", "gauge").endObject(),
+            () -> List.of("routing_field"),
+            dimensionFieldLimit
+        ));
+
+        assertThat(ex.getMessage(), equalTo("Limit of total dimension fields [" + dimensionFieldLimit + "] has been exceeded"));
+    }
+
+    public void testTotalDimensionFieldsSizeLuceneLimit() throws IOException {
+        int dimensionFieldLimit = 21;
+        final List<String> dimensionFieldNames = new ArrayList<>();
+        createTimeSeriesIndex(mapping -> {
+            for (int i = 0; i < dimensionFieldLimit; i++) {
+                String dimensionFieldName = randomAlphaOfLength(10);
+                dimensionFieldNames.add(dimensionFieldName);
+                mapping.startObject(dimensionFieldName).field("type", "keyword").field("time_series_dimension", true).endObject();
+            }
+        },
+            mapping -> mapping.startObject("gauge").field("type", "integer").field("time_series_metric", "gauge").endObject(),
+            () -> List.of(dimensionFieldNames.get(0)),
+            dimensionFieldLimit
+        );
+
+        final Map<String, Object> source = new HashMap<>();
+        source.put("gauge", randomIntBetween(10, 20));
+        source.put("@timestamp", Instant.now().toEpochMilli());
+        for (int i = 0; i < dimensionFieldLimit; i++) {
+            source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1536));
+        }
+        final IndexResponse indexResponse = client().prepareIndex("test").setSource(source).get();
+        assertEquals(RestStatus.CREATED.getStatus(), indexResponse.status().getStatus());
+    }
+
+    public void testTotalDimensionFieldsSizeLuceneLimitPlusOne() throws IOException {
+        int dimensionFieldLimit = 22;
+        final List<String> dimensionFieldNames = new ArrayList<>();
+        createTimeSeriesIndex(mapping -> {
+            for (int i = 0; i < dimensionFieldLimit; i++) {
+                String dimensionFieldName = randomAlphaOfLength(10);
+                dimensionFieldNames.add(dimensionFieldName);
+                mapping.startObject(dimensionFieldName).field("type", "keyword").field("time_series_dimension", true).endObject();
+            }
+        },
+            mapping -> mapping.startObject("gauge").field("type", "integer").field("time_series_metric", "gauge").endObject(),
+            () -> List.of(dimensionFieldNames.get(0)),
+            dimensionFieldLimit
+        );
+
+        final Map<String, Object> source = new HashMap<>();
+        source.put("routing_field", randomAlphaOfLength(1536));
+        source.put("gauge", randomIntBetween(10, 20));
+        source.put("@timestamp", Instant.now().toEpochMilli());
+        for (int i = 0; i < dimensionFieldLimit; i++) {
+            source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1536));
+        }
+        final Exception ex = expectThrows(DocumentParsingException.class, () -> client().prepareIndex("test").setSource(source).get());
+        // NOTE: the number of bytes of the tsid might change slightly, which is why we do not match strings exactly.
+        assertEquals("_tsid longer than [32766] bytes [34104].".substring(0, 30), ex.getCause().getMessage().substring(0, 30));
+    }
+
+    private void createTimeSeriesIndex(
+        final CheckedConsumer<XContentBuilder, IOException> dimensions,
+        final CheckedConsumer<XContentBuilder, IOException> metrics,
+        final Supplier<List<String>> routingPaths,
+        int dimensionsFieldLimit
+    ) throws IOException {
+        XContentBuilder mapping = JsonXContent.contentBuilder();
+        mapping.startObject().startObject("properties");
+        mapping.startObject("@timestamp").field("type", "date").endObject();
+        metrics.accept(mapping);
+        dimensions.accept(mapping);
+        mapping.endObject().endObject();
+
+        Settings settings = Settings.builder()
+            .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+            .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPaths.get())
+            .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-08T23:40:53.384Z")
+            .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z")
+            .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), dimensionsFieldLimit)
+            .build();
+        client().admin().indices().prepareCreate("test").setSettings(settings).setMapping(mapping).get();
+    }
+
+}

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -110,7 +110,7 @@ public enum MergeReason {
     );
     public static final Setting<Long> INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING = Setting.longSetting(
         "index.mapping.dimension_fields.limit",
-        16,
+        21,
         0,
         Property.Dynamic,
         Property.IndexScope

server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java

@@ -67,7 +67,7 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper {
      * comfortable given that dimensions are typically going to be less than a
      * hundred bytes each, but we're being paranoid here.
      */
-    private static final int DIMENSION_VALUE_LIMIT = 1024;
+    private static final int DIMENSION_VALUE_LIMIT = 1536;
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {

server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java

@@ -325,7 +325,7 @@ public void testTooManyDimensionFields() {
         int max;
         Settings settings;
         if (randomBoolean()) {
-            max = 16; // By default no more than 16 dimensions per document are supported
+            max = 21; // By default no more than 21 dimensions per document are supported
             settings = getIndexSettings();
         } else {
             max = between(1, 10000);

server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java

@@ -396,9 +396,9 @@ public void testDimensionExtraLongKeyword() throws IOException {
 
         Exception e = expectThrows(
             DocumentParsingException.class,
-            () -> mapper.parse(source(b -> b.field("field", randomAlphaOfLengthBetween(1025, 2048))))
+            () -> mapper.parse(source(b -> b.field("field", randomAlphaOfLengthBetween(1537, 2048))))
         );
-        assertThat(e.getCause().getMessage(), containsString("Dimension fields must be less than [1024] bytes but was"));
+        assertThat(e.getCause().getMessage(), containsString("Dimension fields must be less than [1536] bytes but was"));
     }
 
     public void testConfigureSimilarity() throws IOException {

server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java

@@ -160,9 +160,9 @@ public void testKeywordTooLong() throws IOException {
 
         Exception e = expectThrows(
             DocumentParsingException.class,
-            () -> parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52)).field("@timestamp", "2021-10-01"))
+            () -> parseDocument(docMapper, b -> b.field("a", "more_than_1536_bytes".repeat(80)).field("@timestamp", "2021-10-01"))
         );
-        assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1040]."));
+        assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1536] bytes but was [1600]."));
     }
 
     public void testKeywordTooLongUtf8() throws IOException {
@@ -173,9 +173,9 @@ public void testKeywordTooLongUtf8() throws IOException {
         String theWordLong = "長い";
         Exception e = expectThrows(
             DocumentParsingException.class,
-            () -> parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200)).field("@timestamp", "2021-10-01"))
+            () -> parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(300)).field("@timestamp", "2021-10-01"))
         );
-        assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1200]."));
+        assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1536] bytes but was [1800]."));
     }
 
     public void testKeywordNull() throws IOException {