Increase the max vector dims to 4096

docs/changelog/99682.yaml

@@ -0,0 +1,5 @@
+pr: 99682
+summary: Increase the max vector dims to 4096
+area: Vector Search
+type: enhancement
+issues: []

docs/reference/mapping/types/dense-vector.asciidoc

@@ -62,7 +62,7 @@ In many cases, a brute-force kNN search is not efficient enough. For this
 reason, the `dense_vector` type supports indexing vectors into a specialized
 data structure to support fast kNN retrieval through the <<search-api-knn, `knn` option>> in the search API
 
-Unmapped array fields of float elements with size between 128 and 2048 are dynamically mapped as `dense_vector` with a default similariy of `cosine`.
+Unmapped array fields of float elements with size between 128 and 4096 are dynamically mapped as `dense_vector` with a default similariy of `cosine`.
 You can override the default similarity by explicitly mapping the field as `dense_vector` with the desired similarity.
 
 Indexing is enabled by default for dense vector fields.
@@ -132,7 +132,7 @@ integer values between -128 to 127, inclusive for both indexing and searching.
 
 `dims`::
 (Optional, integer)
-Number of vector dimensions. Can't exceed `2048`. If `dims` is not specified,
+Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified,
 it will be set to the length of the first vector added to the field.
 
 `index`::

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -74,7 +74,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
     public static final IndexVersion LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION = IndexVersion.V_8_9_0;
 
     public static final String CONTENT_TYPE = "dense_vector";
-    public static short MAX_DIMS_COUNT = 2048; // maximum allowed number of dimensions
+    public static short MAX_DIMS_COUNT = 4096; // maximum allowed number of dimensions
 
     public static short MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING = 128; // minimum number of dims for floats to be dynamically mapped to vector
     public static final int MAGNITUDE_BYTES = 4;

server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java

@@ -19,7 +19,7 @@
  * 1) float[], this is for the ScoreScriptUtils class bindings which have converted a List based query vector into an array
  * 2) List, A painless script will typically use Lists since they are easy to pass as params and have an easy
  *      literal syntax.  Working with Lists directly, instead of converting to a float[], trades off runtime operations against
- *      memory pressure.  Dense Vectors may have high dimensionality, up to 2048.  Allocating a float[] per doc per script API
+ *      memory pressure.  Dense Vectors may have high dimensionality, up to 4096.  Allocating a float[] per doc per script API
  *      call is prohibitively expensive.
  * 3) Object, the whitelisted method for the painless API.  Calls into the float[] or List version based on the
         class of the argument and checks dimensionality.

server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java

@@ -409,7 +409,7 @@ public void testKnnSearchAction() throws IOException {
         assertEquals(2, response.getHits().getHits().length);
     }
 
-    public void testKnnVectorsWith2048Dims() throws IOException {
+    public void testKnnVectorsWith4096Dims() throws IOException {
         int numShards = 1 + randomInt(3);
         Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
 
@@ -418,7 +418,7 @@ public void testKnnVectorsWith2048Dims() throws IOException {
             .startObject("properties")
             .startObject("vector")
             .field("type", "dense_vector")
-            .field("dims", 2048)
+            .field("dims", 4096)
             .field("index", true)
             .field("similarity", "l2_norm")
             .endObject()
@@ -427,18 +427,18 @@ public void testKnnVectorsWith2048Dims() throws IOException {
         createIndex("index", indexSettings, builder);
 
         for (int doc = 0; doc < 10; doc++) {
-            client().prepareIndex("index").setSource("vector", randomVector(2048)).get();
+            client().prepareIndex("index").setSource("vector", randomVector(4096)).get();
         }
 
         indicesAdmin().prepareRefresh("index").get();
 
-        float[] queryVector = randomVector(2048);
+        float[] queryVector = randomVector(4096);
         KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 3, 50, null).boost(5.0f);
         SearchResponse response = client().prepareSearch("index").setKnnSearch(List.of(knnSearch)).addFetchField("*").setSize(10).get();
 
         assertHitCount(response, 3);
         assertEquals(3, response.getHits().getHits().length);
-        assertEquals(2048, response.getHits().getAt(0).field("vector").getValues().size());
+        assertEquals(4096, response.getHits().getAt(0).field("vector").getValues().size());
     }
 
     private float[] randomVector() {

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java

@@ -197,21 +197,21 @@ public void testDims() {
             assertThat(
                 e.getMessage(),
                 equalTo(
-                    "Failed to parse mapping: " + "The number of dimensions for field [field] should be in the range [1, 2048] but was [0]"
+                    "Failed to parse mapping: " + "The number of dimensions for field [field] should be in the range [1, 4096] but was [0]"
                 )
             );
         }
         // test max limit for non-indexed vectors
         {
             Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
                 b.field("type", "dense_vector");
-                b.field("dims", 3000);
+                b.field("dims", 5000);
             })));
             assertThat(
                 e.getMessage(),
                 equalTo(
                     "Failed to parse mapping: "
-                        + "The number of dimensions for field [field] should be in the range [1, 2048] but was [3000]"
+                        + "The number of dimensions for field [field] should be in the range [1, 4096] but was [5000]"
                 )
             );
         }
@@ -220,13 +220,13 @@ public void testDims() {
             Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
                 b.field("type", "dense_vector");
                 b.field("index", "true");
-                b.field("dims", 3000);
+                b.field("dims", 5000);
             })));
             assertThat(
                 e.getMessage(),
                 equalTo(
                     "Failed to parse mapping: "
-                        + "The number of dimensions for field [field] should be in the range [1, 2048] but was [3000]"
+                        + "The number of dimensions for field [field] should be in the range [1, 4096] but was [5000]"
                 )
             );
         }
@@ -597,10 +597,10 @@ public void testDocumentsWithIncorrectDims() throws Exception {
 
     /**
      * Test that max dimensions limit for float dense_vector field
-     * is 2048 as defined by {@link DenseVectorFieldMapper#MAX_DIMS_COUNT}
+     * is 4096 as defined by {@link DenseVectorFieldMapper#MAX_DIMS_COUNT}
      */
     public void testMaxDimsFloatVector() throws IOException {
-        final int dims = 2048;
+        final int dims = 4096;
         VectorSimilarity similarity = VectorSimilarity.COSINE;
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(b -> b.field("type", "dense_vector").field("dims", dims).field("index", true).field("similarity", similarity))
@@ -624,10 +624,10 @@ public void testMaxDimsFloatVector() throws IOException {
 
     /**
      * Test that max dimensions limit for byte dense_vector field
-     * is 2048 as defined by {@link KnnByteVectorField}
+     * is 4096 as defined by {@link KnnByteVectorField}
      */
     public void testMaxDimsByteVector() throws IOException {
-        final int dims = 2048;
+        final int dims = 4096;
         VectorSimilarity similarity = VectorSimilarity.COSINE;
         ;
         DocumentMapper mapper = createDocumentMapper(
@@ -703,7 +703,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val
     @Override
     // TODO: add `byte` element_type tests
     protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException {
-        b.field("type", "dense_vector").field("dims", randomIntBetween(2, 2048)).field("element_type", "float");
+        b.field("type", "dense_vector").field("dims", randomIntBetween(2, 4096)).field("element_type", "float");
         if (randomBoolean()) {
             b.field("index", true).field("similarity", randomFrom(VectorSimilarity.values()).toString());
         }

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java

@@ -158,39 +158,39 @@ public void testFloatCreateKnnQuery() {
     }
 
     public void testCreateKnnQueryMaxDims() {
-        {   // float type with 2048 dims
-            DenseVectorFieldType fieldWith2048dims = new DenseVectorFieldType(
+        {   // float type with 4096 dims
+            DenseVectorFieldType fieldWith4096dims = new DenseVectorFieldType(
                 "f",
                 IndexVersion.current(),
                 DenseVectorFieldMapper.ElementType.FLOAT,
-                2048,
+                4096,
                 true,
                 VectorSimilarity.COSINE,
                 Collections.emptyMap()
             );
-            float[] queryVector = new float[2048];
-            for (int i = 0; i < 2048; i++) {
+            float[] queryVector = new float[4096];
+            for (int i = 0; i < 4096; i++) {
                 queryVector[i] = randomFloat();
             }
-            Query query = fieldWith2048dims.createKnnQuery(queryVector, 10, null, null);
+            Query query = fieldWith4096dims.createKnnQuery(queryVector, 10, null, null);
             assertThat(query, instanceOf(KnnFloatVectorQuery.class));
         }
 
-        {   // byte type with 2048 dims
-            DenseVectorFieldType fieldWith2048dims = new DenseVectorFieldType(
+        {   // byte type with 4096 dims
+            DenseVectorFieldType fieldWith4096dims = new DenseVectorFieldType(
                 "f",
                 IndexVersion.current(),
                 DenseVectorFieldMapper.ElementType.BYTE,
-                2048,
+                4096,
                 true,
                 VectorSimilarity.COSINE,
                 Collections.emptyMap()
             );
-            byte[] queryVector = new byte[2048];
-            for (int i = 0; i < 2048; i++) {
+            byte[] queryVector = new byte[4096];
+            for (int i = 0; i < 4096; i++) {
                 queryVector[i] = randomByte();
             }
-            Query query = fieldWith2048dims.createKnnQuery(queryVector, 10, null, null);
+            Query query = fieldWith4096dims.createKnnQuery(queryVector, 10, null, null);
             assertThat(query, instanceOf(KnnByteVectorQuery.class));
         }
     }