Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…fix_non_snapshot_error_messages

.buildkite/pipelines/periodic.template.yml

@@ -46,7 +46,7 @@ steps:
         matrix:
           setup:
             ES_RUNTIME_JAVA:
-              - openjdk17
+              - openjdk21
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -88,10 +88,7 @@ steps:
         matrix:
           setup:
             ES_RUNTIME_JAVA:
-              - graalvm-ce17
-              - openjdk17
               - openjdk21
-              - openjdk22
               - openjdk23
             GRADLE_TASK:
               - checkPart1
@@ -115,10 +112,7 @@ steps:
         matrix:
           setup:
             ES_RUNTIME_JAVA:
-              - graalvm-ce17
-              - openjdk17
               - openjdk21
-              - openjdk22
               - openjdk23
             BWC_VERSION: $BWC_LIST
         agents:

.buildkite/pipelines/periodic.yml

@@ -407,7 +407,7 @@ steps:
         matrix:
           setup:
             ES_RUNTIME_JAVA:
-              - openjdk17
+              - openjdk21
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -449,10 +449,7 @@ steps:
         matrix:
           setup:
             ES_RUNTIME_JAVA:
-              - graalvm-ce17
-              - openjdk17
               - openjdk21
-              - openjdk22
               - openjdk23
             GRADLE_TASK:
               - checkPart1
@@ -476,10 +473,7 @@ steps:
         matrix:
           setup:
             ES_RUNTIME_JAVA:
-              - graalvm-ce17
-              - openjdk17
               - openjdk21
-              - openjdk22
               - openjdk23
             BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"]
         agents:

TESTING.asciidoc

@@ -472,6 +472,15 @@ You can run a group of YAML test by using wildcards:
   --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=index/*/*}"
 ---------------------------------------------------------------------------
 
+or 
+
+---------------------------------------------------------------------------
+./gradlew :rest-api-spec:yamlRestTest \
+  --tests org.elasticsearch.test.rest.ClientYamlTestSuiteIT -Dtests.method="test {yaml=cat.segments/10_basic/*}"
+---------------------------------------------------------------------------
+
+The latter method is preferable when the YAML suite name contains `.` (period).
+
 Note that if the selected test via the `--tests` filter is not a valid test, i.e., the YAML test
 runner is not able to parse and load it, you might get an error message indicating that the test
 was not found. In such cases, running the whole suite without using the `--tests` could show more

build-tools-internal/src/main/resources/changelog-schema.json

@@ -32,6 +32,7 @@
             "CRUD",
             "Client",
             "Cluster Coordination",
+            "Codec",
             "Data streams",
             "DLM",
             "Discovery-Plugins",

docs/changelog/111684.yaml

@@ -0,0 +1,5 @@
+pr: 111684
+summary: Write downloaded model parts async
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112652.yaml

@@ -0,0 +1,5 @@
+pr: 110399
+summary: "[Inference API] alibabacloud ai search service support chunk infer to support semantic_text field"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112665.yaml

@@ -0,0 +1,14 @@
+pr: 112665
+summary: Remove zstd feature flag for index codec best compression
+area: Codec
+type: enhancement
+issues: []
+highlight:
+  title: Enable ZStandard compression for indices with index.codec set to best_compression
+  body: |-
+    Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to
+    best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with
+    index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a
+    similar indexing throughput depending on what options are used. Experiments with indexing logs have shown that
+    ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput compared to DEFLATE.
+  notable: true

docs/changelog/112834.yaml

@@ -0,0 +1,5 @@
+pr: 112834
+summary: Increase `replica_unassigned_buffer_time` default from 3s to 5s
+area: Health
+type: enhancement
+issues: []

docs/reference/ilm/actions/ilm-forcemerge.asciidoc

@@ -49,7 +49,7 @@ Number of segments to merge to. To fully merge the index, set to `1`.
 `index_codec`::
 (Optional, string)
 Codec used to compress the document store. The only accepted value is
-`best_compression`, which uses {wikipedia}/DEFLATE[DEFLATE] for a higher
+`best_compression`, which uses {wikipedia}/Zstd[ZSTD] for a higher
 compression ratio but slower stored fields performance. To use the default LZ4
 codec, omit this argument.
 +

docs/reference/index-modules.asciidoc

@@ -76,14 +76,16 @@ breaking change].
 
     The +default+ value compresses stored data with LZ4
     compression, but this can be set to +best_compression+
-    which uses {wikipedia}/DEFLATE[DEFLATE] for a higher
-    compression ratio, at the expense of slower stored fields performance.
+    which uses {wikipedia}/Zstd[ZSTD] for a higher
+    compression ratio, at the expense of slower stored fields read performance.
     If you are updating the compression type, the new one will be applied
     after segments are merged. Segment merging can be forced using
     <<indices-forcemerge,force merge>>. Experiments with indexing log datasets
-    have shown that `best_compression` gives up to ~18% lower storage usage in
-    the most ideal scenario compared to `default` while only minimally affecting
-    indexing throughput (~2%).
+    have shown that `best_compression` gives up to ~28% lower storage usage and
+    similar indexing throughput (sometimes a bit slower or faster depending on other used options) compared
+    to `default` while affecting get by id latencies between ~10% and ~33%. The higher get
+    by id latencies is not a concern for many use cases like logging or metrics, since
+    these don't really rely on get by id functionality (Get APIs or searching by _id).
 
 [[index-mode-setting]] `index.mode`::
 +

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml

@@ -9,7 +9,6 @@
             char_filter:
               - type: html_strip
                 escaped_tags: ["xxx", "yyy"]
-                read_ahead: 1024
     - length: { tokens: 1 }
     - match:  { tokens.0.token: "\ntest<yyy>foo</yyy>\n" }
 

server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java

@@ -120,7 +120,7 @@ public class ShardsAvailabilityHealthIndicatorService implements HealthIndicator
      */
     public static final Setting<TimeValue> REPLICA_UNASSIGNED_BUFFER_TIME = Setting.timeSetting(
         "health.shards_availability.replica_unassigned_buffer_time",
-        TimeValue.timeValueSeconds(3),
+        TimeValue.timeValueSeconds(5),
         TimeValue.timeValueSeconds(0),
         TimeValue.timeValueSeconds(20),
         Setting.Property.NodeScope,

server/src/main/java/org/elasticsearch/index/codec/CodecService.java

@@ -53,15 +53,11 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays)
         }
         codecs.put(LEGACY_DEFAULT_CODEC, legacyBestSpeedCodec);
 
+        codecs.put(
+            BEST_COMPRESSION_CODEC,
+            new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
+        );
         Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
-        if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
-            codecs.put(
-                BEST_COMPRESSION_CODEC,
-                new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
-            );
-        } else {
-            codecs.put(BEST_COMPRESSION_CODEC, legacyBestCompressionCodec);
-        }
         codecs.put(LEGACY_BEST_COMPRESSION_CODEC, legacyBestCompressionCodec);
 
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());

server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java

@@ -17,8 +17,6 @@
 public class CodecIntegrationTests extends ESSingleNodeTestCase {
 
     public void testCanConfigureLegacySettings() {
-        assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
-
         createIndex("index1", Settings.builder().put("index.codec", "legacy_default").build());
         var codec = client().admin().indices().prepareGetSettings("index1").execute().actionGet().getSetting("index1", "index.codec");
         assertThat(codec, equalTo("legacy_default"));
@@ -29,8 +27,6 @@ public void testCanConfigureLegacySettings() {
     }
 
     public void testDefaultCodecLogsdb() {
-        assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
-
         var indexService = createIndex("index1", Settings.builder().put("index.mode", "logsdb").build());
         var storedFieldsFormat = (Zstd814StoredFieldsFormat) indexService.getShard(0)
             .getEngineOrNull()

server/src/test/java/org/elasticsearch/index/codec/CodecTests.java

@@ -64,7 +64,6 @@ public void testDefault() throws Exception {
     }
 
     public void testBestCompression() throws Exception {
-        assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
         Codec codec = createCodecService().codec("best_compression");
         assertEquals(
             "Zstd814StoredFieldsFormat(compressionMode=ZSTD(level=3), chunkSize=245760, maxDocsPerChunk=2048, blockShift=10)",

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
 import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionCreator;
 import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
@@ -49,6 +50,7 @@
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceFields.EMBEDDING_MAX_BATCH_SIZE;
 
 public class AlibabaCloudSearchService extends SenderService {
     public static final String NAME = AlibabaCloudSearchUtils.SERVICE_NAME;
@@ -253,7 +255,20 @@ protected void doChunkedInfer(
         TimeValue timeout,
         ActionListener<List<ChunkedInferenceServiceResults>> listener
     ) {
-        listener.onFailure(new ElasticsearchStatusException("Chunking not supported by the {} service", RestStatus.BAD_REQUEST, NAME));
+        if (model instanceof AlibabaCloudSearchModel == false) {
+            listener.onFailure(createInvalidModelException(model));
+            return;
+        }
+
+        AlibabaCloudSearchModel alibabaCloudSearchModel = (AlibabaCloudSearchModel) model;
+        var actionCreator = new AlibabaCloudSearchActionCreator(getSender(), getServiceComponents());
+
+        var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE, EmbeddingRequestChunker.EmbeddingType.FLOAT)
+            .batchRequestsWithListeners(listener);
+        for (var request : batchedRequests) {
+            var action = alibabaCloudSearchModel.accept(actionCreator, taskSettings, inputType);
+            action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener());
+        }
     }
 
     /**

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceFields.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch;
+
+public class AlibabaCloudSearchServiceFields {
+    /**
+     * Taken from https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details
+     */
+    static final int EMBEDDING_MAX_BATCH_SIZE = 32;
+}

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java

@@ -11,28 +11,37 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.ChunkingOptions;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor;
 import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchUtils;
 import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModelTests;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsServiceSettingsTests;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettingsTests;
+import org.hamcrest.CoreMatchers;
 import org.hamcrest.MatcherAssert;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -44,6 +53,7 @@
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap;
 import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.Mockito.mock;
 
@@ -156,6 +166,84 @@ public void doInfer(
         }
     }
 
+    public void testChunkedInfer_Batches() throws IOException {
+        var input = List.of("foo", "bar");
+
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new AlibabaCloudSearchService(senderFactory, createWithEmptySettings(threadPool))) {
+            Map<String, Object> serviceSettingsMap = new HashMap<>();
+            serviceSettingsMap.put(AlibabaCloudSearchServiceSettings.SERVICE_ID, "service_id");
+            serviceSettingsMap.put(AlibabaCloudSearchServiceSettings.HOST, "host");
+            serviceSettingsMap.put(AlibabaCloudSearchServiceSettings.WORKSPACE_NAME, "default");
+            serviceSettingsMap.put(ServiceFields.DIMENSIONS, 1536);
+
+            Map<String, Object> taskSettingsMap = new HashMap<>();
+
+            Map<String, Object> secretSettingsMap = new HashMap<>();
+            secretSettingsMap.put("api_key", "secret");
+
+            var model = new AlibabaCloudSearchEmbeddingsModel(
+                "service",
+                TaskType.TEXT_EMBEDDING,
+                AlibabaCloudSearchUtils.SERVICE_NAME,
+                serviceSettingsMap,
+                taskSettingsMap,
+                secretSettingsMap,
+                null
+            ) {
+                public ExecutableAction accept(
+                    AlibabaCloudSearchActionVisitor visitor,
+                    Map<String, Object> taskSettings,
+                    InputType inputType
+                ) {
+                    return (inferenceInputs, timeout, listener) -> {
+                        InferenceTextEmbeddingFloatResults results = new InferenceTextEmbeddingFloatResults(
+                            List.of(
+                                new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123f, -0.0123f }),
+                                new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0456f, -0.0456f })
+                            )
+                        );
+
+                        listener.onResponse(results);
+                    };
+                }
+            };
+
+            PlainActionFuture<List<ChunkedInferenceServiceResults>> listener = new PlainActionFuture<>();
+            service.chunkedInfer(
+                model,
+                input,
+                new HashMap<>(),
+                InputType.INGEST,
+                new ChunkingOptions(null, null),
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var results = listener.actionGet(TIMEOUT);
+            assertThat(results, hasSize(2));
+
+            // first result
+            {
+                assertThat(results.get(0), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class));
+                var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(0);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertEquals(input.get(0), floatResult.chunks().get(0).matchedText());
+                assertTrue(Arrays.equals(new float[] { 0.0123f, -0.0123f }, floatResult.chunks().get(0).embedding()));
+            }
+
+            // second result
+            {
+                assertThat(results.get(1), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class));
+                var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(1);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertEquals(input.get(1), floatResult.chunks().get(0).matchedText());
+                assertTrue(Arrays.equals(new float[] { 0.0456f, -0.0456f }, floatResult.chunks().get(0).embedding()));
+            }
+        }
+    }
+
     private Map<String, Object> getRequestConfigMap(
         Map<String, Object> serviceSettings,
         Map<String, Object> taskSettings,