Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…fix_CsvTestsDataLoader_classLoading

docs/changelog/109487.yaml

@@ -0,0 +1,5 @@
+pr: 109487
+summary: Start Trained Model Deployment API request query params now override body params
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/109551.yaml

@@ -0,0 +1,5 @@
+pr: 109551
+summary: Avoid `InferenceRunner` deadlock
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/109651.yaml

@@ -0,0 +1,5 @@
+pr: 109651
+summary: Support synthetic source for `geo_point` when `ignore_malformed` is used
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/109794.yaml

@@ -0,0 +1,5 @@
+pr: 109794
+summary: Provide document size reporter with `MapperService`
+area: Infra/Metrics
+type: bug
+issues: []

docs/changelog/109824.yaml

@@ -0,0 +1,6 @@
+pr: 109824
+summary: Check array size before returning array item in script doc values
+area: Infra/Scripting
+type: bug
+issues:
+ - 104998

docs/reference/mapping/types/geo-point.asciidoc

@@ -220,8 +220,7 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `geo_point` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<ignore-malformed,`ignore_malformed`>>, <<copy-to,`copy_to`>>, or with
+default configuration. Synthetic `_source` cannot be used together with <<copy-to,`copy_to`>> or with
 <<doc-values,`doc_values`>> disabled.
 
 Synthetic source always sorts `geo_point` fields (first by latitude and then

docs/reference/rest-api/common-parms.asciidoc

@@ -507,6 +507,10 @@ Return all statistics.
 `completion`::
 <<completion-suggester,Completion suggester>> statistics.
 
+`dense_vector`::
+Total number of dense vectors indexed.
+<<indices-refresh,Index refreshes>> can affect this statistic.
+
 `docs`::
 Number of documents, number of deleted docs which have not yet merged out, and total size in bytes.
 <<indices-refresh,Index refreshes>> can affect this statistic.

libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.xcontent;
+
+import java.io.IOException;
+
+/**
+ * A parser that copies data that was parsed into a {@link XContentBuilder}.
+ * This parser naturally has some memory and runtime overhead to perform said copying.
+ * Use with {@link XContentSubParser} to preserve the entire object.
+ */
+public class CopyingXContentParser extends FilterXContentParserWrapper {
+    private final XContentBuilder builder;
+
+    public CopyingXContentParser(XContentParser delegate) throws IOException {
+        super(delegate);
+        this.builder = XContentBuilder.builder(delegate.contentType().xContent());
+        switch (delegate.currentToken()) {
+            case START_OBJECT -> builder.startObject();
+            case START_ARRAY -> builder.startArray();
+            default -> throw new IllegalArgumentException(
+                "can only copy parsers pointed to START_OBJECT or START_ARRAY but found: " + delegate.currentToken()
+            );
+        }
+    }
+
+    @Override
+    public Token nextToken() throws IOException {
+        XContentParser.Token next = delegate().nextToken();
+        builder.copyCurrentEvent(delegate());
+        return next;
+    }
+
+    public XContentBuilder getBuilder() {
+        return builder;
+    }
+}

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java

@@ -1220,6 +1220,18 @@ public XContentBuilder rawValue(String value) throws IOException {
         return this;
     }
 
+    /**
+     * Copies current event from parser into this builder.
+     * The difference with {@link XContentBuilder#copyCurrentStructure(XContentParser)}
+     * is that this method does not copy sub-objects as a single entity.
+     * @param parser
+     * @throws IOException
+     */
+    public XContentBuilder copyCurrentEvent(XContentParser parser) throws IOException {
+        generator.copyCurrentEvent(parser);
+        return this;
+    }
+
     public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException {
         generator.copyCurrentStructure(parser);
         return this;

modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java

@@ -12,7 +12,7 @@
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -102,7 +102,7 @@ public DocumentSizeObserver newDocumentSizeObserver() {
                 @Override
                 public DocumentSizeReporter newDocumentSizeReporter(
                     String indexName,
-                    IndexMode indexMode,
+                    MapperService mapperService,
                     DocumentSizeAccumulator documentSizeAccumulator
                 ) {
                     return DocumentSizeReporter.EMPTY_INSTANCE;

modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java

@@ -58,7 +58,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -379,18 +378,18 @@ private LegacyGeoShapeParser() {}
         public void parse(
             XContentParser parser,
             CheckedConsumer<ShapeBuilder<?, ?, ?>, IOException> consumer,
-            Consumer<Exception> onMalformed
+            MalformedValueHandler malformedHandler
         ) throws IOException {
             try {
                 if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
                     while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                        parse(parser, consumer, onMalformed);
+                        parse(parser, consumer, malformedHandler);
                     }
                 } else {
                     consumer.accept(ShapeParser.parse(parser));
                 }
             } catch (ElasticsearchParseException e) {
-                onMalformed.accept(e);
+                malformedHandler.notify(e);
             }
         }
 

muted-tests.yml

@@ -10,7 +10,8 @@ tests:
   method: "testGuessIsDayFirstFromLocale"
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
@@ -19,7 +20,8 @@ tests:
   method: "testTrainedModelInference"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
@@ -34,7 +36,8 @@ tests:
   method: "testTimestampFieldTypeExposedByAllIndicesServices"
 - class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109318"
-  method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}"
+  method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling\
+    \ (too complex pattern)}"
 - class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
   issue: "https://github.com/elastic/elasticsearch/issues/101598"
   method: "testFeatureImportanceValues"
@@ -61,6 +64,12 @@ tests:
 - class: org.elasticsearch.action.search.SearchProgressActionListenerIT
   method: testSearchProgressWithHits
   issue: https://github.com/elastic/elasticsearch/issues/109830
+- class: "org.elasticsearch.xpack.shutdown.NodeShutdownReadinessIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109838"
+  method: "testShutdownReadinessService"
+- class: "org.elasticsearch.cluster.PrevalidateShardPathIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/104807"
+  method: "testCheckShards"
 
 # Examples:
 #

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -194,7 +194,7 @@ public void test50Remove() throws Exception {
             }
 
             assertThat(sh.runIgnoreExitCode("systemctl status elasticsearch.service").exitCode(), is(statusExitCode));
-            assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), is(1));
+            assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), not(0));
 
         }
 

qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java

@@ -28,7 +28,7 @@
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
 import static org.elasticsearch.packaging.util.ServerUtils.enableGeoIpDownloader;
-import static org.hamcrest.core.Is.is;
+import static org.hamcrest.Matchers.not;
 import static org.junit.Assume.assumeTrue;
 
 public class RpmPreservationTests extends PackagingTestCase {
@@ -78,7 +78,7 @@ public void test30PreserveConfig() throws Exception {
         assertRemoved(distribution());
 
         if (isSystemd()) {
-            assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), is(1));
+            assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), not(0));
         }
 
         assertPathsDoNotExist(

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java

@@ -56,8 +56,6 @@ public void testScriptByteVectorSearch() throws Exception {
                 """;
             createIndex(SCRIPT_BYTE_INDEX_NAME, Settings.EMPTY, mapping);
             indexVectors(SCRIPT_BYTE_INDEX_NAME);
-            // refresh the index
-            client().performRequest(new Request("POST", "/" + SCRIPT_BYTE_INDEX_NAME + "/_refresh"));
         }
         // search with a script query
         Request searchRequest = new Request("POST", "/" + SCRIPT_BYTE_INDEX_NAME + "/_search");
@@ -107,8 +105,6 @@ public void testScriptVectorSearch() throws Exception {
                 """;
             createIndex(SCRIPT_VECTOR_INDEX_NAME, Settings.EMPTY, mapping);
             indexVectors(SCRIPT_VECTOR_INDEX_NAME);
-            // refresh the index
-            client().performRequest(new Request("POST", "/" + SCRIPT_VECTOR_INDEX_NAME + "/_refresh"));
         }
         // search with a script query
         Request searchRequest = new Request("POST", "/" + SCRIPT_VECTOR_INDEX_NAME + "/_search");
@@ -237,7 +233,6 @@ public void testByteVectorSearch() throws Exception {
             // create index and index 10 random floating point vectors
             createIndex(BYTE_INDEX_NAME, Settings.EMPTY, mapping);
             indexVectors(BYTE_INDEX_NAME);
-            // refresh the index
             // force merge the index
             client().performRequest(new Request("POST", "/" + BYTE_INDEX_NAME + "/_forcemerge?max_num_segments=1"));
         }
@@ -448,6 +443,8 @@ private void indexVectors(String indexName) throws Exception {
             indexRequest.setJsonEntity(vectors[i]);
             assertOK(client().performRequest(indexRequest));
         }
+        // always refresh to ensure the data is visible
+        refresh(indexName);
     }
 
     private static Map<String, Object> search(Request request) throws IOException {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml

@@ -258,14 +258,12 @@
 ---
 "Dense vector stats":
   - requires:
-      cluster_features: [ "gte_v8.10.0" ]
-      reason: "dense vector stats added in 8.10"
+      cluster_features: [ "gte_v8.15.0" ]
+      reason: "dense vector stats reports from primary indices in 8.15"
   - do:
       indices.create:
         index: test1
         body:
-          settings:
-            number_of_replicas: 0
           mappings:
             properties:
               vector:
@@ -283,8 +281,6 @@
       indices.create:
         index: test2
         body:
-          settings:
-            number_of_replicas: 0
           mappings:
             properties:
               vector:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml

@@ -36,10 +36,9 @@
   - exists: indicators.shards_availability.details.started_primaries
   - exists: indicators.shards_availability.details.unassigned_replicas
 
-  - match: { indicators.shards_capacity.status: "green" }
-  - match: { indicators.shards_capacity.symptom: "The cluster has enough room to add new shards." }
-  - exists: indicators.shards_capacity.details.data.max_shards_in_cluster
-  - exists: indicators.shards_capacity.details.frozen.max_shards_in_cluster
+  # The shards_availability indicator is dependent on HealthMetadata being present in the cluster state, which we can't guarantee.
+  - is_true: indicators.shards_capacity.status
+  - is_true: indicators.shards_capacity.symptom
 
   - is_true: indicators.data_stream_lifecycle.status
   - is_true: indicators.data_stream_lifecycle.symptom