Merge branch 'main' into text-date-formatter-warning

thecoop · Sep 6, 2024 · 5e497c0 · 5e497c0
2 parents 60b233d + ee68d0c
commit 5e497c0
Show file tree

Hide file tree

Showing 87 changed files with 1,352 additions and 337 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -594,8 +594,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.1
+      - label: "{{matrix.image}} / 8.15.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -609,7 +609,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.1
+          BWC_VERSION: 8.15.2
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -662,8 +662,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.1#bwcTest
+      - label: 8.15.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -673,7 +673,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.1
+          BWC_VERSION: 8.15.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -821,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -32,5 +32,5 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.1"
+  - "8.15.2"
   - "8.16.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.24"
-  - "8.15.1"
+  - "8.15.2"
   - "8.16.0"
diff --git a/...org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/...org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy
@@ -55,8 +55,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build()
 
         then:
-        // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
@@ -165,7 +164,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         then:
         result.task(':check').outcome == TaskOutcome.UP_TO_DATE
         result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
@@ -41,7 +41,7 @@ develocity {
       if (BuildParams.inFipsJvm) {
         tag 'FIPS'
       }
-      println "onCI = $onCI"
+
       if (onCI) { //Buildkite-specific build scan metadata
         String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL')
         def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH')

diff --git a/...asticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/...asticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
@@ -35,6 +35,7 @@
 import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.testing.Test;
+import org.gradle.language.jvm.tasks.ProcessResources;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -213,6 +214,17 @@ public void apply(Project project) {
             .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK)
             .flatMap(CopyRestTestsTask::getOutputResourceDir);
 
+        // ensure we include other non rest spec related test resources
+        project.getTasks()
+            .withType(ProcessResources.class)
+            .named(yamlCompatTestSourceSet.getProcessResourcesTaskName())
+            .configure(processResources -> {
+                processResources.from(
+                    sourceSets.getByName(YamlRestTestPlugin.YAML_REST_TEST).getResources(),
+                    spec -> { spec.exclude("rest-api-spec/**"); }
+                );
+            });
+
         // setup the test task
         TaskProvider<? extends Test> yamlRestCompatTestTask = registerTestTask(project, yamlCompatTestSourceSet);
         yamlRestCompatTestTask.configure(testTask -> {
@@ -221,7 +233,7 @@ public void apply(Project project) {
             testTask.setTestClassesDirs(
                 yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
             );
-            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false);
+            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getOutput().isEmpty() == false);
             testTask.setClasspath(
                 yamlCompatTestSourceSet.getRuntimeClasspath()
                     // remove the "normal" api and tests

diff --git a/docs/changelog/111161.yaml b/docs/changelog/111161.yaml
@@ -0,0 +1,6 @@
+pr: 111161
+summary: Add support for templates when validating mappings in the simulate ingest
+  API
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/111285.yaml b/docs/changelog/111285.yaml
diff --git a/docs/changelog/111475.yaml b/docs/changelog/111475.yaml
diff --git a/docs/changelog/111548.yaml b/docs/changelog/111548.yaml
@@ -0,0 +1,6 @@
+pr: 111548
+summary: Json parsing exceptions should not cause 500 errors
+area: Infra/Core
+type: bug
+issues:
+ - 111542
diff --git a/docs/changelog/111673.yaml b/docs/changelog/111673.yaml
diff --git a/docs/changelog/111729.yaml b/docs/changelog/111729.yaml
diff --git a/docs/changelog/111756.yaml b/docs/changelog/111756.yaml
diff --git a/docs/changelog/111758.yaml b/docs/changelog/111758.yaml
diff --git a/docs/changelog/111807.yaml b/docs/changelog/111807.yaml
diff --git a/docs/changelog/111843.yaml b/docs/changelog/111843.yaml
diff --git a/docs/changelog/111863.yaml b/docs/changelog/111863.yaml
diff --git a/docs/changelog/111866.yaml b/docs/changelog/111866.yaml
diff --git a/docs/changelog/111943.yaml b/docs/changelog/111943.yaml
diff --git a/docs/changelog/111947.yaml b/docs/changelog/111947.yaml
diff --git a/docs/changelog/111966.yaml b/docs/changelog/111966.yaml
diff --git a/docs/changelog/111983.yaml b/docs/changelog/111983.yaml
diff --git a/docs/changelog/111994.yaml b/docs/changelog/111994.yaml
diff --git a/docs/changelog/112005.yaml b/docs/changelog/112005.yaml
diff --git a/docs/changelog/112038.yaml b/docs/changelog/112038.yaml
diff --git a/docs/changelog/112046.yaml b/docs/changelog/112046.yaml
diff --git a/docs/changelog/112090.yaml b/docs/changelog/112090.yaml
diff --git a/docs/changelog/112135.yaml b/docs/changelog/112135.yaml
diff --git a/docs/changelog/112139.yaml b/docs/changelog/112139.yaml
diff --git a/docs/changelog/112173.yaml b/docs/changelog/112173.yaml
diff --git a/docs/changelog/112178.yaml b/docs/changelog/112178.yaml
diff --git a/docs/changelog/112217.yaml b/docs/changelog/112217.yaml
diff --git a/docs/changelog/112226.yaml b/docs/changelog/112226.yaml
diff --git a/docs/changelog/112230.yaml b/docs/changelog/112230.yaml
diff --git a/docs/changelog/112242.yaml b/docs/changelog/112242.yaml
diff --git a/docs/changelog/112260.yaml b/docs/changelog/112260.yaml
diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc
@@ -119,7 +119,11 @@ as well the same way that a non-simulated ingest would. No data is
 indexed into {es}. Instead, the transformed document is returned,
 along with the list of pipelines that have been executed and the name
 of the index where the document would have been indexed if this were
-not a simulation. This differs from the
+not a simulation. The transformed document is validated against the
+mappings that would apply to this index, and any validation error is
+reported in the result.
+
+This API differs from the
 <<simulate-pipeline-api,simulate pipeline API>> in that you specify a
 single pipeline for that API, and it only runs that one pipeline. The
 simulate pipeline API is more useful for developing a single pipeline,

diff --git a/docs/reference/mapping/types/sparse-vector.asciidoc b/docs/reference/mapping/types/sparse-vector.asciidoc
@@ -91,7 +91,7 @@ NOTE: `sparse_vector` fields can not be included in indices that were *created*
 NOTE: `sparse_vector` fields only support strictly positive values.
 Negative values will be rejected.
 
-NOTE: `sparse_vector` fields do not support querying, sorting or aggregating.
+NOTE: `sparse_vector` fields do not support <<analysis,analyzers>>, querying, sorting or aggregating.
 They may only be used within specialized queries.
 The recommended query to use on these fields are <<query-dsl-sparse-vector-query, `sparse_vector`>> queries.
 They may also be used within legacy <<query-dsl-text-expansion-query,`text_expansion`>> queries.

diff --git a/...ntent/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java b/...ntent/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java
@@ -57,7 +57,8 @@ public Token nextToken() throws IOException {
         try {
             return convertToken(parser.nextToken());
         } catch (JsonEOFException e) {
-            throw new XContentEOFException(e);
+            JsonLocation location = e.getLocation();
+            throw new XContentEOFException(new XContentLocation(location.getLineNr(), location.getColumnNr()), "Unexpected end of file", e);
         } catch (JsonParseException e) {
             throw newXContentParseException(e);
         }

diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java
@@ -8,11 +8,9 @@
 
 package org.elasticsearch.xcontent;
 
-import java.io.IOException;
+public class XContentEOFException extends XContentParseException {
 
-public class XContentEOFException extends IOException {
-
-    public XContentEOFException(IOException cause) {
-        super(cause);
+    public XContentEOFException(XContentLocation location, String message, Exception cause) {
+        super(location, message, cause);
     }
 }
diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle
@@ -54,6 +54,9 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
   task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Filtered test", "Hybrid t-digest produces different results.")
   task.skipTest("search.aggregation/420_percentile_ranks_tdigest_metric/filtered", "Hybrid t-digest produces different results.")
 
+  // Something has changed with response codes
+  task.skipTest("search.aggregation/20_terms/IP test", "Hybrid t-digest produces different results.")
+
   task.addAllowedWarningRegex("\\[types removal\\].*")
 }