Merge branch 'main' into fix/es-6839-serverless-downsampling-replicas

.buildkite/pipelines/intake.yml

@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.2", "8.11.0"]
+            BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1632,6 +1632,22 @@ steps:
         env:
           BWC_VERSION: 8.10.2
 
+      - label: "{{matrix.image}} / 8.10.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.3
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -1002,6 +1002,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.2
+      - label: 8.10.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.3
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300

.buildkite/pull-requests.json

@@ -12,9 +12,26 @@
       "build_on_commit": true,
       "build_on_comment": true,
       "trigger_comment_regex": "buildkite\\W+elasticsearch-ci.+",
-      "labels": ["buildkite-opt-in"],
+      "labels": [
+        "buildkite-opt-in"
+      ],
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-pull-request-check-serverless-submodule",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": true,
+      "build_on_comment": false,
+      "labels": [
+        "test-update-serverless"
+      ]
     }
   ]
 }

.ci/bwcVersions

@@ -99,4 +99,5 @@ BWC_VERSION:
   - "8.10.0"
   - "8.10.1"
   - "8.10.2"
+  - "8.10.3"
   - "8.11.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.14"
-  - "8.10.2"
+  - "8.10.3"
   - "8.11.0"

TRACING.md

@@ -158,6 +158,6 @@ explicitly opening a scope via the `Tracer`.
 [otel]: https://opentelemetry.io/
 [thread-context]: ./server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
 [w3c]: https://www.w3.org/TR/trace-context/
-[tracing]: ./server/src/main/java/org/elasticsearch/tracing
+[tracing]: ./server/src/main/java/org/elasticsearch/telemetry
 [agent-config]: https://www.elastic.co/guide/en/apm/agent/java/master/configuration.html
 [agent]: https://www.elastic.co/guide/en/apm/agent/java/current/index.html

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.benchmark.compute.operator;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.AggregatorMode;
@@ -23,6 +24,7 @@
 import org.elasticsearch.compute.aggregation.SumLongAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.blockhash.BlockHash;
 import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.DoubleArrayVector;
@@ -139,10 +141,11 @@ private static Operator operator(String grouping, String op, String dataType) {
             );
             default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]");
         };
+        DriverContext driverContext = driverContext();
         return new HashAggregationOperator(
             List.of(supplier(op, dataType, groups.size()).groupingAggregatorFactory(AggregatorMode.SINGLE)),
             () -> BlockHash.build(groups, BIG_ARRAYS, 16 * 1024, false),
-            new DriverContext(BigArrays.NON_RECYCLING_INSTANCE)
+            driverContext
         );
     }
 
@@ -576,4 +579,11 @@ private static void run(String grouping, String op, String blockType, int opCoun
         operator.finish();
         checkExpected(grouping, op, blockType, dataType, operator.getOutput(), opCount);
     }
+
+    static DriverContext driverContext() {
+        return new DriverContext(
+            BigArrays.NON_RECYCLING_INSTANCE,
+            BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
+        );
+    }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -8,8 +8,10 @@
 
 package org.elasticsearch.benchmark.compute.operator;
 
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
 import org.elasticsearch.compute.data.IntBlock;
@@ -227,7 +229,7 @@ private static Page page(String operation) {
             case "mv_min", "mv_min_ascending" -> {
                 var builder = LongBlock.newBlockBuilder(BLOCK_LENGTH);
                 if (operation.endsWith("ascending")) {
-                    builder.mvOrdering(Block.MvOrdering.ASCENDING);
+                    builder.mvOrdering(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING);
                 }
                 for (int i = 0; i < BLOCK_LENGTH; i++) {
                     builder.beginPositionEntry();
@@ -262,6 +264,9 @@ private static void run(String operation) {
     }
 
     static DriverContext driverContext() {
-        return new DriverContext(BigArrays.NON_RECYCLING_INSTANCE);
+        return new DriverContext(
+            BigArrays.NON_RECYCLING_INSTANCE,
+            BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
+        );
     }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.DoubleBlock;
@@ -107,6 +108,7 @@ private static Operator operator(String data, int topCount) {
             ClusterSettings.createBuiltInClusterSettings()
         );
         return new TopNOperator(
+            BlockFactory.getNonBreakingInstance(),
             breakerService.getBreaker(CircuitBreaker.REQUEST),
             topCount,
             elementTypes,

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip
+distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy

@@ -55,7 +55,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build()
 
         then:
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
@@ -164,7 +165,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         then:
         result.task(':check').outcome == TaskOutcome.UP_TO_DATE
         result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java

@@ -43,7 +43,8 @@
 /**
  * The Java Module Compile Path Plugin, i.e. --module-path, ---module-version
  */
-public class ElasticsearchJavaModulePathPlugin implements Plugin<Project> {
+public abstract class ElasticsearchJavaModulePathPlugin implements Plugin<Project> {
+
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(JavaPlugin.class);
@@ -53,7 +54,7 @@ public void apply(Project project) {
     // List of root tasks, by name, whose compileJava task should not use the module path. These are test related sources.
     static final Set<String> EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common");
 
-    static void configureCompileModulePath(Project project) {
+    void configureCompileModulePath(Project project) {
         // first disable Gradle's builtin module path inference
         project.getTasks()
             .withType(JavaCompile.class)
@@ -82,12 +83,10 @@ static void configureCompileModulePath(Project project) {
         }).getIncoming().artifactView(it -> {
             it.componentFilter(cf -> {
                 var visited = new HashSet<ComponentIdentifier>();
-                return walkResolvedComponent(
-                    project,
-                    compileClasspath.getIncoming().getResolutionResult().getRoot(),
-                    isModuleProject,
-                    visited
-                ).anyMatch(cf::equals);
+                ResolvedComponentResult root = compileClasspath.getIncoming().getResolutionResult().getRoot();
+                ComponentIdentifier id = root.getId();
+                String currentBuildPath = ((ProjectComponentIdentifier) id).getBuild().getBuildPath();
+                return walkResolvedComponent(currentBuildPath, project, root, isModuleProject, visited).anyMatch(cf::equals);
             });
         }).getFiles();
 
@@ -116,7 +115,8 @@ public void execute(Task t) {
         });
     }
 
-    static Stream<ComponentIdentifier> walkResolvedComponent(
+    Stream<ComponentIdentifier> walkResolvedComponent(
+        String currentBuildPath,
         Project project,
         ResolvedComponentResult result,
         boolean isModuleDependency,
@@ -133,9 +133,10 @@ static Stream<ComponentIdentifier> walkResolvedComponent(
                     return false;
                 }
                 return isModuleDependency
-                    || (it.getId() instanceof ProjectComponentIdentifier projectId && hasModuleInfoDotJava(project, projectId));
+                    || (it.getId() instanceof ProjectComponentIdentifier projectId
+                        && hasModuleInfoDotJava(currentBuildPath, project, projectId));
             })
-            .flatMap(it -> Stream.concat(walkResolvedComponent(project, it, true, visited), Stream.of(it.getId())));
+            .flatMap(it -> Stream.concat(walkResolvedComponent(currentBuildPath, project, it, true, visited), Stream.of(it.getId())));
     }
 
     static class CompileModulePathArgumentProvider implements CommandLineArgumentProvider, Named {
@@ -176,16 +177,16 @@ public String getName() {
         }
     }
 
-    static boolean hasModuleInfoDotJava(Project project) {
+    boolean hasModuleInfoDotJava(Project project) {
         return getJavaMainSourceSet(project).getJava()
             .getSrcDirs()
             .stream()
             .map(dir -> dir.toPath().resolve("module-info.java"))
             .anyMatch(Files::exists);
     }
 
-    static boolean hasModuleInfoDotJava(Project project, ProjectComponentIdentifier id) {
-        return new File(findProjectIdPath(project, id), "src/main/java/module-info.java").exists();
+    boolean hasModuleInfoDotJava(String currentBuildPath, Project project, ProjectComponentIdentifier id) {
+        return new File(findProjectIdPath(currentBuildPath, project, id), "src/main/java/module-info.java").exists();
     }
 
     static SourceSet getJavaMainSourceSet(Project project) {
@@ -209,13 +210,14 @@ static boolean isIdea() {
         return System.getProperty("idea.sync.active", "false").equals("true");
     }
 
-    static File findProjectIdPath(Project project, ProjectComponentIdentifier id) {
-        if (id.getBuild().isCurrentBuild()) {
+    File findProjectIdPath(String currentBuildPath, Project project, ProjectComponentIdentifier id) {
+        if (id.getBuild().getBuildPath().equals(currentBuildPath)) {
             return project.findProject(id.getProjectPath()).getProjectDir();
         } else {
             // For project dependencies sourced from an included build we have to infer the source project path
-            File includedBuildDir = project.getGradle().includedBuild(id.getBuild().getName()).getProjectDir();
-
+            String buildPath = id.getBuild().getBuildPath();
+            String buildName = buildPath.substring(buildPath.lastIndexOf(':') + 1);
+            File includedBuildDir = project.getGradle().includedBuild(buildName).getProjectDir();
             // We have to account for us renaming the :libs projects here
             String[] pathSegments = id.getProjectPath().split(":");
             if (pathSegments[1].equals("libs")) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -66,7 +66,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:multi-cluster-search");
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java

@@ -17,4 +17,4 @@
  * conventional configured tasks of {@link RestIntegTestTask}
  */
 @CacheableTask
-public class RestIntegTestTask extends StandaloneRestIntegTestTask {}
+public abstract class RestIntegTestTask extends StandaloneRestIntegTestTask {}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java

@@ -221,6 +221,7 @@ public void apply(Project project) {
             testTask.setTestClassesDirs(
                 yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
             );
+            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false);
             testTask.setClasspath(
                 yamlCompatTestSourceSet.getRuntimeClasspath()
                     // remove the "normal" api and tests

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -32,7 +32,7 @@
 
   <!-- Intentionally have multi line string for a bulk request, otherwise this needs to fallback to string concatenation  -->
   <suppress files="modules[/\\]data-streams[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]datastreams[/\\]TsdbDataStreamRestIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
+  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.2
+8.3

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 8.11.0
 lucene            = 9.7.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 20.0.2+9@6e380f22cbe7469fa75fb448bd903d8e
+bundled_jdk = 21+35@fd2272bbf8e04c3dbaee13770090416c
 
 # optional dependencies
 spatial4j         = 0.7

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -36,7 +36,7 @@
  * {@link Nested} inputs.
  */
 @CacheableTask
-public class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware {
+public abstract class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware {
 
     private Collection<ElasticsearchCluster> clusters = new HashSet<>();
     private boolean debugServer = false;

distribution/packages/build.gradle

@@ -43,7 +43,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.0.0"
+   id "com.netflix.nebula.ospackage-base" version "11.5.0"
 }
 
 ['deb', 'rpm'].each { type ->

distribution/src/config/jvm.options

@@ -56,7 +56,7 @@
 
 # Leverages accelerated vector hardware instructions; removing this may
 # result in less optimal vector performance
-20:--add-modules=jdk.incubator.vector
+20-:--add-modules=jdk.incubator.vector
 
 ## heap dumps