Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…telemetry_add_commands

.buildkite/scripts/dra-workflow.sh

@@ -13,8 +13,8 @@ fi
 echo --- Preparing
 
 # TODO move this to image
-sudo apt-get update -y
-sudo apt-get install -y libxml2-utils python3.10-venv
+sudo NEEDRESTART_MODE=l apt-get update -y
+sudo NEEDRESTART_MODE=l apt-get install -y libxml2-utils python3.10-venv
 
 RM_BRANCH="$BRANCH"
 if [[ "$BRANCH" == "main" ]]; then

TESTING.asciidoc

@@ -45,7 +45,7 @@ run it using Gradle:
 
 ==== Launching and debugging from an IDE
 
-If you want to run Elasticsearch from your IDE, the `./gradlew run` task
+If you want to run and debug Elasticsearch from your IDE, the `./gradlew run` task
 supports a remote debugging option. Run the following from your terminal:
 
 ---------------------------------------------------------------------------
@@ -55,7 +55,7 @@ supports a remote debugging option. Run the following from your terminal:
 Next start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the IDE to connect to the process and allow debug functionality.
 
 
-As such the IDE needs to be instructed to listen for connections on this port.
+As such the IDE needs to be instructed to listen for connections on the debug port.
 Since we might run multiple JVMs as part of configuring and starting the cluster it's
 recommended to configure the IDE to initiate multiple listening attempts. In case of IntelliJ, this option
 is called "Auto restart" and needs to be checked.
@@ -64,6 +64,22 @@ NOTE: If you have imported the project into IntelliJ according to the instructio
 link:/CONTRIBUTING.md#importing-the-project-into-intellij-idea[CONTRIBUTING.md] then a debug run configuration
 named "Debug Elasticsearch" will be created for you and configured appropriately.
 
+===== Debugging the CLI launcher
+
+The gradle task does not start the Elasticsearch server process directly; like in the Elasticsearch distribution,
+the job of starting the server process is delegated to a launcher CLI tool. If you need to debug the launcher itself,
+add the following option to the `run` task:
+---------------------------------------------------------------------------
+./gradlew run --debug-cli-jvm
+---------------------------------------------------------------------------
+This option can be specified in isolation or combined with `--debug-jvm`. Since the CLI launcher lifespan may overlap
+with the server process lifespan, the CLI launcher process will be started on a different port (5107 for the first node,
+5108 and following for additional cluster nodes).
+
+As with the `--debug-jvm` command, the IDE needs to be instructed to listen for connections on the debug port.
+You need to configure and start an appropriate Remote JVM Debug configuration, e.g. by cloning and editing
+the "Debug Elasticsearch" run configuration to point to the correct debug port.
+
 ==== Disabling assertions
 
 When running Elasticsearch with `./gradlew run`, assertions are enabled by
@@ -103,7 +119,8 @@ password: `elastic-password`.
 - In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
 - In order to use a custom data directory: `--data-dir=/tmp/foo`
 - In order to preserve data in between executions: `--preserve-data`
-- In order to remotely attach a debugger to the process: `--debug-jvm`
+- In order to remotely attach a debugger to the server process: `--debug-jvm`
+- In order to remotely attach a debugger to the CLI launcher process: `--debug-cli-jvm`
 - In order to set a different keystore password: `--keystore-password`
 - In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
 - In order to pass a JVM setting, e.g. to disable assertions: `-Dtests.jvm.argline="-da"`

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -27,11 +27,11 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
-import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.type.EsField;
@@ -61,6 +61,11 @@
 public class EvalBenchmark {
     private static final int BLOCK_LENGTH = 8 * 1024;
 
+    static final DriverContext driverContext = new DriverContext(
+        BigArrays.NON_RECYCLING_INSTANCE,
+        BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
+    );
+
     static {
         // Smoke test all the expected values and force loading subclasses more like prod
         try {
@@ -72,11 +77,6 @@ public class EvalBenchmark {
         }
     }
 
-    static final DriverContext driverContext = new DriverContext(
-        BigArrays.NON_RECYCLING_INSTANCE,
-        BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
-    );
-
     @Param({ "abs", "add", "date_trunc", "equal_to_const", "long_equal_to_long", "long_equal_to_int", "mv_min", "mv_min_ascending" })
     public String operation;
 

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -176,6 +176,11 @@ public SearchLookup lookup() {
                 public Set<String> sourcePaths(String name) {
                     return Set.of(name);
                 }
+
+                @Override
+                public String parentField(String field) {
+                    throw new UnsupportedOperationException();
+                }
             });
         }
         throw new IllegalArgumentException("can't read [" + name + "]");

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip
+distributionSha256Sum=c16d517b50dd28b3f5838f0e844b7520b8f1eb610f2f29de7e4e04a1b7c9c79b
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy

@@ -31,7 +31,8 @@ abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {
         }
         """
 
-        subProject(":distribution:archives:integ-test-zip") << "configurations { extracted }"
+        subProject(":distribution:archives:integ-test-zip") << "configurations.create('extracted')\n"
+        subProject(":distribution:archives:integ-test-zip") << "configurations.create('default')\n"
     }
 
     void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackTests = []) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy

@@ -24,8 +24,6 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTest does nothing when there are no tests"() {
         given:
-        // RestIntegTestTask not cc compatible due to
-        configurationCacheCompatible = false
         buildFile << """
         plugins {
           id 'elasticsearch.legacy-yaml-rest-test'
@@ -43,8 +41,6 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTest executes and copies api and tests to correct source set"() {
         given:
-        // RestIntegTestTask not cc compatible due to
-        configurationCacheCompatible = false
         internalBuild()
         buildFile << """
             apply plugin: 'elasticsearch.legacy-yaml-rest-test'
@@ -56,9 +52,10 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
             // can't actually spin up test cluster from this test
            tasks.withType(Test).configureEach{ enabled = false }
 
+           def clazzpath = sourceSets.yamlRestTest.runtimeClasspath
            tasks.register("printYamlRestTestClasspath").configure {
                doLast {
-                   println sourceSets.yamlRestTest.runtimeClasspath.asPath
+                   println clazzpath.asPath
                }
            }
         """

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -74,13 +74,12 @@ public void doCall() {
                 }
             });
 
+        boolean isModule = GradleUtils.isModuleProject(project.getPath());
+        boolean isXPackModule = isModule && project.getPath().startsWith(":x-pack");
+        if (isModule == false || isXPackModule) {
+            addNoticeGeneration(project, extension);
+        }
         project.afterEvaluate(p -> {
-            boolean isModule = GradleUtils.isModuleProject(p.getPath());
-            boolean isXPackModule = isModule && p.getPath().startsWith(":x-pack");
-            if (isModule == false || isXPackModule) {
-                addNoticeGeneration(p, extension);
-            }
-
             @SuppressWarnings("unchecked")
             NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
                 .getExtensions()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java

@@ -16,11 +16,11 @@
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.file.SourceDirectorySet;
 import org.gradle.api.internal.file.FileOperations;
-import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.PathSensitive;
@@ -43,7 +43,7 @@
  * A task to create a notice file which includes dependencies' notices.
  */
 @CacheableTask
-public class NoticeTask extends DefaultTask {
+public abstract class NoticeTask extends DefaultTask {
 
     @InputFile
     @PathSensitive(PathSensitivity.RELATIVE)
@@ -57,19 +57,17 @@ public class NoticeTask extends DefaultTask {
     /**
      * Directories to include notices from
      */
-    private final ListProperty<File> licensesDirs;
+    @Internal
+    abstract ListProperty<File> getLicenseDirs();
 
     private final FileOperations fileOperations;
-    private ObjectFactory objectFactory;
 
     @Inject
-    public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOperations fileOperations, ObjectFactory objectFactory) {
-        this.objectFactory = objectFactory;
+    public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOperations fileOperations) {
         this.fileOperations = fileOperations;
         setDescription("Create a notice file from dependencies");
         // Default licenses directory is ${projectDir}/licenses (if it exists)
-        licensesDirs = objectFactory.listProperty(File.class);
-        licensesDirs.add(projectLayout.getProjectDirectory().dir("licenses").getAsFile());
+        getLicenseDirs().add(projectLayout.getProjectDirectory().dir("licenses").getAsFile());
         inputFile = new File(buildLayout.getRootDirectory(), "NOTICE.txt");
         outputFile = projectLayout.getBuildDirectory().dir("notices/" + getName()).get().file("NOTICE.txt").getAsFile();
     }
@@ -78,7 +76,7 @@ public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOper
      * Add notices from the specified directory.
      */
     public void licensesDir(File licensesDir) {
-        licensesDirs.add(licensesDir);
+        getLicenseDirs().add(licensesDir);
     }
 
     public void source(Object source) {
@@ -185,7 +183,7 @@ public FileCollection getNoticeFiles() {
     }
 
     private List<File> existingLicenseDirs() {
-        return licensesDirs.get().stream().filter(d -> d.exists()).collect(Collectors.toList());
+        return getLicenseDirs().get().stream().filter(d -> d.exists()).collect(Collectors.toList());
     }
 
     @InputFiles

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -35,7 +35,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":modules:percolator");
         map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval");
         map.put(LegacyRestTestBasePlugin.class, ":modules:reindex");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:repository-s3");
         map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url");
         map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu");
@@ -125,6 +124,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:mixed-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:heap-attack");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:multi-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:single-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet:qa:rest");
@@ -138,11 +138,9 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:multi-cluster-tests-with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:native-multi-node-tests");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:single-node-tests");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:repositories-metering-api:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:minio");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:url");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:tls-basic");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:multi-node");
@@ -151,11 +149,9 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:fs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:minio");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:multi-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:no-sql");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:single-node");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java

@@ -29,11 +29,8 @@
 import org.gradle.api.specs.NotSpec;
 import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.Sync;
-import org.gradle.api.tasks.TaskContainer;
 import org.gradle.api.tasks.bundling.Zip;
 
-import java.util.Collections;
-
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.RestrictedBuildApiService.BUILD_API_RESTRICTIONS_SYS_PROPERTY;
@@ -131,11 +128,7 @@ public void apply(Project project) {
     }
 
     private void configureCacheability(StandaloneRestIntegTestTask testTask) {
-        TaskContainer tasks = project.getTasks();
-        Spec<Task> taskSpec = t -> tasks.withType(StandaloneRestIntegTestTask.class)
-            .stream()
-            .filter(task -> task != testTask)
-            .anyMatch(task -> Collections.disjoint(task.getClusters(), testTask.getClusters()) == false);
+        Spec<Task> taskSpec = task -> testTask.getClusters().stream().anyMatch(ElasticsearchCluster::isShared);
         testTask.getOutputs()
             .doNotCacheIf(
                 "Caching disabled for this task since it uses a cluster shared by other tasks",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java

@@ -85,7 +85,7 @@ public void apply(Project project) {
             NamedDomainObjectContainer<ElasticsearchCluster> clusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
                 .getExtensions()
                 .getByName(TestClustersPlugin.EXTENSION_NAME);
-            clusters.all(c -> {
+            clusters.configureEach(c -> {
                 if (BuildParams.isInFipsJvm()) {
                     c.setting("xpack.security.transport.ssl.key", "test-node.key");
                     c.keystore("xpack.security.transport.ssl.secure_key_passphrase", "test-node-key-password");

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.4
+8.5

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.12.0
-lucene            = 9.8.0
+lucene            = 9.9.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d

build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy

@@ -166,7 +166,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.output.contains("Task ':myTask' is not up-to-date because:\n" +
-                "  Input property 'clusters.myCluster\$0.nodes.\$0.$propertyName'")
+                "  Input property 'clusters.myCluster\$0.$propertyName'")
         result.output.contains("elasticsearch-keystore script executed!")
         assertEsOutputContains("myCluster", "Starting Elasticsearch process")
         assertEsOutputContains("myCluster", "Stopping node")

build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -79,6 +79,7 @@ public void apply(Project project) {
     }
 
     private void setupDistributionContainer(Project project, Property<Boolean> dockerAvailable) {
+
         distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
             Configuration fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name);
             Configuration extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name);

build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -55,7 +55,7 @@ public String toString() {
     private final Property<Boolean> failIfUnavailable;
     private final Property<Boolean> preferArchive;
     private final ConfigurableFileCollection extracted;
-    private Action<ElasticsearchDistribution> distributionFinalizer;
+    private transient Action<ElasticsearchDistribution> distributionFinalizer;
     private boolean frozen = false;
 
     ElasticsearchDistribution(