Merge branch 'main' into otel_plugin

README.asciidoc

@@ -1,6 +1,6 @@
 = Elasticsearch
 
-Elasticsearch is a distributed search and analytics engine optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more.
+Elasticsearch is a distributed search and analytics engine, scalable data store and vector database optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more.
 
 Use cases enabled by Elasticsearch include:
 

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
+distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy

@@ -13,6 +13,7 @@ import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.file.ProjectLayout
+import org.gradle.api.provider.ProviderFactory
 import org.gradle.api.tasks.Internal
 import org.gradle.process.ExecOperations
 
@@ -24,14 +25,17 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
     AntFixture fixture
 
     @Inject
-    AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) {
-       super(projectLayout, execOperations, fileSystemOperations)
+    AntFixtureStop(ProjectLayout projectLayout,
+                   ExecOperations execOperations,
+                   FileSystemOperations fileSystemOperations,
+                   ProviderFactory providerFactory) {
+        super(projectLayout, execOperations, fileSystemOperations, providerFactory)
     }
 
     void setFixture(AntFixture fixture) {
         assert this.fixture == null
         this.fixture = fixture;
-        final Object pid = "${ -> this.fixture.pid }"
+        final Object pid = "${-> this.fixture.pid}"
         onlyIf("pidFile exists") { fixture.pidFile.exists() }
         doFirst {
             logger.info("Shutting down ${fixture.name} with pid ${pid}")

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -26,7 +26,6 @@
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
-import org.gradle.jvm.toolchain.JvmVendorSpec;
 
 import java.io.File;
 import java.io.IOException;
@@ -161,10 +160,8 @@ private static TaskProvider<LoggedExec> createRunBwcGradleTask(
     /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
     private static Provider<String> getJavaHome(ObjectFactory objectFactory, JavaToolchainService toolChainService, final int version) {
         Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
-        return toolChainService.launcherFor(javaToolchainSpec -> {
-            javaToolchainSpec.getLanguageVersion().value(value);
-            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
-        }).map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
+        return toolChainService.launcherFor(javaToolchainSpec -> { javaToolchainSpec.getLanguageVersion().value(value); })
+            .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
     }
 
     private static String readFromFile(File file) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -48,7 +48,6 @@
 import java.nio.file.Files;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
@@ -315,50 +314,15 @@ private Optional<File> findRuntimeJavaHome() {
         return env == null ? Optional.empty() : Optional.of(new File(env));
     }
 
-    @NotNull
-    private String resolveJavaHomeFromEnvVariable(String javaHomeEnvVar) {
-        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
-        // Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet
-        Arrays.stream(javaHomeNames.get().split(","))
-            .filter(s -> s.equals(javaHomeEnvVar))
-            .findFirst()
-            .orElseThrow(
-                () -> new GradleException(
-                    "Environment variable '"
-                        + javaHomeEnvVar
-                        + "' is not registered with Gradle installation supplier. Ensure 'org.gradle.java.installations.fromEnv' is "
-                        + "updated in gradle.properties file."
-                )
-            );
-        String versionedJavaHome = System.getenv(javaHomeEnvVar);
-        if (versionedJavaHome == null) {
-            final String exceptionMessage = String.format(
-                Locale.ROOT,
-                "$%s must be set to build Elasticsearch. "
-                    + "Note that if the variable was just set you "
-                    + "might have to run `./gradlew --stop` for "
-                    + "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details.",
-                javaHomeEnvVar
-            );
-            throw new GradleException(exceptionMessage);
-        }
-        return versionedJavaHome;
-    }
-
     @NotNull
     private File resolveJavaHomeFromToolChainService(String version) {
         Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
         Provider<JavaLauncher> javaLauncherProvider = toolChainService.launcherFor(javaToolchainSpec -> {
             javaToolchainSpec.getLanguageVersion().value(value);
-            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
         });
         return javaLauncherProvider.get().getMetadata().getInstallationPath().getAsFile();
     }
 
-    private static String getJavaHomeEnvVarName(String version) {
-        return "JAVA" + version + "_HOME";
-    }
-
     public static String getResourceContents(String resourcePath) {
         try (
             BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java

@@ -102,7 +102,7 @@ public class LicenseAnalyzer {
             AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
             LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
             OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-            SOFTWARE\\.
+            SOFTWARE\\.?
             """).replaceAll("\\s+", "\\\\s*"), Pattern.DOTALL)),
         new LicenseMatcher(
             "MIT-0",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java

@@ -23,9 +23,12 @@
 import java.util.Map;
 import java.util.Optional;
 
+/**
+ * Resolves released Oracle JDKs that are EOL.
+ */
 public abstract class ArchivedOracleJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
-    private static final Map<Integer, String> ARCHIVED_BASE_VERSIONS = Maps.of(20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1", 17, "17.0.7");
+    private static final Map<Integer, String> ARCHIVED_BASE_VERSIONS = Maps.of(20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1");
 
     @Override
     public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.8
+8.9

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy

@@ -40,12 +40,6 @@ class ArchivedOracleJdkToolchainResolverSpec extends AbstractToolchainResolverSp
                 [18, ORACLE, LINUX, X86_64, "https://download.oracle.com/java/18/archive/jdk-18.0.2.1_linux-x64_bin.tar.gz"],
                 [18, ORACLE, LINUX, AARCH64, "https://download.oracle.com/java/18/archive/jdk-18.0.2.1_linux-aarch64_bin.tar.gz"],
                 [18, ORACLE, WINDOWS, X86_64, "https://download.oracle.com/java/18/archive/jdk-18.0.2.1_windows-x64_bin.zip"],
-
-                [17, ORACLE, MAC_OS, X86_64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_macos-x64_bin.tar.gz"],
-                [17, ORACLE, MAC_OS, AARCH64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_macos-aarch64_bin.tar.gz"],
-                [17, ORACLE, LINUX, X86_64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_linux-x64_bin.tar.gz"],
-                [17, ORACLE, LINUX, AARCH64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_linux-aarch64_bin.tar.gz"],
-                [17, ORACLE, WINDOWS, X86_64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_windows-x64_bin.zip"]
         ]
     }
 

build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -14,6 +14,7 @@
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
@@ -53,6 +54,8 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
 
     private Property<Boolean> dockerAvailability;
 
+    private boolean writingDependencies = false;
+
     @Inject
     public DistributionDownloadPlugin(ObjectFactory objectFactory) {
         this.objectFactory = objectFactory;
@@ -65,6 +68,7 @@ public void setDockerAvailability(Provider<Boolean> dockerAvailability) {
 
     @Override
     public void apply(Project project) {
+        writingDependencies = project.getGradle().getStartParameter().getWriteDependencyVerifications().isEmpty() == false;
         project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
             transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE);
             transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
@@ -87,7 +91,6 @@ private void setupDistributionContainer(Project project) {
             var extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name);
             extractedConfiguration.getAttributes()
                 .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
-
             var distribution = new ElasticsearchDistribution(
                 name,
                 objectFactory,
@@ -96,16 +99,20 @@ private void setupDistributionContainer(Project project) {
                 objectFactory.fileCollection().from(extractedConfiguration)
             );
 
-            registerDistributionDependencies(project, distribution);
+            // when running with --write-dependency-verification to update dependency verification data,
+            // we do not register the dependencies as we ignore elasticsearch internal dependencies anyhow and
+            // want to reduce general resolution time
+            if (writingDependencies == false) {
+                registerDistributionDependencies(project, distribution);
+            }
             return distribution;
         });
         project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
     }
 
     private void registerDistributionDependencies(Project project, ElasticsearchDistribution distribution) {
-        project.getConfigurations()
-            .getByName(DISTRO_CONFIG_PREFIX + distribution.getName())
-            .getDependencies()
+        Configuration distroConfig = project.getConfigurations().getByName(DISTRO_CONFIG_PREFIX + distribution.getName());
+        distroConfig.getDependencies()
             .addLater(
                 project.provider(() -> distribution.maybeFreeze())
                     .map(
@@ -114,9 +121,9 @@ private void registerDistributionDependencies(Project project, ElasticsearchDist
                     )
             );
 
-        project.getConfigurations()
-            .getByName(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName())
-            .getDependencies()
+        Configuration extractedDistroConfig = project.getConfigurations()
+            .getByName(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName());
+        extractedDistroConfig.getDependencies()
             .addAllLater(
                 project.provider(() -> distribution.maybeFreeze())
                     .map(

build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java

@@ -17,6 +17,8 @@
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
@@ -92,17 +94,45 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat
     private String output;
 
     @Inject
-    public LoggedExec(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) {
+    public LoggedExec(
+        ProjectLayout projectLayout,
+        ExecOperations execOperations,
+        FileSystemOperations fileSystemOperations,
+        ProviderFactory providerFactory
+    ) {
         this.projectLayout = projectLayout;
         this.execOperations = execOperations;
         this.fileSystemOperations = fileSystemOperations;
         getWorkingDir().convention(projectLayout.getProjectDirectory().getAsFile());
         // For now mimic default behaviour of Gradle Exec task here
-        getEnvironment().putAll(System.getenv());
+        setupDefaultEnvironment(providerFactory);
         getCaptureOutput().convention(false);
         getSpoolOutput().convention(false);
     }
 
+    /**
+     * We explicitly configure the environment variables that are passed to the executed process.
+     * This is required to make sure that the build cache and Gradle configuration cache is correctly configured
+     * can be reused across different build invocations.
+     * */
+    private void setupDefaultEnvironment(ProviderFactory providerFactory) {
+        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("BUILDKITE"));
+        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("GRADLE_BUILD_CACHE"));
+        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("VAULT"));
+        Provider<String> javaToolchainHome = providerFactory.environmentVariable("JAVA_TOOLCHAIN_HOME");
+        if (javaToolchainHome.isPresent()) {
+            getEnvironment().put("JAVA_TOOLCHAIN_HOME", javaToolchainHome);
+        }
+        Provider<String> javaRuntimeHome = providerFactory.environmentVariable("RUNTIME_JAVA_HOME");
+        if (javaRuntimeHome.isPresent()) {
+            getEnvironment().put("RUNTIME_JAVA_HOME", javaRuntimeHome);
+        }
+        Provider<String> path = providerFactory.environmentVariable("PATH");
+        if (path.isPresent()) {
+            getEnvironment().put("PATH", path);
+        }
+    }
+
     @TaskAction
     public void run() {
         boolean spoolOutput = getSpoolOutput().get();

docs/changelog/108360.yaml

@@ -0,0 +1,6 @@
+pr: 108360
+summary: "ESQL: Fix variable shadowing when pushing down past Project"
+area: ES|QL
+type: bug
+issues:
+ - 108008

docs/changelog/109583.yaml

@@ -0,0 +1,29 @@
+pr: 109583
+summary: "ESQL: INLINESTATS"
+area: ES|QL
+type: feature
+issues:
+ - 107589
+highlight:
+  title: "ESQL: INLINESTATS"
+  body: |-
+    This adds the `INLINESTATS` command to ESQL which performs a STATS and
+    then enriches the results into the output stream. So, this query:
+
+    [source,esql]
+    ----
+    FROM test
+    | INLINESTATS m=MAX(a * b) BY b
+    | WHERE m == a * b
+    | SORT a DESC, b DESC
+    | LIMIT 3
+    ----
+
+    Produces output like:
+
+    |  a  |  b  |   m   |
+    | --- | --- | ----- |
+    |  99 | 999 | 98901 |
+    |  99 | 998 | 98802 |
+    |  99 | 997 | 98703 |
+  notable: true

docs/changelog/110237.yaml

@@ -0,0 +1,7 @@
+pr: 110237
+summary: Optimize the loop processing of URL decoding
+area: Infra/REST API
+type: enhancement
+issues:
+  - 110235
+

docs/changelog/110606.yaml

@@ -0,0 +1,5 @@
+pr: 110606
+summary: Adding mapping validation to the simulate ingest API
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/110630.yaml

@@ -0,0 +1,5 @@
+pr: 110630
+summary: Telemetry for inference adaptive allocations
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/110734.yaml

@@ -0,0 +1,5 @@
+pr: 110734
+summary: Fix bug in ML serverless autoscaling which prevented trained model updates from triggering a scale up
+area: Machine Learning
+type: bug
+issues: [ ]

docs/changelog/110816.yaml

@@ -0,0 +1,6 @@
+pr: 110816
+summary: GET _cluster/settings with include_defaults returns the expected fallback value if defined in elasticsearch.yml
+area: Infra/Settings
+type: bug
+issues:
+ - 110815

docs/changelog/110906.yaml

@@ -0,0 +1,5 @@
+pr: 110906
+summary: "Add comma before charset parameter in WWW-Authenticate response header"
+area: Authentication
+type: bug
+issues: []

docs/changelog/110921.yaml

@@ -0,0 +1,5 @@
+pr: 110921
+summary: "ESQL: Support IP fields in MAX and MIN aggregations"
+area: ES|QL
+type: feature
+issues: []

docs/changelog/110971.yaml

@@ -0,0 +1,5 @@
+pr: 110971
+summary: "Search in ES|QL: Add MATCH operator"
+area: ES|QL
+type: feature
+issues: []

docs/changelog/110974.yaml

@@ -0,0 +1,5 @@
+pr: 110974
+summary: Add custom rule parameters to force time shift
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/110986.yaml

@@ -0,0 +1,6 @@
+pr: 110986
+summary: Fix unnecessary mustache template evaluation
+area: Ingest Node
+type: enhancement
+issues:
+  - 110191