Merge branch 'main' into record-file-extensions

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -11,11 +11,14 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.DoubleVector;
 import org.elasticsearch.compute.data.LongBlock;
@@ -40,9 +43,13 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.planner.Layout;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -56,8 +63,10 @@
 import org.openjdk.jmh.annotations.Warmup;
 
 import java.time.Duration;
+import java.time.ZoneOffset;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -106,7 +115,9 @@ public class EvalBenchmark {
             "long_equal_to_int",
             "mv_min",
             "mv_min_ascending",
-            "rlike" }
+            "rlike",
+            "to_lower",
+            "to_upper" }
     )
     public String operation;
 
@@ -169,7 +180,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     new Coalesce(Source.EMPTY, lhs, List.of(f2)),
                     layout(f1, f2)
                 ).get(driverContext);
-                String desc = operation.endsWith("lazy") ? "CoalesceLazyEvaluator" : "CoalesceEagerEvaluator";
+                String desc = operation.endsWith("lazy") ? "CoalesceLongLazyEvaluator" : "CoalesceLongEagerEvaluator";
                 if (evaluator.toString().contains(desc) == false) {
                     throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
                 }
@@ -214,6 +225,16 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
+            case "to_lower" -> {
+                FieldAttribute keywordField = keywordField();
+                ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, toLower, layout(keywordField)).get(driverContext);
+            }
+            case "to_upper" -> {
+                FieldAttribute keywordField = keywordField();
+                ToUpper toUpper = new ToUpper(Source.EMPTY, keywordField, configuration());
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, toUpper, layout(keywordField)).get(driverContext);
+            }
             default -> throw new UnsupportedOperationException();
         };
     }
@@ -234,6 +255,23 @@ private static FieldAttribute keywordField() {
         return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
     }
 
+    private static Configuration configuration() {
+        return new Configuration(
+            ZoneOffset.UTC,
+            Locale.ROOT,
+            null,
+            null,
+            null,
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY),
+            null,
+            false,
+            Map.of(),
+            0,
+            false
+        );
+    }
+
     private static Layout layout(FieldAttribute... fields) {
         Layout.Builder layout = new Layout.Builder();
         layout.append(Arrays.asList(fields));
@@ -366,10 +404,24 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
+            case "to_lower" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
+            case "to_upper" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
             default -> throw new UnsupportedOperationException(operation);
         }
     }
 
+    private static void checkBytes(String operation, Page actual, BytesRef[] expectedVals) {
+        BytesRef scratch = new BytesRef();
+        BytesRefVector v = actual.<BytesRefBlock>getBlock(1).asVector();
+        for (int i = 0; i < BLOCK_LENGTH; i++) {
+            BytesRef expected = expectedVals[i % 2];
+            BytesRef b = v.getBytesRef(i, scratch);
+            if (b.equals(expected) == false) {
+                throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + b + "]");
+            }
+        }
+    }
+
     private static Page page(String operation) {
         return switch (operation) {
             case "abs", "add", "date_trunc", "equal_to_const" -> {
@@ -440,7 +492,7 @@ private static Page page(String operation) {
                 }
                 yield new Page(builder.build());
             }
-            case "rlike" -> {
+            case "rlike", "to_lower", "to_upper" -> {
                 var builder = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
                 BytesRef[] values = new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") };
                 for (int i = 0; i < BLOCK_LENGTH; i++) {

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java

@@ -15,6 +15,7 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Transformer;
+import org.gradle.api.invocation.Gradle;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.Copy;
@@ -38,6 +39,15 @@ public class EclipseConventionPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
         project.getPlugins().apply(EclipsePlugin.class);
+        Gradle gradle = project.getGradle();
+
+        boolean isEclipse = project.getProviders().systemProperty("eclipse.launcher").isPresent() ||   // Gradle launched from Eclipse
+            project.getProviders().systemProperty("eclipse.application").isPresent() || // Gradle launched from the Eclipse compiler server
+            gradle.getStartParameter().getTaskNames().contains("eclipse") ||  // Gradle launched from the command line to do eclipse stuff
+            gradle.getStartParameter().getTaskNames().contains("cleanEclipse");
+        // for eclipse ide specific hacks...
+        project.getExtensions().add("isEclipse", isEclipse);
+
         EclipseModel eclipseModel = project.getExtensions().getByType(EclipseModel.class);
         EclipseProject eclipseProject = eclipseModel.getProject();
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:d74b1fda6b7fee2c90b410df258e005c049e0672fe16d79d00e58f14fb69f90b",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c66fdafe581a6ab1668a962015de4ce4666a60ed601d24f019f03bb4aaab8eeb",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java

@@ -33,11 +33,15 @@ interface JdkBuild {
         String url(String os, String arch, String extension);
     }
 
-    record ReleasedJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) implements JdkBuild {
+    record ReleaseJdkBuild(JavaLanguageVersion languageVersion, String host, String version, String buildNumber, String hash)
+        implements
+            JdkBuild {
 
         @Override
         public String url(String os, String arch, String extension) {
-            return "https://download.oracle.com/java/GA/jdk"
+            return "https://"
+                + host
+                + "/java/GA/jdk"
                 + version
                 + "/"
                 + hash
@@ -111,7 +115,8 @@ private static String resolveBuildNumber(int version) {
     // package private so it can be replaced by tests
     List<JdkBuild> builds = List.of(
         getBundledJdkBuild(),
-        new EarlyAccessJdkBuild(JavaLanguageVersion.of(24)),
+        // release candidate of JDK 24
+        new ReleaseJdkBuild(JavaLanguageVersion.of(24), "download.java.net", "24", "36", "1f9ff9062db4449d8ca828c504ffae90"),
         new EarlyAccessJdkBuild(JavaLanguageVersion.of(25))
     );
 
@@ -125,7 +130,7 @@ private JdkBuild getBundledJdkBuild() {
         String baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
         String build = jdkVersionMatcher.group(3);
         String hash = jdkVersionMatcher.group(5);
-        return new ReleasedJdkBuild(bundledJdkMajorVersion, baseVersion, build, hash);
+        return new ReleaseJdkBuild(bundledJdkMajorVersion, "download.oracle.com", baseVersion, build, hash);
     }
 
     /**

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy

@@ -30,9 +30,10 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
                 return null
             }
         }
-        toolChain.builds = toolChain.builds.findAll { it instanceof OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild } + [
-            new OracleOpenJdkToolchainResolver.ReleasedJdkBuild(
+        toolChain.builds = toolChain.builds + [
+            new OracleOpenJdkToolchainResolver.ReleaseJdkBuild(
                 JavaLanguageVersion.of(20),
+                "download.oracle.com",
                 "20",
                 "36",
                 "bdc68b4b9cbc4ebcb30745c85038d91d"
@@ -52,16 +53,16 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
          [20, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-aarch64_bin.tar.gz"],
          [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"],
          // https://download.java.net/java/early_access/jdk23/23/GPL/openjdk-23-ea+23_macos-aarch64_bin.tar.gz
-         [24, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"],
-         [24, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"],
-         [24, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"],
-         [24, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"],
-         [24, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"],
-         [24, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"],
-         [24, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"],
-         [24, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"],
-         [24, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"],
-         [24, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"]]
+         [24, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-x64_bin.tar.gz"],
+         [24, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-aarch64_bin.tar.gz"],
+         [24, ORACLE, LINUX, X86_64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-x64_bin.tar.gz"],
+         [24, ORACLE, LINUX, AARCH64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-aarch64_bin.tar.gz"],
+         [24, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_windows-x64_bin.zip"],
+         [24, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-x64_bin.tar.gz"],
+         [24, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-aarch64_bin.tar.gz"],
+         [24, anyVendor(), LINUX, X86_64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-x64_bin.tar.gz"],
+         [24, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-aarch64_bin.tar.gz"],
+         [24, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_windows-x64_bin.zip"]]
     }
 
     @RestoreSystemProperties
@@ -85,16 +86,6 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
 
         where:
         version | vendor      | os      | arch    | expectedUrl
-        24      | ORACLE      | MAC_OS  | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz"
-        24      | ORACLE      | MAC_OS  | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz"
-        24      | ORACLE      | LINUX   | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz"
-        24      | ORACLE      | LINUX   | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz"
-        24      | ORACLE      | WINDOWS | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip"
-        24      | anyVendor() | MAC_OS  | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz"
-        24      | anyVendor() | MAC_OS  | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz"
-        24      | anyVendor() | LINUX   | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz"
-        24      | anyVendor() | LINUX   | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz"
-        24      | anyVendor() | WINDOWS | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip"
         25      | ORACLE      | MAC_OS  | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz"
         25      | ORACLE      | MAC_OS  | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz"
         25      | ORACLE      | LINUX   | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz"

build.gradle

@@ -247,15 +247,6 @@ allprojects {
     }
   }
 
-  // injecting groovy property variables into all projects
-  project.ext {
-    // for ide hacks...
-    isEclipse = providers.systemProperty("eclipse.launcher").isPresent() ||   // Detects gradle launched from Eclipse's IDE
-      providers.systemProperty("eclipse.application").isPresent() ||    // Detects gradle launched from the Eclipse compiler server
-      gradle.startParameter.taskNames.contains('eclipse') ||  // Detects gradle launched from the command line to do eclipse stuff
-      gradle.startParameter.taskNames.contains('cleanEclipse')
-  }
-
   ext.bwc_tests_enabled = bwc_tests_enabled
 
   // eclipse configuration

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -28,7 +28,7 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-        boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "true"));
         // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
         boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
         return Stream.of(

docs/changelog/121827.yaml

@@ -0,0 +1,6 @@
+pr: 121827
+summary: Updates to allow using Cohere binary embedding response in semantic search
+  queries
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/121920.yaml

@@ -0,0 +1,6 @@
+pr: 121920
+summary: Account for the `SearchHit` source in circuit breaker
+area: Search
+type: enhancement
+issues:
+ - 89656

docs/changelog/122860.yaml

@@ -0,0 +1,5 @@
+pr: 122860
+summary: Improved error message when index field type is invalid
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/122886.yaml

@@ -0,0 +1,6 @@
+pr: 122886
+summary: Add support to VALUES aggregation for spatial types
+area: ES|QL
+type: bug
+issues:
+ - 122413

docs/changelog/122960.yaml

@@ -0,0 +1,10 @@
+pr: 122960
+summary: Deprecate Behavioral Analytics CRUD apis
+area: Search
+type: deprecation
+issues: [ ]
+deprecation:
+  title: Deprecate Behavioral Analytics CRUD apis
+  area: Search
+  details: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated.
+  impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release.

docs/changelog/122999.yaml

@@ -0,0 +1,5 @@
+pr: 122999
+summary: Store arrays offsets for ip fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/123079.yaml

@@ -0,0 +1,5 @@
+pr: 123079
+summary: Register `IngestGeoIpMetadata` as a NamedXContent
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/123105.yaml

@@ -0,0 +1,6 @@
+pr: 123105
+summary: fix stale data in synthetic source for string stored field
+area: Mapping
+type: bug
+issues:
+  - 123110

docs/changelog/123197.yaml

@@ -0,0 +1,5 @@
+pr: 123197
+summary: Fix early termination in `LuceneSourceOperator`
+area: ES|QL
+type: bug
+issues: []

docs/changelog/123272.yaml

@@ -0,0 +1,5 @@
+pr: 123272
+summary: Set Connect Timeout to 5s
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/123346.yaml

@@ -0,0 +1,6 @@
+pr: 123346
+summary: Reduce license checks in `LicensedWriteLoadForecaster`
+area: CRUD
+type: bug
+issues:
+ - 123247

docs/changelog/123384.yaml

@@ -0,0 +1,5 @@
+pr: 123384
+summary: Fixing serialization of `ScriptStats` `cache_evictions_history`
+area: Stats
+type: bug
+issues: []

docs/changelog/123403.yaml

@@ -0,0 +1,5 @@
+pr: 123403
+summary: Use ordered maps for `PipelineConfiguration` xcontent deserialization
+area: Ingest Node
+type: bug
+issues: []