Merge branch 'main' into exclusive2

docs/changelog/119995.yaml

@@ -0,0 +1,5 @@
+pr: 119995
+summary: "apm-data: Use representative count as event.success_count if available"
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/122134.yaml

@@ -0,0 +1,5 @@
+pr: 122134
+summary: Adding integration for VoyageAI embeddings and rerank models
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/122586.yaml

@@ -0,0 +1,6 @@
+pr: 122586
+summary: "ESQL: Fix inconsistent results in using scaled_float field"
+area: ES|QL
+type: bug
+issues:
+  - 122547

docs/changelog/122601.yaml

@@ -0,0 +1,6 @@
+pr: 122601
+summary: Implicit numeric casting for CASE/GREATEST/LEAST
+area: ES|QL
+type: bug
+issues:
+ - 121890

docs/changelog/122938.yaml

@@ -0,0 +1,5 @@
+pr: 122938
+summary: Fix geoip databases index access after system feature migration (again)
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/123010.yaml

@@ -0,0 +1,6 @@
+pr: 123010
+summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)`
+area: Engine
+type: bug
+issues:
+ - 122974

docs/changelog/123085.yaml

@@ -0,0 +1,5 @@
+pr: 123085
+summary: Remove duplicated nested commands
+area: ES|QL
+type: bug
+issues: []

docs/changelog/123155.yaml

@@ -0,0 +1,5 @@
+pr: 123155
+summary: Add `ElasticInferenceServiceCompletionServiceSettings`
+area: Machine Learning
+type: bug
+issues: []

libs/entitlement/bridge/src/main/java/module-info.java

@@ -12,6 +12,7 @@
 module org.elasticsearch.entitlement.bridge {
     requires java.net.http;
     requires jdk.net;
+    requires java.logging;
 
     exports org.elasticsearch.entitlement.bridge;
 }

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java

@@ -88,6 +88,7 @@
 import java.util.concurrent.ForkJoinPool;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
+import java.util.logging.FileHandler;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
@@ -882,9 +883,34 @@ public interface EntitlementChecker {
 
     void check$java_nio_file_Files$$lines(Class<?> callerClass, Path path);
 
-    // file system providers
     void check$java_nio_file_spi_FileSystemProvider$(Class<?> callerClass);
 
+    void check$java_util_logging_FileHandler$(Class<?> callerClass);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, boolean append);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count, boolean append);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, long limit, int count, boolean append);
+
+    void check$java_util_logging_FileHandler$close(Class<?> callerClass, FileHandler that);
+
+    void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class<?> callerClass, Path path);
+
+    void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path);
+
+    void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path, OpenOption... options);
+
+    void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload(Class<?> callerClass, Path directory, OpenOption... openOptions);
+
+    void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory);
+
+    void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory, OpenOption... openOptions);
+
     void checkNewFileSystem(Class<?> callerClass, FileSystemProvider that, URI uri, Map<String, ?> env);
 
     void checkNewFileSystem(Class<?> callerClass, FileSystemProvider that, Path path, Map<String, ?> env);

libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java

@@ -12,6 +12,7 @@
     requires org.elasticsearch.entitlement;
     requires org.elasticsearch.base; // SuppressForbidden
     requires org.elasticsearch.logging;
+    requires java.logging;
 
     exports org.elasticsearch.entitlement.qa.entitled; // Must be unqualified so non-modular IT tests can call us
 }

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java

@@ -22,17 +22,22 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
 import java.util.Scanner;
 import java.util.jar.JarFile;
+import java.util.logging.FileHandler;
 import java.util.zip.ZipException;
 import java.util.zip.ZipFile;
 
 import static java.nio.charset.Charset.defaultCharset;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.WRITE;
 import static java.util.zip.ZipFile.OPEN_DELETE;
 import static java.util.zip.ZipFile.OPEN_READ;
 import static org.elasticsearch.entitlement.qa.entitled.EntitledActions.createTempFileForWrite;
@@ -477,5 +482,86 @@ static void createScannerFileWithCharsetName() throws FileNotFoundException {
         new Scanner(readFile().toFile(), "UTF-8");
     }
 
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler() throws IOException {
+        new FileHandler();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_String() throws IOException {
+        new FileHandler(readFile().toString());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringBoolean() throws IOException {
+        new FileHandler(readFile().toString(), false);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringIntInt() throws IOException {
+        new FileHandler(readFile().toString(), 1, 2);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringIntIntBoolean() throws IOException {
+        new FileHandler(readFile().toString(), 1, 2, false);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringLongIntBoolean() throws IOException {
+        new FileHandler(readFile().toString(), 1L, 2, false);
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpRequestBodyPublishersOfFile() throws IOException {
+        HttpRequest.BodyPublishers.ofFile(readFile());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodyHandlersOfFile() {
+        HttpResponse.BodyHandlers.ofFile(readWriteFile());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodyHandlersOfFile_readOnly() {
+        HttpResponse.BodyHandlers.ofFile(readFile());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodyHandlersOfFileDownload() {
+        HttpResponse.BodyHandlers.ofFileDownload(readWriteDir());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodyHandlersOfFileDownload_readOnly() {
+        HttpResponse.BodyHandlers.ofFileDownload(readDir());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodySubscribersOfFile_File() {
+        HttpResponse.BodySubscribers.ofFile(readWriteFile());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodySubscribersOfFile_File_readOnly() {
+        HttpResponse.BodySubscribers.ofFile(readFile());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodySubscribersOfFile_FileOpenOptions() {
+        // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed
+        // an overload distinct from ofFile with no OpenOptions, and so it needs its
+        // own instrumentation and its own test.
+        HttpResponse.BodySubscribers.ofFile(readWriteFile(), CREATE, WRITE);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodySubscribersOfFile_FileOpenOptions_readOnly() {
+        // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed
+        // an overload distinct from ofFile with no OpenOptions, and so it needs its
+        // own instrumentation and its own test.
+        HttpResponse.BodySubscribers.ofFile(readFile(), CREATE, WRITE);
+    }
+
     private FileCheckActions() {}
 }

libs/entitlement/src/main/java/module-info.java

@@ -8,12 +8,13 @@
  */
 
 module org.elasticsearch.entitlement {
+    requires org.elasticsearch.base;
     requires org.elasticsearch.xcontent;
     requires org.elasticsearch.logging;
     requires java.instrument;
-    requires org.elasticsearch.base;
-    requires jdk.attach;
+    requires java.logging;
     requires java.net.http;
+    requires jdk.attach;
     requires jdk.net;
 
     requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -97,6 +97,7 @@
 import java.util.concurrent.ForkJoinPool;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
+import java.util.logging.FileHandler;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
@@ -1845,6 +1846,78 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
         policyManager.checkChangeJVMGlobalState(callerClass);
     }
 
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, boolean append) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count, boolean append) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, long limit, int count, boolean append) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$close(Class<?> callerClass, FileHandler that) {
+        // Note that there's no IT test for this one, because there's no way to create
+        // a FileHandler. However, we have this check just in case someone does manage
+        // to get their hands on a FileHandler and uses close() to cause its lock file to be deleted.
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class<?> callerClass, Path path) {
+        policyManager.checkFileRead(callerClass, path);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path) {
+        policyManager.checkFileWrite(callerClass, path);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path, OpenOption... options) {
+        policyManager.checkFileWrite(callerClass, path);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload(
+        Class<?> callerClass,
+        Path directory,
+        OpenOption... openOptions
+    ) {
+        policyManager.checkFileWrite(callerClass, directory);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory) {
+        policyManager.checkFileWrite(callerClass, directory);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory, OpenOption... openOptions) {
+        policyManager.checkFileWrite(callerClass, directory);
+    }
+
     @Override
     public void checkNewFileSystem(Class<?> callerClass, FileSystemProvider that, URI uri, Map<String, ?> env) {
         policyManager.checkChangeJVMGlobalState(callerClass);

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -295,6 +295,10 @@ public void checkChangeJVMGlobalState(Class<?> callerClass) {
         neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("change JVM global state"));
     }
 
+    public void checkLoggingFileHandler(Class<?> callerClass) {
+        neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("create logging file handler"));
+    }
+
     private Optional<String> walkStackForCheckMethodName() {
         // Look up the check$ method to compose an informative error message.
         // This way, we don't need to painstakingly describe every individual global-state change.

modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java

@@ -123,13 +123,19 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
 
             // as should a normal get *
             assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex));
+
+            // and getting data streams
+            assertBusy(() -> testGetDatastreams());
         } else {
             // after the upgrade, but before the migration, Kibana should work
             assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex));
 
             // as should a normal get *
             assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex));
 
+            // and getting data streams
+            assertBusy(() -> testGetDatastreams());
+
             // migrate the system features and give the cluster a moment to settle
             Request migrateSystemFeatures = new Request("POST", "/_migration/system_features");
             assertOK(client().performRequest(migrateSystemFeatures));
@@ -144,6 +150,9 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
             // as should a normal get *
             assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndexReindexed));
 
+            // and getting data streams
+            assertBusy(() -> testGetDatastreams());
+
             Request disableDownloader = new Request("PUT", "/_cluster/settings");
             disableDownloader.setJsonEntity("""
                 {"persistent": {"ingest.geoip.downloader.enabled": false}}
@@ -257,4 +266,15 @@ private void testGetStarAsKibana(List<String> indexNames, @Nullable List<String>
         Map<String, Object> map = responseAsMap(response);
         assertThat(map.keySet(), is(new HashSet<>(indexNames)));
     }
+
+    private void testGetDatastreams() throws IOException {
+        Request getStar = new Request("GET", "_data_stream");
+        getStar.setOptions(
+            RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
+        );
+        Response response = client().performRequest(getStar);
+        assertOK(response);
+
+        // note: we don't actually care about the response, just that there was one and that it didn't error out on us
+    }
 }

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java

@@ -50,6 +50,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchResponseUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -341,7 +342,7 @@ private String mockSearches(String databaseName, int firstChunk, int lastChunk)
             }
 
             SearchHits hits = SearchHits.unpooled(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f);
-            SearchResponse searchResponse = new SearchResponse(hits, null, null, false, null, null, 0, null, 1, 1, 0, 1L, null, null);
+            SearchResponse searchResponse = SearchResponseUtils.successfulResponse(hits);
             toRelease.add(searchResponse::decRef);
             @SuppressWarnings("unchecked")
             ActionFuture<SearchResponse> actionFuture = mock(ActionFuture.class);