Merge branch 'main' into source-mem-accounting

andreidan · Feb 23, 2025 · acf2f63 · acf2f63
2 parents abacdb1 + 5b90305
commit acf2f63
Show file tree

Hide file tree

Showing 164 changed files with 3,944 additions and 4,211 deletions.
diff --git a/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -110,6 +110,8 @@ private List<File> resolveProjectLogs(File projectDir) {
         projectDirFiles.include("**/build/testrun/*/temp/**");
         projectDirFiles.include("**/build/**/hs_err_pid*.log");
         projectDirFiles.include("**/build/**/replay_pid*.log");
+        // core dump files are in the working directory of the installation, which is not project specific
+        projectDirFiles.include("distribution/**/build/install/*/core.*");
         projectDirFiles.exclude("**/build/testclusters/**/data/**");
         projectDirFiles.exclude("**/build/testclusters/**/distro/**");
         projectDirFiles.exclude("**/build/testclusters/**/repo/**");

diff --git a/docs/changelog/121392.yaml b/docs/changelog/121392.yaml
@@ -0,0 +1,5 @@
+pr: 121392
+summary: Include data streams when converting an existing resource to a system resource
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/122390.yaml b/docs/changelog/122390.yaml
@@ -0,0 +1,5 @@
+pr: 122390
+summary: Add health indicator impact to `HealthPeriodicLogger`
+area: Health
+type: enhancement
+issues: []
diff --git a/docs/changelog/122938.yaml b/docs/changelog/122938.yaml
@@ -0,0 +1,5 @@
+pr: 122938
+summary: Fix geoip databases index access after system feature migration (again)
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/123085.yaml b/docs/changelog/123085.yaml
@@ -0,0 +1,5 @@
+pr: 123085
+summary: Remove duplicated nested commands
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/123155.yaml b/docs/changelog/123155.yaml
@@ -0,0 +1,5 @@
+pr: 123155
+summary: Add `ElasticInferenceServiceCompletionServiceSettings`
+area: Machine Learning
+type: bug
+issues: []
diff --git a/libs/entitlement/bridge/src/main/java/module-info.java b/libs/entitlement/bridge/src/main/java/module-info.java
@@ -12,6 +12,7 @@
 module org.elasticsearch.entitlement.bridge {
     requires java.net.http;
     requires jdk.net;
+    requires java.logging;
 
     exports org.elasticsearch.entitlement.bridge;
 }
diff --git a/...tlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/...tlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java
@@ -88,6 +88,7 @@
 import java.util.concurrent.ForkJoinPool;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
+import java.util.logging.FileHandler;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
@@ -882,9 +883,34 @@ public interface EntitlementChecker {
 
     void check$java_nio_file_Files$$lines(Class<?> callerClass, Path path);
 
-    // file system providers
     void check$java_nio_file_spi_FileSystemProvider$(Class<?> callerClass);
 
+    void check$java_util_logging_FileHandler$(Class<?> callerClass);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, boolean append);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count, boolean append);
+
+    void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, long limit, int count, boolean append);
+
+    void check$java_util_logging_FileHandler$close(Class<?> callerClass, FileHandler that);
+
+    void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class<?> callerClass, Path path);
+
+    void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path);
+
+    void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path, OpenOption... options);
+
+    void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload(Class<?> callerClass, Path directory, OpenOption... openOptions);
+
+    void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory);
+
+    void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory, OpenOption... openOptions);
+
     void checkNewFileSystem(Class<?> callerClass, FileSystemProvider that, URI uri, Map<String, ?> env);
 
     void checkNewFileSystem(Class<?> callerClass, FileSystemProvider that, Path path, Map<String, ?> env);

diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java b/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java
@@ -12,6 +12,7 @@
     requires org.elasticsearch.entitlement;
     requires org.elasticsearch.base; // SuppressForbidden
     requires org.elasticsearch.logging;
+    requires java.logging;
 
     exports org.elasticsearch.entitlement.qa.entitled; // Must be unqualified so non-modular IT tests can call us
 }
diff --git a/...ent-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java b/...ent-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java
@@ -22,17 +22,22 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
 import java.util.Scanner;
 import java.util.jar.JarFile;
+import java.util.logging.FileHandler;
 import java.util.zip.ZipException;
 import java.util.zip.ZipFile;
 
 import static java.nio.charset.Charset.defaultCharset;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.WRITE;
 import static java.util.zip.ZipFile.OPEN_DELETE;
 import static java.util.zip.ZipFile.OPEN_READ;
 import static org.elasticsearch.entitlement.qa.entitled.EntitledActions.createTempFileForWrite;
@@ -477,5 +482,86 @@ static void createScannerFileWithCharsetName() throws FileNotFoundException {
         new Scanner(readFile().toFile(), "UTF-8");
     }
 
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler() throws IOException {
+        new FileHandler();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_String() throws IOException {
+        new FileHandler(readFile().toString());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringBoolean() throws IOException {
+        new FileHandler(readFile().toString(), false);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringIntInt() throws IOException {
+        new FileHandler(readFile().toString(), 1, 2);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringIntIntBoolean() throws IOException {
+        new FileHandler(readFile().toString(), 1, 2, false);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void fileHandler_StringLongIntBoolean() throws IOException {
+        new FileHandler(readFile().toString(), 1L, 2, false);
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpRequestBodyPublishersOfFile() throws IOException {
+        HttpRequest.BodyPublishers.ofFile(readFile());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodyHandlersOfFile() {
+        HttpResponse.BodyHandlers.ofFile(readWriteFile());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodyHandlersOfFile_readOnly() {
+        HttpResponse.BodyHandlers.ofFile(readFile());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodyHandlersOfFileDownload() {
+        HttpResponse.BodyHandlers.ofFileDownload(readWriteDir());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodyHandlersOfFileDownload_readOnly() {
+        HttpResponse.BodyHandlers.ofFileDownload(readDir());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodySubscribersOfFile_File() {
+        HttpResponse.BodySubscribers.ofFile(readWriteFile());
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodySubscribersOfFile_File_readOnly() {
+        HttpResponse.BodySubscribers.ofFile(readFile());
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void httpResponseBodySubscribersOfFile_FileOpenOptions() {
+        // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed
+        // an overload distinct from ofFile with no OpenOptions, and so it needs its
+        // own instrumentation and its own test.
+        HttpResponse.BodySubscribers.ofFile(readWriteFile(), CREATE, WRITE);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void httpResponseBodySubscribersOfFile_FileOpenOptions_readOnly() {
+        // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed
+        // an overload distinct from ofFile with no OpenOptions, and so it needs its
+        // own instrumentation and its own test.
+        HttpResponse.BodySubscribers.ofFile(readFile(), CREATE, WRITE);
+    }
+
     private FileCheckActions() {}
 }
diff --git a/libs/entitlement/src/main/java/module-info.java b/libs/entitlement/src/main/java/module-info.java
@@ -8,12 +8,13 @@
  */
 
 module org.elasticsearch.entitlement {
+    requires org.elasticsearch.base;
     requires org.elasticsearch.xcontent;
     requires org.elasticsearch.logging;
     requires java.instrument;
-    requires org.elasticsearch.base;
-    requires jdk.attach;
+    requires java.logging;
     requires java.net.http;
+    requires jdk.attach;
     requires jdk.net;
 
     requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base

diff --git a/.../main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/.../main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java
@@ -97,6 +97,7 @@
 import java.util.concurrent.ForkJoinPool;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
+import java.util.logging.FileHandler;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
@@ -1845,6 +1846,78 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
         policyManager.checkChangeJVMGlobalState(callerClass);
     }
 
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, boolean append) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, int limit, int count, boolean append) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$(Class<?> callerClass, String pattern, long limit, int count, boolean append) {
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_FileHandler$close(Class<?> callerClass, FileHandler that) {
+        // Note that there's no IT test for this one, because there's no way to create
+        // a FileHandler. However, we have this check just in case someone does manage
+        // to get their hands on a FileHandler and uses close() to cause its lock file to be deleted.
+        policyManager.checkLoggingFileHandler(callerClass);
+    }
+
+    @Override
+    public void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class<?> callerClass, Path path) {
+        policyManager.checkFileRead(callerClass, path);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path) {
+        policyManager.checkFileWrite(callerClass, path);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class<?> callerClass, Path path, OpenOption... options) {
+        policyManager.checkFileWrite(callerClass, path);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload(
+        Class<?> callerClass,
+        Path directory,
+        OpenOption... openOptions
+    ) {
+        policyManager.checkFileWrite(callerClass, directory);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory) {
+        policyManager.checkFileWrite(callerClass, directory);
+    }
+
+    @Override
+    public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class<?> callerClass, Path directory, OpenOption... openOptions) {
+        policyManager.checkFileWrite(callerClass, directory);
+    }
+
     @Override
     public void checkNewFileSystem(Class<?> callerClass, FileSystemProvider that, URI uri, Map<String, ?> env) {
         policyManager.checkChangeJVMGlobalState(callerClass);

diff --git a/...ntitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java b/...ntitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java
@@ -49,8 +49,24 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         readPaths.sort(String::compareTo);
         writePaths.sort(String::compareTo);
 
-        this.readPaths = readPaths.toArray(new String[0]);
-        this.writePaths = writePaths.toArray(new String[0]);
+        this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
+        this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
+    }
+
+    private static List<String> pruneSortedPaths(List<String> paths) {
+        List<String> prunedReadPaths = new ArrayList<>();
+        if (paths.isEmpty() == false) {
+            String currentPath = paths.get(0);
+            prunedReadPaths.add(currentPath);
+            for (int i = 1; i < paths.size(); ++i) {
+                String nextPath = paths.get(i);
+                if (nextPath.startsWith(currentPath) == false) {
+                    prunedReadPaths.add(nextPath);
+                    currentPath = nextPath;
+                }
+            }
+        }
+        return prunedReadPaths;
     }
 
     public static FileAccessTree of(FilesEntitlement filesEntitlement, PathLookup pathLookup) {

diff --git a/...entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/...entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
@@ -240,6 +240,10 @@ public void checkChangeJVMGlobalState(Class<?> callerClass) {
         neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("change JVM global state"));
     }
 
+    public void checkLoggingFileHandler(Class<?> callerClass) {
+        neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("create logging file handler"));
+    }
+
     private Optional<String> walkStackForCheckMethodName() {
         // Look up the check$ method to compose an informative error message.
         // This way, we don't need to painstakingly describe every individual global-state change.

diff --git a/...ement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java b/...ement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java
@@ -96,6 +96,27 @@ public void testReadWriteUnderRead() {
         assertThat(tree.canWrite(path("foo/bar")), is(true));
     }
 
+    public void testPrunedPaths() {
+        var tree = accessTree(entitlement("foo", "read", "foo/baz", "read", "foo/bar", "read"));
+        assertThat(tree.canRead(path("foo")), is(true));
+        assertThat(tree.canWrite(path("foo")), is(false));
+        assertThat(tree.canRead(path("foo/bar")), is(true));
+        assertThat(tree.canWrite(path("foo/bar")), is(false));
+        assertThat(tree.canRead(path("foo/baz")), is(true));
+        assertThat(tree.canWrite(path("foo/baz")), is(false));
+        // also test a non-existent subpath
+        assertThat(tree.canRead(path("foo/barf")), is(true));
+        assertThat(tree.canWrite(path("foo/barf")), is(false));
+
+        tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write"));
+        assertThat(tree.canRead(path("foo")), is(true));
+        assertThat(tree.canWrite(path("foo")), is(false));
+        assertThat(tree.canRead(path("foo/bar")), is(true));
+        assertThat(tree.canWrite(path("foo/bar")), is(true));
+        assertThat(tree.canRead(path("foo/baz")), is(true));
+        assertThat(tree.canWrite(path("foo/baz")), is(false));
+    }
+
     public void testReadWithRelativePath() {
         for (var dir : List.of("config", "home")) {
             var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read", "relative_to", dir)));