Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…esql-migrate-sql-tests-for-simplify-comparison-arithmetics

distribution/tools/cli-launcher/src/main/java/org/elasticsearch/launcher/CliToolLauncher.java

@@ -91,7 +91,7 @@ static Thread createShutdownHook(Terminal terminal, Closeable closeable) {
             try {
                 closeable.close();
             } catch (final IOException e) {
-                e.printStackTrace(terminal.getErrorWriter());
+                terminal.errorPrintln(e);
             }
             terminal.flush(); // make sure to flush whatever the close or error might have written
         }, "elasticsearch-cli-shutdown");

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java

@@ -9,12 +9,14 @@
 package org.elasticsearch.server.cli;
 
 import org.elasticsearch.bootstrap.BootstrapInfo;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.Terminal.Verbosity;
 
 import java.io.BufferedReader;
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
@@ -29,9 +31,9 @@
  * {@link BootstrapInfo#SERVER_READY_MARKER} signals the server is ready and the cli may
  * detach if daemonizing. All other messages are passed through to stderr.
  */
-class ErrorPumpThread extends Thread {
+class ErrorPumpThread extends Thread implements Closeable {
     private final BufferedReader reader;
-    private final PrintWriter writer;
+    private final Terminal terminal;
 
     // a latch which changes state when the server is ready or has had a bootstrap error
     private final CountDownLatch readyOrDead = new CountDownLatch(1);
@@ -42,10 +44,24 @@ class ErrorPumpThread extends Thread {
     // an unexpected io failure that occurred while pumping stderr
     private volatile IOException ioFailure;
 
-    ErrorPumpThread(PrintWriter errOutput, InputStream errInput) {
+    ErrorPumpThread(Terminal terminal, InputStream errInput) {
         super("server-cli[stderr_pump]");
         this.reader = new BufferedReader(new InputStreamReader(errInput, StandardCharsets.UTF_8));
-        this.writer = errOutput;
+        this.terminal = terminal;
+    }
+
+    private void checkForIoFailure() throws IOException {
+        IOException failure = ioFailure;
+        ioFailure = null;
+        if (failure != null) {
+            throw failure;
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        assert isAlive() == false : "Pump thread must be drained first";
+        checkForIoFailure();
     }
 
     /**
@@ -56,9 +72,7 @@ class ErrorPumpThread extends Thread {
      */
     boolean waitUntilReady() throws IOException {
         nonInterruptibleVoid(readyOrDead::await);
-        if (ioFailure != null) {
-            throw ioFailure;
-        }
+        checkForIoFailure();
         return ready;
     }
 
@@ -81,13 +95,13 @@ public void run() {
                     ready = true;
                     readyOrDead.countDown();
                 } else if (filter.contains(line) == false) {
-                    writer.println(line);
+                    terminal.errorPrintln(Verbosity.SILENT, line, false);
                 }
             }
         } catch (IOException e) {
             ioFailure = e;
         } finally {
-            writer.flush();
+            terminal.flush();
             readyOrDead.countDown();
         }
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeystorePasswordTerminal.java

@@ -23,7 +23,7 @@ class KeystorePasswordTerminal extends Terminal implements Closeable {
     private final SecureString password;
 
     KeystorePasswordTerminal(Terminal delegate, SecureString password) {
-        super(delegate.getReader(), delegate.getWriter(), delegate.getErrorWriter());
+        super(delegate);
         this.delegate = delegate;
         this.password = password;
         setVerbosity(delegate.getVerbosity());

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -231,7 +232,7 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings
     }
 
     @Override
-    public void close() {
+    public void close() throws IOException {
         if (server != null) {
             server.stop();
         }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java

@@ -61,16 +61,21 @@ public long pid() {
      */
     public synchronized void detach() throws IOException {
         errorPump.drain();
-        IOUtils.close(jvmProcess.getOutputStream(), jvmProcess.getInputStream(), jvmProcess.getErrorStream());
-        detached = true;
+        try {
+            IOUtils.close(jvmProcess.getOutputStream(), jvmProcess.getInputStream(), jvmProcess.getErrorStream(), errorPump);
+        } finally {
+            detached = true;
+        }
     }
 
     /**
      * Waits for the subprocess to exit.
      */
-    public int waitFor() {
+    public int waitFor() throws IOException {
         errorPump.drain();
-        return nonInterruptible(jvmProcess::waitFor);
+        int exitCode = nonInterruptible(jvmProcess::waitFor);
+        errorPump.close();
+        return exitCode;
     }
 
     /**
@@ -81,7 +86,7 @@ public int waitFor() {
      *
      * <p> Note that if {@link #detach()} has been called, this method is a no-op.
      */
-    public synchronized void stop() {
+    public synchronized void stop() throws IOException {
         if (detached) {
             return;
         }
@@ -93,7 +98,7 @@ public synchronized void stop() {
     /**
      * Stop the subprocess, sending a SIGKILL.
      */
-    public void forceStop() {
+    public void forceStop() throws IOException {
         assert detached == false;
         jvmProcess.destroyForcibly();
         waitFor();

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java

@@ -154,7 +154,7 @@ ServerProcess start(ProcessStarter processStarter) throws UserException {
         boolean success = false;
         try {
             jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), processStarter);
-            errorPump = new ErrorPumpThread(terminal.getErrorWriter(), jvmProcess.getErrorStream());
+            errorPump = new ErrorPumpThread(terminal, jvmProcess.getErrorStream());
             errorPump.start();
             sendArgs(serverArgs, jvmProcess.getOutputStream());
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java

@@ -33,6 +33,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -43,8 +44,11 @@
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.emptyString;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesRegex;
 import static org.hamcrest.Matchers.not;
 
 public class ServerCliTests extends CommandTestCase {
@@ -321,11 +325,16 @@ protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo,
                 throw new InterruptedException("interrupted while get jvm options");
             }
         };
-        var e = expectThrows(
-            InterruptedException.class,
-            () -> command.main(new String[0], terminal, new ProcessInfo(sysprops, envVars, esHomeDir))
-        );
-        assertThat(e.getMessage(), equalTo("interrupted while get jvm options"));
+
+        int exitCode = command.main(new String[0], terminal, new ProcessInfo(sysprops, envVars, esHomeDir));
+        assertThat(exitCode, is(ExitCodes.CODE_ERROR));
+
+        String[] lines = terminal.getErrorOutput().split(System.lineSeparator());
+        assertThat(List.of(lines), hasSize(greaterThan(10))); // at least decent sized stacktrace
+        assertThat(lines[0], is("java.lang.InterruptedException: interrupted while get jvm options"));
+        assertThat(lines[1], matchesRegex("\\tat org.elasticsearch.server.cli.ServerCliTests.+startServer\\(ServerCliTests.java:\\d+\\)"));
+        assertThat(lines[lines.length - 1], matchesRegex("\tat java.base/java.lang.Thread.run\\(Thread.java:\\d+\\)"));
+
         command.close();
     }
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java

@@ -38,6 +38,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -393,15 +394,24 @@ public void testWaitFor() throws Exception {
             stderr.println("final message");
         };
         var server = startProcess(false, false);
+
+        CompletableFuture<Void> stopping = new CompletableFuture<>();
         new Thread(() -> {
-            // simulate stop run as shutdown hook in another thread, eg from Ctrl-C
-            nonInterruptibleVoid(mainReady::await);
-            server.stop();
+            try {
+                // simulate stop run as shutdown hook in another thread, eg from Ctrl-C
+                nonInterruptibleVoid(mainReady::await);
+                server.stop();
+                stopping.complete(null);
+            } catch (Throwable e) {
+                stopping.completeExceptionally(e);
+            }
         }).start();
         int exitCode = server.waitFor();
         assertThat(process.main.isDone(), is(true));
         assertThat(exitCode, equalTo(0));
         assertThat(terminal.getErrorOutput(), containsString("final message"));
+        // rethrow any potential exception observed while stopping
+        stopping.get();
     }
 
     public void testProcessDies() throws Exception {

distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java

@@ -23,6 +23,8 @@
 import org.elasticsearch.server.cli.ServerProcessBuilder;
 import org.elasticsearch.server.cli.ServerProcessUtils;
 
+import java.io.IOException;
+
 /**
  * Starts an Elasticsearch process, but does not wait for it to exit.
  * <p>
@@ -55,7 +57,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
     }
 
     @Override
-    public void close() {
+    public void close() throws IOException {
         if (server != null) {
             server.stop();
         }

distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/ProcrunCommandTests.java

@@ -22,6 +22,8 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 
 public class ProcrunCommandTests extends WindowsServiceCliTestCase {
 
@@ -111,8 +113,10 @@ protected String getDefaultFailureMessage() {
 
     public void testMissingExe() throws Exception {
         Files.delete(serviceExe);
-        var e = expectThrows(IllegalStateException.class, () -> executeMain("install"));
-        assertThat(e.getMessage(), containsString("Missing procrun exe"));
+        int exitCode = executeMain("install");
+
+        assertThat(exitCode, is(ExitCodes.CODE_ERROR));
+        assertThat(terminal.getErrorOutput(), startsWith("java.lang.IllegalStateException: Missing procrun exe"));
     }
 
     public void testServiceId() throws Exception {

docs/changelog/106820.yaml

@@ -0,0 +1,5 @@
+pr: 106820
+summary: Add a capabilities API to check node and cluster capabilities
+area: Infra/REST API
+type: feature
+issues: []

docs/changelog/107088.yaml

@@ -0,0 +1,5 @@
+pr: 107088
+summary: Introduce role description field
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/107891.yaml

@@ -0,0 +1,6 @@
+pr: 107891
+summary: Fix `startOffset` must be non-negative error in XLMRoBERTa tokenizer
+area: Machine Learning
+type: bug
+issues:
+ - 104626

docs/changelog/108238.yaml

@@ -0,0 +1,6 @@
+pr: 108238
+summary: "Nativeaccess: try to load all located libsystemds"
+area: Infra/Core
+type: bug
+issues:
+ - 107878

docs/changelog/108300.yaml

@@ -0,0 +1,5 @@
+pr: 108300
+summary: "ESQL: Add more time span units"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/108340.yaml

@@ -0,0 +1,5 @@
+pr: 108340
+summary: "Apm-data: increase version for templates"
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/108349.yaml

@@ -0,0 +1,6 @@
+pr: 108349
+summary: "Ecs@mappings: reduce scope for `ecs_geo_point`"
+area: Data streams
+type: bug
+issues:
+ - 108338

docs/changelog/108365.yaml

@@ -0,0 +1,5 @@
+pr: 108365
+summary: "[Bugfix] Connector API - fix status serialisation issue in termquery"
+area: Application
+type: bug
+issues: []

docs/changelog/108379.yaml

@@ -0,0 +1,5 @@
+pr: 108379
+summary: Create a new `NodeRequest` for every `NodesDataTiersUsageTransport` use
+area: Indices APIs
+type: bug
+issues: []

docs/changelog/108396.yaml

@@ -0,0 +1,6 @@
+pr: 108396
+summary: "Apm-data: improve default pipeline performance"
+area: Data streams
+type: enhancement
+issues:
+ - 108290

docs/changelog/108431.yaml

@@ -0,0 +1,5 @@
+pr: 108431
+summary: "ESQL: Disable quoting in FROM command"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/108444.yaml

@@ -0,0 +1,5 @@
+pr: 108444
+summary: "Apm-data: ignore malformed fields, and too many dynamic fields"
+area: Data streams
+type: enhancement
+issues: []