diff --git a/quarkus-test-cli/src/main/java/io/quarkus/test/bootstrap/QuarkusCliClient.java b/quarkus-test-cli/src/main/java/io/quarkus/test/bootstrap/QuarkusCliClient.java
index c55c10704..00cccad67 100644
--- a/quarkus-test-cli/src/main/java/io/quarkus/test/bootstrap/QuarkusCliClient.java
+++ b/quarkus-test-cli/src/main/java/io/quarkus/test/bootstrap/QuarkusCliClient.java
@@ -38,7 +38,7 @@ public QuarkusCliClient(ScenarioContext context) {
     }
 
     public Result run(Path servicePath, String... args) {
-        return runCliAndWait(servicePath, args);
+        return runCliAndWait(servicePath, false, args);
     }
 
     public Result run(String... args) {
@@ -49,7 +49,7 @@ public Result buildApplicationOnJvm(Path serviceFolder, String... extraArgs) {
         List<String> args = new ArrayList<>();
         args.add(BUILD);
         args.addAll(Arrays.asList(extraArgs));
-        return runCliAndWait(serviceFolder, args.toArray(new String[args.size()]));
+        return runCliAndWait(serviceFolder, true, args.toArray(new String[args.size()]));
     }
 
     public Result buildApplicationOnNative(Path serviceFolder, String... extraArgs) {
@@ -57,7 +57,7 @@ public Result buildApplicationOnNative(Path serviceFolder, String... extraArgs)
         args.add(BUILD);
         args.add("--native");
         args.addAll(Arrays.asList(extraArgs));
-        return runCliAndWait(serviceFolder, args.toArray(new String[args.size()]));
+        return runCliAndWait(serviceFolder, true, args.toArray(new String[args.size()]));
     }
 
     public Process runOnDev(Path servicePath, File logOutput, Map<String, String> arguments) {
@@ -66,7 +66,7 @@ public Process runOnDev(Path servicePath, File logOutput, Map<String, String> ar
         cmd.addAll(arguments.entrySet().stream()
                 .map(e -> "-D" + e.getKey() + "=" + e.getValue())
                 .collect(Collectors.toList()));
-        return runCli(servicePath, logOutput, cmd.toArray(new String[cmd.size()]));
+        return runCli(servicePath, logOutput, true, cmd.toArray(new String[cmd.size()]));
     }
 
     public QuarkusCliRestService createApplication(String name) {
@@ -102,7 +102,8 @@ public QuarkusCliRestService createApplication(String name, CreateApplicationReq
             Stream.of(request.extraArgs).forEach(args::add);
         }
 
-        Result result = runCliAndWait(serviceContext.getServiceFolder().getParent(), args.toArray(new String[args.size()]));
+        Result result = runCliAndWait(serviceContext.getServiceFolder().getParent(), false,
+                args.toArray(new String[args.size()]));
         assertTrue(result.isSuccessful(), "The application was not created. Output: " + result.getOutput());
 
         return service;
@@ -139,23 +140,24 @@ public QuarkusCliDefaultService createExtension(String name, CreateExtensionRequ
             Stream.of(request.extraArgs).forEach(args::add);
         }
 
-        Result result = runCliAndWait(serviceContext.getServiceFolder().getParent(), args.toArray(new String[args.size()]));
+        Result result = runCliAndWait(serviceContext.getServiceFolder().getParent(), false,
+                args.toArray(new String[args.size()]));
         assertTrue(result.isSuccessful(), "The extension was not created. Output: " + result.getOutput());
 
         return service;
     }
 
     private Result runCliAndWait(String... args) {
-        return runCliAndWait(TARGET, args);
+        return runCliAndWait(TARGET, false, args);
     }
 
-    private Result runCliAndWait(Path workingDirectory, String... args) {
+    private Result runCliAndWait(Path workingDirectory, boolean disableAnalytics, String... args) {
         Result result = new Result();
         File output = workingDirectory.resolve(COMMAND_LOG_FILE).toFile();
 
         try (FileLoggingHandler loggingHandler = new FileLoggingHandler(output)) {
             loggingHandler.startWatching();
-            Process process = runCli(workingDirectory, output, args);
+            Process process = runCli(workingDirectory, output, disableAnalytics, args);
             result.exitCode = process.waitFor();
         } catch (Exception e) {
             e.printStackTrace();
@@ -168,12 +170,12 @@ private Result runCliAndWait(Path workingDirectory, String... args) {
         return result;
     }
 
-    private Process runCli(Path workingDirectory, File logOutput, String... args) {
+    private Process runCli(Path workingDirectory, File logOutput, boolean disableAnalytics, String... args) {
         List<String> cmd = new ArrayList<>();
         cmd.addAll(Arrays.asList(COMMAND.get().split(" ")));
         cmd.addAll(Arrays.asList(args));
 
-        if (QuarkusProperties.disableBuildAnalytics()) {
+        if (QuarkusProperties.disableBuildAnalytics() && disableAnalytics) {
             cmd.add(format("-D%s=%s", QUARKUS_ANALYTICS_DISABLED_LOCAL_PROP_KEY, Boolean.TRUE));
         }
 
diff --git a/quarkus-test-cli/src/test/java/io/quarkus/test/QuarkusCliClientIT.java b/quarkus-test-cli/src/test/java/io/quarkus/test/QuarkusCliClientIT.java
index b9e23c4f9..45b7dcb7f 100644
--- a/quarkus-test-cli/src/test/java/io/quarkus/test/QuarkusCliClientIT.java
+++ b/quarkus-test-cli/src/test/java/io/quarkus/test/QuarkusCliClientIT.java
@@ -14,6 +14,7 @@
 import jakarta.inject.Inject;
 
 import org.apache.http.HttpStatus;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
@@ -46,6 +47,7 @@ public void shouldVersionMatchQuarkusVersion() {
     }
 
     @Test
+    @Disabled("Causing StringIndexOutOfBoundsException due to https://github.com/quarkusio/quarkus/pull/37808")
     public void shouldCreateApplicationOnJvm() {
         // Create application
         QuarkusCliRestService app = cliClient.createApplication("app", defaults());