Attempt to fix ThreadLeakControl issues in S3BlobStoreRepositoryTests (opensearch-project#18317)

Similar to opensearch-project#18201 but applied via the TestPlugin used in
S3BlobStoreRepositoryTests.

Closes opensearch-project#14299

Signed-off-by: David Causse <dcausse@wikimedia.org>

Author: nomoa

plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -74,6 +74,9 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
 import fixture.s3.S3HttpHandler;
@@ -234,9 +237,13 @@ public void testRequestStats() throws Exception {
      * S3RepositoryPlugin that allows to disable chunked encoding and to set a low threshold between single upload and multipart upload.
      */
     public static class TestS3RepositoryPlugin extends S3RepositoryPlugin {
-
         public TestS3RepositoryPlugin(final Settings settings, final Path configPath) {
-            super(settings, configPath);
+            super(
+                settings,
+                configPath,
+                new S3Service(configPath, Executors.newSingleThreadScheduledExecutor()),
+                new S3AsyncService(configPath, Executors.newSingleThreadScheduledExecutor())
+            );
         }
 
         @Override
@@ -246,6 +253,13 @@ public List<Setting<?>> getSettings() {
             return settings;
         }
 
+        @Override
+        public void close() throws IOException {
+            super.close();
+            Stream.of(service.getClientExecutorService(), s3AsyncService.getClientExecutorService())
+                .forEach(e -> assertTrue(ThreadPool.terminate(e, 5, TimeUnit.SECONDS)));
+        }
+
         @Override
         protected S3Repository createRepository(
             RepositoryMetadata metadata,

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java

@@ -455,4 +455,9 @@ public void close() {
         releaseCachedClients();
 
     }
+
+    @Nullable
+    ScheduledExecutorService getClientExecutorService() {
+        return clientExecutorService;
+    }
 }

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java

@@ -527,4 +527,9 @@ public AwsCredentials resolveCredentials() {
     public void close() {
         releaseCachedClients();
     }
+
+    @Nullable
+    ScheduledExecutorService getClientExecutorService() {
+        return clientExecutorService;
+    }
 }