Merge branch 'main' into fix_template

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

Author: sandeshkr419

CHANGELOG.md

@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactor the ThreadPoolStats.Stats class to use the Builder pattern instead of constructors ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317))
 - Refactor the IndexingStats.Stats class to use the Builder pattern instead of constructors ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
 - Remove FeatureFlag.MERGED_SEGMENT_WARMER_EXPERIMENTAL_FLAG. ([#19715](https://github.com/opensearch-project/OpenSearch/pull/19715))
+- Replace java.security.AccessController with org.opensearch.secure_sm.AccessController in sub projects with SocketAccess class ([#19803](https://github.com/opensearch-project/OpenSearch/pull/19803))
 - Replace java.security.AccessController with org.opensearch.secure_sm.AccessController in discovery plugins ([#19802](https://github.com/opensearch-project/OpenSearch/pull/19802))
 - Change the default value of doc_values in WildcardFieldMapper to true. ([#19796](https://github.com/opensearch-project/OpenSearch/pull/19796))
 - Make Engine#loadHistoryUUID() protected and Origin#isFromTranslog() public ([#19753](https://github.com/opensearch-project/OpenSearch/pull/19752))
@@ -89,6 +90,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix GRPC Bulk ([#19937](https://github.com/opensearch-project/OpenSearch/pull/19937))
 - Fix node bootstrap error when enable stream transport and remote cluster state ([#19948](https://github.com/opensearch-project/OpenSearch/pull/19948))
 - Fix deletion failure/error of unused index template; case when an index template matches a data stream but has a lower priority. ([#20102](https://github.com/opensearch-project/OpenSearch/pull/20102))
+- Fix toBuilder method in EngineConfig to include mergedSegmentTransferTracker([20105](https://github.com/opensearch-project/OpenSearch/pull/20105))
 
 ### Dependencies
 - Bump Apache Lucene from 10.3.1 to 10.3.2 ([#20026](https://github.com/opensearch-project/OpenSearch/pull/20026))
@@ -122,6 +124,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.cloud:google-cloud-storage` from 2.55.0 to 2.60.0 ([#20023](https://github.com/opensearch-project/OpenSearch/pull/20023))
 - Bump `commons-cli:commons-cli` from 1.10.0 to 1.11.0 ([#20022](https://github.com/opensearch-project/OpenSearch/pull/20022))
 - Bump `com.squareup.okio:okio` from 3.16.0 to 3.16.3 ([#20025](https://github.com/opensearch-project/OpenSearch/pull/20025))
+- Bump `org.tukaani:xz` from 1.10 to 1.11 ([#20082](https://github.com/opensearch-project/OpenSearch/pull/20082))
 - Bump `com.google.api:api-common` from 2.52.0 to 2.55.1 ([#20083](https://github.com/opensearch-project/OpenSearch/pull/20083))
 - Bump `actions/upload-artifact` from 4 to 5 ([#20081](https://github.com/opensearch-project/OpenSearch/pull/20081))
 - Bump `com.nimbusds:nimbus-jose-jwt` from 10.5 to 10.6 ([#20084](https://github.com/opensearch-project/OpenSearch/pull/20084))

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java

@@ -136,6 +136,22 @@ public Void call(Object... names) {
                     return null;
                 }
             });
+            // Add a closure to allow projects to optionally call `forbidSleep()` which will add the signatures
+            // to forbid all usages of `Thread.sleep`
+            ext.set("forbidSleep", new Closure<Void>(t) {
+                @Override
+                public Void call(Object... unused) {
+                    final List<String> signatures = new ArrayList<>();
+                    signatures.addAll(t.getSignatures());
+                    signatures.add(
+                        "java.lang.Thread#sleep(**) @ Fixed sleeps lead to non-deterministic test failures."
+                            + " Poll for whatever condition you're waiting for."
+                            + " Use helpers like `assertBusy` or the awaitility lib."
+                    );
+                    t.setSignatures(signatures);
+                    return null;
+                }
+            });
             // Use of the deprecated security manager APIs are pervasive so set them to warn
             // globally for all projects. Replacements for (most of) these APIs are available
             // so usages can move to the non-deprecated variants to avoid the warnings.

libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/AccessController.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.secure_sm;
 
-import java.util.concurrent.Callable;
 import java.util.function.Supplier;
 
 /**
@@ -78,38 +77,37 @@ public static <T> T doPrivileged(Supplier<T> action) {
     }
 
     /**
-     * Performs the specified action.
+     * Performs the specified action in a privileged block.
      *
-     * <p> If the action's {@code run} method throws an <i>unchecked</i>
+     * <p> If the action's {@code run} method throws an (unchecked)
      * exception, it will propagate through this method.
      *
-     * @param <T> the type of the value returned by the
-     *                  PrivilegedExceptionAction's {@code run} method
-     *
      * @param action the action to be performed
      *
-     * @return the value returned by the action's {@code run} method
-     *
-     * @throws Exception if the specified action's
+     * @throws T if the specified action's
      *         {@code call} method threw a <i>checked</i> exception
      */
-    public static <T> T doPrivilegedChecked(Callable<T> action) throws Exception {
-        return action.call();
+    public static <T extends Exception> void doPrivilegedChecked(CheckedRunnable<T> action) throws T {
+        action.run();
     }
 
     /**
-     * Performs the specified action in a privileged block.
+     * Performs the specified action in a privileged block and returns a value.
      *
-     * <p> If the action's {@code run} method throws an (unchecked)
-     * exception, it will propagate through this method.
+     * <p> If the action's {@code call} method throws an exception,
+     * it will propagate through this method.
      *
+     * @param <R> the type of the value returned by the action
+     * @param <T> the type of the exception that can be thrown
      * @param action the action to be performed
      *
+     * @return the value returned by the action's {@code call} method
+     *
      * @throws T if the specified action's
      *         {@code call} method threw a <i>checked</i> exception
      */
-    public static <T extends Exception> void doPrivilegedChecked(CheckedRunnable<T> action) throws T {
-        action.run();
+    public static <R, T extends Exception> R doPrivilegedChecked(CheckedSupplier<R, T> action) throws T {
+        return action.get();
     }
 
     /**
@@ -126,4 +124,21 @@ public interface CheckedRunnable<E extends Exception> {
          */
         void run() throws E;
     }
+
+    /**
+     * A functional interface that represents a supplier action that can throw a checked exception.
+     *
+     * @param <R> the type of the value returned
+     * @param <E> the type of the exception that can be thrown
+     */
+    public interface CheckedSupplier<R, E extends Exception> {
+
+        /**
+         * Gets a result.
+         *
+         * @return a result
+         * @throws E
+         */
+        R get() throws E;
+    }
 }

plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/CredentialProviderFactory.java

@@ -15,6 +15,8 @@
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.core.SdkSystemSetting;
 
+import org.opensearch.secure_sm.AccessController;
+
 import java.util.function.Supplier;
 
 /**
@@ -44,7 +46,7 @@ private AwsCredentialsProvider initializeProvider() {
 
         @Override
         public AwsCredentials resolveCredentials() {
-            return SocketAccess.doPrivileged(credentials::resolveCredentials);
+            return AccessController.doPrivileged(credentials::resolveCredentials);
         }
     }
 

plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsMasterKeyProvider.java

@@ -19,6 +19,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.crypto.DataKeyPair;
 import org.opensearch.common.crypto.MasterKeyProvider;
+import org.opensearch.secure_sm.AccessController;
 
 import java.util.Map;
 import java.util.function.Supplier;
@@ -51,7 +52,7 @@ public DataKeyPair generateDataPair() {
                 .keySpec(DataKeySpec.AES_256)
                 .keyId(keyArn)
                 .build();
-            GenerateDataKeyResponse dataKeyPair = SocketAccess.doPrivileged(() -> clientReference.get().generateDataKey(request));
+            GenerateDataKeyResponse dataKeyPair = AccessController.doPrivileged(() -> clientReference.get().generateDataKey(request));
             return new DataKeyPair(dataKeyPair.plaintext().asByteArray(), dataKeyPair.ciphertextBlob().asByteArray());
         }
     }
@@ -63,7 +64,7 @@ public byte[] decryptKey(byte[] encryptedKey) {
                 .ciphertextBlob(SdkBytes.fromByteArray(encryptedKey))
                 .encryptionContext(encryptionContext)
                 .build();
-            DecryptResponse decryptResponse = SocketAccess.doPrivileged(() -> clientReference.get().decrypt(decryptRequest));
+            DecryptResponse decryptResponse = AccessController.doPrivileged(() -> clientReference.get().decrypt(decryptRequest));
             return decryptResponse.plaintext().asByteArray();
         }
     }

plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java

@@ -29,6 +29,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
+import org.opensearch.secure_sm.AccessController;
 
 import java.io.Closeable;
 import java.net.URI;
@@ -70,10 +71,10 @@ public KmsService() {
     }
 
     private KmsClient buildClient(KmsClientSettings clientSettings) {
-        SocketAccess.doPrivilegedVoid(KmsService::setDefaultAwsProfilePath);
+        AccessController.doPrivileged(KmsService::setDefaultAwsProfilePath);
         final AwsCredentialsProvider awsCredentialsProvider = buildCredentials(clientSettings);
         final ClientOverrideConfiguration overrideConfiguration = buildOverrideConfiguration();
-        final ProxyConfiguration proxyConfiguration = SocketAccess.doPrivileged(() -> buildProxyConfiguration(clientSettings));
+        final ProxyConfiguration proxyConfiguration = AccessController.doPrivileged(() -> buildProxyConfiguration(clientSettings));
         return buildClient(
             awsCredentialsProvider,
             proxyConfiguration,
@@ -113,7 +114,7 @@ protected KmsClient buildClient(
             builder.region(Region.of(region));
         }
 
-        return SocketAccess.doPrivileged(builder::build);
+        return AccessController.doPrivileged(builder::build);
     }
 
     ProxyConfiguration buildProxyConfiguration(KmsClientSettings clientSettings) {
@@ -166,7 +167,7 @@ public AmazonKmsClientReference client(CryptoMetadata cryptoMetadata) {
                 return existing;
             }
             final AmazonKmsClientReference clientReference = new AmazonKmsClientReference(
-                SocketAccess.doPrivileged(() -> buildClient(clientSettings))
+                AccessController.doPrivileged(() -> buildClient(clientSettings))
             );
             clientReference.incRef();
             clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientSettings, clientReference).immutableMap();

plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/SocketAccess.java

@@ -12,6 +12,7 @@
 
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.io.PathUtils;
+import org.opensearch.secure_sm.AccessController;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.nio.file.Path;
@@ -42,13 +43,15 @@ private Path configPath() {
 
     @SuppressForbidden(reason = "set predictable aws defaults")
     private void setUpAwsProfile() throws Exception {
-        previousOpenSearchPathConf = SocketAccess.doPrivileged(() -> System.setProperty("opensearch.path.conf", configPath().toString()));
-        awsRegion = SocketAccess.doPrivileged(() -> System.setProperty("aws.region", "us-west-2"));
-        awsAccessKeyId = SocketAccess.doPrivileged(() -> System.setProperty("aws.accessKeyId", "aws-access-key-id"));
-        awsSecretAccessKey = SocketAccess.doPrivileged(() -> System.setProperty("aws.secretAccessKey", "aws-secret-access-key"));
+        previousOpenSearchPathConf = AccessController.doPrivileged(
+            () -> System.setProperty("opensearch.path.conf", configPath().toString())
+        );
+        awsRegion = AccessController.doPrivileged(() -> System.setProperty("aws.region", "us-west-2"));
+        awsAccessKeyId = AccessController.doPrivileged(() -> System.setProperty("aws.accessKeyId", "aws-access-key-id"));
+        awsSecretAccessKey = AccessController.doPrivileged(() -> System.setProperty("aws.secretAccessKey", "aws-secret-access-key"));
         awsSharedCredentialsFile = System.getProperty(ProfileFileSystemSetting.AWS_SHARED_CREDENTIALS_FILE.property());
         awsConfigFile = System.getProperty(ProfileFileSystemSetting.AWS_CONFIG_FILE.property());
-        SocketAccess.doPrivilegedVoid(KmsService::setDefaultAwsProfilePath);
+        AccessController.doPrivileged(KmsService::setDefaultAwsProfilePath);
     }
 
     @SuppressForbidden(reason = "reset aws settings")
@@ -64,9 +67,9 @@ private void resetAwsProfile() throws Exception {
     @SuppressForbidden(reason = "reset aws settings")
     private void resetPropertyValue(String key, String value) {
         if (value != null) {
-            SocketAccess.doPrivileged(() -> System.setProperty(key, value));
+            AccessController.doPrivileged(() -> System.setProperty(key, value));
         } else {
-            SocketAccess.doPrivileged(() -> System.clearProperty(key));
+            AccessController.doPrivileged(() -> System.clearProperty(key));
         }
     }
 }

plugins/crypto-kms/src/test/java/org/opensearch/crypto/kms/AbstractAwsTestCase.java

@@ -20,6 +20,7 @@
 import org.opensearch.cluster.metadata.CryptoMetadata;
 import org.opensearch.common.settings.MockSecureSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.secure_sm.AccessController;
 
 public class KmsServiceTests extends AbstractAwsTestCase {
     private final CryptoMetadata cryptoMetadata = new CryptoMetadata("kp1", "kp2", Settings.EMPTY);
@@ -38,11 +39,11 @@ public void testAWSDefaultConfiguration() {
             assertNull(proxyConfiguration.password());
 
             // retry policy
-            RetryPolicy retryPolicyConfiguration = SocketAccess.doPrivileged(kmsService::buildRetryPolicy);
+            RetryPolicy retryPolicyConfiguration = AccessController.doPrivileged(kmsService::buildRetryPolicy);
 
             assertEquals(retryPolicyConfiguration.numRetries().intValue(), 10);
 
-            ClientOverrideConfiguration clientOverrideConfiguration = SocketAccess.doPrivileged(kmsService::buildOverrideConfiguration);
+            ClientOverrideConfiguration clientOverrideConfiguration = AccessController.doPrivileged(kmsService::buildOverrideConfiguration);
             assertTrue(clientOverrideConfiguration.retryPolicy().isPresent());
             assertEquals(clientOverrideConfiguration.retryPolicy().get().numRetries().intValue(), 10);
         }
@@ -63,7 +64,7 @@ public void testAWSConfigurationWithAwsSettings() {
 
         try (KmsService kmsService = new KmsService()) {
             // proxy configuration
-            final ProxyConfiguration proxyConfiguration = SocketAccess.doPrivileged(
+            final ProxyConfiguration proxyConfiguration = AccessController.doPrivileged(
                 () -> kmsService.buildProxyConfiguration(KmsClientSettings.getClientSettings(settings))
             );
 
@@ -73,10 +74,10 @@ public void testAWSConfigurationWithAwsSettings() {
             assertEquals(proxyConfiguration.password(), "aws_proxy_password");
 
             // retry policy
-            RetryPolicy retryPolicyConfiguration = SocketAccess.doPrivileged(kmsService::buildRetryPolicy);
+            RetryPolicy retryPolicyConfiguration = AccessController.doPrivileged(kmsService::buildRetryPolicy);
             assertEquals(retryPolicyConfiguration.numRetries().intValue(), 10);
 
-            ClientOverrideConfiguration clientOverrideConfiguration = SocketAccess.doPrivileged(kmsService::buildOverrideConfiguration);
+            ClientOverrideConfiguration clientOverrideConfiguration = AccessController.doPrivileged(kmsService::buildOverrideConfiguration);
             assertTrue(clientOverrideConfiguration.retryPolicy().isPresent());
             assertEquals(clientOverrideConfiguration.retryPolicy().get().numRetries().intValue(), 10);
         }

plugins/crypto-kms/src/test/java/org/opensearch/crypto/kms/KmsServiceTests.java

@@ -160,9 +160,9 @@ static Settings getAvailabilityZoneNodeAttributes(Settings settings, String azMe
             logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
             urlConnection = AccessController.doPrivilegedChecked(() -> url.openConnection());
             urlConnection.setConnectTimeout(2000);
-        } catch (final Exception e) {
+        } catch (final IOException e) {
             // should not happen, we know the url is not malformed, and openConnection does not actually hit network
-            throw new UncheckedIOException((IOException) e);
+            throw new UncheckedIOException(e);
         }
 
         try (
@@ -176,10 +176,7 @@ static Settings getAvailabilityZoneNodeAttributes(Settings settings, String azMe
             } else {
                 attrs.put(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone", metadataResult);
             }
-        } catch (final Exception e) {
-            if (e instanceof IllegalStateException ise) {
-                throw ise;
-            }
+        } catch (final IOException e) {
             // this is lenient so the plugin does not fail when installed outside of ec2
             logger.error("failed to get metadata for [placement/availability-zone]", e);
         }