Support use of IRSA for repository-s3 plugin credentials (#3475)

* Support use of IRSA for repository-s3 plugin credentials

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Address code review comments

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Address code review comments

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

plugins/repository-s3/build.gradle

@@ -51,6 +51,7 @@ versions << [
 dependencies {
  api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
  api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
+ api "com.amazonaws:aws-java-sdk-sts:${versions.aws}"
  api "com.amazonaws:jmespath-java:${versions.aws}"
  api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
  api "org.apache.httpcomponents:httpcore:${versions.httpcore}"

plugins/repository-s3/licenses/aws-java-sdk-sts-1.11.749.jar.sha1

@@ -0,0 +1 @@
+724bd22c0ff41c496469e18f9bea12bdfb2f7540

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonS3Reference.java

@@ -32,17 +32,39 @@
 
 package org.opensearch.repositories.s3;
 
+import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
+
+import org.opensearch.common.Nullable;
 import org.opensearch.common.concurrent.RefCountedReleasable;
 
+import java.io.Closeable;
+import java.io.IOException;
+
 /**
  * Handles the shutdown of the wrapped {@link AmazonS3Client} using reference
  * counting.
  */
 public class AmazonS3Reference extends RefCountedReleasable<AmazonS3> {
-
  AmazonS3Reference(AmazonS3 client) {
- super("AWS_S3_CLIENT", client, client::shutdown);
+ this(client, null);
+ }
+
+ AmazonS3Reference(AmazonS3WithCredentials client) {
+ this(client.client(), client.credentials());
+ }
+
+ AmazonS3Reference(AmazonS3 client, @Nullable AWSCredentialsProvider credentials) {
+ super("AWS_S3_CLIENT", client, () -> {
+ client.shutdown();
+ if (credentials instanceof Closeable) {
+ try {
+ ((Closeable) credentials).close();
+ } catch (IOException e) {
+ /* Do nothing here */
+ }
+ }
+ });
  }
 }

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonS3WithCredentials.java

@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+
+import org.opensearch.common.Nullable;
+
+/**
+ * The holder of the AmazonS3 and AWSCredentialsProvider
+ */
+final class AmazonS3WithCredentials {
+ private final AmazonS3 client;
+ private final AWSCredentialsProvider credentials;
+
+ private AmazonS3WithCredentials(final AmazonS3 client, @Nullable final AWSCredentialsProvider credentials) {
+ this.client = client;
+ this.credentials = credentials;
+ }
+
+ AmazonS3 client() {
+ return client;
+ }
+
+ AWSCredentialsProvider credentials() {
+ return credentials;
+ }
+
+ static AmazonS3WithCredentials create(final AmazonS3 client, @Nullable final AWSCredentialsProvider credentials) {
+ return new AmazonS3WithCredentials(client, credentials);
+ }
+}

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java

@@ -67,6 +67,29 @@ final class S3ClientSettings {
  /** Placeholder client name for normalizing client settings in the repository settings. */
  private static final String PLACEHOLDER_CLIENT = "placeholder";
 
+ // Properties to support using IAM Roles for Service Accounts (IRSA)
+
+ /** The identity token file for connecting to s3. */
+ static final Setting.AffixSetting<String> IDENTITY_TOKEN_FILE_SETTING = Setting.affixKeySetting(
+ PREFIX,
+ "identity_token_file",
+ key -> SecureSetting.simpleString(key, Property.NodeScope)
+ );
+
+ /** The role ARN (Amazon Resource Name) for connecting to s3. */
+ static final Setting.AffixSetting<SecureString> ROLE_ARN_SETTING = Setting.affixKeySetting(
+ PREFIX,
+ "role_arn",
+ key -> SecureSetting.secureString(key, null)
+ );
+
+ /** The role session name for connecting to s3. */
+ static final Setting.AffixSetting<SecureString> ROLE_SESSION_NAME_SETTING = Setting.affixKeySetting(
+ PREFIX,
+ "role_session_name",
+ key -> SecureSetting.secureString(key, null)
+ );
+
  /** The access key (ie login id) for connecting to s3. */
  static final Setting.AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(
  PREFIX,
@@ -189,6 +212,9 @@ final class S3ClientSettings {
  /** Credentials to authenticate with s3. */
  final S3BasicCredentials credentials;
 
+ /** Credentials to authenticate with s3 using IAM Roles for Service Accounts (IRSA). */
+ final IrsaCredentials irsaCredentials;
+
  /** The s3 endpoint the client should talk to, or empty string to use the default. */
  final String endpoint;
 
@@ -221,6 +247,7 @@ final class S3ClientSettings {
 
  private S3ClientSettings(
  S3BasicCredentials credentials,
+ IrsaCredentials irsaCredentials,
  String endpoint,
  Protocol protocol,
  int readTimeoutMillis,
@@ -233,6 +260,7 @@ private S3ClientSettings(
  ProxySettings proxySettings
  ) {
  this.credentials = credentials;
+ this.irsaCredentials = irsaCredentials;
  this.endpoint = endpoint;
  this.protocol = protocol;
  this.readTimeoutMillis = readTimeoutMillis;
@@ -301,6 +329,7 @@ S3ClientSettings refine(Settings repositorySettings) {
  validateInetAddressFor(newProxyHost);
  return new S3ClientSettings(
  newCredentials,
+ irsaCredentials,
  newEndpoint,
  newProtocol,
  newReadTimeoutMillis,
@@ -396,12 +425,27 @@ private static S3BasicCredentials loadCredentials(Settings settings, String clie
  }
  }
 
+ private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName) {
+ String identityTokenFile = getConfigValue(settings, clientName, IDENTITY_TOKEN_FILE_SETTING);
+ try (
+ SecureString roleArn = getConfigValue(settings, clientName, ROLE_ARN_SETTING);
+ SecureString roleSessionName = getConfigValue(settings, clientName, ROLE_SESSION_NAME_SETTING)
+ ) {
+ if (identityTokenFile.length() != 0 || roleArn.length() != 0 || roleSessionName.length() != 0) {
+ return new IrsaCredentials(identityTokenFile.toString(), roleArn.toString(), roleSessionName.toString());
+ }
+
+ return null;
+ }
+ }
+
  // pkg private for tests
  /** Parse settings for a single client. */
  static S3ClientSettings getClientSettings(final Settings settings, final String clientName) {
  final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING);
  return new S3ClientSettings(
  S3ClientSettings.loadCredentials(settings, clientName),
+ S3ClientSettings.loadIrsaCredentials(settings, clientName),
  getConfigValue(settings, clientName, ENDPOINT_SETTING),
  awsProtocol,
  Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()),
@@ -482,7 +526,8 @@ public boolean equals(final Object o) {
  && proxySettings.equals(that.proxySettings)
  && Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding)
  && Objects.equals(region, that.region)
- && Objects.equals(signerOverride, that.signerOverride);
+ && Objects.equals(signerOverride, that.signerOverride)
+ && Objects.equals(irsaCredentials, that.irsaCredentials);
  }
 
  @Override
@@ -512,4 +557,51 @@ private static <T> T getRepoSettingOrDefault(Setting.AffixSetting<T> setting, Se
  }
  return defaultValue;
  }
+
+ /**
+ * Class to store IAM Roles for Service Accounts (IRSA) credentials
+ * See please: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
+ */
+ static class IrsaCredentials {
+ private final String identityTokenFile;
+ private final String roleArn;
+ private final String roleSessionName;
+
+ IrsaCredentials(String identityTokenFile, String roleArn, String roleSessionName) {
+ this.identityTokenFile = Strings.isNullOrEmpty(identityTokenFile) ? null : identityTokenFile;
+ this.roleArn = Strings.isNullOrEmpty(roleArn) ? null : roleArn;
+ this.roleSessionName = Strings.isNullOrEmpty(roleSessionName) ? "s3-sdk-java-" + System.currentTimeMillis() : roleSessionName;
+ }
+
+ public String getIdentityTokenFile() {
+ return identityTokenFile;
+ }
+
+ public String getRoleArn() {
+ return roleArn;
+ }
+
+ public String getRoleSessionName() {
+ return roleSessionName;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ final IrsaCredentials that = (IrsaCredentials) o;
+ return Objects.equals(identityTokenFile, that.identityTokenFile)
+ && Objects.equals(roleArn, that.roleArn)
+ && Objects.equals(roleSessionName, that.roleSessionName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(identityTokenFile, roleArn, roleSessionName);
+ }
+ }
 }

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java

@@ -132,7 +132,10 @@ public List<Setting<?>> getSettings() {
  S3Repository.ACCESS_KEY_SETTING,
  S3Repository.SECRET_KEY_SETTING,
  S3ClientSettings.SIGNER_OVERRIDE,
- S3ClientSettings.REGION
+ S3ClientSettings.REGION,
+ S3ClientSettings.ROLE_ARN_SETTING,
+ S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING,
+ S3ClientSettings.ROLE_SESSION_NAME_SETTING
  );
  }