Skip to content

Commit

Permalink
Support use of IRSA for repository-s3 plugin credentials
Browse files Browse the repository at this point in the history 
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
  • Loading branch information
@reta
reta committed Jun 1, 2022
1 parent e9b19a0 commit 2e330af
Show file tree
Hide file tree
Showing 7 changed files with 380 additions and 7 deletions.
1 change: 1 addition & 0 deletions plugins/repository-s3/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ versions << [
dependencies {
api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
api "com.amazonaws:aws-java-sdk-sts:${versions.aws}"
api "com.amazonaws:jmespath-java:${versions.aws}"
api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
Expand Down
1 change: 1 addition & 0 deletions plugins/repository-s3/licenses/aws-java-sdk-sts-1.11.749.jar.sha1
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
724bd22c0ff41c496469e18f9bea12bdfb2f7540
94 changes: 93 additions & 1 deletion plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,29 @@ final class S3ClientSettings {
/** Placeholder client name for normalizing client settings in the repository settings. */
private static final String PLACEHOLDER_CLIENT = "placeholder";

// Properties to support using IAM Roles for Service Accounts (IRSA)

/** The identity token file for connecting to s3. */
static final Setting.AffixSetting<String> IDENTITY_TOKEN_FILE_SETTING = Setting.affixKeySetting(
PREFIX,
"identity_token_file",
key -> SecureSetting.simpleString(key, Property.NodeScope)
);

/** The role ARN (Amazon Resource Name) for connecting to s3. */
static final Setting.AffixSetting<SecureString> ROLE_ARN_SETTING = Setting.affixKeySetting(
PREFIX,
"role_arn",
key -> SecureSetting.secureString(key, null)
);

/** The role session name for connecting to s3. */
static final Setting.AffixSetting<SecureString> ROLE_SESSION_NAME_SETTING = Setting.affixKeySetting(
PREFIX,
"role_session_name",
key -> SecureSetting.secureString(key, null)
);

/** The access key (ie login id) for connecting to s3. */
static final Setting.AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(
PREFIX,
Expand Down Expand Up @@ -189,6 +212,9 @@ final class S3ClientSettings {
/** Credentials to authenticate with s3. */
final S3BasicCredentials credentials;

/** Credentials to authenticate with s3 using IAM Roles for Service Accounts (IRSA). */
final IrsaCredentials irsaCredentials;

/** The s3 endpoint the client should talk to, or empty string to use the default. */
final String endpoint;

Expand Down Expand Up @@ -221,6 +247,7 @@ final class S3ClientSettings {

private S3ClientSettings(
S3BasicCredentials credentials,
IrsaCredentials irsaCredentials,
String endpoint,
Protocol protocol,
int readTimeoutMillis,
Expand All @@ -233,6 +260,7 @@ private S3ClientSettings(
ProxySettings proxySettings
) {
this.credentials = credentials;
this.irsaCredentials = irsaCredentials;
this.endpoint = endpoint;
this.protocol = protocol;
this.readTimeoutMillis = readTimeoutMillis;
Expand Down Expand Up @@ -301,6 +329,7 @@ S3ClientSettings refine(Settings repositorySettings) {
validateInetAddressFor(newProxyHost);
return new S3ClientSettings(
newCredentials,
irsaCredentials,
newEndpoint,
newProtocol,
newReadTimeoutMillis,
Expand Down Expand Up @@ -396,12 +425,27 @@ private static S3BasicCredentials loadCredentials(Settings settings, String clie
}
}

private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName) {
String identityTokenFile = getConfigValue(settings, clientName, IDENTITY_TOKEN_FILE_SETTING);
try (
SecureString roleArn = getConfigValue(settings, clientName, ROLE_ARN_SETTING);
SecureString roleSessionName = getConfigValue(settings, clientName, ROLE_SESSION_NAME_SETTING)
) {
if (identityTokenFile.length() != 0 || roleArn.length() != 0 || roleSessionName.length() != 0) {
return new IrsaCredentials(identityTokenFile.toString(), roleArn.toString(), roleSessionName.toString());
}

return null;
}
}

// pkg private for tests
/** Parse settings for a single client. */
static S3ClientSettings getClientSettings(final Settings settings, final String clientName) {
final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING);
return new S3ClientSettings(
S3ClientSettings.loadCredentials(settings, clientName),
S3ClientSettings.loadIrsaCredentials(settings, clientName),
getConfigValue(settings, clientName, ENDPOINT_SETTING),
awsProtocol,
Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()),
Expand Down Expand Up @@ -482,7 +526,8 @@ public boolean equals(final Object o) {
&& proxySettings.equals(that.proxySettings)
&& Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding)
&& Objects.equals(region, that.region)
&& Objects.equals(signerOverride, that.signerOverride);
&& Objects.equals(signerOverride, that.signerOverride)
&& Objects.equals(irsaCredentials, that.irsaCredentials);
}

@Override
Expand Down Expand Up @@ -512,4 +557,51 @@ private static <T> T getRepoSettingOrDefault(Setting.AffixSetting<T> setting, Se
}
return defaultValue;
}

/**
* Class to store IAM Roles for Service Accounts (IRSA) credentials
* See please: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
*/
static class IrsaCredentials {
private final String identityTokenFile;
private final String roleArn;
private final String roleSessionName;

IrsaCredentials(String identityTokenFile, String roleArn, String roleSessionName) {
this.identityTokenFile = Strings.isNullOrEmpty(identityTokenFile) ? null : identityTokenFile;
this.roleArn = Strings.isNullOrEmpty(roleArn) ? null : roleArn;
this.roleSessionName = Strings.isNullOrEmpty(roleSessionName) ? "s3-sdk-java-" + System.currentTimeMillis() : roleSessionName;
}

public String getIdentityTokenFile() {
return identityTokenFile;
}

public String getRoleArn() {
return roleArn;
}

public String getRoleSessionName() {
return roleSessionName;
}

@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
final IrsaCredentials that = (IrsaCredentials) o;
return Objects.equals(identityTokenFile, that.identityTokenFile)
&& Objects.equals(roleArn, that.roleArn)
&& Objects.equals(roleSessionName, that.roleSessionName);
}

@Override
public int hashCode() {
return Objects.hash(identityTokenFile, roleArn, roleSessionName);
}
}
}
5 changes: 4 additions & 1 deletion plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,10 @@ public List<Setting<?>> getSettings() {
S3Repository.ACCESS_KEY_SETTING,
S3Repository.SECRET_KEY_SETTING,
S3ClientSettings.SIGNER_OVERRIDE,
S3ClientSettings.REGION
S3ClientSettings.REGION,
S3ClientSettings.ROLE_ARN_SETTING,
S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING,
S3ClientSettings.ROLE_SESSION_NAME_SETTING
);
}

Expand Down
134 changes: 129 additions & 5 deletions plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,11 @@
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSSessionCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.auth.STSAssumeRoleWithWebIdentitySessionCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.http.SystemPropertyTlsKeyManagersProvider;
Expand All @@ -45,16 +48,20 @@
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.internal.Constants;
import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;

import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.protocol.HttpContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.common.Nullable;
import org.opensearch.common.Strings;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.common.settings.Settings;
import org.opensearch.repositories.s3.S3ClientSettings.IrsaCredentials;

import javax.net.ssl.SSLContext;
import java.io.Closeable;
Expand All @@ -66,13 +73,19 @@
import java.net.Socket;
import java.security.SecureRandom;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import static com.amazonaws.SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR;
import static com.amazonaws.SDKGlobalConfiguration.AWS_ROLE_SESSION_NAME_ENV_VAR;
import static com.amazonaws.SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR;
import static java.util.Collections.emptyMap;

class S3Service implements Closeable {
private static final Logger logger = LogManager.getLogger(S3Service.class);

private volatile Map<S3ClientSettings, AmazonS3Reference> clientsCache = emptyMap();
private Set<Closeable> credentialsCache = ConcurrentHashMap.newKeySet();

/**
* Client settings calculated from static configuration and settings in the keystore.
Expand Down Expand Up @@ -165,7 +178,13 @@ S3ClientSettings settings(RepositoryMetadata repositoryMetadata) {
// proxy for testing
AmazonS3 buildClient(final S3ClientSettings clientSettings) {
final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
builder.withCredentials(buildCredentials(logger, clientSettings));

final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
if (credentials instanceof Closeable) {
credentialsCache.add((Closeable) credentials);
}

builder.withCredentials(credentials);
builder.withClientConfiguration(buildConfiguration(clientSettings));

String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME;
Expand Down Expand Up @@ -259,23 +278,91 @@ public Socket createSocket(final HttpContext ctx) throws IOException {
// pkg private for tests
static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) {
final S3BasicCredentials credentials = clientSettings.credentials;
if (credentials == null) {
logger.debug("Using instance profile credentials");
return new PrivilegedInstanceProfileCredentialsProvider();
} else {
final IrsaCredentials irsaCredentials = buildFromEnviroment(clientSettings.irsaCredentials);

// If IAM Roles for Service Accounts (IRSA) credentials are configured, start with them first
if (irsaCredentials != null) {
logger.debug("Using IRSA credentials");

AWSSecurityTokenService securityTokenService = null;
final String region = Strings.hasLength(clientSettings.region) ? clientSettings.region : null;
if (region != null || credentials != null) {
securityTokenService = SocketAccess.doPrivileged(
() -> AWSSecurityTokenServiceClientBuilder.standard()
.withCredentials((credentials != null) ? new AWSStaticCredentialsProvider(credentials) : null)
.withRegion(region)
.build()
);
}

if (irsaCredentials.getIdentityTokenFile() == null) {
return new PrivilegedSTSAssumeRoleSessionCredentialsProvider<>(
securityTokenService,
new STSAssumeRoleSessionCredentialsProvider.Builder(irsaCredentials.getRoleArn(), irsaCredentials.getRoleSessionName())
.withStsClient(securityTokenService)
.build()
);
} else {
return new PrivilegedSTSAssumeRoleSessionCredentialsProvider<>(
securityTokenService,
new STSAssumeRoleWithWebIdentitySessionCredentialsProvider.Builder(
irsaCredentials.getRoleArn(),
irsaCredentials.getRoleSessionName(),
irsaCredentials.getIdentityTokenFile()
).withStsClient(securityTokenService).build()
);
}
} else if (credentials != null) {
logger.debug("Using basic key/secret credentials");
return new AWSStaticCredentialsProvider(credentials);
} else {
logger.debug("Using instance profile credentials");
return new PrivilegedInstanceProfileCredentialsProvider();
}
}

private static IrsaCredentials buildFromEnviroment(IrsaCredentials defaults) {
if (defaults == null) {
return null;
}

String webIdentityTokenFile = defaults.getIdentityTokenFile();
if (webIdentityTokenFile == null) {
webIdentityTokenFile = System.getenv(AWS_WEB_IDENTITY_ENV_VAR);
}

String roleArn = defaults.getRoleArn();
if (roleArn == null) {
roleArn = System.getenv(AWS_ROLE_ARN_ENV_VAR);
}

String roleSessionName = defaults.getRoleSessionName();
if (roleSessionName == null) {
roleSessionName = System.getenv(AWS_ROLE_SESSION_NAME_ENV_VAR);
}

return new IrsaCredentials(webIdentityTokenFile, roleArn, roleSessionName);
}

private synchronized void releaseCachedClients() {
// the clients will shutdown when they will not be used anymore
for (final AmazonS3Reference clientReference : clientsCache.values()) {
clientReference.decRef();
}

for (final Closeable closeable : credentialsCache) {
try {
closeable.close();
} catch (IOException e) {
/* Ignoring */
}
}

// clear previously cached clients, they will be build lazily
clientsCache = emptyMap();
derivedClientSettings = emptyMap();
credentialsCache.clear();

// shutdown IdleConnectionReaper background thread
// it will be restarted on new client usage
IdleConnectionReaper.shutdown();
Expand All @@ -300,6 +387,43 @@ public void refresh() {
}
}

static class PrivilegedSTSAssumeRoleSessionCredentialsProvider<P extends AWSSessionCredentialsProvider & Closeable>
implements
AWSCredentialsProvider,
Closeable {
private final P credentials;
private final AWSSecurityTokenService securityTokenService;

private PrivilegedSTSAssumeRoleSessionCredentialsProvider(
@Nullable final AWSSecurityTokenService securityTokenService,
final P credentials
) {
this.securityTokenService = securityTokenService;
this.credentials = credentials;
}

@Override
public AWSCredentials getCredentials() {
return SocketAccess.doPrivileged(credentials::getCredentials);
}

@Override
public void refresh() {
SocketAccess.doPrivilegedVoid(credentials::refresh);
}

@Override
public void close() throws IOException {
SocketAccess.doPrivilegedIOException(() -> {
credentials.close();
if (securityTokenService != null) {
securityTokenService.shutdown();
}
return null;
});
};
}

@Override
public void close() {
releaseCachedClients();
Expand Down
Loading

0 comments on commit 2e330af

Please sign in to comment.