Merge pull request #80 from civitaspo/update-sdk

Update sdk

README.md

@@ -50,7 +50,10 @@ Define the below options on properties (which is indicated by `-c`, `--config`).
 - **athena.allow_auth_method_instance**: Indicates whether users can use **auth_method** `"instance"` (boolean, default: `false`)
 - **athena.allow_auth_method_profile**: Indicates whether users can use **auth_method** `"profile"` (boolean, default: `false`)
 - **athena.allow_auth_method_properties**: Indicates whether users can use **auth_method** `"properties"` (boolean, default: `false`)
+- **athena.allow_auth_method_web_identity_token**: Indicates whether users can use **auth_method** `"web_identity_token"` (boolean, default: `false`)
 - **athena.assume_role_timeout_duration**: Maximum duration which server administer allows when users assume **role_arn**. (`DurationParam`, default: `1h`)
+- **athena.default_web_identity_token_file**: Path to a web identity token file. (string, optional)
+- **athena.default_web_identity_role_arn**:  AWS Role when using a web identity token. (string, optional)
 
 ### Secrets
 
@@ -77,6 +80,9 @@ Define the below options on properties (which is indicated by `-c`, `--config`).
   - `"properties"`: uses aws.accessKeyId and aws.secretKey Java system properties.
   - `"anonymous"`: uses anonymous access. This auth method can access only public files.
   - `"session"`: uses temporary-generated access_key_id, secret_access_key and session_token.
+  - `"web_identity_token"`: uses web identity token.
+    - **web_identity_token_file**: path to a web identity token file. (string, default: given by **athena.default_web_identity_token_file**)
+    - **web_identity_role_arn**:  aws role arn when using a web identity token. (string, default: given by **athena.default_web_identity_role_arn**)
 - **use_http_proxy**: Indicate whether using when accessing AWS via http proxy. (boolean, default: `false`)
 - **region**: The AWS region to use for Athena service. (string, optional)
 - **endpoint**: The Amazon Athena endpoint address to use. (string, optional)
@@ -170,7 +176,7 @@ Nothing
 - **database**: The name of the database. (string, optional)
 - **workgroup**: The name of the workgroup in which the query is being started. (string, optional)
 - **timeout**: Specify timeout period. (`DurationParam`, default: `"10m"`)
-- **preview**: Call `athena.preview>` operator after run `athena.query>`. (boolean, default: `true`)
+- **preview**: Call `athena.preview>` operator after run `athena.query>`. (boolean, default: `false`)
 
 ### Output Parameters
 

build.gradle

@@ -7,8 +7,8 @@ plugins {
 group = 'pro.civitaspo'
 version = '0.3.2'
 
-def digdagVersion = '0.9.37'
-def awsSdkVersion = "1.11.587"
+def digdagVersion = '0.9.39'
+def awsSdkVersion = "1.11.653"
 def scalaSemanticVersion = "2.13.0"
 def depScalaVersion = "2.13"
 

src/main/scala/pro/civitaspo/digdag/plugin/athena/AbstractAthenaOperator.scala

@@ -39,12 +39,17 @@ abstract class AbstractAthenaOperator(operatorName: String,
             isAllowedAuthMethodInstance = systemConfig.get("athena.allow_auth_method_instance", classOf[Boolean], false),
             isAllowedAuthMethodProfile = systemConfig.get("athena.allow_auth_method_profile", classOf[Boolean], false),
             isAllowedAuthMethodProperties = systemConfig.get("athena.allow_auth_method_properties", classOf[Boolean], false),
+            isAllowedAuthMethodWebIdentityToken = systemConfig.get("athena.allow_auth_method_web_identity_token", classOf[Boolean], false),
             assumeRoleTimeoutDuration = systemConfig.get("athena.assume_role_timeout_duration", classOf[DurationParam], DurationParam.parse("1h")),
             accessKeyId = secrets.getSecretOptional("access_key_id"),
             secretAccessKey = secrets.getSecretOptional("secret_access_key"),
             sessionToken = secrets.getSecretOptional("session_token"),
             roleArn = secrets.getSecretOptional("role_arn"),
             roleSessionName = secrets.getSecretOptional("role_session_name").or(s"digdag-athena-$sessionUuid"),
+            defaultWebIdentityTokenFile = systemConfig.getOptional("athena.default_web_identity_token_file", classOf[String]),
+            webIdentityTokenFile = params.getOptional("web_identity_token_file", classOf[String]),
+            defaultWebIdentityRoleArn = systemConfig.getOptional("athena.default_web_identity_role_arn", classOf[String]),
+            webIdentityRoleArn = params.getOptional("web_identity_role_arn", classOf[String]),
             httpProxy = secrets.getSecrets("http_proxy"),
             authMethod = params.get("auth_method", classOf[String], "basic"),
             profileName = params.get("profile_name", classOf[String], "default"),

src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/Aws.scala

@@ -2,7 +2,7 @@ package pro.civitaspo.digdag.plugin.athena.aws
 
 
 import com.amazonaws.{ClientConfiguration, Protocol}
-import com.amazonaws.auth.{AnonymousAWSCredentials, AWSCredentials, AWSCredentialsProvider, AWSStaticCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, EC2ContainerCredentialsProviderWrapper, EnvironmentVariableCredentialsProvider, SystemPropertiesCredentialsProvider}
+import com.amazonaws.auth.{AnonymousAWSCredentials, AWSCredentials, AWSCredentialsProvider, AWSStaticCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, EC2ContainerCredentialsProviderWrapper, EnvironmentVariableCredentialsProvider, SystemPropertiesCredentialsProvider, WebIdentityTokenCredentialsProvider}
 import com.amazonaws.auth.profile.{ProfileCredentialsProvider, ProfilesConfigFile}
 import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
@@ -78,14 +78,15 @@ case class Aws(conf: AwsConf)
     private def standardCredentialsProvider: AWSCredentialsProvider =
     {
         conf.authMethod match {
-            case "basic"      => basicAuthMethodAWSCredentialsProvider
-            case "env"        => envAuthMethodAWSCredentialsProvider
-            case "instance"   => instanceAuthMethodAWSCredentialsProvider
-            case "profile"    => profileAuthMethodAWSCredentialsProvider
-            case "properties" => propertiesAuthMethodAWSCredentialsProvider
-            case "anonymous"  => anonymousAuthMethodAWSCredentialsProvider
-            case "session"    => sessionAuthMethodAWSCredentialsProvider
-            case _            =>
+            case "basic"              => basicAuthMethodAWSCredentialsProvider
+            case "env"                => envAuthMethodAWSCredentialsProvider
+            case "instance"           => instanceAuthMethodAWSCredentialsProvider
+            case "profile"            => profileAuthMethodAWSCredentialsProvider
+            case "properties"         => propertiesAuthMethodAWSCredentialsProvider
+            case "anonymous"          => anonymousAuthMethodAWSCredentialsProvider
+            case "session"            => sessionAuthMethodAWSCredentialsProvider
+            case "web_identity_token" => webIdentityTokenAuthMethodAWSCredentialsProvider
+            case _                    =>
                 throw new ConfigException(
                     s"""auth_method: "${conf.authMethod}" is not supported. available `auth_method`s are "basic", "env", "instance", "profile", "properties", "anonymous", or "session"."""
                     )
@@ -151,6 +152,18 @@ case class Aws(conf: AwsConf)
         new AWSStaticCredentialsProvider(credentials)
     }
 
+    private def webIdentityTokenAuthMethodAWSCredentialsProvider: AWSCredentialsProvider =
+    {
+        if (!conf.isAllowedAuthMethodWebIdentityToken) throw new ConfigException(s"""auth_method: "${conf.authMethod}" is not allowed.""")
+        if (!conf.webIdentityTokenFile.or(conf.defaultWebIdentityTokenFile).isPresent) throw new ConfigException(s"""`web_identity_token_file` or `athena.allow_auth_method_web_identity_token` (system) must be set when `auth_method` is "${conf.authMethod}".""")
+        if (!conf.webIdentityRoleArn.or(conf.defaultWebIdentityRoleArn).isPresent) throw new ConfigException(s"""`web_identity_role_arn` or `athena.allow_auth_method_web_identity_role_arn` (system) must be set when `auth_method` is "${conf.authMethod}".""")
+        WebIdentityTokenCredentialsProvider.builder()
+            .webIdentityTokenFile(conf.webIdentityTokenFile.or(conf.defaultWebIdentityTokenFile).get())
+            .roleArn(conf.webIdentityRoleArn.or(conf.defaultWebIdentityRoleArn).get())
+            .roleSessionName(conf.roleSessionName)
+            .build()
+    }
+
     private def clientConfiguration: ClientConfiguration =
     {
         if (!conf.useHttpProxy) return new ClientConfiguration()

src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/AwsConf.scala

@@ -10,6 +10,7 @@ case class AwsConf(
     isAllowedAuthMethodInstance: Boolean,
     isAllowedAuthMethodProfile: Boolean,
     isAllowedAuthMethodProperties: Boolean,
+    isAllowedAuthMethodWebIdentityToken: Boolean,
     assumeRoleTimeoutDuration: DurationParam,
     accessKeyId: Optional[String],
     secretAccessKey: Optional[String],
@@ -20,6 +21,10 @@ case class AwsConf(
     authMethod: String,
     profileName: String,
     profileFile: Optional[String],
+    defaultWebIdentityTokenFile: Optional[String],
+    webIdentityTokenFile: Optional[String],
+    defaultWebIdentityRoleArn: Optional[String],
+    webIdentityRoleArn: Optional[String],
     useHttpProxy: Boolean,
     region: Optional[String],
     endpoint: Optional[String]