From 3732bdc7ec06adfa0f642a32d7ae7adf5740069c Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Fri, 18 Dec 2020 15:06:03 +0900 Subject: [PATCH 01/25] Add mwaa_environment Co-authored-by: Christoph Caprano --- .../generators/servicetags/main.go | 1 + .../generators/updatetags/main.go | 1 + .../service_generation_customizations.go | 5 + aws/internal/keyvaluetags/service_tags_gen.go | 10 + aws/internal/keyvaluetags/update_tags_gen.go | 37 + aws/provider.go | 1 + aws/resource_aws_mwaa_environment.go | 753 ++++++++++++++++++ 7 files changed, 808 insertions(+) create mode 100644 aws/resource_aws_mwaa_environment.go diff --git a/aws/internal/keyvaluetags/generators/servicetags/main.go b/aws/internal/keyvaluetags/generators/servicetags/main.go index 761772d520d..5f338cc23b2 100644 --- a/aws/internal/keyvaluetags/generators/servicetags/main.go +++ b/aws/internal/keyvaluetags/generators/servicetags/main.go @@ -132,6 +132,7 @@ var mapServiceNames = []string{ "medialive", "mediapackage", "mq", + "mwaa", "opsworks", "qldb", "pinpoint", diff --git a/aws/internal/keyvaluetags/generators/updatetags/main.go b/aws/internal/keyvaluetags/generators/updatetags/main.go index 584e64a08b7..9a80448dcb8 100644 --- a/aws/internal/keyvaluetags/generators/updatetags/main.go +++ b/aws/internal/keyvaluetags/generators/updatetags/main.go @@ -94,6 +94,7 @@ var serviceNames = []string{ "mediapackage", "mediastore", "mq", + "mwaa", "neptune", "networkfirewall", "networkmanager", diff --git a/aws/internal/keyvaluetags/service_generation_customizations.go b/aws/internal/keyvaluetags/service_generation_customizations.go index 48841befb37..75dafbd8732 100644 --- a/aws/internal/keyvaluetags/service_generation_customizations.go +++ b/aws/internal/keyvaluetags/service_generation_customizations.go @@ -84,6 +84,7 @@ import ( "github.com/aws/aws-sdk-go/service/mediapackage" "github.com/aws/aws-sdk-go/service/mediastore" "github.com/aws/aws-sdk-go/service/mq" + "github.com/aws/aws-sdk-go/service/mwaa" "github.com/aws/aws-sdk-go/service/neptune" "github.com/aws/aws-sdk-go/service/networkfirewall" "github.com/aws/aws-sdk-go/service/networkmanager" @@ -283,6 +284,8 @@ func ServiceClientType(serviceName string) string { funcType = reflect.TypeOf(mediastore.New) case "mq": funcType = reflect.TypeOf(mq.New) + case "mwaa": + funcType = reflect.TypeOf(mwaa.New) case "neptune": funcType = reflect.TypeOf(neptune.New) case "networkfirewall": @@ -409,6 +412,8 @@ func ServiceListTagsFunction(serviceName string) string { return "ListTags" case "mq": return "ListTags" + case "mwaa": + return "ListTags" case "opsworks": return "ListTags" case "redshift": diff --git a/aws/internal/keyvaluetags/service_tags_gen.go b/aws/internal/keyvaluetags/service_tags_gen.go index 0f8a6481e11..c138816121c 100644 --- a/aws/internal/keyvaluetags/service_tags_gen.go +++ b/aws/internal/keyvaluetags/service_tags_gen.go @@ -382,6 +382,16 @@ func MqKeyValueTags(tags map[string]*string) KeyValueTags { return New(tags) } +// MwaaTags returns mwaa service tags. +func (tags KeyValueTags) MwaaTags() map[string]*string { + return aws.StringMap(tags.Map()) +} + +// MwaaKeyValueTags creates KeyValueTags from mwaa service tags. +func MwaaKeyValueTags(tags map[string]*string) KeyValueTags { + return New(tags) +} + // OpsworksTags returns opsworks service tags. func (tags KeyValueTags) OpsworksTags() map[string]*string { return aws.StringMap(tags.Map()) diff --git a/aws/internal/keyvaluetags/update_tags_gen.go b/aws/internal/keyvaluetags/update_tags_gen.go index 754c49b3133..1d7396faaf8 100644 --- a/aws/internal/keyvaluetags/update_tags_gen.go +++ b/aws/internal/keyvaluetags/update_tags_gen.go @@ -83,6 +83,7 @@ import ( "github.com/aws/aws-sdk-go/service/mediapackage" "github.com/aws/aws-sdk-go/service/mediastore" "github.com/aws/aws-sdk-go/service/mq" + "github.com/aws/aws-sdk-go/service/mwaa" "github.com/aws/aws-sdk-go/service/neptune" "github.com/aws/aws-sdk-go/service/networkfirewall" "github.com/aws/aws-sdk-go/service/networkmanager" @@ -2891,6 +2892,42 @@ func MqUpdateTags(conn *mq.MQ, identifier string, oldTagsMap interface{}, newTag return nil } +// MwaaUpdateTags updates mwaa service tags. +// The identifier is typically the Amazon Resource Name (ARN), although +// it may also be a different identifier depending on the service. +func MwaaUpdateTags(conn *mwaa.MWAA, identifier string, oldTagsMap interface{}, newTagsMap interface{}) error { + oldTags := New(oldTagsMap) + newTags := New(newTagsMap) + + if removedTags := oldTags.Removed(newTags); len(removedTags) > 0 { + input := &mwaa.UntagResourceInput{ + ResourceArn: aws.String(identifier), + TagKeys: aws.StringSlice(removedTags.IgnoreAws().Keys()), + } + + _, err := conn.UntagResource(input) + + if err != nil { + return fmt.Errorf("error untagging resource (%s): %w", identifier, err) + } + } + + if updatedTags := oldTags.Updated(newTags); len(updatedTags) > 0 { + input := &mwaa.TagResourceInput{ + ResourceArn: aws.String(identifier), + Tags: updatedTags.IgnoreAws().MwaaTags(), + } + + _, err := conn.TagResource(input) + + if err != nil { + return fmt.Errorf("error tagging resource (%s): %w", identifier, err) + } + } + + return nil +} + // NeptuneUpdateTags updates neptune service tags. // The identifier is typically the Amazon Resource Name (ARN), although // it may also be a different identifier depending on the service. diff --git a/aws/provider.go b/aws/provider.go index 06cc9a18dd8..2d659d67294 100644 --- a/aws/provider.go +++ b/aws/provider.go @@ -786,6 +786,7 @@ func Provider() *schema.Provider { "aws_msk_cluster": resourceAwsMskCluster(), "aws_msk_configuration": resourceAwsMskConfiguration(), "aws_msk_scram_secret_association": resourceAwsMskScramSecretAssociation(), + "aws_mwaa_environment": resourceAwsMwaaEnvironment(), "aws_nat_gateway": resourceAwsNatGateway(), "aws_network_acl": resourceAwsNetworkAcl(), "aws_default_network_acl": resourceAwsDefaultNetworkAcl(), diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go new file mode 100644 index 00000000000..7e7be38594d --- /dev/null +++ b/aws/resource_aws_mwaa_environment.go @@ -0,0 +1,753 @@ +package aws + +import ( + "fmt" + "log" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/mwaa" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags" +) + +func resourceAwsMwaaEnvironment() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsMwaaEnvironmentCreate, + Read: resourceAwsMwaaEnvironmentRead, + Update: resourceAwsMwaaEnvironmentUpdate, + Delete: resourceAwsMwaaEnvironmentDelete, + Importer: &schema.ResourceImporter{ + State: schema.ImportStatePassthrough, + }, + + Schema: map[string]*schema.Schema{ + "airflow_configuration_options": { + Type: schema.TypeMap, + Optional: true, + Sensitive: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "airflow_version": { + Type: schema.TypeString, + Computed: true, + Optional: true, + }, + "arn": { + Type: schema.TypeString, + Computed: true, + }, + "created_at": { + Type: schema.TypeString, + Computed: true, + }, + "dag_s3_path": { + Type: schema.TypeString, + Required: true, + }, + "environment_class": { + Type: schema.TypeString, + Optional: true, + Default: "mw1.small", + }, + "execution_role_arn": { + Type: schema.TypeString, + Required: true, + }, + "kms_key": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validateArn, + }, + "last_updated": { + Type: schema.TypeList, + Computed: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "created_at": { + Type: schema.TypeString, + Computed: true, + }, + "error": { + Type: schema.TypeList, + Computed: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "error_code": { + Type: schema.TypeString, + Computed: true, + }, + "error_message": { + Type: schema.TypeString, + Computed: true, + }, + }, + }, + }, + "status": { + Type: schema.TypeString, + Computed: true, + }, + }, + }, + }, + "logging_configuration": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "dag_processing_logs": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + }, + "scheduler_logs": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + }, + "task_logs": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(true), + }, + "webserver_logs": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + }, + "worker_logs": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + }, + }, + }, + }, + "max_workers": { + Type: schema.TypeInt, + Optional: true, + Default: 10, + ValidateFunc: validation.IntAtLeast(1), + }, + "name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "network_configuration": { + Type: schema.TypeList, + Required: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "security_group_ids": { + Type: schema.TypeSet, + Required: true, + MinItems: 1, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "subnet_ids": { + Type: schema.TypeSet, + Required: true, + ForceNew: true, + MinItems: 2, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + }, + }, + }, + "plugins_s3_object_version": { + Type: schema.TypeString, + Optional: true, + }, + "plugins_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + "requirements_s3_object_version": { + Type: schema.TypeString, + Optional: true, + }, + "requirements_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + "service_role_arn": { + Type: schema.TypeString, + Computed: true, + }, + "source_bucket_arn": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validateArn, + }, + "status": { + Type: schema.TypeString, + Computed: true, + }, + "webserver_access_mode": { + Type: schema.TypeString, + Optional: true, + Default: mwaa.WebserverAccessModePrivateOnly, + ValidateFunc: validation.StringInSlice([]string{ + mwaa.WebserverAccessModePrivateOnly, + mwaa.WebserverAccessModePublicOnly, + }, false), + }, + "webserver_url": { + Type: schema.TypeString, + Computed: true, + }, + "weekly_maintenance_window_start": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + "tags": tagsSchema(), + }, + } +} + +func resourceAwsMwaaEnvironmentCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).mwaaconn + + input := mwaa.CreateEnvironmentInput{ + DagS3Path: aws.String(d.Get("dag_s3_path").(string)), + ExecutionRoleArn: aws.String(d.Get("execution_role_arn").(string)), + Name: aws.String(d.Get("name").(string)), + NetworkConfiguration: expandMwaaEnvironmentNetworkConfigurationCreate(d.Get("network_configuration").([]interface{})), + SourceBucketArn: aws.String(d.Get("source_bucket_arn").(string)), + } + + if v, ok := d.GetOk("airflow_configuration_options"); ok { + input.AirflowConfigurationOptions = stringMapToPointers(v.(map[string]interface{})) + } + + if v, ok := d.GetOk("airflow_version"); ok { + input.AirflowVersion = aws.String(v.(string)) + } + + if v, ok := d.GetOk("environment_class"); ok { + input.EnvironmentClass = aws.String(v.(string)) + } + + if v, ok := d.GetOk("kms_key"); ok { + input.KmsKey = aws.String(v.(string)) + } + + if v, ok := d.GetOk("logging_configuration"); ok { + input.LoggingConfiguration = expandMwaaEnvironmentLoggingConfiguration(v.([]interface{})) + } + + if v, ok := d.GetOk("max_workers"); ok { + input.MaxWorkers = aws.Int64(int64(v.(int))) + } + + if v, ok := d.GetOk("plugins_s3_object_version"); ok { + input.PluginsS3ObjectVersion = aws.String(v.(string)) + } + + if v, ok := d.GetOk("plugins_s3_path"); ok { + input.PluginsS3Path = aws.String(v.(string)) + } + + if v, ok := d.GetOk("requirements_s3_object_version"); ok { + input.RequirementsS3ObjectVersion = aws.String(v.(string)) + } + + if v, ok := d.GetOk("requirements_s3_path"); ok { + input.RequirementsS3Path = aws.String(v.(string)) + } + + if v, ok := d.GetOk("webserver_access_mode"); ok { + input.WebserverAccessMode = aws.String(v.(string)) + } + + if v, ok := d.GetOk("weekly_maintenance_window_start"); ok { + input.WeeklyMaintenanceWindowStart = aws.String(v.(string)) + } + + if v, ok := d.GetOk("tags"); ok { + input.Tags = keyvaluetags.New(v.(map[string]interface{})).IgnoreAws().MwaaTags() + } + + log.Printf("[INFO] Creating MWAA Environment: %s", input) + out, err := conn.CreateEnvironment(&input) + if err != nil { + return err + } + + d.SetId(*input.Name) + d.Set("arn", out.Arn) + + if err := waitForMwaaEnvironmentCreation(conn, d.Id()); err != nil { + return err + } + + return resourceAwsMwaaEnvironmentRead(d, meta) +} + +func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).mwaaconn + ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig + + log.Printf("[INFO] Reading MWAA Environment: %s", d.Id()) + + out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ + Name: aws.String(d.Id()), + }) + + if err != nil { + if isAWSErr(err, mwaa.ErrCodeResourceNotFoundException, "") && !d.IsNewResource() { + log.Printf("[WARN] MWAA Environment %q not found, removing from state", d.Id()) + d.SetId("") + return nil + } + + return err + } + + d.Set("airflow_configuration_options", aws.StringValueMap(out.Environment.AirflowConfigurationOptions)) + d.Set("airflow_version", out.Environment.AirflowVersion) + d.Set("arn", out.Environment.Arn) + d.Set("created_at", aws.TimeValue(out.Environment.CreatedAt).String()) + d.Set("dag_s3_path", out.Environment.DagS3Path) + d.Set("environment_class", out.Environment.EnvironmentClass) + d.Set("execution_role_arn", out.Environment.ExecutionRoleArn) + d.Set("kms_key", out.Environment.KmsKey) + if err := d.Set("last_updated", flattenMwaaLastUpdate(out.Environment.LastUpdate)); err != nil { + return err + } + if err := d.Set("logging_configuration", flattenMwaaLoggingConfiguration(out.Environment.LoggingConfiguration)); err != nil { + return err + } + d.Set("max_workers", out.Environment.MaxWorkers) + d.Set("name", out.Environment.Name) + if err := d.Set("network_configuration", flattenMwaaNetworkConfiguration(out.Environment.NetworkConfiguration)); err != nil { + return err + } + d.Set("plugins_s3_object_version", out.Environment.PluginsS3ObjectVersion) + d.Set("plugins_s3_path", out.Environment.PluginsS3Path) + d.Set("requirements_s3_object_version", out.Environment.RequirementsS3ObjectVersion) + d.Set("requirements_s3_path", out.Environment.RequirementsS3Path) + d.Set("service_role_arn", out.Environment.ServiceRoleArn) + d.Set("source_bucket_arn", out.Environment.SourceBucketArn) + d.Set("status", out.Environment.Status) + d.Set("webserver_access_mode", out.Environment.WebserverAccessMode) + d.Set("webserver_url", out.Environment.WebserverUrl) + d.Set("weekly_maintenance_window_start", out.Environment.WeeklyMaintenanceWindowStart) + + if err := d.Set("tags", keyvaluetags.MwaaKeyValueTags(out.Environment.Tags).IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil { + return fmt.Errorf("error setting tags: %s", err) + } + + return nil +} + +func resourceAwsMwaaEnvironmentUpdate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).mwaaconn + + needsUpdate := false + + input := mwaa.UpdateEnvironmentInput{ + Name: aws.String(d.Get("name").(string)), + } + + if d.HasChange("airflow_configuration_options") { + needsUpdate = true + + options, ok := d.GetOk("airflow_configuration_options") + if !ok { + options = map[string]interface{}{} + } + + input.AirflowConfigurationOptions = stringMapToPointers(options.(map[string]interface{})) + } + + if d.HasChange("airflow_version") { + needsUpdate = true + input.AirflowVersion = aws.String(d.Get("airflow_version").(string)) + } + + if d.HasChange("dag_s3_path") { + needsUpdate = true + input.DagS3Path = aws.String(d.Get("dag_s3_path").(string)) + } + + if d.HasChange("environment_class") { + needsUpdate = true + input.EnvironmentClass = aws.String(d.Get("environment_class").(string)) + } + + if d.HasChange("execution_role_arn") { + needsUpdate = true + input.ExecutionRoleArn = aws.String(d.Get("execution_role_arn").(string)) + } + + if d.HasChange("logging_configuration") { + needsUpdate = true + input.LoggingConfiguration = expandMwaaEnvironmentLoggingConfiguration(d.Get("logging_configuration").([]interface{})) + } + + if d.HasChange("max_workers") { + needsUpdate = true + input.MaxWorkers = aws.Int64(int64(d.Get("max_workers").(int))) + } + + if d.HasChange("network_configuration") { + needsUpdate = true + input.NetworkConfiguration = expandMwaaEnvironmentNetworkConfigurationUpdate(d.Get("network_configuration").([]interface{})) + } + + if d.HasChange("plugins_s3_path") { + needsUpdate = true + input.PluginsS3Path = aws.String(d.Get("plugins_s3_path").(string)) + } + + if d.HasChange("requirements_s3_object_version") { + needsUpdate = true + input.RequirementsS3ObjectVersion = aws.String(d.Get("requirements_s3_object_version").(string)) + } + + if d.HasChange("requirements_s3_path") { + needsUpdate = true + input.RequirementsS3Path = aws.String(d.Get("requirements_s3_path").(string)) + } + + if d.HasChange("source_bucket_arn") { + needsUpdate = true + input.SourceBucketArn = aws.String(d.Get("source_bucket_arn").(string)) + } + + if d.HasChange("webserver_access_mode") { + needsUpdate = true + input.WebserverAccessMode = aws.String(d.Get("webserver_access_mode").(string)) + } + + if d.HasChange("weekly_maintenance_window_start") { + needsUpdate = true + input.WeeklyMaintenanceWindowStart = aws.String(d.Get("weekly_maintenance_window_start").(string)) + } + + if needsUpdate { + log.Printf("[INFO] Updating MWAA Environment: %s", input) + _, err := conn.UpdateEnvironment(&input) + + if err != nil { + return err + } + + if err := waitForMwaaEnvironmentUpdate(conn, d.Id()); err != nil { + return err + } + } + + if d.HasChange("tags") { + o, n := d.GetChange("tags") + + if err := keyvaluetags.MwaaUpdateTags(conn, d.Get("arn").(string), o, n); err != nil { + return fmt.Errorf("error updating MWAA Environment (%s) tags: %s", d.Get("arn").(string), err) + } + } + + return resourceAwsMwaaEnvironmentRead(d, meta) +} + +func resourceAwsMwaaEnvironmentDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).mwaaconn + + log.Printf("[INFO] Deleting MWAA Environment: %s", d.Id()) + _, err := conn.DeleteEnvironment(&mwaa.DeleteEnvironmentInput{ + Name: aws.String(d.Id()), + }) + if err != nil { + if isAWSErr(err, mwaa.ErrCodeResourceNotFoundException, "") { + return nil + } + + return err + } + + return waitForMwaaEnvironmentDeletion(conn, d.Id()) +} + +func mwaaEnvironmentModuleLoggingConfigurationSchema(defaultEnabled bool) *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "cloud_watch_log_group_arn": { + Type: schema.TypeString, + Computed: true, + }, + "enabled": { + Type: schema.TypeBool, + Optional: true, + Default: defaultEnabled, + }, + "log_level": { + Type: schema.TypeString, + Optional: true, + Default: mwaa.LoggingLevelInfo, + ValidateFunc: validation.StringInSlice([]string{ + mwaa.LoggingLevelCritical, + mwaa.LoggingLevelError, + mwaa.LoggingLevelWarning, + mwaa.LoggingLevelInfo, + }, false), + }, + }, + } +} + +func expandMwaaEnvironmentLoggingConfiguration(l []interface{}) *mwaa.LoggingConfigurationInput { + if len(l) == 0 || l[0] == nil { + return nil + } + + input := &mwaa.LoggingConfigurationInput{} + + m := l[0].(map[string]interface{}) + + if v, ok := m["dag_processing_logs"]; ok { + input.DagProcessingLogs = expandMwaaEnvironmentModuleLoggingConfiguration(v.([]interface{})) + } + + if v, ok := m["scheduler_logs"]; ok { + input.SchedulerLogs = expandMwaaEnvironmentModuleLoggingConfiguration(v.([]interface{})) + } + + if v, ok := m["task_logs"]; ok { + input.TaskLogs = expandMwaaEnvironmentModuleLoggingConfiguration(v.([]interface{})) + } + + if v, ok := m["webserver_logs"]; ok { + input.WebserverLogs = expandMwaaEnvironmentModuleLoggingConfiguration(v.([]interface{})) + } + + if v, ok := m["worker_logs"]; ok { + input.WorkerLogs = expandMwaaEnvironmentModuleLoggingConfiguration(v.([]interface{})) + } + + return input +} + +func expandMwaaEnvironmentModuleLoggingConfiguration(l []interface{}) *mwaa.ModuleLoggingConfigurationInput { + if len(l) == 0 || l[0] == nil { + return nil + } + + input := &mwaa.ModuleLoggingConfigurationInput{} + m := l[0].(map[string]interface{}) + + input.Enabled = aws.Bool(m["enabled"].(bool)) + input.LogLevel = aws.String(m["log_level"].(string)) + + return input +} + +func expandMwaaEnvironmentNetworkConfigurationCreate(l []interface{}) *mwaa.NetworkConfiguration { + m := l[0].(map[string]interface{}) + + return &mwaa.NetworkConfiguration{ + SecurityGroupIds: expandStringSet(m["security_group_ids"].(*schema.Set)), + SubnetIds: expandStringSet(m["subnet_ids"].(*schema.Set)), + } +} + +func expandMwaaEnvironmentNetworkConfigurationUpdate(l []interface{}) *mwaa.UpdateNetworkConfigurationInput { + m := l[0].(map[string]interface{}) + + return &mwaa.UpdateNetworkConfigurationInput{ + SecurityGroupIds: expandStringSet(m["security_group_ids"].(*schema.Set)), + } +} + +func waitForMwaaEnvironmentCreation(conn *mwaa.MWAA, id string) error { + stateConf := resource.StateChangeConf{ + Pending: []string{ + mwaa.EnvironmentStatusCreating, + }, + Target: []string{ + mwaa.EnvironmentStatusAvailable, + }, + Timeout: 90 * time.Minute, + Delay: 1 * time.Minute, + Refresh: func() (interface{}, string, error) { + out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ + Name: aws.String(id), + }) + + if err != nil { + return nil, "", err + } + + return out, *out.Environment.Status, nil + }, + } + _, err := stateConf.WaitForState() + return err +} + +func waitForMwaaEnvironmentUpdate(conn *mwaa.MWAA, id string) error { + stateConf := resource.StateChangeConf{ + Pending: []string{ + mwaa.EnvironmentStatusUpdating, + }, + Target: []string{ + mwaa.EnvironmentStatusAvailable, + }, + Timeout: 90 * time.Minute, + Delay: 1 * time.Minute, + Refresh: func() (interface{}, string, error) { + out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ + Name: aws.String(id), + }) + + if err != nil { + return nil, "", err + } + + return out, *out.Environment.Status, nil + }, + } + _, err := stateConf.WaitForState() + return err +} + +func waitForMwaaEnvironmentDeletion(conn *mwaa.MWAA, id string) error { + stateConf := resource.StateChangeConf{ + Pending: []string{ + mwaa.EnvironmentStatusDeleting, + }, + Target: []string{}, + Timeout: 90 * time.Minute, + Delay: 1 * time.Minute, + Refresh: func() (interface{}, string, error) { + out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ + Name: aws.String(id), + }) + if err != nil { + + if isAWSErr(err, mwaa.ErrCodeResourceNotFoundException, "") { + return nil, "", nil + } + return nil, "", err + } + + return out, *out.Environment.Status, nil + }, + } + _, err := stateConf.WaitForState() + return err +} + +func flattenMwaaLastUpdate(lastUpdate *mwaa.LastUpdate) []interface{} { + if lastUpdate == nil { + return []interface{}{} + } + + m := map[string]interface{}{} + + if lastUpdate.CreatedAt != nil { + m["created_at"] = aws.TimeValue(lastUpdate.CreatedAt).String() + } + + if lastUpdate.Error != nil { + m["error"] = flattenMwaaLastUpdateError(lastUpdate.Error) + } + + if lastUpdate.Status != nil { + m["status"] = lastUpdate.Status + } + + return []interface{}{m} +} + +func flattenMwaaLastUpdateError(error *mwaa.UpdateError) []interface{} { + if error == nil { + return []interface{}{} + } + + m := map[string]interface{}{} + + if error.ErrorCode != nil { + m["error_code"] = error.ErrorCode + } + + if error.ErrorMessage != nil { + m["error_message"] = error.ErrorMessage + } + + return []interface{}{m} +} + +func flattenMwaaLoggingConfiguration(loggingConfiguration *mwaa.LoggingConfiguration) []interface{} { + if loggingConfiguration == nil { + return []interface{}{} + } + + m := map[string]interface{}{} + + if loggingConfiguration.DagProcessingLogs != nil { + m["dag_processing_logs"] = flattenMwaaModuleLoggingConfiguration(loggingConfiguration.DagProcessingLogs) + } + + if loggingConfiguration.SchedulerLogs != nil { + m["scheduler_logs"] = flattenMwaaModuleLoggingConfiguration(loggingConfiguration.SchedulerLogs) + } + + if loggingConfiguration.TaskLogs != nil { + m["task_logs"] = flattenMwaaModuleLoggingConfiguration(loggingConfiguration.TaskLogs) + } + + if loggingConfiguration.WebserverLogs != nil { + m["webserver_logs"] = flattenMwaaModuleLoggingConfiguration(loggingConfiguration.WebserverLogs) + } + + if loggingConfiguration.WorkerLogs != nil { + m["worker_logs"] = flattenMwaaModuleLoggingConfiguration(loggingConfiguration.WorkerLogs) + } + + return []interface{}{m} +} + +func flattenMwaaModuleLoggingConfiguration(moduleLoggingConfiguration *mwaa.ModuleLoggingConfiguration) []interface{} { + if moduleLoggingConfiguration == nil { + return []interface{}{} + } + + m := map[string]interface{}{ + "cloud_watch_log_group_arn": aws.StringValue(moduleLoggingConfiguration.CloudWatchLogGroupArn), + "enabled": aws.BoolValue(moduleLoggingConfiguration.Enabled), + "log_level": aws.StringValue(moduleLoggingConfiguration.LogLevel), + } + + return []interface{}{m} +} + +func flattenMwaaNetworkConfiguration(networkConfiguration *mwaa.NetworkConfiguration) []interface{} { + if networkConfiguration == nil { + return []interface{}{} + } + + m := map[string]interface{}{ + "security_group_ids": flattenStringSet(networkConfiguration.SecurityGroupIds), + "subnet_ids": flattenStringSet(networkConfiguration.SubnetIds), + } + + return []interface{}{m} +} From 19ab800dcf91efa607027284e0a3ea10ad4380d5 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Fri, 18 Dec 2020 15:06:27 +0900 Subject: [PATCH 02/25] Add tests for mwaa_environment Co-authored-by: Christoph Caprano --- aws/resource_aws_mwaa_environment_test.go | 780 ++++++++++++++++++++++ 1 file changed, 780 insertions(+) create mode 100644 aws/resource_aws_mwaa_environment_test.go diff --git a/aws/resource_aws_mwaa_environment_test.go b/aws/resource_aws_mwaa_environment_test.go new file mode 100644 index 00000000000..81eed273974 --- /dev/null +++ b/aws/resource_aws_mwaa_environment_test.go @@ -0,0 +1,780 @@ +package aws + +import ( + "fmt" + "log" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/mwaa" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +func init() { + resource.AddTestSweepers("aws_mwaa_environment", &resource.Sweeper{ + Name: "aws_mwaa_environment", + F: testSweepMwaaEnvironment, + }) +} + +func testSweepMwaaEnvironment(region string) error { + client, err := sharedClientForRegion(region) + if err != nil { + return fmt.Errorf("error getting client: %s", err) + } + conn := client.(*AWSClient).mwaaconn + + listOutput, err := conn.ListEnvironments(&mwaa.ListEnvironmentsInput{}) + if err != nil { + if testSweepSkipSweepError(err) || isAWSErr(err, "InternalFailure", "") { + log.Printf("[WARN] Skipping MWAA Environment sweep for %s: %s", region, err) + return nil + } + return fmt.Errorf("Error retrieving MWAA Environment: %s", err) + } + for _, environment := range listOutput.Environments { + name := aws.StringValue(environment) + r := resourceAwsMwaaEnvironment() + d := r.Data(nil) + d.SetId(name) + + err := r.Delete(d, client) + if err != nil { + log.Printf("[ERROR] Failed to delete MWAA Environment %s: %s", name, err) + } + } + return nil +} + +func TestAccAWSMwaaEnvironment_basic(t *testing.T) { + var environment mwaa.GetEnvironmentOutput + + rName := acctest.RandomWithPrefix("tf-acc-test") + // The bucket name should start with "airflow-" + bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") + resourceName := "aws_mwaa_environment.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSMwssEnvironmentBasicConfig(rName, bucketName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + resource.TestCheckResourceAttrSet(resourceName, "airflow_version"), + testAccCheckResourceAttrRegionalARN(resourceName, "arn", "airflow", "environment/"+rName), + resource.TestCheckResourceAttrSet(resourceName, "created_at"), + resource.TestCheckResourceAttr(resourceName, "dag_s3_path", "dags/"), + resource.TestCheckResourceAttr(resourceName, "environment_class", "mw1.small"), + testAccCheckResourceAttrGlobalARN(resourceName, "execution_role_arn", "iam", "role/service-role/"+rName), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.log_level", "INFO"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.log_level", "INFO"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.task_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.log_level", "INFO"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.log_level", "INFO"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.log_level", "INFO"), + resource.TestCheckResourceAttr(resourceName, "max_workers", "10"), + resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestCheckResourceAttr(resourceName, "network_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "network_configuration.0.security_group_ids.#", "1"), + resource.TestCheckResourceAttr(resourceName, "network_configuration.0.subnet_ids.#", "2"), + resource.TestCheckResourceAttrSet(resourceName, "service_role_arn"), + testAccCheckResourceAttrGlobalARNNoAccount(resourceName, "source_bucket_arn", "s3", bucketName), + resource.TestCheckResourceAttrSet(resourceName, "status"), + resource.TestCheckResourceAttr(resourceName, "webserver_access_mode", mwaa.WebserverAccessModePrivateOnly), + resource.TestCheckResourceAttrSet(resourceName, "webserver_url"), + resource.TestCheckResourceAttrSet(resourceName, "weekly_maintenance_window_start"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccAWSMwaaEnvironment_disappears(t *testing.T) { + var environment mwaa.GetEnvironmentOutput + + rName := acctest.RandomWithPrefix("tf-acc-test") + // The bucket name should start with "airflow-" + bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") + resourceName := "aws_mwaa_environment.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSMwssEnvironmentBasicConfig(rName, bucketName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + testAccCheckResourceDisappears(testAccProvider, resourceAwsMwaaEnvironment(), resourceName), + ), + ExpectNonEmptyPlan: true, + }, + }, + }) +} + +func TestAccAWSMwaaEnvironment_AirflowConfigurationOptions(t *testing.T) { + var environment mwaa.GetEnvironmentOutput + + rName := acctest.RandomWithPrefix("tf-acc-test") + // The bucket name should start with "airflow-" + bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") + resourceName := "aws_mwaa_environment.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, bucketName, "1", "16"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "2"), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.core.default_task_retries", "1"), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.core.parallelism", "16"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, bucketName, "2", "32"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "2"), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.core.default_task_retries", "2"), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.core.parallelism", "32"), + ), + }, + { + Config: testAccAWSMwssEnvironmentBasicConfig(rName, bucketName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "0"), + ), + }, + }, + }) +} + +func TestAccAWSMwaaEnvironment_LogConfiguration(t *testing.T) { + var environment mwaa.GetEnvironmentOutput + + rName := acctest.RandomWithPrefix("tf-acc-test") + // The bucket name should start with "airflow-" + bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") + resourceName := "aws_mwaa_environment.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, bucketName, "true", mwaa.LoggingLevelCritical), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.dag_processing_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.log_level", mwaa.LoggingLevelCritical), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.scheduler_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.log_level", mwaa.LoggingLevelCritical), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.task_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.log_level", mwaa.LoggingLevelCritical), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.webserver_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.log_level", mwaa.LoggingLevelCritical), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.worker_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.log_level", mwaa.LoggingLevelCritical), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, bucketName, "false", mwaa.LoggingLevelInfo), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.cloud_watch_log_group_arn", ""), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.log_level", mwaa.LoggingLevelInfo), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.cloud_watch_log_group_arn", ""), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.log_level", mwaa.LoggingLevelInfo), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.cloud_watch_log_group_arn", ""), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.log_level", mwaa.LoggingLevelInfo), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.cloud_watch_log_group_arn", ""), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.log_level", mwaa.LoggingLevelInfo), + + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.cloud_watch_log_group_arn", ""), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.log_level", mwaa.LoggingLevelInfo), + ), + }, + }, + }) +} + +func TestAccAWSMwaaEnvironment_full(t *testing.T) { + var environment mwaa.GetEnvironmentOutput + + rName := acctest.RandomWithPrefix("tf-acc-test") + // The bucket name should start with "airflow-" + bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") + resourceName := "aws_mwaa_environment.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSMwssEnvironmentFullConfig(rName, bucketName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "2"), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.core.default_task_retries", "1"), + resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.core.parallelism", "16"), + resource.TestCheckResourceAttr(resourceName, "airflow_version", "1.10.12"), + testAccCheckResourceAttrRegionalARN(resourceName, "arn", "airflow", "environment/"+rName), + resource.TestCheckResourceAttrSet(resourceName, "created_at"), + resource.TestCheckResourceAttr(resourceName, "dag_s3_path", "dags/"), + resource.TestCheckResourceAttr(resourceName, "environment_class", "mw1.medium"), + testAccCheckResourceAttrGlobalARN(resourceName, "execution_role_arn", "iam", "role/service-role/"+rName), + resource.TestCheckResourceAttrSet(resourceName, "kms_key"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.dag_processing_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.0.log_level", "INFO"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.scheduler_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.scheduler_logs.0.log_level", "WARNING"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.task_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.task_logs.0.log_level", "ERROR"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.webserver_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.webserver_logs.0.log_level", "CRITICAL"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.#", "1"), + resource.TestCheckResourceAttrSet(resourceName, "logging_configuration.0.worker_logs.0.cloud_watch_log_group_arn"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.log_level", "WARNING"), + resource.TestCheckResourceAttr(resourceName, "max_workers", "20"), + resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestCheckResourceAttr(resourceName, "network_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "network_configuration.0.security_group_ids.#", "1"), + resource.TestCheckResourceAttr(resourceName, "network_configuration.0.subnet_ids.#", "2"), + resource.TestCheckResourceAttr(resourceName, "plugins_s3_object_version", ""), + resource.TestCheckResourceAttr(resourceName, "plugins_s3_path", "plugins.zip"), + resource.TestCheckResourceAttr(resourceName, "requirements_s3_object_version", ""), + resource.TestCheckResourceAttr(resourceName, "requirements_s3_path", "requirements.txt"), + resource.TestCheckResourceAttrSet(resourceName, "service_role_arn"), + testAccCheckResourceAttrGlobalARNNoAccount(resourceName, "source_bucket_arn", "s3", bucketName), + resource.TestCheckResourceAttrSet(resourceName, "status"), + resource.TestCheckResourceAttr(resourceName, "webserver_access_mode", mwaa.WebserverAccessModePublicOnly), + resource.TestCheckResourceAttrSet(resourceName, "webserver_url"), + resource.TestCheckResourceAttr(resourceName, "weekly_maintenance_window_start", "SAT:03:00"), + resource.TestCheckResourceAttr(resourceName, "tags.Name", rName), + resource.TestCheckResourceAttr(resourceName, "tags.Environment", "production"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccCheckAWSMwaaEnvironmentExists(resourceName string, environment *mwaa.GetEnvironmentOutput) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[resourceName] + if !ok { + return fmt.Errorf("Not found: %s", resourceName) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No MWAA Environment ID is set") + } + + conn := testAccProvider.Meta().(*AWSClient).mwaaconn + resp, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ + Name: aws.String(rs.Primary.ID), + }) + + if err != nil { + return fmt.Errorf("Error getting MWAA Environment: %s", err.Error()) + } + + *environment = *resp + + return nil + } +} + +func testAccCheckAWSMwaaEnvironmentDestroy(s *terraform.State) error { + conn := testAccProvider.Meta().(*AWSClient).mwaaconn + + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_mwaa_environment" { + continue + } + + _, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ + Name: aws.String(rs.Primary.ID), + }) + + if err != nil { + if isAWSErr(err, mwaa.ErrCodeResourceNotFoundException, "") { + return nil + } + return err + } + + return fmt.Errorf("Expected MWAA Environment to be destroyed, %s found", rs.Primary.ID) + } + + return nil +} + +func testAccAWSMwaaEnvironmentBase(rName, bucketName string) string { + return fmt.Sprintf(` +data "aws_availability_zones" "available" { + state = "available" + + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +resource "aws_vpc" "test" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + Name = %[1]q + } +} + +resource "aws_internet_gateway" "test" { + vpc_id = aws_vpc.test.id +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.test.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.test.id + } +} + +resource "aws_main_route_table_association" "test" { + route_table_id = aws_route_table.public.id + vpc_id = aws_vpc.test.id +} + +resource "aws_subnet" "private" { + count = 2 + + availability_zone = data.aws_availability_zones.available.names[count.index] + cidr_block = cidrsubnet(aws_vpc.test.cidr_block, 8, count.index + 2) + vpc_id = aws_vpc.test.id + + tags = { + Name = "%[1]s-private-${count.index}" + } +} + +resource "aws_eip" "private" { + count = 2 + + vpc = true +} + +resource "aws_nat_gateway" "private" { + count = 2 + + allocation_id = aws_eip.private[count.index].id + subnet_id = aws_subnet.public[count.index].id +} + +resource "aws_route_table" "private" { + count = 2 + + vpc_id = aws_vpc.test.id + + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.private[count.index].id + } +} + +resource "aws_route_table_association" "private" { + count = 2 + + subnet_id = aws_subnet.private[count.index].id + route_table_id = aws_route_table.private[count.index].id +} + +resource "aws_subnet" "public" { + count = 2 + + availability_zone = data.aws_availability_zones.available.names[count.index] + cidr_block = cidrsubnet(aws_vpc.test.cidr_block, 8, count.index) + vpc_id = aws_vpc.test.id + + tags = { + Name = "%[1]s-public-${count.index}" + } +} + +resource "aws_security_group" "test" { + vpc_id = aws_vpc.test.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = %[1]q + } +} + +resource "aws_s3_bucket" "test" { + bucket = %[2]q + acl = "private" + + versioning { + enabled = true + } +} + +resource "aws_s3_bucket_object" "dags" { + bucket = aws_s3_bucket.test.id + acl = "private" + key = "dags/" + content_type = "application/x-directory" + + depends_on = [aws_s3_bucket.test] +} + +resource "aws_iam_role" "test" { + name = %[1]q + path = "/service-role/" + + assume_role_policy = < Date: Fri, 18 Dec 2020 15:06:44 +0900 Subject: [PATCH 03/25] Add a documentation for mwaa_environment Co-authored-by: Christoph Caprano --- website/docs/r/mwaa_environment.html.markdown | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 website/docs/r/mwaa_environment.html.markdown diff --git a/website/docs/r/mwaa_environment.html.markdown b/website/docs/r/mwaa_environment.html.markdown new file mode 100644 index 00000000000..7557b2e72d4 --- /dev/null +++ b/website/docs/r/mwaa_environment.html.markdown @@ -0,0 +1,190 @@ +--- +subcategory: "Managed Workflows for Apache Airflow (MWAA)" +layout: "aws" +page_title: "AWS: aws_mwaa_environment" +description: |- + Creates a MWAA Environment +--- + +# Resource: aws_mwaa_environment + +Creates a MWAA Environment resource. + +## Example Usage + +A MWAA Environment requires an IAM role (`aws_iam_role`), two subnets in the private zone (`aws_subnet`) and a versioned S3 bucket (`aws_s3_bucket`). + +### Basic Usage + +```hcl +resource "aws_mwaa_environment" "example" { + dag_s3_path = "dags/" + execution_role_arn = aws_iam_role.example.arn + name = "example" + + network_configuration { + security_group_ids = [aws_security_group.example.id] + subnet_ids = aws_subnet.private[*].id + } + + source_bucket_arn = aws_s3_bucket.example.arn +} +``` + +### Example with Airflow configuration options + +```hcl +resource "aws_mwaa_environment" "example" { + airflow_configuration_options = { + "core.default_task_retries" = 16 + "core.parallelism" = 1 + } + + dag_s3_path = "dags/" + execution_role_arn = aws_iam_role.example.arn + name = "example" + + network_configuration { + security_group_ids = [aws_security_group.example.id] + subnet_ids = aws_subnet.private[*].id + } + + source_bucket_arn = aws_s3_bucket.example.arn +} +``` + +### Example with logging configurations + +Note that Airflow task logs are enabled by default with the `INFO` log level. + +```hcl +resource "aws_mwaa_environment" "example" { + dag_s3_path = "dags/" + execution_role_arn = aws_iam_role.example.arn + + logging_configuration { + dag_processing_logs { + enabled = true + log_level = "DEBUG" + } + + scheduler_logs { + enabled = true + log_level = "INFO" + } + + task_logs { + enabled = true + log_level = "WARNING" + } + + webserver_logs { + enabled = true + log_level = "ERROR" + } + + worker_logs { + enabled = true + log_level = "CRITICAL" + } + } + + name = "example" + + network_configuration { + security_group_ids = [aws_security_group.example.id] + subnet_ids = aws_subnet.private[*].id + } + + source_bucket_arn = aws_s3_bucket.example.arn +} +``` + +### Example with tags + +```hcl +resource "aws_mwaa_environment" "example" { + dag_s3_path = "dags/" + execution_role_arn = aws_iam_role.example.arn + name = "example" + + network_configuration { + security_group_ids = [aws_security_group.example.id] + subnet_ids = aws_subnet.private[*].id + } + + source_bucket_arn = aws_s3_bucket.example.arn + + tags = { + Name = "example" + Environment = "production" + } +} +``` + +## Argument Reference + +The following arguments are supported: + +* `airflow_configuration_options` - (Optional) The `airflow_configuration_options` parameter specifies airflow override options. Check the [Official documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-env-variables.html#configuring-env-variables-reference) for all possible configuration options. +* `airflow_version` - (Optional) Airflow version of your environment, will be set by default to the latest version that MWAA supports. +* `dag_s3_path` - (Required) The relative path to the DAG folder on your Amazon S3 storage bucket. For example, dags. For more information, see [Importing DAGs on Amazon MWAA](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-dag-import.html). +* `environment_class` - (Optional) Environment class for the cluster. Possible options are `mw1.small`, `mw1.medium`, `mw1.large`. Will be set by default to `mw1.small`. Please check the [AWS Pricing](https://aws.amazon.com/de/managed-workflows-for-apache-airflow/pricing/) for more information about the environment classes. +* `execution_role_arn` - (Required) The Amazon Resource Name (ARN) of the task execution role that the Amazon MWAA and its environment can assume. Check the [official AWS documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-create-role.html) for the detailed role specification. +* `kms_key` - (Optional) The Amazon Resource Name (ARN) of your KMS key that you want to use for encryption. Will be set to the ARN of the managed KMS key `aws/airflow` by default. Please check the [Official Documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/custom-keys-certs.html) for more information. +* `logging_configuration` - (Optional) The Apache Airflow logs you want to send to Amazon CloudWatch Logs. +* `max_workers` - (Optional) The maximum number of workers that can be automatically scaled up. Value need to be between `1` and `25`. Will be `10` by default. +* `name` - (Required) The name of the Apache Airflow Environment +* `network_configuration` - (Required) Specifies the network configuration for your Apache Airflow Environment. This includes two private subnets as well as security groups for the Airflow environment. Each subnet requires internet connection, otherwise the deployment will fail. See [Network configuration](#network) below for details. +* `plugins_s3_object_version` - (Optional) The plugins.zip file version you want to use. +* `plugins_s3_path` - (Optional) The relative path to the plugins.zip file on your Amazon S3 storage bucket. For example, plugins.zip. If a relative path is provided in the request, then plugins_s3_object_version is required. For more information, see [Importing DAGs on Amazon MWAA](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-dag-import.html). +* `requirements_s3_object_version` - (Optional) The requirements.txt file version you want to use. +* `requirements_s3_path` - (Optional) The relative path to the requirements.txt file on your Amazon S3 storage bucket. For example, requirements.txt. If a relative path is provided in the request, then requirements_s3_object_version is required. For more information, see [Importing DAGs on Amazon MWAA](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-dag-import.html). +* `source_bucket_arn` - (Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname. +* `webserver_access_mode` - (Optional) Specifies whether the webserver should be accessible over the internet or via your specified VPC. Possible options: `PRIVATE_ONLY` (default) and `PUBLIC_ONLY`. +* `weekly_maintenance_window_start` - (Optional) Specifies the start date for the weekly maintenance window. +* `tags` - (Optional) An array of key:value pairs to associate with the resource. + +### Logging configurations + +The `logging_configuration` block supports the following arguments. + +* `dag_processing_logs` - (Optional) (Optional) Log configuration options for processing DAGs. See [Module logging configuration](#module-logging-configuration) for more information. Disabled by default. +* `scheduler_logs` - (Optional) Log configuration options for the schedulers. See [Module logging configuration](#module-logging-configuration) for more information. Disabled by default. +* `task_logs` - (Optional) Log configuration options for DAG tasks. See [Module logging configuration](#module-logging-configuration) for more information. Enabled by default with `INFO` log level. +* `webserver_logs` - (Optional) Log configuration options for the webservers. See [Module logging configuration](#module-logging-configuration) for more information. Disabled by default. +* `worker_logs` - (Optional) Log configuration options for the workers. See [Module logging configuration](#module-logging-configuration) for more information. Disabled by default. + +### Module logging configuration + +A configuration block to use for logging with respect to the various Apache Airflow services: DagProcessingLogs, SchedulerLogs, TaskLogs, WebserverLogs, and WorkerLogs. It supports the following arguments. + +* `enabled` - (Required) Enabling or disabling the collection of logs +* `log_level` - (Optional) Logging level. Valid values: `CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG`. Will be `INFO` by default. + +### Network configuration + +The `network_configuration` block supports the following arguments. More information about the required subnet and security group settings can be found in the [official AWS documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/vpc-create.html). + +* `security_group_ids` - (Required) Security groups IDs for the environment. At least one of the security group needs to allow MWAA resources to talk to each other. Otherwise MWAA can't be provisioned and will fail. See [Base Usage](#base-usage) for an exemplary security group. +* `subnet_ids` - (Required) The private subnet IDs in which the environment should be created. MWAA requires two subnets. + +## Attributes Reference + +In addition to all arguments above, the following attributes are exported: + +* `arn` - The ARN of the MWAA Environment +* `created_at` - The Created At date of the MWAA Environment +* `logging_configuration..cloud_watch_log_group_arn` - Provides the ARN for the CloudWatch group where the logs will be published +* `service_role_arn` - The Service Role ARN of the Amazon MWAA Environment +* `status` - The status of the Amazon MWAA Environment +* `webserver_url` - The webserver URL of the MWAA Environment + + +## Import + +MWAA Environment can be imported using `Name` e.g. + +``` +$ terraform import aws_mwaa_environment.example MyAirflowEnvironment +``` From 021743212d77ec3e1b439aa80d0b89cf0f2e0720 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Fri, 18 Dec 2020 15:12:04 +0900 Subject: [PATCH 04/25] Use aws.StringValue instead Co-authored-by: Christoph Caprano --- aws/resource_aws_mwaa_environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 7e7be38594d..6996c33153f 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -292,7 +292,7 @@ func resourceAwsMwaaEnvironmentCreate(d *schema.ResourceData, meta interface{}) return err } - d.SetId(*input.Name) + d.SetId(aws.StringValue(input.Name)) d.Set("arn", out.Arn) if err := waitForMwaaEnvironmentCreation(conn, d.Id()); err != nil { From bb92fe1903f4768582d4f113ff4b2bff5f3d159e Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 16 Jan 2021 16:18:58 +0900 Subject: [PATCH 05/25] Use values functions instead for mwaa_environment --- aws/resource_aws_mwaa_environment.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 6996c33153f..748511532fa 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -204,10 +204,7 @@ func resourceAwsMwaaEnvironment() *schema.Resource { Type: schema.TypeString, Optional: true, Default: mwaa.WebserverAccessModePrivateOnly, - ValidateFunc: validation.StringInSlice([]string{ - mwaa.WebserverAccessModePrivateOnly, - mwaa.WebserverAccessModePublicOnly, - }, false), + ValidateFunc: validation.StringInSlice(mwaa.WebserverAccessMode_Values(), false), }, "webserver_url": { Type: schema.TypeString, @@ -502,12 +499,7 @@ func mwaaEnvironmentModuleLoggingConfigurationSchema(defaultEnabled bool) *schem Type: schema.TypeString, Optional: true, Default: mwaa.LoggingLevelInfo, - ValidateFunc: validation.StringInSlice([]string{ - mwaa.LoggingLevelCritical, - mwaa.LoggingLevelError, - mwaa.LoggingLevelWarning, - mwaa.LoggingLevelInfo, - }, false), + ValidateFunc: validation.StringInSlice(mwaa.LoggingLevel_Values(), false), }, }, } From 05aaf6e5b5be574675b42717f4c1c955d09c2db8 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 16 Jan 2021 16:19:53 +0900 Subject: [PATCH 06/25] Move tags attribute up to keep alphabetical ordering for mwaa_environment --- aws/resource_aws_mwaa_environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 748511532fa..44b763d2cbb 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -200,6 +200,7 @@ func resourceAwsMwaaEnvironment() *schema.Resource { Type: schema.TypeString, Computed: true, }, + "tags": tagsSchema(), "webserver_access_mode": { Type: schema.TypeString, Optional: true, @@ -215,7 +216,6 @@ func resourceAwsMwaaEnvironment() *schema.Resource { Optional: true, Computed: true, }, - "tags": tagsSchema(), }, } } From b7d75adfc6d69840824fd7a1a714b55a445c5560 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 16 Jan 2021 16:21:38 +0900 Subject: [PATCH 07/25] Remove unnecessary arn set from mwaa_environment read function --- aws/resource_aws_mwaa_environment.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 44b763d2cbb..2d81b850886 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -284,13 +284,12 @@ func resourceAwsMwaaEnvironmentCreate(d *schema.ResourceData, meta interface{}) } log.Printf("[INFO] Creating MWAA Environment: %s", input) - out, err := conn.CreateEnvironment(&input) + _, err := conn.CreateEnvironment(&input) if err != nil { return err } d.SetId(aws.StringValue(input.Name)) - d.Set("arn", out.Arn) if err := waitForMwaaEnvironmentCreation(conn, d.Id()); err != nil { return err From a26f8dce883c7c2ebdea83ca6e191b6a58148df8 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 16 Jan 2021 16:26:27 +0900 Subject: [PATCH 08/25] Wrap error messages with context for mwaa_environment --- aws/resource_aws_mwaa_environment.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 2d81b850886..3beddc2c062 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -286,13 +286,13 @@ func resourceAwsMwaaEnvironmentCreate(d *schema.ResourceData, meta interface{}) log.Printf("[INFO] Creating MWAA Environment: %s", input) _, err := conn.CreateEnvironment(&input) if err != nil { - return err + return fmt.Errorf("error creating MWAA Environment: %w", err) } d.SetId(aws.StringValue(input.Name)) if err := waitForMwaaEnvironmentCreation(conn, d.Id()); err != nil { - return err + return fmt.Errorf("error creating MWAA Environment (%s): %w", d.Id(), err) } return resourceAwsMwaaEnvironmentRead(d, meta) @@ -315,7 +315,7 @@ func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) er return nil } - return err + return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } d.Set("airflow_configuration_options", aws.StringValueMap(out.Environment.AirflowConfigurationOptions)) @@ -327,15 +327,15 @@ func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) er d.Set("execution_role_arn", out.Environment.ExecutionRoleArn) d.Set("kms_key", out.Environment.KmsKey) if err := d.Set("last_updated", flattenMwaaLastUpdate(out.Environment.LastUpdate)); err != nil { - return err + return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } if err := d.Set("logging_configuration", flattenMwaaLoggingConfiguration(out.Environment.LoggingConfiguration)); err != nil { - return err + return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } d.Set("max_workers", out.Environment.MaxWorkers) d.Set("name", out.Environment.Name) if err := d.Set("network_configuration", flattenMwaaNetworkConfiguration(out.Environment.NetworkConfiguration)); err != nil { - return err + return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } d.Set("plugins_s3_object_version", out.Environment.PluginsS3ObjectVersion) d.Set("plugins_s3_path", out.Environment.PluginsS3Path) @@ -445,11 +445,11 @@ func resourceAwsMwaaEnvironmentUpdate(d *schema.ResourceData, meta interface{}) _, err := conn.UpdateEnvironment(&input) if err != nil { - return err + return fmt.Errorf("error updating MWAA Environment (%s): %w", d.Id(), err) } if err := waitForMwaaEnvironmentUpdate(conn, d.Id()); err != nil { - return err + return fmt.Errorf("error updating MWAA Environment (%s): %w", d.Id(), err) } } @@ -476,7 +476,7 @@ func resourceAwsMwaaEnvironmentDelete(d *schema.ResourceData, meta interface{}) return nil } - return err + return fmt.Errorf("error deleting MWAA Environment (%s): %w", d.Id(), err) } return waitForMwaaEnvironmentDeletion(conn, d.Id()) From 1e0ac6ae4244b68fe3b57b40c8114e0571b34097 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 16 Jan 2021 16:34:45 +0900 Subject: [PATCH 09/25] Use HasChangesExcept instead of HasChange for meaa_environment --- aws/resource_aws_mwaa_environment.go | 59 +--------------------------- 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 3beddc2c062..c7b7f015c20 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -358,89 +358,32 @@ func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) er func resourceAwsMwaaEnvironmentUpdate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).mwaaconn - needsUpdate := false - input := mwaa.UpdateEnvironmentInput{ Name: aws.String(d.Get("name").(string)), } - if d.HasChange("airflow_configuration_options") { - needsUpdate = true - + if d.HasChangesExcept("tags") { options, ok := d.GetOk("airflow_configuration_options") if !ok { options = map[string]interface{}{} } input.AirflowConfigurationOptions = stringMapToPointers(options.(map[string]interface{})) - } - if d.HasChange("airflow_version") { - needsUpdate = true input.AirflowVersion = aws.String(d.Get("airflow_version").(string)) - } - - if d.HasChange("dag_s3_path") { - needsUpdate = true input.DagS3Path = aws.String(d.Get("dag_s3_path").(string)) - } - - if d.HasChange("environment_class") { - needsUpdate = true input.EnvironmentClass = aws.String(d.Get("environment_class").(string)) - } - - if d.HasChange("execution_role_arn") { - needsUpdate = true input.ExecutionRoleArn = aws.String(d.Get("execution_role_arn").(string)) - } - - if d.HasChange("logging_configuration") { - needsUpdate = true input.LoggingConfiguration = expandMwaaEnvironmentLoggingConfiguration(d.Get("logging_configuration").([]interface{})) - } - - if d.HasChange("max_workers") { - needsUpdate = true input.MaxWorkers = aws.Int64(int64(d.Get("max_workers").(int))) - } - - if d.HasChange("network_configuration") { - needsUpdate = true input.NetworkConfiguration = expandMwaaEnvironmentNetworkConfigurationUpdate(d.Get("network_configuration").([]interface{})) - } - - if d.HasChange("plugins_s3_path") { - needsUpdate = true input.PluginsS3Path = aws.String(d.Get("plugins_s3_path").(string)) - } - - if d.HasChange("requirements_s3_object_version") { - needsUpdate = true input.RequirementsS3ObjectVersion = aws.String(d.Get("requirements_s3_object_version").(string)) - } - - if d.HasChange("requirements_s3_path") { - needsUpdate = true input.RequirementsS3Path = aws.String(d.Get("requirements_s3_path").(string)) - } - - if d.HasChange("source_bucket_arn") { - needsUpdate = true input.SourceBucketArn = aws.String(d.Get("source_bucket_arn").(string)) - } - - if d.HasChange("webserver_access_mode") { - needsUpdate = true input.WebserverAccessMode = aws.String(d.Get("webserver_access_mode").(string)) - } - - if d.HasChange("weekly_maintenance_window_start") { - needsUpdate = true input.WeeklyMaintenanceWindowStart = aws.String(d.Get("weekly_maintenance_window_start").(string)) - } - if needsUpdate { log.Printf("[INFO] Updating MWAA Environment: %s", input) _, err := conn.UpdateEnvironment(&input) From ed8adb913bb95d60be1dbb972f8179e03ce0bc48 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 16 Jan 2021 17:09:15 +0900 Subject: [PATCH 10/25] Split the waiters into separate packages for meaa_environment --- aws/internal/service/mwaa/finder/finder.go | 25 ++++ aws/internal/service/mwaa/waiter/status.go | 35 +++++ aws/internal/service/mwaa/waiter/waiter.go | 82 +++++++++++ aws/resource_aws_mwaa_environment.go | 157 +++++---------------- 4 files changed, 181 insertions(+), 118 deletions(-) create mode 100644 aws/internal/service/mwaa/finder/finder.go create mode 100644 aws/internal/service/mwaa/waiter/status.go create mode 100644 aws/internal/service/mwaa/waiter/waiter.go diff --git a/aws/internal/service/mwaa/finder/finder.go b/aws/internal/service/mwaa/finder/finder.go new file mode 100644 index 00000000000..da514fce335 --- /dev/null +++ b/aws/internal/service/mwaa/finder/finder.go @@ -0,0 +1,25 @@ +package finder + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/mwaa" +) + +// EnvironmentByName returns the MWAA Environment corresponding to the specified Name. +// Returns nil if no environment is found. +func EnvironmentByName(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) { + input := &mwaa.GetEnvironmentInput{ + Name: aws.String(name), + } + + output, err := conn.GetEnvironment(input) + if err != nil { + return nil, err + } + + if output == nil { + return nil, nil + } + + return output.Environment, nil +} diff --git a/aws/internal/service/mwaa/waiter/status.go b/aws/internal/service/mwaa/waiter/status.go new file mode 100644 index 00000000000..19d1843a071 --- /dev/null +++ b/aws/internal/service/mwaa/waiter/status.go @@ -0,0 +1,35 @@ +package waiter + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/mwaa" + "github.com/hashicorp/aws-sdk-go-base/tfawserr" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/mwaa/finder" +) + +const ( + environmentStatusNotFound = "NotFound" + environmentStatusUnknown = "Unknown" +) + +// EnvironmentStatus fetches the Environment and its Status +func EnvironmentStatus(conn *mwaa.MWAA, name string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + environment, err := finder.EnvironmentByName(conn, name) + + if tfawserr.ErrCodeEquals(err, mwaa.ErrCodeResourceNotFoundException) { + return nil, environmentStatusNotFound, nil + } + + if err != nil { + return nil, environmentStatusUnknown, err + } + + if environment == nil { + return nil, environmentStatusNotFound, nil + } + + return environment, aws.StringValue(environment.Status), nil + } +} diff --git a/aws/internal/service/mwaa/waiter/waiter.go b/aws/internal/service/mwaa/waiter/waiter.go new file mode 100644 index 00000000000..96c461903b5 --- /dev/null +++ b/aws/internal/service/mwaa/waiter/waiter.go @@ -0,0 +1,82 @@ +package waiter + +import ( + "github.com/aws/aws-sdk-go/service/mwaa" + "time" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +const ( + // Maximum amount of time to wait for an environment creation + EnvironmentCreatedTimeout = 90 * time.Minute + // Amount of delay to check an environment status + EnvironmentCreatedDelay = 1 * time.Minute + + // Maximum amount of time to wait for an environment update + EnvironmentUpdatedTimeout = 90 * time.Minute + // Amount of delay to check an environment status + EnvironmentUpdatedDelay = 1 * time.Minute + + // Maximum amount of time to wait for an environment deletion + EnvironmentDeletedTimeout = 90 * time.Minute + // Amount of delay to check an environment status + EnvironmentDeletedDelay = 1 * time.Minute +) + +// EnvironmentCreated waits for a Environment to return AVAILABLE +func EnvironmentCreated(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) { + stateConf := &resource.StateChangeConf{ + Pending: []string{mwaa.EnvironmentStatusCreating}, + Target: []string{mwaa.EnvironmentStatusAvailable}, + Refresh: EnvironmentStatus(conn, name), + Timeout: EnvironmentCreatedTimeout, + Delay: EnvironmentCreatedDelay, + } + + outputRaw, err := stateConf.WaitForState() + + if v, ok := outputRaw.(*mwaa.Environment); ok { + return v, err + } + + return nil, err +} + +// EnvironmentUpdated waits for a Environment to return AVAILABLE +func EnvironmentUpdated(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) { + stateConf := &resource.StateChangeConf{ + Pending: []string{mwaa.EnvironmentStatusUpdating}, + Target: []string{mwaa.EnvironmentStatusAvailable}, + Refresh: EnvironmentStatus(conn, name), + Timeout: EnvironmentUpdatedTimeout, + Delay: EnvironmentUpdatedDelay, + } + + outputRaw, err := stateConf.WaitForState() + + if v, ok := outputRaw.(*mwaa.Environment); ok { + return v, err + } + + return nil, err +} + +// EnvironmentDeleted waits for a Environment to return AVAILABLE +func EnvironmentDeleted(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) { + stateConf := &resource.StateChangeConf{ + Pending: []string{mwaa.EnvironmentStatusDeleting}, + Target: []string{environmentStatusNotFound}, + Refresh: EnvironmentStatus(conn, name), + Timeout: EnvironmentDeletedTimeout, + Delay: EnvironmentDeletedDelay, + } + + outputRaw, err := stateConf.WaitForState() + + if v, ok := outputRaw.(*mwaa.Environment); ok { + return v, err + } + + return nil, err +} diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index c7b7f015c20..f2be3fda642 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -2,15 +2,15 @@ package aws import ( "fmt" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/mwaa/finder" "log" - "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/mwaa" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" "github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/mwaa/waiter" ) func resourceAwsMwaaEnvironment() *schema.Resource { @@ -202,9 +202,9 @@ func resourceAwsMwaaEnvironment() *schema.Resource { }, "tags": tagsSchema(), "webserver_access_mode": { - Type: schema.TypeString, - Optional: true, - Default: mwaa.WebserverAccessModePrivateOnly, + Type: schema.TypeString, + Optional: true, + Default: mwaa.WebserverAccessModePrivateOnly, ValidateFunc: validation.StringInSlice(mwaa.WebserverAccessMode_Values(), false), }, "webserver_url": { @@ -291,7 +291,7 @@ func resourceAwsMwaaEnvironmentCreate(d *schema.ResourceData, meta interface{}) d.SetId(aws.StringValue(input.Name)) - if err := waitForMwaaEnvironmentCreation(conn, d.Id()); err != nil { + if _, err := waiter.EnvironmentCreated(conn, d.Id()); err != nil { return fmt.Errorf("error creating MWAA Environment (%s): %w", d.Id(), err) } @@ -304,9 +304,7 @@ func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) er log.Printf("[INFO] Reading MWAA Environment: %s", d.Id()) - out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ - Name: aws.String(d.Id()), - }) + environment, err := finder.EnvironmentByName(conn, d.Id()) if err != nil { if isAWSErr(err, mwaa.ErrCodeResourceNotFoundException, "") && !d.IsNewResource() { @@ -318,37 +316,37 @@ func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) er return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } - d.Set("airflow_configuration_options", aws.StringValueMap(out.Environment.AirflowConfigurationOptions)) - d.Set("airflow_version", out.Environment.AirflowVersion) - d.Set("arn", out.Environment.Arn) - d.Set("created_at", aws.TimeValue(out.Environment.CreatedAt).String()) - d.Set("dag_s3_path", out.Environment.DagS3Path) - d.Set("environment_class", out.Environment.EnvironmentClass) - d.Set("execution_role_arn", out.Environment.ExecutionRoleArn) - d.Set("kms_key", out.Environment.KmsKey) - if err := d.Set("last_updated", flattenMwaaLastUpdate(out.Environment.LastUpdate)); err != nil { + d.Set("airflow_configuration_options", aws.StringValueMap(environment.AirflowConfigurationOptions)) + d.Set("airflow_version", environment.AirflowVersion) + d.Set("arn", environment.Arn) + d.Set("created_at", aws.TimeValue(environment.CreatedAt).String()) + d.Set("dag_s3_path", environment.DagS3Path) + d.Set("environment_class", environment.EnvironmentClass) + d.Set("execution_role_arn", environment.ExecutionRoleArn) + d.Set("kms_key", environment.KmsKey) + if err := d.Set("last_updated", flattenMwaaLastUpdate(environment.LastUpdate)); err != nil { return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } - if err := d.Set("logging_configuration", flattenMwaaLoggingConfiguration(out.Environment.LoggingConfiguration)); err != nil { + if err := d.Set("logging_configuration", flattenMwaaLoggingConfiguration(environment.LoggingConfiguration)); err != nil { return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } - d.Set("max_workers", out.Environment.MaxWorkers) - d.Set("name", out.Environment.Name) - if err := d.Set("network_configuration", flattenMwaaNetworkConfiguration(out.Environment.NetworkConfiguration)); err != nil { + d.Set("max_workers", environment.MaxWorkers) + d.Set("name", environment.Name) + if err := d.Set("network_configuration", flattenMwaaNetworkConfiguration(environment.NetworkConfiguration)); err != nil { return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } - d.Set("plugins_s3_object_version", out.Environment.PluginsS3ObjectVersion) - d.Set("plugins_s3_path", out.Environment.PluginsS3Path) - d.Set("requirements_s3_object_version", out.Environment.RequirementsS3ObjectVersion) - d.Set("requirements_s3_path", out.Environment.RequirementsS3Path) - d.Set("service_role_arn", out.Environment.ServiceRoleArn) - d.Set("source_bucket_arn", out.Environment.SourceBucketArn) - d.Set("status", out.Environment.Status) - d.Set("webserver_access_mode", out.Environment.WebserverAccessMode) - d.Set("webserver_url", out.Environment.WebserverUrl) - d.Set("weekly_maintenance_window_start", out.Environment.WeeklyMaintenanceWindowStart) - - if err := d.Set("tags", keyvaluetags.MwaaKeyValueTags(out.Environment.Tags).IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil { + d.Set("plugins_s3_object_version", environment.PluginsS3ObjectVersion) + d.Set("plugins_s3_path", environment.PluginsS3Path) + d.Set("requirements_s3_object_version", environment.RequirementsS3ObjectVersion) + d.Set("requirements_s3_path", environment.RequirementsS3Path) + d.Set("service_role_arn", environment.ServiceRoleArn) + d.Set("source_bucket_arn", environment.SourceBucketArn) + d.Set("status", environment.Status) + d.Set("webserver_access_mode", environment.WebserverAccessMode) + d.Set("webserver_url", environment.WebserverUrl) + d.Set("weekly_maintenance_window_start", environment.WeeklyMaintenanceWindowStart) + + if err := d.Set("tags", keyvaluetags.MwaaKeyValueTags(environment.Tags).IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil { return fmt.Errorf("error setting tags: %s", err) } @@ -391,7 +389,7 @@ func resourceAwsMwaaEnvironmentUpdate(d *schema.ResourceData, meta interface{}) return fmt.Errorf("error updating MWAA Environment (%s): %w", d.Id(), err) } - if err := waitForMwaaEnvironmentUpdate(conn, d.Id()); err != nil { + if _, err := waiter.EnvironmentUpdated(conn, d.Id()); err != nil { return fmt.Errorf("error updating MWAA Environment (%s): %w", d.Id(), err) } } @@ -422,7 +420,9 @@ func resourceAwsMwaaEnvironmentDelete(d *schema.ResourceData, meta interface{}) return fmt.Errorf("error deleting MWAA Environment (%s): %w", d.Id(), err) } - return waitForMwaaEnvironmentDeletion(conn, d.Id()) + _, err = waiter.EnvironmentDeleted(conn, d.Id()) + + return err } func mwaaEnvironmentModuleLoggingConfigurationSchema(defaultEnabled bool) *schema.Resource { @@ -438,9 +438,9 @@ func mwaaEnvironmentModuleLoggingConfigurationSchema(defaultEnabled bool) *schem Default: defaultEnabled, }, "log_level": { - Type: schema.TypeString, - Optional: true, - Default: mwaa.LoggingLevelInfo, + Type: schema.TypeString, + Optional: true, + Default: mwaa.LoggingLevelInfo, ValidateFunc: validation.StringInSlice(mwaa.LoggingLevel_Values(), false), }, }, @@ -510,85 +510,6 @@ func expandMwaaEnvironmentNetworkConfigurationUpdate(l []interface{}) *mwaa.Upda } } -func waitForMwaaEnvironmentCreation(conn *mwaa.MWAA, id string) error { - stateConf := resource.StateChangeConf{ - Pending: []string{ - mwaa.EnvironmentStatusCreating, - }, - Target: []string{ - mwaa.EnvironmentStatusAvailable, - }, - Timeout: 90 * time.Minute, - Delay: 1 * time.Minute, - Refresh: func() (interface{}, string, error) { - out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ - Name: aws.String(id), - }) - - if err != nil { - return nil, "", err - } - - return out, *out.Environment.Status, nil - }, - } - _, err := stateConf.WaitForState() - return err -} - -func waitForMwaaEnvironmentUpdate(conn *mwaa.MWAA, id string) error { - stateConf := resource.StateChangeConf{ - Pending: []string{ - mwaa.EnvironmentStatusUpdating, - }, - Target: []string{ - mwaa.EnvironmentStatusAvailable, - }, - Timeout: 90 * time.Minute, - Delay: 1 * time.Minute, - Refresh: func() (interface{}, string, error) { - out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ - Name: aws.String(id), - }) - - if err != nil { - return nil, "", err - } - - return out, *out.Environment.Status, nil - }, - } - _, err := stateConf.WaitForState() - return err -} - -func waitForMwaaEnvironmentDeletion(conn *mwaa.MWAA, id string) error { - stateConf := resource.StateChangeConf{ - Pending: []string{ - mwaa.EnvironmentStatusDeleting, - }, - Target: []string{}, - Timeout: 90 * time.Minute, - Delay: 1 * time.Minute, - Refresh: func() (interface{}, string, error) { - out, err := conn.GetEnvironment(&mwaa.GetEnvironmentInput{ - Name: aws.String(id), - }) - if err != nil { - - if isAWSErr(err, mwaa.ErrCodeResourceNotFoundException, "") { - return nil, "", nil - } - return nil, "", err - } - - return out, *out.Environment.Status, nil - }, - } - _, err := stateConf.WaitForState() - return err -} - func flattenMwaaLastUpdate(lastUpdate *mwaa.LastUpdate) []interface{} { if lastUpdate == nil { return []interface{}{} From 4250e2bd04b36a2a28d50f89b9050a458189910f Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 16 Jan 2021 20:27:57 +0900 Subject: [PATCH 11/25] Remove an airflow prefix from meaa_environment test S3 bucket names --- aws/resource_aws_mwaa_environment_test.go | 52 +++++++++-------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/aws/resource_aws_mwaa_environment_test.go b/aws/resource_aws_mwaa_environment_test.go index 81eed273974..76c9abf254c 100644 --- a/aws/resource_aws_mwaa_environment_test.go +++ b/aws/resource_aws_mwaa_environment_test.go @@ -52,8 +52,6 @@ func TestAccAWSMwaaEnvironment_basic(t *testing.T) { var environment mwaa.GetEnvironmentOutput rName := acctest.RandomWithPrefix("tf-acc-test") - // The bucket name should start with "airflow-" - bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") resourceName := "aws_mwaa_environment.test" resource.ParallelTest(t, resource.TestCase{ @@ -62,7 +60,7 @@ func TestAccAWSMwaaEnvironment_basic(t *testing.T) { CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ { - Config: testAccAWSMwssEnvironmentBasicConfig(rName, bucketName), + Config: testAccAWSMwssEnvironmentBasicConfig(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), resource.TestCheckResourceAttrSet(resourceName, "airflow_version"), @@ -94,7 +92,7 @@ func TestAccAWSMwaaEnvironment_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "network_configuration.0.security_group_ids.#", "1"), resource.TestCheckResourceAttr(resourceName, "network_configuration.0.subnet_ids.#", "2"), resource.TestCheckResourceAttrSet(resourceName, "service_role_arn"), - testAccCheckResourceAttrGlobalARNNoAccount(resourceName, "source_bucket_arn", "s3", bucketName), + testAccCheckResourceAttrGlobalARNNoAccount(resourceName, "source_bucket_arn", "s3", rName), resource.TestCheckResourceAttrSet(resourceName, "status"), resource.TestCheckResourceAttr(resourceName, "webserver_access_mode", mwaa.WebserverAccessModePrivateOnly), resource.TestCheckResourceAttrSet(resourceName, "webserver_url"), @@ -114,8 +112,6 @@ func TestAccAWSMwaaEnvironment_disappears(t *testing.T) { var environment mwaa.GetEnvironmentOutput rName := acctest.RandomWithPrefix("tf-acc-test") - // The bucket name should start with "airflow-" - bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") resourceName := "aws_mwaa_environment.test" resource.ParallelTest(t, resource.TestCase{ @@ -124,7 +120,7 @@ func TestAccAWSMwaaEnvironment_disappears(t *testing.T) { CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ { - Config: testAccAWSMwssEnvironmentBasicConfig(rName, bucketName), + Config: testAccAWSMwssEnvironmentBasicConfig(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), testAccCheckResourceDisappears(testAccProvider, resourceAwsMwaaEnvironment(), resourceName), @@ -139,8 +135,6 @@ func TestAccAWSMwaaEnvironment_AirflowConfigurationOptions(t *testing.T) { var environment mwaa.GetEnvironmentOutput rName := acctest.RandomWithPrefix("tf-acc-test") - // The bucket name should start with "airflow-" - bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") resourceName := "aws_mwaa_environment.test" resource.ParallelTest(t, resource.TestCase{ @@ -149,7 +143,7 @@ func TestAccAWSMwaaEnvironment_AirflowConfigurationOptions(t *testing.T) { CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ { - Config: testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, bucketName, "1", "16"), + Config: testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, "1", "16"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "2"), @@ -163,7 +157,7 @@ func TestAccAWSMwaaEnvironment_AirflowConfigurationOptions(t *testing.T) { ImportStateVerify: true, }, { - Config: testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, bucketName, "2", "32"), + Config: testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, "2", "32"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "2"), @@ -172,7 +166,7 @@ func TestAccAWSMwaaEnvironment_AirflowConfigurationOptions(t *testing.T) { ), }, { - Config: testAccAWSMwssEnvironmentBasicConfig(rName, bucketName), + Config: testAccAWSMwssEnvironmentBasicConfig(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "0"), @@ -186,8 +180,6 @@ func TestAccAWSMwaaEnvironment_LogConfiguration(t *testing.T) { var environment mwaa.GetEnvironmentOutput rName := acctest.RandomWithPrefix("tf-acc-test") - // The bucket name should start with "airflow-" - bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") resourceName := "aws_mwaa_environment.test" resource.ParallelTest(t, resource.TestCase{ @@ -196,7 +188,7 @@ func TestAccAWSMwaaEnvironment_LogConfiguration(t *testing.T) { CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ { - Config: testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, bucketName, "true", mwaa.LoggingLevelCritical), + Config: testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, "true", mwaa.LoggingLevelCritical), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), @@ -233,7 +225,7 @@ func TestAccAWSMwaaEnvironment_LogConfiguration(t *testing.T) { ImportStateVerify: true, }, { - Config: testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, bucketName, "false", mwaa.LoggingLevelInfo), + Config: testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, "false", mwaa.LoggingLevelInfo), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), @@ -272,8 +264,6 @@ func TestAccAWSMwaaEnvironment_full(t *testing.T) { var environment mwaa.GetEnvironmentOutput rName := acctest.RandomWithPrefix("tf-acc-test") - // The bucket name should start with "airflow-" - bucketName := acctest.RandomWithPrefix("airflow-tf-acc-test") resourceName := "aws_mwaa_environment.test" resource.ParallelTest(t, resource.TestCase{ @@ -282,7 +272,7 @@ func TestAccAWSMwaaEnvironment_full(t *testing.T) { CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ { - Config: testAccAWSMwssEnvironmentFullConfig(rName, bucketName), + Config: testAccAWSMwssEnvironmentFullConfig(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSMwaaEnvironmentExists(resourceName, &environment), resource.TestCheckResourceAttr(resourceName, "airflow_configuration_options.%", "2"), @@ -326,7 +316,7 @@ func TestAccAWSMwaaEnvironment_full(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "requirements_s3_object_version", ""), resource.TestCheckResourceAttr(resourceName, "requirements_s3_path", "requirements.txt"), resource.TestCheckResourceAttrSet(resourceName, "service_role_arn"), - testAccCheckResourceAttrGlobalARNNoAccount(resourceName, "source_bucket_arn", "s3", bucketName), + testAccCheckResourceAttrGlobalARNNoAccount(resourceName, "source_bucket_arn", "s3", rName), resource.TestCheckResourceAttrSet(resourceName, "status"), resource.TestCheckResourceAttr(resourceName, "webserver_access_mode", mwaa.WebserverAccessModePublicOnly), resource.TestCheckResourceAttrSet(resourceName, "webserver_url"), @@ -395,7 +385,7 @@ func testAccCheckAWSMwaaEnvironmentDestroy(s *terraform.State) error { return nil } -func testAccAWSMwaaEnvironmentBase(rName, bucketName string) string { +func testAccAWSMwaaEnvironmentBase(rName string) string { return fmt.Sprintf(` data "aws_availability_zones" "available" { state = "available" @@ -512,7 +502,7 @@ resource "aws_security_group" "test" { } resource "aws_s3_bucket" "test" { - bucket = %[2]q + bucket = %[1]q acl = "private" versioning { @@ -569,11 +559,11 @@ resource "aws_iam_role_policy" "test" { POLICY } -`, rName, bucketName) +`, rName) } -func testAccAWSMwssEnvironmentBasicConfig(rName, bucketName string) string { - return testAccAWSMwaaEnvironmentBase(rName, bucketName) + fmt.Sprintf(` +func testAccAWSMwssEnvironmentBasicConfig(rName string) string { + return testAccAWSMwaaEnvironmentBase(rName) + fmt.Sprintf(` resource "aws_mwaa_environment" "test" { dag_s3_path = aws_s3_bucket_object.dags.key execution_role_arn = aws_iam_role.test.arn @@ -591,8 +581,8 @@ resource "aws_mwaa_environment" "test" { `, rName) } -func testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, bucketName, retries, parallelism string) string { - return testAccAWSMwaaEnvironmentBase(rName, bucketName) + fmt.Sprintf(` +func testAccAWSMwssEnvironmentAirflowConfigurationOptionsConfig(rName, retries, parallelism string) string { + return testAccAWSMwaaEnvironmentBase(rName) + fmt.Sprintf(` resource "aws_mwaa_environment" "test" { airflow_configuration_options = { "core.default_task_retries" = %[2]q @@ -615,8 +605,8 @@ resource "aws_mwaa_environment" "test" { `, rName, retries, parallelism) } -func testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, bucketName, logEnabled, logLevel string) string { - return testAccAWSMwaaEnvironmentBase(rName, bucketName) + fmt.Sprintf(` +func testAccAWSMwssEnvironmentLoggingConfigurationConfig(rName, logEnabled, logLevel string) string { + return testAccAWSMwaaEnvironmentBase(rName) + fmt.Sprintf(` resource "aws_mwaa_environment" "test" { dag_s3_path = aws_s3_bucket_object.dags.key execution_role_arn = aws_iam_role.test.arn @@ -662,8 +652,8 @@ resource "aws_mwaa_environment" "test" { `, rName, logEnabled, logLevel) } -func testAccAWSMwssEnvironmentFullConfig(rName, bucketName string) string { - return testAccAWSMwaaEnvironmentBase(rName, bucketName) + fmt.Sprintf(` +func testAccAWSMwssEnvironmentFullConfig(rName string) string { + return testAccAWSMwaaEnvironmentBase(rName) + fmt.Sprintf(` resource "aws_mwaa_environment" "test" { airflow_configuration_options = { "core.default_task_retries" = 1 From 02c098b5732b46763790de3a12dcf0958b305b71 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sun, 17 Jan 2021 08:09:08 +0900 Subject: [PATCH 12/25] Format files --- aws/internal/service/mwaa/waiter/waiter.go | 2 +- aws/resource_aws_mwaa_environment.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/internal/service/mwaa/waiter/waiter.go b/aws/internal/service/mwaa/waiter/waiter.go index 96c461903b5..5e9ae0896e3 100644 --- a/aws/internal/service/mwaa/waiter/waiter.go +++ b/aws/internal/service/mwaa/waiter/waiter.go @@ -1,9 +1,9 @@ package waiter import ( - "github.com/aws/aws-sdk-go/service/mwaa" "time" + "github.com/aws/aws-sdk-go/service/mwaa" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" ) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index f2be3fda642..fed8e535c24 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -2,7 +2,6 @@ package aws import ( "fmt" - "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/mwaa/finder" "log" "github.com/aws/aws-sdk-go/aws" @@ -10,6 +9,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" "github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/mwaa/finder" "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/mwaa/waiter" ) From dc37143ca6a2121a51c9f0a536e9178108d6631c Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sun, 17 Jan 2021 08:42:35 +0900 Subject: [PATCH 13/25] Fix EnvironmentDeleted waiter target statuses --- aws/internal/service/mwaa/waiter/waiter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/internal/service/mwaa/waiter/waiter.go b/aws/internal/service/mwaa/waiter/waiter.go index 5e9ae0896e3..19b57fa6f81 100644 --- a/aws/internal/service/mwaa/waiter/waiter.go +++ b/aws/internal/service/mwaa/waiter/waiter.go @@ -62,11 +62,11 @@ func EnvironmentUpdated(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) return nil, err } -// EnvironmentDeleted waits for a Environment to return AVAILABLE +// EnvironmentDeleted waits for a Environment to be deleted func EnvironmentDeleted(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) { stateConf := &resource.StateChangeConf{ Pending: []string{mwaa.EnvironmentStatusDeleting}, - Target: []string{environmentStatusNotFound}, + Target: []string{}, Refresh: EnvironmentStatus(conn, name), Timeout: EnvironmentDeletedTimeout, Delay: EnvironmentDeletedDelay, From 456e595650ddd4aacc3236d8fb29726967ac426c Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sun, 17 Jan 2021 09:08:23 +0900 Subject: [PATCH 14/25] Improve mwaa_environment error messages --- aws/resource_aws_mwaa_environment.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index fed8e535c24..7ec3b677051 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -292,7 +292,7 @@ func resourceAwsMwaaEnvironmentCreate(d *schema.ResourceData, meta interface{}) d.SetId(aws.StringValue(input.Name)) if _, err := waiter.EnvironmentCreated(conn, d.Id()); err != nil { - return fmt.Errorf("error creating MWAA Environment (%s): %w", d.Id(), err) + return fmt.Errorf("error waiting for MWAA Environment (%s) creation: %w", d.Id(), err) } return resourceAwsMwaaEnvironmentRead(d, meta) @@ -390,7 +390,7 @@ func resourceAwsMwaaEnvironmentUpdate(d *schema.ResourceData, meta interface{}) } if _, err := waiter.EnvironmentUpdated(conn, d.Id()); err != nil { - return fmt.Errorf("error updating MWAA Environment (%s): %w", d.Id(), err) + return fmt.Errorf("error waiting for MWAA Environment (%s) update: %w", d.Id(), err) } } @@ -422,7 +422,11 @@ func resourceAwsMwaaEnvironmentDelete(d *schema.ResourceData, meta interface{}) _, err = waiter.EnvironmentDeleted(conn, d.Id()) - return err + if err != nil { + return fmt.Errorf("error waiting for MWAA Environment (%s) deletion: %w", d.Id(), err) + } + + return nil } func mwaaEnvironmentModuleLoggingConfigurationSchema(defaultEnabled bool) *schema.Resource { From 8110889cd7588d53778112b19c6c3d993c1ba436 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Mon, 18 Jan 2021 16:57:52 +0900 Subject: [PATCH 15/25] Add back HasChange to check mwaa_environment updates --- aws/resource_aws_mwaa_environment.go | 76 +++++++++++++++++++++------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 7ec3b677051..80d5ae20e4c 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -361,26 +361,66 @@ func resourceAwsMwaaEnvironmentUpdate(d *schema.ResourceData, meta interface{}) } if d.HasChangesExcept("tags") { - options, ok := d.GetOk("airflow_configuration_options") - if !ok { - options = map[string]interface{}{} + if d.HasChange("airflow_configuration_options") { + options, ok := d.GetOk("airflow_configuration_options") + if !ok { + options = map[string]interface{}{} + } + + input.AirflowConfigurationOptions = stringMapToPointers(options.(map[string]interface{})) + } + + if d.HasChange("airflow_version") { + input.AirflowVersion = aws.String(d.Get("airflow_version").(string)) + } + + if d.HasChange("dag_s3_path") { + input.DagS3Path = aws.String(d.Get("dag_s3_path").(string)) + } + + if d.HasChange("environment_class") { + input.EnvironmentClass = aws.String(d.Get("environment_class").(string)) + } + + if d.HasChange("execution_role_arn") { + input.ExecutionRoleArn = aws.String(d.Get("execution_role_arn").(string)) + } + + if d.HasChange("logging_configuration") { + input.LoggingConfiguration = expandMwaaEnvironmentLoggingConfiguration(d.Get("logging_configuration").([]interface{})) + } + + if d.HasChange("max_workers") { + input.MaxWorkers = aws.Int64(int64(d.Get("max_workers").(int))) } - input.AirflowConfigurationOptions = stringMapToPointers(options.(map[string]interface{})) - - input.AirflowVersion = aws.String(d.Get("airflow_version").(string)) - input.DagS3Path = aws.String(d.Get("dag_s3_path").(string)) - input.EnvironmentClass = aws.String(d.Get("environment_class").(string)) - input.ExecutionRoleArn = aws.String(d.Get("execution_role_arn").(string)) - input.LoggingConfiguration = expandMwaaEnvironmentLoggingConfiguration(d.Get("logging_configuration").([]interface{})) - input.MaxWorkers = aws.Int64(int64(d.Get("max_workers").(int))) - input.NetworkConfiguration = expandMwaaEnvironmentNetworkConfigurationUpdate(d.Get("network_configuration").([]interface{})) - input.PluginsS3Path = aws.String(d.Get("plugins_s3_path").(string)) - input.RequirementsS3ObjectVersion = aws.String(d.Get("requirements_s3_object_version").(string)) - input.RequirementsS3Path = aws.String(d.Get("requirements_s3_path").(string)) - input.SourceBucketArn = aws.String(d.Get("source_bucket_arn").(string)) - input.WebserverAccessMode = aws.String(d.Get("webserver_access_mode").(string)) - input.WeeklyMaintenanceWindowStart = aws.String(d.Get("weekly_maintenance_window_start").(string)) + if d.HasChange("network_configuration") { + input.NetworkConfiguration = expandMwaaEnvironmentNetworkConfigurationUpdate(d.Get("network_configuration").([]interface{})) + } + + if d.HasChange("plugins_s3_path") { + input.PluginsS3Path = aws.String(d.Get("plugins_s3_path").(string)) + } + + if d.HasChange("requirements_s3_object_version") { + input.RequirementsS3ObjectVersion = aws.String(d.Get("requirements_s3_object_version").(string)) + } + + if d.HasChange("requirements_s3_path") { + input.RequirementsS3Path = aws.String(d.Get("requirements_s3_path").(string)) + } + + if d.HasChange("source_bucket_arn") { + input.SourceBucketArn = aws.String(d.Get("source_bucket_arn").(string)) + } + + if d.HasChange("webserver_access_mode") { + input.WebserverAccessMode = aws.String(d.Get("webserver_access_mode").(string)) + } + + if d.HasChange("weekly_maintenance_window_start") { + input.WeeklyMaintenanceWindowStart = aws.String(d.Get("weekly_maintenance_window_start").(string)) + } log.Printf("[INFO] Updating MWAA Environment: %s", input) _, err := conn.UpdateEnvironment(&input) From d6074e9ca0ebcfd5229b3bcfb532201dfd800ae9 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 11:47:00 +0900 Subject: [PATCH 16/25] Remove the delay on creation --- aws/internal/service/mwaa/waiter/waiter.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/aws/internal/service/mwaa/waiter/waiter.go b/aws/internal/service/mwaa/waiter/waiter.go index 19b57fa6f81..c156eb3ce67 100644 --- a/aws/internal/service/mwaa/waiter/waiter.go +++ b/aws/internal/service/mwaa/waiter/waiter.go @@ -10,8 +10,6 @@ import ( const ( // Maximum amount of time to wait for an environment creation EnvironmentCreatedTimeout = 90 * time.Minute - // Amount of delay to check an environment status - EnvironmentCreatedDelay = 1 * time.Minute // Maximum amount of time to wait for an environment update EnvironmentUpdatedTimeout = 90 * time.Minute @@ -31,7 +29,6 @@ func EnvironmentCreated(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) Target: []string{mwaa.EnvironmentStatusAvailable}, Refresh: EnvironmentStatus(conn, name), Timeout: EnvironmentCreatedTimeout, - Delay: EnvironmentCreatedDelay, } outputRaw, err := stateConf.WaitForState() From dfbfb5da11da5847dfac2eee216c2c2c421742ef Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 11:48:02 +0900 Subject: [PATCH 17/25] Remove the delay on update and deletion too --- aws/internal/service/mwaa/waiter/waiter.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/aws/internal/service/mwaa/waiter/waiter.go b/aws/internal/service/mwaa/waiter/waiter.go index c156eb3ce67..ef707f416a7 100644 --- a/aws/internal/service/mwaa/waiter/waiter.go +++ b/aws/internal/service/mwaa/waiter/waiter.go @@ -13,13 +13,9 @@ const ( // Maximum amount of time to wait for an environment update EnvironmentUpdatedTimeout = 90 * time.Minute - // Amount of delay to check an environment status - EnvironmentUpdatedDelay = 1 * time.Minute // Maximum amount of time to wait for an environment deletion EnvironmentDeletedTimeout = 90 * time.Minute - // Amount of delay to check an environment status - EnvironmentDeletedDelay = 1 * time.Minute ) // EnvironmentCreated waits for a Environment to return AVAILABLE @@ -47,7 +43,6 @@ func EnvironmentUpdated(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) Target: []string{mwaa.EnvironmentStatusAvailable}, Refresh: EnvironmentStatus(conn, name), Timeout: EnvironmentUpdatedTimeout, - Delay: EnvironmentUpdatedDelay, } outputRaw, err := stateConf.WaitForState() @@ -66,7 +61,6 @@ func EnvironmentDeleted(conn *mwaa.MWAA, name string) (*mwaa.Environment, error) Target: []string{}, Refresh: EnvironmentStatus(conn, name), Timeout: EnvironmentDeletedTimeout, - Delay: EnvironmentDeletedDelay, } outputRaw, err := stateConf.WaitForState() From 24c109266813935a55a0f6819f50c560217114ca Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 11:51:37 +0900 Subject: [PATCH 18/25] Update the markdown format from hcl to terraform --- website/docs/r/mwaa_environment.html.markdown | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/docs/r/mwaa_environment.html.markdown b/website/docs/r/mwaa_environment.html.markdown index 7557b2e72d4..2021fa9ccfe 100644 --- a/website/docs/r/mwaa_environment.html.markdown +++ b/website/docs/r/mwaa_environment.html.markdown @@ -16,7 +16,7 @@ A MWAA Environment requires an IAM role (`aws_iam_role`), two subnets in the pri ### Basic Usage -```hcl +```terraform resource "aws_mwaa_environment" "example" { dag_s3_path = "dags/" execution_role_arn = aws_iam_role.example.arn @@ -33,7 +33,7 @@ resource "aws_mwaa_environment" "example" { ### Example with Airflow configuration options -```hcl +```terraform resource "aws_mwaa_environment" "example" { airflow_configuration_options = { "core.default_task_retries" = 16 @@ -57,7 +57,7 @@ resource "aws_mwaa_environment" "example" { Note that Airflow task logs are enabled by default with the `INFO` log level. -```hcl +```terraform resource "aws_mwaa_environment" "example" { dag_s3_path = "dags/" execution_role_arn = aws_iam_role.example.arn @@ -102,7 +102,7 @@ resource "aws_mwaa_environment" "example" { ### Example with tags -```hcl +```terraform resource "aws_mwaa_environment" "example" { dag_s3_path = "dags/" execution_role_arn = aws_iam_role.example.arn From 68d70a7e4134efe97b05b68f6edde20fc1c064a7 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 11:59:32 +0900 Subject: [PATCH 19/25] Update the documentation on the security_group_ids arguments --- website/docs/r/mwaa_environment.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/r/mwaa_environment.html.markdown b/website/docs/r/mwaa_environment.html.markdown index 2021fa9ccfe..35092624ab1 100644 --- a/website/docs/r/mwaa_environment.html.markdown +++ b/website/docs/r/mwaa_environment.html.markdown @@ -166,7 +166,7 @@ A configuration block to use for logging with respect to the various Apache Airf The `network_configuration` block supports the following arguments. More information about the required subnet and security group settings can be found in the [official AWS documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/vpc-create.html). -* `security_group_ids` - (Required) Security groups IDs for the environment. At least one of the security group needs to allow MWAA resources to talk to each other. Otherwise MWAA can't be provisioned and will fail. See [Base Usage](#base-usage) for an exemplary security group. +* `security_group_ids` - (Required) Security groups IDs for the environment. At least one of the security group needs to allow MWAA resources to talk to each other, otherwise MWAA cannot be provisioned. * `subnet_ids` - (Required) The private subnet IDs in which the environment should be created. MWAA requires two subnets. ## Attributes Reference From 43ab58ab12e0f7d6e4946a78bf62e28f2a44c108 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 12:00:01 +0900 Subject: [PATCH 20/25] Add validation to execution_role_arn --- aws/resource_aws_mwaa_environment.go | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 80d5ae20e4c..35476830fcb 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -55,6 +55,7 @@ func resourceAwsMwaaEnvironment() *schema.Resource { "execution_role_arn": { Type: schema.TypeString, Required: true, + ValidateFunc: validateArn, }, "kms_key": { Type: schema.TypeString, From 84825bc96bfe508f8ecf5dc4e4ef34912565d384 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 12:36:01 +0900 Subject: [PATCH 21/25] Support the min_workers argument --- aws/resource_aws_mwaa_environment.go | 18 ++++++++++++++++-- aws/resource_aws_mwaa_environment_test.go | 10 ++++++++++ website/docs/r/mwaa_environment.html.markdown | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 35476830fcb..06f43c0cb67 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -53,8 +53,8 @@ func resourceAwsMwaaEnvironment() *schema.Resource { Default: "mw1.small", }, "execution_role_arn": { - Type: schema.TypeString, - Required: true, + Type: schema.TypeString, + Required: true, ValidateFunc: validateArn, }, "kms_key": { @@ -145,6 +145,11 @@ func resourceAwsMwaaEnvironment() *schema.Resource { Default: 10, ValidateFunc: validation.IntAtLeast(1), }, + "min_workers": { + Type: schema.TypeInt, + Optional: true, + ValidateFunc: validation.IntAtLeast(1), + }, "name": { Type: schema.TypeString, Required: true, @@ -256,6 +261,10 @@ func resourceAwsMwaaEnvironmentCreate(d *schema.ResourceData, meta interface{}) input.MaxWorkers = aws.Int64(int64(v.(int))) } + if v, ok := d.GetOk("min_workers"); ok { + input.MinWorkers = aws.Int64(int64(v.(int))) + } + if v, ok := d.GetOk("plugins_s3_object_version"); ok { input.PluginsS3ObjectVersion = aws.String(v.(string)) } @@ -332,6 +341,7 @@ func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) er return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } d.Set("max_workers", environment.MaxWorkers) + d.Set("min_workers", environment.MinWorkers) d.Set("name", environment.Name) if err := d.Set("network_configuration", flattenMwaaNetworkConfiguration(environment.NetworkConfiguration)); err != nil { return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) @@ -395,6 +405,10 @@ func resourceAwsMwaaEnvironmentUpdate(d *schema.ResourceData, meta interface{}) input.MaxWorkers = aws.Int64(int64(d.Get("max_workers").(int))) } + if d.HasChange("min_workers") { + input.MinWorkers = aws.Int64(int64(d.Get("min_workers").(int))) + } + if d.HasChange("network_configuration") { input.NetworkConfiguration = expandMwaaEnvironmentNetworkConfigurationUpdate(d.Get("network_configuration").([]interface{})) } diff --git a/aws/resource_aws_mwaa_environment_test.go b/aws/resource_aws_mwaa_environment_test.go index 76c9abf254c..e28c6fa57c0 100644 --- a/aws/resource_aws_mwaa_environment_test.go +++ b/aws/resource_aws_mwaa_environment_test.go @@ -87,6 +87,7 @@ func TestAccAWSMwaaEnvironment_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.enabled", "false"), resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.log_level", "INFO"), resource.TestCheckResourceAttr(resourceName, "max_workers", "10"), + resource.TestCheckResourceAttr(resourceName, "min_workers", "1"), resource.TestCheckResourceAttr(resourceName, "name", rName), resource.TestCheckResourceAttr(resourceName, "network_configuration.#", "1"), resource.TestCheckResourceAttr(resourceName, "network_configuration.0.security_group_ids.#", "1"), @@ -307,6 +308,7 @@ func TestAccAWSMwaaEnvironment_full(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.enabled", "true"), resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.worker_logs.0.log_level", "WARNING"), resource.TestCheckResourceAttr(resourceName, "max_workers", "20"), + resource.TestCheckResourceAttr(resourceName, "min_workers", "15"), resource.TestCheckResourceAttr(resourceName, "name", rName), resource.TestCheckResourceAttr(resourceName, "network_configuration.#", "1"), resource.TestCheckResourceAttr(resourceName, "network_configuration.0.security_group_ids.#", "1"), @@ -510,6 +512,13 @@ resource "aws_s3_bucket" "test" { } } +resource "aws_s3_bucket_public_access_block" "test" { + bucket = aws_s3_bucket.test.bucket + + block_public_acls = true + block_public_policy = true +} + resource "aws_s3_bucket_object" "dags" { bucket = aws_s3_bucket.test.id acl = "private" @@ -694,6 +703,7 @@ resource "aws_mwaa_environment" "test" { } max_workers = 20 + min_workers = 15 name = %[1]q network_configuration { diff --git a/website/docs/r/mwaa_environment.html.markdown b/website/docs/r/mwaa_environment.html.markdown index 35092624ab1..2154748fc17 100644 --- a/website/docs/r/mwaa_environment.html.markdown +++ b/website/docs/r/mwaa_environment.html.markdown @@ -134,6 +134,7 @@ The following arguments are supported: * `kms_key` - (Optional) The Amazon Resource Name (ARN) of your KMS key that you want to use for encryption. Will be set to the ARN of the managed KMS key `aws/airflow` by default. Please check the [Official Documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/custom-keys-certs.html) for more information. * `logging_configuration` - (Optional) The Apache Airflow logs you want to send to Amazon CloudWatch Logs. * `max_workers` - (Optional) The maximum number of workers that can be automatically scaled up. Value need to be between `1` and `25`. Will be `10` by default. +* `min_workers` - (Optional) The minimum number of workers that you want to run in your environment. Will be `1` by default. * `name` - (Required) The name of the Apache Airflow Environment * `network_configuration` - (Required) Specifies the network configuration for your Apache Airflow Environment. This includes two private subnets as well as security groups for the Airflow environment. Each subnet requires internet connection, otherwise the deployment will fail. See [Network configuration](#network) below for details. * `plugins_s3_object_version` - (Optional) The plugins.zip file version you want to use. From 4c2cb357cb77133b2d15bfdfab7d1a31b148aab6 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 12:38:25 +0900 Subject: [PATCH 22/25] Add nil check on the environment --- aws/resource_aws_mwaa_environment.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 06f43c0cb67..5738e0dc4f1 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -326,6 +326,10 @@ func resourceAwsMwaaEnvironmentRead(d *schema.ResourceData, meta interface{}) er return fmt.Errorf("error reading MWAA Environment (%s): %w", d.Id(), err) } + if environment == nil { + return fmt.Errorf("error reading MWAA Environment (%s): empty response", d.Id()) + } + d.Set("airflow_configuration_options", aws.StringValueMap(environment.AirflowConfigurationOptions)) d.Set("airflow_version", environment.AirflowVersion) d.Set("arn", environment.Arn) From 7902e1e18ef63a168a504009ed8ccaad92283fb9 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 12:41:47 +0900 Subject: [PATCH 23/25] Add ErrorChecks to the tests --- aws/resource_aws_mwaa_environment_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aws/resource_aws_mwaa_environment_test.go b/aws/resource_aws_mwaa_environment_test.go index e28c6fa57c0..9ac1505cc1a 100644 --- a/aws/resource_aws_mwaa_environment_test.go +++ b/aws/resource_aws_mwaa_environment_test.go @@ -56,6 +56,7 @@ func TestAccAWSMwaaEnvironment_basic(t *testing.T) { resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, + ErrorCheck: testAccErrorCheck(t, mwaa.EndpointsID), Providers: testAccProviders, CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ @@ -117,6 +118,7 @@ func TestAccAWSMwaaEnvironment_disappears(t *testing.T) { resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, + ErrorCheck: testAccErrorCheck(t, mwaa.EndpointsID), Providers: testAccProviders, CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ @@ -140,6 +142,7 @@ func TestAccAWSMwaaEnvironment_AirflowConfigurationOptions(t *testing.T) { resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, + ErrorCheck: testAccErrorCheck(t, mwaa.EndpointsID), Providers: testAccProviders, CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ @@ -185,6 +188,7 @@ func TestAccAWSMwaaEnvironment_LogConfiguration(t *testing.T) { resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, + ErrorCheck: testAccErrorCheck(t, mwaa.EndpointsID), Providers: testAccProviders, CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ @@ -269,6 +273,7 @@ func TestAccAWSMwaaEnvironment_full(t *testing.T) { resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, + ErrorCheck: testAccErrorCheck(t, mwaa.EndpointsID), Providers: testAccProviders, CheckDestroy: testAccCheckAWSMwaaEnvironmentDestroy, Steps: []resource.TestStep{ From 07d84e9c29d7c3e98ac017ccfc25c2327901e269 Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 12:49:19 +0900 Subject: [PATCH 24/25] Miscellaneous changes on the tests --- aws/resource_aws_mwaa_environment_test.go | 24 ++++++----------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/aws/resource_aws_mwaa_environment_test.go b/aws/resource_aws_mwaa_environment_test.go index 9ac1505cc1a..4cb56d0b9f7 100644 --- a/aws/resource_aws_mwaa_environment_test.go +++ b/aws/resource_aws_mwaa_environment_test.go @@ -381,7 +381,7 @@ func testAccCheckAWSMwaaEnvironmentDestroy(s *terraform.State) error { if err != nil { if isAWSErr(err, mwaa.ErrCodeResourceNotFoundException, "") { - return nil + continue } return err } @@ -403,6 +403,8 @@ data "aws_availability_zones" "available" { } } +data "aws_partition" "current" {} + resource "aws_vpc" "test" { cidr_block = "10.0.0.0/16" enable_dns_hostnames = true @@ -529,8 +531,6 @@ resource "aws_s3_bucket_object" "dags" { acl = "private" key = "dags/" content_type = "application/x-directory" - - depends_on = [aws_s3_bucket.test] } resource "aws_iam_role" "test" { @@ -545,8 +545,8 @@ resource "aws_iam_role" "test" { "Effect": "Allow", "Principal": { "Service": [ - "airflow.amazonaws.com", - "airflow-env.amazonaws.com" + "airflow.${data.aws_partition.current.dns_suffix}", + "airflow-env.${data.aws_partition.current.dns_suffix}" ] }, "Action": "sts:AssumeRole" @@ -589,8 +589,6 @@ resource "aws_mwaa_environment" "test" { } source_bucket_arn = aws_s3_bucket.test.arn - - depends_on = [aws_s3_bucket_object.dags] } `, rName) } @@ -613,8 +611,6 @@ resource "aws_mwaa_environment" "test" { } source_bucket_arn = aws_s3_bucket.test.arn - - depends_on = [aws_s3_bucket_object.dags] } `, rName, retries, parallelism) } @@ -660,8 +656,6 @@ resource "aws_mwaa_environment" "test" { } source_bucket_arn = aws_s3_bucket.test.arn - - depends_on = [aws_s3_bucket_object.dags] } `, rName, logEnabled, logLevel) } @@ -728,8 +722,6 @@ resource "aws_mwaa_environment" "test" { Name = %[1]q Environment = "production" } - - depends_on = [aws_s3_bucket_object.dags, aws_s3_bucket_object.plugins, aws_s3_bucket_object.requirements] } data "aws_region" "current" {} @@ -753,7 +745,7 @@ resource "aws_kms_key" "test" { { "Effect": "Allow", "Principal": { - "Service": "logs.${data.aws_region.current.name}.amazonaws.com" + "Service": "logs.${data.aws_region.current.name}.${data.aws_partition.current.dns_suffix}" }, "Action": "kms:*", "Resource": "*" @@ -768,8 +760,6 @@ resource "aws_s3_bucket_object" "plugins" { acl = "private" key = "plugins.zip" content = "" - - depends_on = [aws_s3_bucket.test] } resource "aws_s3_bucket_object" "requirements" { @@ -777,8 +767,6 @@ resource "aws_s3_bucket_object" "requirements" { acl = "private" key = "requirements.txt" content = "" - - depends_on = [aws_s3_bucket.test] } `, rName) From fde9ee09dd13e8349a099d3df1414f7943ff6c8b Mon Sep 17 00:00:00 2001 From: shuheiktgw Date: Sat, 3 Apr 2021 16:12:08 +0900 Subject: [PATCH 25/25] Make some arguments computed --- aws/resource_aws_mwaa_environment.go | 25 +++++++++++++---------- aws/resource_aws_mwaa_environment_test.go | 4 ---- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/aws/resource_aws_mwaa_environment.go b/aws/resource_aws_mwaa_environment.go index 5738e0dc4f1..f10ec81473e 100644 --- a/aws/resource_aws_mwaa_environment.go +++ b/aws/resource_aws_mwaa_environment.go @@ -50,7 +50,7 @@ func resourceAwsMwaaEnvironment() *schema.Resource { "environment_class": { Type: schema.TypeString, Optional: true, - Default: "mw1.small", + Computed: true, }, "execution_role_arn": { Type: schema.TypeString, @@ -106,35 +106,35 @@ func resourceAwsMwaaEnvironment() *schema.Resource { Optional: true, Computed: true, MaxItems: 1, - Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(), }, "scheduler_logs": { Type: schema.TypeList, Optional: true, Computed: true, MaxItems: 1, - Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(), }, "task_logs": { Type: schema.TypeList, Optional: true, Computed: true, MaxItems: 1, - Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(true), + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(), }, "webserver_logs": { Type: schema.TypeList, Optional: true, Computed: true, MaxItems: 1, - Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(), }, "worker_logs": { Type: schema.TypeList, Optional: true, Computed: true, MaxItems: 1, - Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(false), + Elem: mwaaEnvironmentModuleLoggingConfigurationSchema(), }, }, }, @@ -142,12 +142,13 @@ func resourceAwsMwaaEnvironment() *schema.Resource { "max_workers": { Type: schema.TypeInt, Optional: true, - Default: 10, + Computed: true, ValidateFunc: validation.IntAtLeast(1), }, "min_workers": { Type: schema.TypeInt, Optional: true, + Computed: true, ValidateFunc: validation.IntAtLeast(1), }, "name": { @@ -180,6 +181,7 @@ func resourceAwsMwaaEnvironment() *schema.Resource { "plugins_s3_object_version": { Type: schema.TypeString, Optional: true, + Computed: true, }, "plugins_s3_path": { Type: schema.TypeString, @@ -188,6 +190,7 @@ func resourceAwsMwaaEnvironment() *schema.Resource { "requirements_s3_object_version": { Type: schema.TypeString, Optional: true, + Computed: true, }, "requirements_s3_path": { Type: schema.TypeString, @@ -210,7 +213,7 @@ func resourceAwsMwaaEnvironment() *schema.Resource { "webserver_access_mode": { Type: schema.TypeString, Optional: true, - Default: mwaa.WebserverAccessModePrivateOnly, + Computed: true, ValidateFunc: validation.StringInSlice(mwaa.WebserverAccessMode_Values(), false), }, "webserver_url": { @@ -488,7 +491,7 @@ func resourceAwsMwaaEnvironmentDelete(d *schema.ResourceData, meta interface{}) return nil } -func mwaaEnvironmentModuleLoggingConfigurationSchema(defaultEnabled bool) *schema.Resource { +func mwaaEnvironmentModuleLoggingConfigurationSchema() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ "cloud_watch_log_group_arn": { @@ -498,12 +501,12 @@ func mwaaEnvironmentModuleLoggingConfigurationSchema(defaultEnabled bool) *schem "enabled": { Type: schema.TypeBool, Optional: true, - Default: defaultEnabled, + Computed: true, }, "log_level": { Type: schema.TypeString, Optional: true, - Default: mwaa.LoggingLevelInfo, + Computed: true, ValidateFunc: validation.StringInSlice(mwaa.LoggingLevel_Values(), false), }, }, diff --git a/aws/resource_aws_mwaa_environment_test.go b/aws/resource_aws_mwaa_environment_test.go index 4cb56d0b9f7..2e58405b671 100644 --- a/aws/resource_aws_mwaa_environment_test.go +++ b/aws/resource_aws_mwaa_environment_test.go @@ -318,9 +318,7 @@ func TestAccAWSMwaaEnvironment_full(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "network_configuration.#", "1"), resource.TestCheckResourceAttr(resourceName, "network_configuration.0.security_group_ids.#", "1"), resource.TestCheckResourceAttr(resourceName, "network_configuration.0.subnet_ids.#", "2"), - resource.TestCheckResourceAttr(resourceName, "plugins_s3_object_version", ""), resource.TestCheckResourceAttr(resourceName, "plugins_s3_path", "plugins.zip"), - resource.TestCheckResourceAttr(resourceName, "requirements_s3_object_version", ""), resource.TestCheckResourceAttr(resourceName, "requirements_s3_path", "requirements.txt"), resource.TestCheckResourceAttrSet(resourceName, "service_role_arn"), testAccCheckResourceAttrGlobalARNNoAccount(resourceName, "source_bucket_arn", "s3", rName), @@ -710,9 +708,7 @@ resource "aws_mwaa_environment" "test" { subnet_ids = aws_subnet.private[*].id } - plugins_s3_object_version = "" plugins_s3_path = aws_s3_bucket_object.plugins.key - requirements_s3_object_version = "" requirements_s3_path = aws_s3_bucket_object.requirements.key source_bucket_arn = aws_s3_bucket.test.arn webserver_access_mode = "PUBLIC_ONLY"