From e6af72e5f9fc4c8d80e308e9bce4ff863dece5dd Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 20:58:45 +0200 Subject: [PATCH 01/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 08936cba639..5c0cb416de3 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -113,6 +113,12 @@ func resourceAwsEMRCluster() *schema.Resource { Optional: true, ForceNew: true, }, + "subnet_ids": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, "additional_master_security_groups": { Type: schema.TypeString, Optional: true, @@ -782,6 +788,9 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error if v, ok := attributes["subnet_id"]; ok { instanceConfig.Ec2SubnetId = aws.String(v.(string)) } + if v, ok := attributes["subnet_ids"]; ok { + instanceConfig.Ec2SubnetIds = aws.String(v.(string)) + } if v, ok := attributes["additional_master_security_groups"]; ok { strSlice := strings.Split(v.(string), ",") @@ -1444,6 +1453,9 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } + if ia.Ec2SubnetIds != nil { + attrs["subnet_ids"] = *ia.Ec2SubnetIds + } if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile } From 1841fb6792ed1fc6909c73018a6230543cdb8bbc Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 21:15:32 +0200 Subject: [PATCH 02/17] Update documentation --- website/docs/r/emr_cluster.html.markdown | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/r/emr_cluster.html.markdown b/website/docs/r/emr_cluster.html.markdown index 9f33d1193d9..4bda364b5ae 100644 --- a/website/docs/r/emr_cluster.html.markdown +++ b/website/docs/r/emr_cluster.html.markdown @@ -386,6 +386,7 @@ Attributes for the Amazon EC2 instances running the job flow * `key_name` - (Optional) Amazon EC2 key pair that can be used to ssh to the master node as the user called `hadoop` * `subnet_id` - (Optional) VPC subnet id where you want the job flow to launch. Cannot specify the `cc1.4xlarge` instance type for nodes of a job flow launched in a Amazon VPC +* `subnet_ids` - (Optional) List of VPC subnet id-s where you want the job flow to launch. Amazon EMR identifies the best Availability Zone to launch instances according to your fleet specifications * `additional_master_security_groups` - (Optional) String containing a comma separated list of additional Amazon EC2 security group IDs for the master node * `additional_slave_security_groups` - (Optional) String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string * `emr_managed_master_security_group` - (Optional) Identifier of the Amazon EC2 EMR-Managed security group for the master node From c60332ba629351230c058f0dfaacee94bfaa9bce Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 21:39:07 +0200 Subject: [PATCH 03/17] Add testAccAWSEmrClusterConfigMultipleSubnets --- aws/resource_aws_emr_cluster_test.go | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index 44392bbf04a..98a12190955 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -1907,6 +1907,63 @@ resource "aws_emr_cluster" "tf-test-cluster" { ) } +func testAccAWSEmrClusterConfigMultipleSubnets(r string) string { + return testAccAWSEmrComposeConfig(false, + testAccAWSEmrClusterConfigCurrentPartition(), + testAccAWSEmrClusterConfigIAMServiceRoleBase(r), + testAccAWSEmrClusterConfigIAMInstanceProfileBase(r), + testAccAWSEmrClusterConfigIAMAutoscalingRole(r), + fmt.Sprintf(` +resource "aws_emr_cluster" "tf-test-cluster" { + name = "%[1]s" + release_label = "emr-4.6.0" + applications = ["Spark"] + + ec2_attributes { + subnet_ids = [aws_subnet.test.id] + emr_managed_master_security_group = aws_security_group.test.id + emr_managed_slave_security_group = aws_security_group.test.id + instance_profile = aws_iam_instance_profile.emr_instance_profile.arn + } + + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } + + tags = { + role = "rolename" + dns_zone = "env_zone" + env = "env" + name = "name-env" + } + + keep_job_flow_alive_when_no_steps = true + termination_protection = false + + scale_down_behavior = "TERMINATE_AT_TASK_COMPLETION" + + configurations = "test-fixtures/emr_configurations.json" + + depends_on = [ + aws_route_table_association.test, + aws_iam_role_policy_attachment.emr_service, + aws_iam_role_policy_attachment.emr_instance_profile, + aws_iam_role_policy_attachment.emr_autoscaling_role, + ] + + service_role = aws_iam_role.emr_service.arn + autoscaling_role = aws_iam_role.emr_autoscaling_role.arn + ebs_root_volume_size = 21 +} +`, r), + ) +} + func testAccAWSEmrClusterConfigAdditionalInfo(r string) string { return testAccAWSEmrComposeConfig(false, testAccAWSEmrClusterConfigCurrentPartition(), From 9ce2cca0cae189029e3e9fcfdaa09156e63e5e28 Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 22:04:30 +0200 Subject: [PATCH 04/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 5c0cb416de3..300b1b8fde7 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -789,7 +789,11 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.Ec2SubnetId = aws.String(v.(string)) } if v, ok := attributes["subnet_ids"]; ok { - instanceConfig.Ec2SubnetIds = aws.String(v.(string)) + strSlice := strings.Split(v.(string), ",") + for i, s := range strSlice { + strSlice[i] = strings.TrimSpace(s) + } + instanceConfig.Ec2SubnetIds = aws.StringSlice(v.(string)) } if v, ok := attributes["additional_master_security_groups"]; ok { @@ -1453,8 +1457,9 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } - if ia.Ec2SubnetIds != nil { - attrs["subnet_ids"] = *ia.Ec2SubnetIds + if len(ia.Ec2SubnetIds) > 0 { + strs := aws.StringValueSlice(ia.Ec2SubnetIds) + attrs["subnet_ids"] = strings.Join(strs, ",") } if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile From c3f01a5eb3280cdb7dbe54c8b53f67f6d159f3bc Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 22:09:53 +0200 Subject: [PATCH 05/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 300b1b8fde7..abaa72f6ae2 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -793,7 +793,7 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error for i, s := range strSlice { strSlice[i] = strings.TrimSpace(s) } - instanceConfig.Ec2SubnetIds = aws.StringSlice(v.(string)) + instanceConfig.Ec2SubnetIds = aws.StringSlice(strSlice) } if v, ok := attributes["additional_master_security_groups"]; ok { From b2b3652df3b46fab3bc221a13aa3e97a1c2fd289 Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 22:19:15 +0200 Subject: [PATCH 06/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index abaa72f6ae2..7bbf5bfbc8a 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -114,10 +114,9 @@ func resourceAwsEMRCluster() *schema.Resource { ForceNew: true, }, "subnet_ids": { - Type: schema.TypeList, + Type: schema.TypeString, Optional: true, ForceNew: true, - Elem: &schema.Schema{Type: schema.TypeString}, }, "additional_master_security_groups": { Type: schema.TypeString, From fb5bfd0eb7c128f9808a6629e7619c4e38f8ec1e Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 22:35:44 +0200 Subject: [PATCH 07/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 7bbf5bfbc8a..f45ca5d8b23 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -114,9 +114,11 @@ func resourceAwsEMRCluster() *schema.Resource { ForceNew: true, }, "subnet_ids": { - Type: schema.TypeString, + Type: schema.TypeSet, Optional: true, ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, }, "additional_master_security_groups": { Type: schema.TypeString, @@ -788,11 +790,7 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.Ec2SubnetId = aws.String(v.(string)) } if v, ok := attributes["subnet_ids"]; ok { - strSlice := strings.Split(v.(string), ",") - for i, s := range strSlice { - strSlice[i] = strings.TrimSpace(s) - } - instanceConfig.Ec2SubnetIds = aws.StringSlice(strSlice) + instanceConfig.Ec2SubnetIds = expandStringList(v.(*schema.Set).List()) } if v, ok := attributes["additional_master_security_groups"]; ok { @@ -1457,8 +1455,7 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ attrs["subnet_id"] = *ia.Ec2SubnetId } if len(ia.Ec2SubnetIds) > 0 { - strs := aws.StringValueSlice(ia.Ec2SubnetIds) - attrs["subnet_ids"] = strings.Join(strs, ",") + attrs["subnet_id"] = *ia.Ec2SubnetIds } if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile From 5bd6664e8843de76ea5cd9167b95d6188d06ac3b Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 22:45:23 +0200 Subject: [PATCH 08/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index f45ca5d8b23..bfbd7543afb 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -790,7 +790,7 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.Ec2SubnetId = aws.String(v.(string)) } if v, ok := attributes["subnet_ids"]; ok { - instanceConfig.Ec2SubnetIds = expandStringList(v.(*schema.Set).List()) + instanceConfig.Ec2SubnetIds = expandStringSet(v.(*schema.Set) } if v, ok := attributes["additional_master_security_groups"]; ok { From 7f8b7e35543047dc01633920e56e51fa663bf1cf Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 22:48:58 +0200 Subject: [PATCH 09/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index bfbd7543afb..5424d2e6008 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -790,7 +790,7 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.Ec2SubnetId = aws.String(v.(string)) } if v, ok := attributes["subnet_ids"]; ok { - instanceConfig.Ec2SubnetIds = expandStringSet(v.(*schema.Set) + instanceConfig.Ec2SubnetIds = expandStringSet(v.(*schema.Set)) } if v, ok := attributes["additional_master_security_groups"]; ok { From 9628e89af87c74595f05214947bf4ca86ee64e95 Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 23:03:28 +0200 Subject: [PATCH 10/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 5424d2e6008..79f655a80e4 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -1454,8 +1454,8 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } - if len(ia.Ec2SubnetIds) > 0 { - attrs["subnet_id"] = *ia.Ec2SubnetIds + if len(ia.Ec2SubnetIds) != nil { + attrs["subnet_ids"] = *ia.Ec2SubnetIds } if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile From 91a71f66cd447029abec9c38a55b572e81dab5d9 Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 23:10:55 +0200 Subject: [PATCH 11/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 79f655a80e4..ab4b7d2919d 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -1454,7 +1454,7 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } - if len(ia.Ec2SubnetIds) != nil { + if ia.Ec2SubnetIds != nil { attrs["subnet_ids"] = *ia.Ec2SubnetIds } if ia.IamInstanceProfile != nil { From 7b14e151a00dec7b0e13ec3c9f3e3cbe5beaa135 Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Thu, 21 Jan 2021 23:58:29 +0200 Subject: [PATCH 12/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index ab4b7d2919d..8066a4153c6 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -1454,9 +1454,6 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } - if ia.Ec2SubnetIds != nil { - attrs["subnet_ids"] = *ia.Ec2SubnetIds - } if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile } From f38482864c7372371bacd02712db0c96f90c7fcc Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Fri, 22 Jan 2021 00:21:19 +0200 Subject: [PATCH 13/17] Update resource_aws_emr_cluster_test.go --- aws/resource_aws_emr_cluster_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index 98a12190955..8603cb824ce 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -98,6 +98,17 @@ func TestAccAWSEMRCluster_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "bootstrap_action.#", "0"), ), }, + { + Config: testAccAWSEmrClusterConfigMultipleSubnets(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster), + resource.TestCheckResourceAttr(resourceName, "scale_down_behavior", "TERMINATE_AT_TASK_COMPLETION"), + resource.TestCheckResourceAttr(resourceName, "step.#", "0"), + resource.TestCheckResourceAttrSet(resourceName, "arn"), + resource.TestCheckNoResourceAttr(resourceName, "additional_info"), + resource.TestCheckResourceAttr(resourceName, "bootstrap_action.#", "0"), + ), + }, { ResourceName: resourceName, ImportState: true, From 6b96ad5da595e62a27221532d125f3d2a45fd739 Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Fri, 22 Jan 2021 14:15:49 +0200 Subject: [PATCH 14/17] Add ConflictsWith to subnet_id and subnet_ids --- aws/resource_aws_emr_cluster.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 8066a4153c6..2ae8635ab4a 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -109,16 +109,18 @@ func resourceAwsEMRCluster() *schema.Resource { ForceNew: true, }, "subnet_id": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"subnet_ids"}, }, "subnet_ids": { - Type: schema.TypeSet, - Optional: true, - ForceNew: true, - Elem: &schema.Schema{Type: schema.TypeString}, - Set: schema.HashString, + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + ConflictsWith: []string{"subnet_id"}, }, "additional_master_security_groups": { Type: schema.TypeString, From f14a5b15c10f119e632f1541ed2590c9273f398c Mon Sep 17 00:00:00 2001 From: kristerjaanhold Date: Fri, 22 Jan 2021 15:12:17 +0200 Subject: [PATCH 15/17] Update resource_aws_emr_cluster.go --- aws/resource_aws_emr_cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 2ae8635ab4a..ba7f9fd169a 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -112,7 +112,7 @@ func resourceAwsEMRCluster() *schema.Resource { Type: schema.TypeString, Optional: true, ForceNew: true, - ConflictsWith: []string{"subnet_ids"}, + ConflictsWith: []string{"ec2_attributes.0.subnet_ids"}, }, "subnet_ids": { Type: schema.TypeSet, @@ -120,7 +120,7 @@ func resourceAwsEMRCluster() *schema.Resource { ForceNew: true, Elem: &schema.Schema{Type: schema.TypeString}, Set: schema.HashString, - ConflictsWith: []string{"subnet_id"}, + ConflictsWith: []string{"ec2_attributes.0.subnet_id"}, }, "additional_master_security_groups": { Type: schema.TypeString, From a81c258276d42547f999be546ac12d311d5f8037 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Thu, 15 Apr 2021 16:08:23 -0700 Subject: [PATCH 16/17] Sets `subnet_id` and `subnet_ids` to Computed and moves multiple subnet test to instance fleet tests --- aws/resource_aws_emr_cluster.go | 5 + aws/resource_aws_emr_cluster_test.go | 236 ++++++++++++++++++--------- 2 files changed, 166 insertions(+), 75 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index a110dbe3506..9c0cfe7a61b 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -112,12 +112,14 @@ func resourceAwsEMRCluster() *schema.Resource { "subnet_id": { Type: schema.TypeString, Optional: true, + Computed: true, ForceNew: true, ConflictsWith: []string{"ec2_attributes.0.subnet_ids"}, }, "subnet_ids": { Type: schema.TypeSet, Optional: true, + Computed: true, ForceNew: true, Elem: &schema.Schema{Type: schema.TypeString}, Set: schema.HashString, @@ -1457,6 +1459,9 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } + if ia.RequestedEc2SubnetIds != nil && len(ia.RequestedEc2SubnetIds) > 0 { + attrs["subnet_ids"] = flattenStringSet(ia.RequestedEc2SubnetIds) + } if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile } diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index 0fdbe4c1451..caec9a25f56 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -99,17 +99,6 @@ func TestAccAWSEMRCluster_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "bootstrap_action.#", "0"), ), }, - { - Config: testAccAWSEmrClusterConfigMultipleSubnets(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - resource.TestCheckResourceAttr(resourceName, "scale_down_behavior", "TERMINATE_AT_TASK_COMPLETION"), - resource.TestCheckResourceAttr(resourceName, "step.#", "0"), - resource.TestCheckResourceAttrSet(resourceName, "arn"), - resource.TestCheckNoResourceAttr(resourceName, "additional_info"), - resource.TestCheckResourceAttr(resourceName, "bootstrap_action.#", "0"), - ), - }, { ResourceName: resourceName, ImportState: true, @@ -1398,11 +1387,14 @@ func TestAccAWSEMRCluster_custom_ami_id(t *testing.T) { }) } -func TestAccAWSEMRCluster_instance_fleet(t *testing.T) { - var cluster emr.Cluster +func TestAccAWSEMRCluster_InstanceFleet_basic(t *testing.T) { + var cluster1, cluster2 emr.Cluster resourceName := "aws_emr_cluster.tf-test-cluster" + subnetResourceName := "aws_subnet.test" + subnet2ResourceName := "aws_subnet.test2" rName := acctest.RandomWithPrefix("tf-acc-test") + resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, ErrorCheck: testAccErrorCheck(t, emr.EndpointsID), @@ -1410,18 +1402,59 @@ func TestAccAWSEMRCluster_instance_fleet(t *testing.T) { CheckDestroy: testAccCheckAWSEmrDestroy, Steps: []resource.TestStep{ { - Config: testAccAWSEmrClusterConfigInstanceFleets(rName), + Config: testAccAWSEmrClusterConfig_InstanceFleets(rName), Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "master_instance_fleet.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_fleet.#", "1"), + resource.TestCheckResourceAttr(resourceName, "ec2_attributes.#", "1"), + resource.TestCheckResourceAttrPair(resourceName, "ec2_attributes.0.subnet_id", subnetResourceName, "id"), + resource.TestCheckResourceAttr(resourceName, "ec2_attributes.0.subnet_ids.#", "1"), + resource.TestCheckTypeSetElemAttrPair(resourceName, "ec2_attributes.0.subnet_ids.*", subnetResourceName, "id"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "0"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "0"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfig_InstanceFleet_MultipleSubnets(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterRecreated(&cluster1, &cluster2), resource.TestCheckResourceAttr(resourceName, "master_instance_fleet.#", "1"), resource.TestCheckResourceAttr(resourceName, "core_instance_fleet.#", "1"), + resource.TestCheckResourceAttr(resourceName, "ec2_attributes.#", "1"), + resource.TestCheckResourceAttr(resourceName, "ec2_attributes.0.subnet_ids.#", "2"), + resource.TestCheckTypeSetElemAttrPair(resourceName, "ec2_attributes.0.subnet_ids.*", subnetResourceName, "id"), + resource.TestCheckTypeSetElemAttrPair(resourceName, "ec2_attributes.0.subnet_ids.*", subnet2ResourceName, "id"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "0"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "0"), ), }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, }, }) } -func TestAccAWSEMRCluster_instance_fleet_master_only(t *testing.T) { +func TestAccAWSEMRCluster_InstanceFleet_master_only(t *testing.T) { var cluster emr.Cluster resourceName := "aws_emr_cluster.tf-test-cluster" @@ -1437,8 +1470,19 @@ func TestAccAWSEMRCluster_instance_fleet_master_only(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckAWSEmrClusterExists(resourceName, &cluster), resource.TestCheckResourceAttr(resourceName, "master_instance_fleet.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_fleet.#", "0"), ), }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, }, }) } @@ -1949,63 +1993,6 @@ resource "aws_emr_cluster" "tf-test-cluster" { ) } -func testAccAWSEmrClusterConfigMultipleSubnets(r string) string { - return testAccAWSEmrComposeConfig(false, - testAccAWSEmrClusterConfigCurrentPartition(), - testAccAWSEmrClusterConfigIAMServiceRoleBase(r), - testAccAWSEmrClusterConfigIAMInstanceProfileBase(r), - testAccAWSEmrClusterConfigIAMAutoscalingRole(r), - fmt.Sprintf(` -resource "aws_emr_cluster" "tf-test-cluster" { - name = "%[1]s" - release_label = "emr-4.6.0" - applications = ["Spark"] - - ec2_attributes { - subnet_ids = [aws_subnet.test.id] - emr_managed_master_security_group = aws_security_group.test.id - emr_managed_slave_security_group = aws_security_group.test.id - instance_profile = aws_iam_instance_profile.emr_instance_profile.arn - } - - master_instance_group { - instance_type = "c4.large" - } - - core_instance_group { - instance_count = 1 - instance_type = "c4.large" - } - - tags = { - role = "rolename" - dns_zone = "env_zone" - env = "env" - name = "name-env" - } - - keep_job_flow_alive_when_no_steps = true - termination_protection = false - - scale_down_behavior = "TERMINATE_AT_TASK_COMPLETION" - - configurations = "test-fixtures/emr_configurations.json" - - depends_on = [ - aws_route_table_association.test, - aws_iam_role_policy_attachment.emr_service, - aws_iam_role_policy_attachment.emr_instance_profile, - aws_iam_role_policy_attachment.emr_autoscaling_role, - ] - - service_role = aws_iam_role.emr_service.arn - autoscaling_role = aws_iam_role.emr_autoscaling_role.arn - ebs_root_volume_size = 21 -} -`, r), - ) -} - func testAccAWSEmrClusterConfigAdditionalInfo(r string) string { return testAccAWSEmrComposeConfig(false, testAccAWSEmrClusterConfigCurrentPartition(), @@ -3253,7 +3240,7 @@ resource "aws_security_group" "test" { resource "aws_subnet" "test" { availability_zone = data.aws_availability_zones.available.names[0] - cidr_block = "10.0.0.0/24" + cidr_block = cidrsubnet(aws_vpc.test.cidr_block, 8, 0) map_public_ip_on_launch = %[1]t vpc_id = aws_vpc.test.id @@ -3571,7 +3558,7 @@ resource "aws_emr_cluster" "tf-test-cluster" { ) } -func testAccAWSEmrClusterConfigInstanceFleets(r string) string { +func testAccAWSEmrClusterConfig_InstanceFleets(r string) string { return testAccAWSEmrComposeConfig(false, testAccAWSEmrClusterConfigCurrentPartition(), testAccAWSEmrClusterConfigIAMServiceRoleBase(r), @@ -3658,6 +3645,105 @@ resource "aws_emr_cluster" "tf-test-cluster" { ) } +func testAccAWSEmrClusterConfig_InstanceFleet_MultipleSubnets(r string) string { + return testAccAWSEmrComposeConfig(false, + testAccAWSEmrClusterConfigCurrentPartition(), + testAccAWSEmrClusterConfigIAMServiceRoleBase(r), + testAccAWSEmrClusterConfigIAMInstanceProfileBase(r), + testAccAWSEmrClusterConfigBootstrapActionBucket(r), + fmt.Sprintf(` +resource "aws_emr_cluster" "tf-test-cluster" { + name = "%[1]s" + release_label = "emr-5.30.1" + applications = ["Hadoop", "Hive"] + log_uri = "s3n://terraform/testlog/" + + master_instance_fleet { + instance_type_configs { + instance_type = "m3.xlarge" + } + + target_on_demand_capacity = 1 + } + core_instance_fleet { + instance_type_configs { + bid_price_as_percentage_of_on_demand_price = 80 + ebs_config { + size = 100 + type = "gp2" + volumes_per_instance = 1 + } + instance_type = "m3.xlarge" + weighted_capacity = 1 + } + instance_type_configs { + bid_price_as_percentage_of_on_demand_price = 100 + ebs_config { + size = 100 + type = "gp2" + volumes_per_instance = 1 + } + instance_type = "m4.xlarge" + weighted_capacity = 1 + } + instance_type_configs { + bid_price_as_percentage_of_on_demand_price = 100 + ebs_config { + size = 100 + type = "gp2" + volumes_per_instance = 1 + } + instance_type = "m4.2xlarge" + weighted_capacity = 2 + } + launch_specifications { + spot_specification { + allocation_strategy = "capacity-optimized" + block_duration_minutes = 0 + timeout_action = "SWITCH_TO_ON_DEMAND" + timeout_duration_minutes = 10 + } + } + name = "core fleet" + target_on_demand_capacity = 0 + target_spot_capacity = 2 + } + service_role = aws_iam_role.emr_service.arn + depends_on = [ + aws_route_table_association.test, + aws_iam_role_policy_attachment.emr_service, + aws_iam_role_policy_attachment.emr_instance_profile, + ] + + ec2_attributes { + subnet_ids = [aws_subnet.test.id,aws_subnet.test2.id] + emr_managed_master_security_group = aws_security_group.test.id + emr_managed_slave_security_group = aws_security_group.test.id + instance_profile = aws_iam_instance_profile.emr_instance_profile.arn + } + + bootstrap_action { + path = "s3://elasticmapreduce/bootstrap-actions/run-if" + name = "runif" + args = ["instance.isMaster=true", "echo running on master node"] + } +} + +resource "aws_subnet" "test2" { + availability_zone = data.aws_availability_zones.available.names[1] + cidr_block = cidrsubnet(aws_vpc.test.cidr_block, 8, 1) + map_public_ip_on_launch = false + vpc_id = aws_vpc.test.id +} + +resource "aws_route_table_association" "test2" { + route_table_id = aws_route_table.test.id + subnet_id = aws_subnet.test2.id +} +`, r), + ) +} + func testAccAWSEmrClusterConfigInstanceFleetsMasterOnly(r string) string { return testAccAWSEmrComposeConfig(false, testAccAWSEmrClusterConfigCurrentPartition(), From 9b4932692bc758c7d79486f1455b07823387782a Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Thu, 15 Apr 2021 16:10:58 -0700 Subject: [PATCH 17/17] Adds CHANGELOG --- .changelog/17219.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/17219.txt diff --git a/.changelog/17219.txt b/.changelog/17219.txt new file mode 100644 index 00000000000..2cd8bafab50 --- /dev/null +++ b/.changelog/17219.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +resource/aws_emr_cluster: Adds support for multiple subnets +```