diff --git a/aws/resource_aws_appautoscaling_scheduled_action_test.go b/aws/resource_aws_appautoscaling_scheduled_action_test.go index 0d5645d1f7e8..0090e3ea3b6d 100644 --- a/aws/resource_aws_appautoscaling_scheduled_action_test.go +++ b/aws/resource_aws_appautoscaling_scheduled_action_test.go @@ -256,9 +256,14 @@ resource "aws_emr_cluster" "hoge" { instance_profile = "${aws_iam_instance_profile.instance_profile.arn}" } - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 2 + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 2 + instance_type = "c4.large" + } tags = { role = "rolename" diff --git a/aws/resource_aws_appautoscaling_target_test.go b/aws/resource_aws_appautoscaling_target_test.go index 6b5f04a3167a..ce5927d83cbf 100644 --- a/aws/resource_aws_appautoscaling_target_test.go +++ b/aws/resource_aws_appautoscaling_target_test.go @@ -345,9 +345,14 @@ resource "aws_emr_cluster" "tf-test-cluster" { instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" } - master_instance_type = "m3.xlarge" - core_instance_type = "m3.xlarge" - core_instance_count = 2 + master_instance_group { + instance_type = "m3.xlarge" + } + + core_instance_group { + instance_count = 2 + instance_type = "m3.xlarge" + } tags = { role = "rolename" diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index de696e96baf0..54a7b1cf7f64 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -33,57 +33,6 @@ func resourceAwsEMRCluster() *schema.Resource { State: schema.ImportStatePassthrough, }, - CustomizeDiff: func(diff *schema.ResourceDiff, v interface{}) error { - if diff.HasChange("instance_group") { - o, n := diff.GetChange("instance_group") - oSet := o.(*schema.Set).List() - nSet := n.(*schema.Set).List() - - // Everything in instance group needs to be set to forcenew if the autoscaling policy doesn't change - if len(oSet) != len(nSet) { - return nil - } - for _, currInstanceGroup := range oSet { - for _, nextInstanceGroup := range nSet { - oInstanceGroup := currInstanceGroup.(map[string]interface{}) - nInstanceGroup := nextInstanceGroup.(map[string]interface{}) - - if oInstanceGroup["instance_role"].(string) != nInstanceGroup["instance_role"].(string) || oInstanceGroup["name"].(string) != nInstanceGroup["name"].(string) { - continue - } - - oAutoScalingPolicy := oInstanceGroup["autoscaling_policy"].(string) - nAutoScalingPolicy := nInstanceGroup["autoscaling_policy"].(string) - - if oAutoScalingPolicy == "" && nAutoScalingPolicy == "" { - continue - } - - oJSON, err := structure.NormalizeJsonString(oAutoScalingPolicy) - if err != nil { - return fmt.Errorf("error reading old json value: %s", err) - } - nJSON, err := structure.NormalizeJsonString(nAutoScalingPolicy) - if err != nil { - return fmt.Errorf("error reading new json value: %s", err) - } - - if oJSON != nJSON { - continue - } - for _, k := range diff.GetChangedKeysPrefix(fmt.Sprintf("instance_group.%d", resourceAwsEMRClusterInstanceGroupHash(oInstanceGroup))) { - if strings.HasSuffix(k, ".#") { - k = strings.TrimSuffix(k, ".#") - } - diff.ForceNew(k) - } - break - } - } - } - return nil - }, - Schema: map[string]*schema.Schema{ "arn": { Type: schema.TypeString, @@ -99,14 +48,6 @@ func resourceAwsEMRCluster() *schema.Resource { ForceNew: true, Required: true, }, - "master_instance_type": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Computed: true, - ConflictsWith: []string{"master_instance_group"}, - Deprecated: "use `master_instance_group` configuration block `instance_type` argument instead", - }, "additional_info": { Type: schema.TypeString, Optional: true, @@ -118,22 +59,6 @@ func resourceAwsEMRCluster() *schema.Resource { return json }, }, - "core_instance_type": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Computed: true, - ConflictsWith: []string{"core_instance_group"}, - Deprecated: "use `core_instance_group` configuration block `instance_type` argument instead", - }, - "core_instance_count": { - Type: schema.TypeInt, - Optional: true, - ValidateFunc: validation.IntAtLeast(1), - Computed: true, - ConflictsWith: []string{"core_instance_group"}, - Deprecated: "use `core_instance_group` configuration block `instance_count` argument instead", - }, "cluster_state": { Type: schema.TypeString, Computed: true, @@ -264,12 +189,11 @@ func resourceAwsEMRCluster() *schema.Resource { }, }, "core_instance_group": { - Type: schema.TypeList, - Optional: true, - Computed: true, - ForceNew: true, - MaxItems: 1, - ConflictsWith: []string{"core_instance_count", "core_instance_type", "instance_group"}, + Type: schema.TypeList, + Optional: true, + Computed: true, + ForceNew: true, + MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "autoscaling_policy": { @@ -340,12 +264,11 @@ func resourceAwsEMRCluster() *schema.Resource { }, }, "master_instance_group": { - Type: schema.TypeList, - Optional: true, - Computed: true, - ForceNew: true, - MaxItems: 1, - ConflictsWith: []string{"master_instance_type", "instance_group"}, + Type: schema.TypeList, + Optional: true, + Computed: true, + ForceNew: true, + MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "bid_price": { @@ -410,87 +333,6 @@ func resourceAwsEMRCluster() *schema.Resource { }, }, }, - "instance_group": { - Type: schema.TypeSet, - Optional: true, - ForceNew: true, - Computed: true, - ConflictsWith: []string{"core_instance_group", "master_instance_group"}, - Deprecated: "use `master_instance_group` configuration block, `core_instance_group` configuration block, and `aws_emr_instance_group` resource(s) instead", - Elem: &schema.Resource{ - Schema: map[string]*schema.Schema{ - "bid_price": { - Type: schema.TypeString, - Optional: true, - }, - "ebs_config": { - Type: schema.TypeSet, - Optional: true, - Computed: true, - Elem: &schema.Resource{ - Schema: map[string]*schema.Schema{ - "iops": { - Type: schema.TypeInt, - Optional: true, - }, - "size": { - Type: schema.TypeInt, - Required: true, - }, - "type": { - Type: schema.TypeString, - Required: true, - ValidateFunc: validateAwsEmrEbsVolumeType(), - }, - "volumes_per_instance": { - Type: schema.TypeInt, - Optional: true, - Default: 1, - }, - }, - }, - Set: resourceAwsEMRClusterEBSConfigHash, - }, - "instance_count": { - Type: schema.TypeInt, - Optional: true, - Default: 0, - }, - "autoscaling_policy": { - Type: schema.TypeString, - Optional: true, - DiffSuppressFunc: suppressEquivalentJsonDiffs, - ValidateFunc: validation.StringIsJSON, - StateFunc: func(v interface{}) string { - jsonString, _ := structure.NormalizeJsonString(v) - return jsonString - }, - }, - "instance_role": { - Type: schema.TypeString, - Required: true, - ValidateFunc: validation.StringInSlice([]string{ - emr.InstanceFleetTypeMaster, - emr.InstanceFleetTypeCore, - emr.InstanceFleetTypeTask, - }, false), - }, - "instance_type": { - Type: schema.TypeString, - Required: true, - }, - "name": { - Type: schema.TypeString, - Optional: true, - }, - "id": { - Type: schema.TypeString, - Computed: true, - }, - }, - }, - Set: resourceAwsEMRClusterInstanceGroupHash, - }, "bootstrap_action": { Type: schema.TypeList, Optional: true, @@ -721,36 +563,6 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.InstanceGroups = append(instanceConfig.InstanceGroups, instanceGroup) } - // DEPRECATED: Remove in a future major version - if v, ok := d.GetOk("master_instance_type"); ok { - masterInstanceGroupConfig := &emr.InstanceGroupConfig{ - InstanceRole: aws.String(emr.InstanceRoleTypeMaster), - InstanceType: aws.String(v.(string)), - InstanceCount: aws.Int64(1), - } - instanceConfig.InstanceGroups = append(instanceConfig.InstanceGroups, masterInstanceGroupConfig) - } - - // DEPRECATED: Remove in a future major version - var coreInstanceType string - if v, ok := d.GetOk("core_instance_type"); ok { - coreInstanceType = v.(string) - } - var coreInstanceCount int64 - if v, ok := d.GetOk("core_instance_count"); ok { - coreInstanceCount = int64(v.(int)) - } - if (coreInstanceCount == 0 && coreInstanceType != "") || (coreInstanceCount > 0 && coreInstanceType == "") { - return fmt.Errorf("Must specify both `core_instance_count` and `core_instance_type`") - } else if coreInstanceCount > 0 && coreInstanceType != "" { - coreInstanceGroupConfig := &emr.InstanceGroupConfig{ - InstanceCount: aws.Int64(int64(d.Get("core_instance_count").(int))), - InstanceRole: aws.String(emr.InstanceRoleTypeCore), - InstanceType: aws.String(d.Get("core_instance_type").(string)), - } - instanceConfig.InstanceGroups = append(instanceConfig.InstanceGroups, coreInstanceGroupConfig) - } - var instanceProfile string if a, ok := d.GetOk("ec2_attributes"); ok { ec2Attributes := a.([]interface{}) @@ -795,18 +607,6 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error } } - // DEPRECATED: Remove in a future major version - if v, ok := d.GetOk("instance_group"); ok { - instanceGroupConfigs := v.(*schema.Set).List() - instanceGroups, err := expandInstanceGroupConfigs(instanceGroupConfigs) - - if err != nil { - return fmt.Errorf("error parsing EMR instance groups configuration: %s", err) - } - - instanceConfig.InstanceGroups = instanceGroups - } - emrApps := expandApplications(applications) params := &emr.RunJobFlowInput{ @@ -1004,27 +804,6 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error { coreGroup := emrCoreInstanceGroup(instanceGroups) masterGroup := findMasterGroup(instanceGroups) - d.Set("core_instance_count", 0) - d.Set("core_instance_type", "") - d.Set("master_instance_type", "") - - if coreGroup != nil { - d.Set("core_instance_type", coreGroup.InstanceType) - d.Set("core_instance_count", coreGroup.RequestedInstanceCount) - } - - if masterGroup != nil { - d.Set("master_instance_type", masterGroup.InstanceType) - } - - flattenedInstanceGroups, err := flattenInstanceGroups(instanceGroups) - if err != nil { - return fmt.Errorf("error flattening instance groups: %s", err) - } - if err := d.Set("instance_group", flattenedInstanceGroups); err != nil { - return fmt.Errorf("error setting instance_group: %s", err) - } - flattenedCoreInstanceGroup, err := flattenEmrCoreInstanceGroup(coreGroup) if err != nil { @@ -1127,59 +906,6 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error { func resourceAwsEMRClusterUpdate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).emrconn - if d.HasChange("core_instance_count") { - log.Printf("[DEBUG] Modify EMR cluster") - groups, err := fetchAllEMRInstanceGroups(conn, d.Id()) - if err != nil { - log.Printf("[DEBUG] Error finding all instance groups: %s", err) - return err - } - - coreInstanceCount := d.Get("core_instance_count").(int) - coreGroup := emrCoreInstanceGroup(groups) - if coreGroup == nil { - return fmt.Errorf("Error finding core group") - } - - params := &emr.ModifyInstanceGroupsInput{ - InstanceGroups: []*emr.InstanceGroupModifyConfig{ - { - InstanceGroupId: coreGroup.Id, - InstanceCount: aws.Int64(int64(coreInstanceCount) - 1), - }, - }, - } - _, errModify := conn.ModifyInstanceGroups(params) - if errModify != nil { - log.Printf("[ERROR] %s", errModify) - return errModify - } - - log.Printf("[DEBUG] Modify EMR Cluster done...") - - log.Println("[INFO] Waiting for EMR Cluster to be available") - - stateConf := &resource.StateChangeConf{ - Pending: []string{ - emr.ClusterStateBootstrapping, - emr.ClusterStateStarting, - }, - Target: []string{ - emr.ClusterStateRunning, - emr.ClusterStateWaiting, - }, - Refresh: resourceAwsEMRClusterStateRefreshFunc(d, meta), - Timeout: 40 * time.Minute, - MinTimeout: 10 * time.Second, - Delay: 5 * time.Second, - } - - _, err = stateConf.WaitForState() - if err != nil { - return fmt.Errorf("Error waiting for EMR Cluster state to be \"WAITING\" or \"RUNNING\" after modification: %s", err) - } - } - if d.HasChange("visible_to_all_users") { _, errModify := conn.SetVisibleToAllUsers(&emr.SetVisibleToAllUsersInput{ JobFlowIds: []*string{aws.String(d.Id())}, @@ -1685,49 +1411,6 @@ func flattenEmrStepSummary(stepSummary *emr.StepSummary) map[string]interface{} return m } -func flattenInstanceGroup(ig *emr.InstanceGroup) (map[string]interface{}, error) { - attrs := map[string]interface{}{} - if ig.BidPrice != nil { - attrs["bid_price"] = *ig.BidPrice - } - - attrs["id"] = *ig.Id - attrs["ebs_config"] = flattenEBSConfig(ig.EbsBlockDevices) - attrs["instance_count"] = int(*ig.RequestedInstanceCount) - attrs["instance_role"] = *ig.InstanceGroupType - attrs["instance_type"] = *ig.InstanceType - if ig.Name != nil { - attrs["name"] = *ig.Name - } - - autoscalingPolicy, err := flattenEmrAutoScalingPolicyDescription(ig.AutoScalingPolicy) - - if err != nil { - return nil, err - } - - attrs["autoscaling_policy"] = autoscalingPolicy - - if attrs["name"] != nil { - attrs["name"] = *ig.Name - } - - return attrs, nil -} - -func flattenInstanceGroups(igs []*emr.InstanceGroup) (*schema.Set, error) { - instanceGroupSet := []interface{}{} - for _, ig := range igs { - flattenedInstanceGroup, err := flattenInstanceGroup(ig) - if err != nil { - return nil, err - } - instanceGroupSet = append(instanceGroupSet, flattenedInstanceGroup) - } - - return schema.NewSet(resourceAwsEMRClusterInstanceGroupHash, instanceGroupSet), nil -} - func flattenEBSConfig(ebsBlockDevices []*emr.EbsBlockDevice) *schema.Set { ebsConfig := make([]interface{}, 0) @@ -1870,54 +1553,6 @@ func expandEmrStepConfigs(l []interface{}) []*emr.StepConfig { return stepConfigs } -func expandInstanceGroupConfigs(instanceGroupConfigs []interface{}) ([]*emr.InstanceGroupConfig, error) { - instanceGroupConfig := []*emr.InstanceGroupConfig{} - - for _, raw := range instanceGroupConfigs { - configAttributes := raw.(map[string]interface{}) - configInstanceRole := configAttributes["instance_role"].(string) - configInstanceCount := configAttributes["instance_count"].(int) - configInstanceType := configAttributes["instance_type"].(string) - configName := configAttributes["name"].(string) - config := &emr.InstanceGroupConfig{ - Name: aws.String(configName), - InstanceRole: aws.String(configInstanceRole), - InstanceType: aws.String(configInstanceType), - InstanceCount: aws.Int64(int64(configInstanceCount)), - } - - expandBidPrice(config, configAttributes) - expandEbsConfig(configAttributes, config) - - if v, ok := configAttributes["autoscaling_policy"]; ok && v.(string) != "" { - var autoScalingPolicy *emr.AutoScalingPolicy - - err := json.Unmarshal([]byte(v.(string)), &autoScalingPolicy) - - if err != nil { - return []*emr.InstanceGroupConfig{}, fmt.Errorf("error parsing EMR Auto Scaling Policy JSON: %s", err) - } - - config.AutoScalingPolicy = autoScalingPolicy - } - - instanceGroupConfig = append(instanceGroupConfig, config) - } - - return instanceGroupConfig, nil -} - -func expandBidPrice(config *emr.InstanceGroupConfig, configAttributes map[string]interface{}) { - if bidPrice, ok := configAttributes["bid_price"]; ok { - if bidPrice != "" { - config.BidPrice = aws.String(bidPrice.(string)) - config.Market = aws.String("SPOT") - } else { - config.Market = aws.String("ON_DEMAND") - } - } -} - func expandEbsConfig(configAttributes map[string]interface{}, config *emr.InstanceGroupConfig) { if rawEbsConfigs, ok := configAttributes["ebs_config"]; ok { ebsConfig := &emr.EbsConfiguration{} @@ -2065,42 +1700,6 @@ func findMasterGroup(instanceGroups []*emr.InstanceGroup) *emr.InstanceGroup { return nil } -// EMRCluster always has an instance role of either master, core, or task -// Name is optional for core and master(only group allowed for this type) but needed for task -// since you can have multiple task instance groups. -func resourceAwsEMRClusterInstanceGroupHash(v interface{}) int { - var buf bytes.Buffer - m := v.(map[string]interface{}) - buf.WriteString(fmt.Sprintf("%s-", m["instance_role"].(string))) - buf.WriteString(fmt.Sprintf("%s-", m["instance_type"].(string))) - buf.WriteString(fmt.Sprintf("%d-", m["instance_count"].(int))) - if v, ok := m["name"]; ok { - buf.WriteString(fmt.Sprintf("%s-", v.(string))) - } - if v, ok := m["bid_price"]; ok { - buf.WriteString(fmt.Sprintf("%s-", v.(string))) - } - - if v, ok := m["autoscaling_policy"]; ok { - pleaseWork, _ := structure.NormalizeJsonString(v.(string)) - buf.WriteString(fmt.Sprintf("%s-", pleaseWork)) - } - - if v, ok := m["ebs_config"]; ok { - configs := v.(*schema.Set).List() - - // There is an issue where an `ebs_config` is automatically configured when not specified in Terraform and - // this causes the hashcode to change. Instead, we'll ignore that configuration when setting up the hashcode. - if len(configs) > 1 { - for _, ebsConfigs := range configs { - buf.WriteString(fmt.Sprintf("%d-", resourceAwsEMRClusterEBSConfigHash(ebsConfigs.(map[string]interface{})))) - } - } - } - - return hashcode.String(buf.String()) -} - func resourceAwsEMRClusterEBSConfigHash(v interface{}) int { var buf bytes.Buffer m := v.(map[string]interface{}) diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index 1aa10b98d613..115150ec0445 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -99,10 +99,14 @@ func TestAccAWSEMRCluster_basic(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -135,10 +139,15 @@ func TestAccAWSEMRCluster_additionalInfo(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps", "additional_info"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "additional_info", + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -185,10 +194,14 @@ func TestAccAWSEMRCluster_configurationsJson(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -279,10 +292,14 @@ func TestAccAWSEMRCluster_CoreInstanceGroup_AutoscalingPolicy(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigCoreInstanceGroupAutoscalingPolicy(rName, autoscalingPolicy2), @@ -325,10 +342,14 @@ func TestAccAWSEMRCluster_CoreInstanceGroup_BidPrice(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigCoreInstanceGroupBidPrice(rName, "0.51"), @@ -362,10 +383,14 @@ func TestAccAWSEMRCluster_CoreInstanceGroup_InstanceCount(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceCount(rName, 1), @@ -408,10 +433,14 @@ func TestAccAWSEMRCluster_CoreInstanceGroup_InstanceType(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceType(rName, "m4.xlarge"), @@ -426,78 +455,6 @@ func TestAccAWSEMRCluster_CoreInstanceGroup_InstanceType(t *testing.T) { }) } -func TestAccAWSEMRCluster_CoreInstanceGroup_Migration_CoreInstanceType(t *testing.T) { - var cluster1, cluster2 emr.Cluster - rName := acctest.RandomWithPrefix("tf-acc-test") - resourceName := "aws_emr_cluster.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigCoreInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster1), - resource.TestCheckResourceAttr(resourceName, "core_instance_type", "m4.large"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - { - Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster2), - testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), - resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), - resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_type", "m4.large"), - ), - }, - }, - }) -} - -func TestAccAWSEMRCluster_CoreInstanceGroup_Migration_InstanceGroup(t *testing.T) { - var cluster1, cluster2 emr.Cluster - rName := acctest.RandomWithPrefix("tf-acc-test") - resourceName := "aws_emr_cluster.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigInstanceGroupCoreInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster1), - resource.TestCheckResourceAttr(resourceName, "instance_group.#", "2"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - { - Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster2), - testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), - resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), - resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_type", "m4.large"), - ), - }, - }, - }) -} - func TestAccAWSEMRCluster_CoreInstanceGroup_Name(t *testing.T) { var cluster1, cluster2 emr.Cluster rName := acctest.RandomWithPrefix("tf-acc-test") @@ -517,10 +474,14 @@ func TestAccAWSEMRCluster_CoreInstanceGroup_Name(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigCoreInstanceGroupName(rName, "name2"), @@ -535,153 +496,6 @@ func TestAccAWSEMRCluster_CoreInstanceGroup_Name(t *testing.T) { }) } -func TestAccAWSEMRCluster_instance_group(t *testing.T) { - var cluster emr.Cluster - - resourceName := "aws_emr_cluster.tf-test-cluster" - rName := acctest.RandomWithPrefix("tf-acc-test") - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigInstanceGroups(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - resource.TestCheckResourceAttr(resourceName, "instance_group.#", "2"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - }, - }) -} - -func TestAccAWSEMRCluster_instance_group_names(t *testing.T) { - var cluster emr.Cluster - - resourceName := "aws_emr_cluster.tf-test-cluster" - rName := acctest.RandomWithPrefix("tf-acc-test") - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigInstanceGroupsName(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - resource.TestCheckResourceAttr(resourceName, "instance_group.#", "3"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - }, - }) -} - -func TestAccAWSEMRCluster_instance_group_update(t *testing.T) { - var cluster emr.Cluster - - resourceName := "aws_emr_cluster.tf-test-cluster" - rName := acctest.RandomWithPrefix("tf-acc-test") - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigInstanceGroups(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - resource.TestCheckResourceAttr(resourceName, "instance_group.#", "2"), - ), - }, - { - Config: testAccAWSEmrClusterConfigInstanceGroupsUpdate(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - resource.TestCheckResourceAttr(resourceName, "instance_group.#", "2"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - }, - }) -} - -func TestAccAWSEMRCluster_instance_group_EBSVolumeType_st1(t *testing.T) { - var cluster emr.Cluster - - resourceName := "aws_emr_cluster.tf-test-cluster" - rName := acctest.RandomWithPrefix("tf-acc-test") - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigInstanceGroups_st1(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - resource.TestCheckResourceAttr(resourceName, "instance_group.#", "2"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - }, - }) -} - -func TestAccAWSEMRCluster_updateAutoScalingPolicy(t *testing.T) { - var cluster emr.Cluster - - resourceName := "aws_emr_cluster.tf-test-cluster" - rName := acctest.RandomWithPrefix("tf-acc-test") - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigInstanceGroups_st1(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - ), - }, - { - Config: testAccAWSEmrClusterConfigInstanceGroups_updateAutoScalingPolicy(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - }, - }) -} - func TestAccAWSEMRCluster_Ec2Attributes_DefaultManagedSecurityGroups(t *testing.T) { var cluster emr.Cluster var vpc ec2.Vpc @@ -745,10 +559,15 @@ func TestAccAWSEMRCluster_Kerberos_ClusterDedicatedKdc(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps", "kerberos_attributes.0.kdc_admin_password"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + "kerberos_attributes.0.kdc_admin_password", + }, }, }, }) @@ -773,10 +592,14 @@ func TestAccAWSEMRCluster_MasterInstanceGroup_BidPrice(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigMasterInstanceGroupBidPrice(rName, "0.51"), @@ -810,10 +633,14 @@ func TestAccAWSEMRCluster_MasterInstanceGroup_InstanceCount(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceCount(rName, 1), @@ -847,10 +674,14 @@ func TestAccAWSEMRCluster_MasterInstanceGroup_InstanceType(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, "m4.xlarge"), @@ -865,78 +696,6 @@ func TestAccAWSEMRCluster_MasterInstanceGroup_InstanceType(t *testing.T) { }) } -func TestAccAWSEMRCluster_MasterInstanceGroup_Migration_InstanceGroup(t *testing.T) { - var cluster1, cluster2 emr.Cluster - rName := acctest.RandomWithPrefix("tf-acc-test") - resourceName := "aws_emr_cluster.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigInstanceGroupMasterInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster1), - resource.TestCheckResourceAttr(resourceName, "instance_group.#", "1"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - { - Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster2), - testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), - resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), - resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.instance_type", "m4.large"), - ), - }, - }, - }) -} - -func TestAccAWSEMRCluster_MasterInstanceGroup_Migration_MasterInstanceType(t *testing.T) { - var cluster1, cluster2 emr.Cluster - rName := acctest.RandomWithPrefix("tf-acc-test") - resourceName := "aws_emr_cluster.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckAWSEmrDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAWSEmrClusterConfigMasterInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster1), - resource.TestCheckResourceAttr(resourceName, "master_instance_type", "m4.large"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, - }, - { - Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, "m4.large"), - Check: resource.ComposeTestCheckFunc( - testAccCheckAWSEmrClusterExists(resourceName, &cluster2), - testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), - resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), - resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.instance_type", "m4.large"), - ), - }, - }, - }) -} - func TestAccAWSEMRCluster_MasterInstanceGroup_Name(t *testing.T) { var cluster1, cluster2 emr.Cluster rName := acctest.RandomWithPrefix("tf-acc-test") @@ -956,10 +715,14 @@ func TestAccAWSEMRCluster_MasterInstanceGroup_Name(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigMasterInstanceGroupName(rName, "name2"), @@ -992,10 +755,14 @@ func TestAccAWSEMRCluster_security_config(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1025,10 +792,14 @@ func TestAccAWSEMRCluster_Step_Basic(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1052,10 +823,14 @@ func TestAccAWSEMRCluster_Step_ConfigMode(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"cluster_state", "configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfig_Step_NoBlocks(rName), @@ -1065,10 +840,14 @@ func TestAccAWSEMRCluster_Step_ConfigMode(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"cluster_state", "configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfig_Step_Zeroed(rName), @@ -1078,10 +857,14 @@ func TestAccAWSEMRCluster_Step_ConfigMode(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"cluster_state", "configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1115,10 +898,14 @@ func TestAccAWSEMRCluster_Step_Multiple(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1150,10 +937,14 @@ func TestAccAWSEMRCluster_bootstrap_ordering(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfig_bootstrapAdd(rName), @@ -1176,10 +967,14 @@ func TestAccAWSEMRCluster_bootstrap_ordering(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfig_bootstrapReorder(rName), @@ -1202,10 +997,14 @@ func TestAccAWSEMRCluster_bootstrap_ordering(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1236,10 +1035,14 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { //Need to turn off termination_protection to allow the job to be deleted @@ -1250,10 +1053,14 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1277,10 +1084,14 @@ func TestAccAWSEMRCluster_keepJob(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1304,10 +1115,14 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, { Config: testAccAWSEmrClusterConfigVisibleToAllUsersUpdated(rName), @@ -1317,10 +1132,14 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1345,10 +1164,14 @@ func TestAccAWSEMRCluster_s3Logging(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1385,10 +1208,14 @@ func TestAccAWSEMRCluster_tags(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1419,10 +1246,14 @@ func TestAccAWSEMRCluster_root_volume_size(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1452,10 +1283,14 @@ func TestAccAWSEMRCluster_step_concurrency_level(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1479,10 +1314,14 @@ func TestAccAWSEMRCluster_custom_ami_id(t *testing.T) { ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "cluster_state", // Ignore RUNNING versus WAITING changes + "configurations", + "keep_job_flow_alive_when_no_steps", + }, }, }, }) @@ -1741,9 +1580,15 @@ resource "aws_emr_cluster" "test" { release_label = "emr-5.0.0" applications = ["Hadoop", "Hive"] log_uri = "s3n://terraform/testlog/" - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 1 + + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } service_role = "${aws_iam_role.emr_service.arn}" depends_on = [ @@ -1797,9 +1642,15 @@ resource "aws_emr_cluster" "test" { release_label = "emr-5.0.0" applications = ["Hadoop", "Hive"] log_uri = "s3n://terraform/testlog/" - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 1 + + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } service_role = "${aws_iam_role.emr_service.arn}" depends_on = [ @@ -1857,9 +1708,15 @@ resource "aws_emr_cluster" "test" { release_label = "emr-5.0.0" applications = ["Hadoop", "Hive"] log_uri = "s3n://terraform/testlog/" - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 1 + + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } service_role = "${aws_iam_role.emr_service.arn}" depends_on = [ @@ -1926,9 +1783,14 @@ resource "aws_emr_cluster" "tf-test-cluster" { instance_profile = "${aws_iam_instance_profile.emr_instance_profile.arn}" } - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 1 + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } tags = { role = "rolename" @@ -1986,9 +1848,14 @@ EOF instance_profile = "${aws_iam_instance_profile.emr_instance_profile.arn}" } - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 1 + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } tags = { role = "rolename" @@ -2036,9 +1903,14 @@ resource "aws_emr_cluster" "tf-test-cluster" { instance_profile = "${aws_iam_instance_profile.emr_instance_profile.arn}" } - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 1 + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } keep_job_flow_alive_when_no_steps = true termination_protection = false @@ -2104,16 +1976,22 @@ EOF resource "aws_emr_cluster" "tf-test-cluster" { applications = ["Spark"] - core_instance_count = 1 - core_instance_type = "c4.large" keep_job_flow_alive_when_no_steps = true - master_instance_type = "c4.large" name = "%[1]s" release_label = "emr-5.12.0" security_configuration = "${aws_emr_security_configuration.foo.name}" service_role = "EMR_DefaultRole" termination_protection = false + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } + ec2_attributes { emr_managed_master_security_group = "${aws_security_group.test.id}" emr_managed_slave_security_group = "${aws_security_group.test.id}" @@ -2149,9 +2027,14 @@ resource "aws_emr_cluster" "tf-test-cluster" { instance_profile = "${aws_iam_instance_profile.emr_instance_profile.arn}" } - master_instance_type = "c4.large" - core_instance_type = "c4.large" - core_instance_count = 1 + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } security_configuration = "${aws_emr_security_configuration.foo.name}" @@ -2270,16 +2153,22 @@ func testAccAWSEmrClusterConfig_Step(r string, stepConfig string) string { return testAccAWSEmrComposeConfig(false, fmt.Sprintf(` resource "aws_emr_cluster" "tf-test-cluster" { applications = ["Spark"] - core_instance_count = 1 - core_instance_type = "c4.large" keep_job_flow_alive_when_no_steps = true log_uri = "s3://${aws_s3_bucket.test.bucket}/" - master_instance_type = "c4.large" name = "%[1]s" release_label = "emr-5.12.0" service_role = "EMR_DefaultRole" termination_protection = false + master_instance_group { + instance_type = "c4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } + ec2_attributes { emr_managed_master_security_group = "${aws_security_group.test.id}" emr_managed_slave_security_group = "${aws_security_group.test.id}" @@ -2547,13 +2436,35 @@ resource "aws_emr_cluster" "test" { ) } -func testAccAWSEmrClusterConfigCoreInstanceType(rName, coreInstanceType string) string { +func testAccAWSEmrClusterConfigEc2AttributesDefaultManagedSecurityGroups(rName string) string { + return testAccAWSEmrComposeConfig(false, fmt.Sprintf(` +resource "aws_emr_cluster" "tf-test-cluster" { + applications = ["Spark"] + keep_job_flow_alive_when_no_steps = true + name = %[1]q + release_label = "emr-5.28.0" + service_role = "EMR_DefaultRole" + + ec2_attributes { + instance_profile = "EMR_EC2_DefaultRole" + subnet_id = "${aws_subnet.test.id}" + } + + master_instance_group { + instance_type = "m4.large" + } + + depends_on = ["aws_route_table_association.test"] +} +`, rName), + ) +} + +func testAccAWSEmrClusterConfigMasterInstanceGroupBidPrice(rName, bidPrice string) string { return testAccAWSEmrComposeConfig(false, fmt.Sprintf(` resource "aws_emr_cluster" "test" { applications = ["Spark"] keep_job_flow_alive_when_no_steps = true - core_instance_count = 1 - core_instance_type = %[2]q name = %[1]q release_label = "emr-5.12.0" service_role = "EMR_DefaultRole" @@ -2566,24 +2477,28 @@ resource "aws_emr_cluster" "test" { } master_instance_group { + bid_price = %[2]q instance_type = "m4.large" } depends_on = ["aws_route_table_association.test"] } -`, rName, coreInstanceType), +`, rName, bidPrice), ) } -func testAccAWSEmrClusterConfigInstanceGroupCoreInstanceType(rName, coreInstanceType string) string { - return testAccAWSEmrComposeConfig(false, fmt.Sprintf(` +func testAccAWSEmrClusterConfigMasterInstanceGroupInstanceCount(rName string, instanceCount int) string { + return testAccAWSEmrComposeConfig(true, fmt.Sprintf(` resource "aws_emr_cluster" "test" { applications = ["Spark"] keep_job_flow_alive_when_no_steps = true name = %[1]q - release_label = "emr-5.12.0" + release_label = "emr-5.24.1" service_role = "EMR_DefaultRole" + # Termination protection is automatically enabled for multiple master clusters + termination_protection = false + ec2_attributes { emr_managed_master_security_group = "${aws_security_group.test.id}" emr_managed_slave_security_group = "${aws_security_group.test.id}" @@ -2591,25 +2506,23 @@ resource "aws_emr_cluster" "test" { subnet_id = "${aws_subnet.test.id}" } - instance_group { - instance_count = 1 - instance_role = "MASTER" + master_instance_group { + instance_count = %[2]d instance_type = "m4.large" } - instance_group { - instance_count = 1 - instance_role = "CORE" - instance_type = %[2]q + # core_instance_group is required with multiple masters + core_instance_group { + instance_type = "m4.large" } depends_on = ["aws_route_table_association.test"] } -`, rName, coreInstanceType), +`, rName, instanceCount), ) } -func testAccAWSEmrClusterConfigInstanceGroupMasterInstanceType(rName, masterInstanceType string) string { +func testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, instanceType string) string { return testAccAWSEmrComposeConfig(false, fmt.Sprintf(` resource "aws_emr_cluster" "test" { applications = ["Spark"] @@ -2625,19 +2538,44 @@ resource "aws_emr_cluster" "test" { subnet_id = "${aws_subnet.test.id}" } - instance_group { - instance_count = 1 - instance_role = "MASTER" - instance_type = %[2]q + master_instance_group { + instance_type = %[2]q + } + + depends_on = ["aws_route_table_association.test"] +} +`, rName, instanceType), + ) +} + +func testAccAWSEmrClusterConfigMasterInstanceGroupName(rName, instanceGroupName string) string { + return testAccAWSEmrComposeConfig(false, fmt.Sprintf(` +resource "aws_emr_cluster" "test" { + applications = ["Spark"] + keep_job_flow_alive_when_no_steps = true + name = %[1]q + release_label = "emr-5.12.0" + service_role = "EMR_DefaultRole" + + ec2_attributes { + emr_managed_master_security_group = "${aws_security_group.test.id}" + emr_managed_slave_security_group = "${aws_security_group.test.id}" + instance_profile = "EMR_EC2_DefaultRole" + subnet_id = "${aws_subnet.test.id}" + } + + master_instance_group { + instance_type = "m4.large" + name = %[2]q } depends_on = ["aws_route_table_association.test"] } -`, rName, masterInstanceType), +`, rName, instanceGroupName), ) } -func testAccAWSEmrClusterConfigInstanceGroups(r string) string { +func testAccAWSEmrClusterConfigTerminationPolicy(r string, term string) string { return testAccAWSEmrComposeConfig(false, testAccAWSEmrClusterConfigIAMServiceRoleBase(r), testAccAWSEmrClusterConfigIAMInstanceProfileBase(r), @@ -2655,58 +2593,13 @@ resource "aws_emr_cluster" "tf-test-cluster" { instance_profile = "${aws_iam_instance_profile.emr_instance_profile.arn}" } - instance_group { - instance_role = "CORE" - instance_type = "c4.large" - instance_count = "1" - - ebs_config { - size = "40" - type = "gp2" - volumes_per_instance = 1 - } - - bid_price = "0.30" - - autoscaling_policy = <