Skip to content

Commit

Permalink
Merge pull request #9704 from terraform-providers/rfd-retry-ecs
Browse files Browse the repository at this point in the history 
Final retries after timeouts for ECS resources
  • Loading branch information
@ryndaniels
ryndaniels authored Aug 12, 2019
2 parents 3270b94 + a3beae3 commit 9303676
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 40 deletions.
78 changes: 45 additions & 33 deletions aws/resource_aws_ecs_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/helper/schema"
Expand Down Expand Up @@ -97,6 +96,9 @@ func resourceAwsEcsClusterRead(d *schema.ResourceData, meta interface{}) error {

return nil
})
if isResourceTimeoutError(err) {
out, err = conn.DescribeClusters(input)
}

if isResourceNotFoundError(err) {
log.Printf("[WARN] ECS Cluster (%s) not found, removing from state", d.Id())
Expand Down Expand Up @@ -185,66 +187,76 @@ func resourceAwsEcsClusterDelete(d *schema.ResourceData, meta interface{}) error
conn := meta.(*AWSClient).ecsconn

log.Printf("[DEBUG] Deleting ECS cluster %s", d.Id())

input := &ecs.DeleteClusterInput{
Cluster: aws.String(d.Id()),
}
err := resource.Retry(10*time.Minute, func() *resource.RetryError {
out, err := conn.DeleteCluster(&ecs.DeleteClusterInput{
Cluster: aws.String(d.Id()),
})
_, err := conn.DeleteCluster(input)

if err == nil {
log.Printf("[DEBUG] ECS cluster %s deleted: %s", d.Id(), out)
log.Printf("[DEBUG] ECS cluster %s deleted", d.Id())
return nil
}

awsErr, ok := err.(awserr.Error)
if !ok {
return resource.NonRetryableError(err)
}

if awsErr.Code() == "ClusterContainsContainerInstancesException" {
log.Printf("[TRACE] Retrying ECS cluster %q deletion after %q", d.Id(), awsErr.Code())
if isAWSErr(err, "ClusterContainsContainerInstancesException", "") {
log.Printf("[TRACE] Retrying ECS cluster %q deletion after %s", d.Id(), err)
return resource.RetryableError(err)
}

if awsErr.Code() == "ClusterContainsServicesException" {
log.Printf("[TRACE] Retrying ECS cluster %q deletion after %q", d.Id(), awsErr.Code())
if isAWSErr(err, "ClusterContainsServicesException", "") {
log.Printf("[TRACE] Retrying ECS cluster %q deletion after %s", d.Id(), err)
return resource.RetryableError(err)
}

return resource.NonRetryableError(err)
})
if isResourceTimeoutError(err) {
_, err = conn.DeleteCluster(input)
}
if err != nil {
return err
return fmt.Errorf("Error deleting ECS cluster: %s", err)
}

clusterName := d.Get("name").(string)
dcInput := &ecs.DescribeClustersInput{
Clusters: []*string{aws.String(clusterName)},
}
var out *ecs.DescribeClustersOutput
err = resource.Retry(5*time.Minute, func() *resource.RetryError {
log.Printf("[DEBUG] Checking if ECS Cluster %q is INACTIVE", d.Id())
out, err := conn.DescribeClusters(&ecs.DescribeClustersInput{
Clusters: []*string{aws.String(clusterName)},
})

for _, c := range out.Clusters {
if *c.ClusterName == clusterName {
if *c.Status == "INACTIVE" {
return nil
}

return resource.RetryableError(
fmt.Errorf("ECS Cluster %q is still %q", clusterName, *c.Status))
}
}
out, err = conn.DescribeClusters(dcInput)

if err != nil {
return resource.NonRetryableError(err)
}
if !ecsClusterInactive(out, clusterName) {
return resource.RetryableError(fmt.Errorf("ECS Cluster %q is not inactive", clusterName))
}

return nil
})
if isResourceTimeoutError(err) {
out, err = conn.DescribeClusters(dcInput)
if err != nil {
return fmt.Errorf("Error waiting for ECS cluster to become inactive: %s", err)
}
if !ecsClusterInactive(out, clusterName) {
return fmt.Errorf("ECS Cluster %q is still not inactive", clusterName)
}
}
if err != nil {
return err
return fmt.Errorf("Error waiting for ECS cluster to become inactive: %s", err)
}

log.Printf("[DEBUG] ECS cluster %q deleted", d.Id())
return nil
}

func ecsClusterInactive(out *ecs.DescribeClustersOutput, clusterName string) bool {
for _, c := range out.Clusters {
if aws.StringValue(c.ClusterName) == clusterName {
if *c.Status == "INACTIVE" {
return true
}
}
}
return false
}
26 changes: 19 additions & 7 deletions aws/resource_aws_ecs_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,9 @@ func resourceAwsEcsServiceCreate(d *schema.ResourceData, meta interface{}) error

return nil
})
if isResourceTimeoutError(err) {
out, err = conn.CreateService(&input)
}
if err != nil {
return fmt.Errorf("%s %q", err, d.Get("name").(string))
}
Expand Down Expand Up @@ -535,11 +538,17 @@ func resourceAwsEcsServiceRead(d *schema.ResourceData, meta interface{}) error {

return nil
})
if isResourceTimeoutError(err) {
out, err = conn.DescribeServices(&input)
}
if err != nil {
return err
return fmt.Errorf("Error reading ECS service: %s", err)
}

if len(out.Services) < 1 {
if d.IsNewResource() {
return fmt.Errorf("ECS service not created: %q", d.Id())
}
log.Printf("[WARN] Removing ECS service %s (%s) because it's gone", d.Get("name").(string), d.Id())
d.SetId("")
return nil
Expand Down Expand Up @@ -837,7 +846,7 @@ func resourceAwsEcsServiceUpdate(d *schema.ResourceData, meta interface{}) error
log.Printf("[DEBUG] Updating ECS Service (%s): %s", d.Id(), input)
// Retry due to IAM eventual consistency
err := resource.Retry(2*time.Minute, func() *resource.RetryError {
out, err := conn.UpdateService(&input)
_, err := conn.UpdateService(&input)
if err != nil {
if isAWSErr(err, ecs.ErrCodeInvalidParameterException, "Please verify that the ECS service role being passed has the proper permissions.") {
return resource.RetryableError(err)
Expand All @@ -847,12 +856,13 @@ func resourceAwsEcsServiceUpdate(d *schema.ResourceData, meta interface{}) error
}
return resource.NonRetryableError(err)
}

log.Printf("[DEBUG] Updated ECS service %s", out.Service)
return nil
})
if isResourceTimeoutError(err) {
_, err = conn.UpdateService(&input)
}
if err != nil {
return fmt.Errorf("error updating ECS Service (%s): %s", d.Id(), err)
return fmt.Errorf("Error updating ECS Service (%s): %s", d.Id(), err)
}
}

Expand Down Expand Up @@ -951,9 +961,11 @@ func resourceAwsEcsServiceDelete(d *schema.ResourceData, meta interface{}) error
}
return nil
})

if isResourceTimeoutError(err) {
_, err = conn.DeleteService(&input)
}
if err != nil {
return err
return fmt.Errorf("Error deleting ECS service: %s", err)
}

// Wait until it's deleted
Expand Down

0 comments on commit 9303676

Please sign in to comment.