Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleaning up some resource retries for cloudwatch functions #9065

Merged
merged 6 commits into from
Jul 17, 2019
Merged

Cleaning up some resource retries for cloudwatch functions #9065

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cba76d0
Cleaning up some resource retries for cloudwatch functions
ryndaniels Jun 20, 2019
5184446
Clean up the old error handling
ryndaniels Jun 20, 2019
86263d2
Update aws/resource_aws_cloudwatch_log_destination.go
ryndaniels Jun 21, 2019
98e09fd
changes to AWSErr handling from code review
ryndaniels Jun 21, 2019
c823a73
Update event permission read based on code review
ryndaniels Jul 5, 2019
f6e475b
Appease the linting gods
ryndaniels Jul 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 27 additions & 11 deletions aws/resource_aws_cloudwatch_event_permission.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,38 +103,54 @@ func resourceAwsCloudWatchEventPermissionRead(d *schema.ResourceData, meta inter
var policyStatement *CloudWatchEventPermissionPolicyStatement

// Especially with concurrent PutPermission calls there can be a slight delay
err := resource.Retry(1*time.Minute, func() *resource.RetryError {
var out *events.DescribeEventBusOutput
var err error
err = resource.Retry(1*time.Minute, func() *resource.RetryError {
log.Printf("[DEBUG] Reading CloudWatch Events bus: %s", input)
debo, err := conn.DescribeEventBus(&input)
out, err = conn.DescribeEventBus(&input)

if err != nil {
return resource.NonRetryableError(fmt.Errorf("Reading CloudWatch Events permission '%s' failed: %s", d.Id(), err.Error()))
}

if debo.Policy == nil {
if out.Policy == nil {
return resource.RetryableError(&resource.NotFoundError{
Message: fmt.Sprintf("CloudWatch Events permission %q not found"+
"in given results from DescribeEventBus", d.Id()),
LastResponse: debo,
LastResponse: out,
LastRequest: input,
})
}
return nil
})
if isResourceTimeoutError(err) {
out, err = conn.DescribeEventBus(&input)
}
if err != nil {
return fmt.Errorf("Reading CloudWatch Events permission '%s' failed: %s", d.Id(), err.Error())
}

err = json.Unmarshal([]byte(*debo.Policy), &policyDoc)
if err != nil {
return resource.NonRetryableError(fmt.Errorf("Reading CloudWatch Events permission '%s' failed: %s", d.Id(), err.Error()))
}
err = json.Unmarshal([]byte(*out.Policy), &policyDoc)
if err != nil {
return fmt.Errorf("Reading CloudWatch Events permission '%s' failed: %s", d.Id(), err.Error())
}

err = resource.Retry(1*time.Minute, func() *resource.RetryError {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure how I missed this the first lookthrough, but the addition of this new resource.Retry() is extraneous here and will never pass after retries. The original intention of the retry logic was to treat completely a missing policy and a policy missing the new statement as the same for retries. This logic should all be contained in one big retry function as it was originally or split into its own function so the isResourceTimeoutError() implementation is easier. 👍

policyStatement, err = findCloudWatchEventPermissionPolicyStatementByID(&policyDoc, d.Id())
return resource.RetryableError(err)
if err != nil {
return resource.RetryableError(err)
}
return nil
})
if isResourceTimeoutError(err) {
policyStatement, err = findCloudWatchEventPermissionPolicyStatementByID(&policyDoc, d.Id())
}
if err != nil {
// Missing statement inside valid policy
if nfErr, ok := err.(*resource.NotFoundError); ok {
log.Printf("[WARN] %s", nfErr)
d.SetId("")
return nil
}

return err
}

Expand Down
36 changes: 19 additions & 17 deletions aws/resource_aws_cloudwatch_event_rule.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package aws
import (
"fmt"
"log"
"regexp"
"time"

"github.com/aws/aws-sdk-go/aws"
Expand Down Expand Up @@ -99,22 +98,23 @@ func resourceAwsCloudWatchEventRuleCreate(d *schema.ResourceData, meta interface
// IAM Roles take some time to propagate
var out *events.PutRuleOutput
err = resource.Retry(30*time.Second, func() *resource.RetryError {
var err error
out, err = conn.PutRule(input)
pattern := regexp.MustCompile(`cannot be assumed by principal '[a-z]+\.amazonaws\.com'\.$`)

if isAWSErr(err, "ValidationException", "cannot be assumed by principal") {
log.Printf("[DEBUG] Retrying update of CloudWatch Event Rule %q", *input.Name)
return resource.RetryableError(err)
}
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == "ValidationException" && pattern.MatchString(awsErr.Message()) {
log.Printf("[DEBUG] Retrying creation of CloudWatch Event Rule %q", *input.Name)
return resource.RetryableError(err)
}
}
return resource.NonRetryableError(err)
}
return nil
})
if isResourceTimeoutError(err) {
_, err = conn.PutRule(input)
}

if err != nil {
return fmt.Errorf("Creating CloudWatch Event Rule failed: %s", err)
return fmt.Errorf("Updating CloudWatch Event Rule failed: %s", err)
}

d.Set("arn", out.RuleArn)
Expand Down Expand Up @@ -197,18 +197,20 @@ func resourceAwsCloudWatchEventRuleUpdate(d *schema.ResourceData, meta interface
// IAM Roles take some time to propagate
err = resource.Retry(30*time.Second, func() *resource.RetryError {
_, err := conn.PutRule(input)
pattern := regexp.MustCompile(`cannot be assumed by principal '[a-z]+\.amazonaws\.com'\.$`)

if isAWSErr(err, "ValidationException", "cannot be assumed by principal") {
log.Printf("[DEBUG] Retrying update of CloudWatch Event Rule %q", *input.Name)
return resource.RetryableError(err)
}
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == "ValidationException" && pattern.MatchString(awsErr.Message()) {
log.Printf("[DEBUG] Retrying update of CloudWatch Event Rule %q", *input.Name)
return resource.RetryableError(err)
}
}
return resource.NonRetryableError(err)
}
return nil
})
if isResourceTimeoutError(err) {
_, err = conn.PutRule(input)
}

if err != nil {
return fmt.Errorf("Updating CloudWatch Event Rule failed: %s", err)
}
Expand Down
40 changes: 22 additions & 18 deletions aws/resource_aws_cloudwatch_log_destination.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,9 @@ package aws

import (
"fmt"
"strings"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/helper/schema"
Expand Down Expand Up @@ -64,28 +62,34 @@ func resourceAwsCloudWatchLogDestinationPut(d *schema.ResourceData, meta interfa
TargetArn: aws.String(target_arn),
}

return resource.Retry(3*time.Minute, func() *resource.RetryError {
resp, err := conn.PutDestination(params)
var resp *cloudwatchlogs.PutDestinationOutput
var err error
err = resource.Retry(3*time.Minute, func() *resource.RetryError {
resp, err = conn.PutDestination(params)

if err == nil {
d.SetId(name)
d.Set("arn", *resp.Destination.Arn)
}

awsErr, ok := err.(awserr.Error)
if !ok {
if isAWSErr(err, cloudwatchlogs.ErrCodeInvalidParameterException, "Could not deliver test message to specified") {
return resource.RetryableError(err)
}

if awsErr.Code() == "InvalidParameterException" {
if strings.Contains(awsErr.Message(), "Could not deliver test message to specified") {
return resource.RetryableError(err)
}
if isAWSErr(err, cloudwatchlogs.ErrCodeInvalidParameterException, "") {
ryndaniels marked this conversation as resolved.
Show resolved Hide resolved
return resource.NonRetryableError(err)
}

return resource.NonRetryableError(err)
if isAWSErr(err, "", "") {
return resource.NonRetryableError(err)
}
if err != nil {
return resource.RetryableError(err)
}
return nil
})
if isResourceTimeoutError(err) {
resp, err = conn.PutDestination(params)
}
if err != nil {
return fmt.Errorf("Error putting cloudwatch log destination: %s", err)
}
d.SetId(name)
d.Set("arn", *resp.Destination.Arn)
ryndaniels marked this conversation as resolved.
Show resolved Hide resolved
return nil
}

func resourceAwsCloudWatchLogDestinationRead(d *schema.ResourceData, meta interface{}) error {
Expand Down
41 changes: 22 additions & 19 deletions aws/resource_aws_cloudwatch_log_subscription_filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,32 +64,35 @@ func resourceAwsCloudwatchLogSubscriptionFilterCreate(d *schema.ResourceData, me
params := getAwsCloudWatchLogsSubscriptionFilterInput(d)
log.Printf("[DEBUG] Creating SubscriptionFilter %#v", params)

return resource.Retry(5*time.Minute, func() *resource.RetryError {
err := resource.Retry(5*time.Minute, func() *resource.RetryError {
_, err := conn.PutSubscriptionFilter(&params)

if err == nil {
d.SetId(cloudwatchLogsSubscriptionFilterId(d.Get("log_group_name").(string)))
log.Printf("[DEBUG] Cloudwatch logs subscription %q created", d.Id())
if isAWSErr(err, cloudwatchlogs.ErrCodeInvalidParameterException, "Could not deliver test message to specified") {
return resource.RetryableError(err)
}

awsErr, ok := err.(awserr.Error)
if !ok {
if isAWSErr(err, cloudwatchlogs.ErrCodeInvalidParameterException, "Could not execute the lambda function") {
return resource.RetryableError(err)
}

if awsErr.Code() == "InvalidParameterException" {
log.Printf("[DEBUG] Caught message: %q, code: %q: Retrying", awsErr.Message(), awsErr.Code())
if strings.Contains(awsErr.Message(), "Could not deliver test message to specified") {
return resource.RetryableError(err)
}
if strings.Contains(awsErr.Message(), "Could not execute the lambda function") {
return resource.RetryableError(err)
}
resource.NonRetryableError(err)
if isAWSErr(err, cloudwatchlogs.ErrCodeInvalidParameterException, "") {
ryndaniels marked this conversation as resolved.
Show resolved Hide resolved
return resource.NonRetryableError(err)
}

return resource.NonRetryableError(err)
if isAWSErr(err, "", "") {
return resource.NonRetryableError(err)
}
return resource.RetryableError(err)
})

if isResourceTimeoutError(err) {
_, err = conn.PutSubscriptionFilter(&params)
}

if err != nil {
return fmt.Errorf("Error creating Cloudwatch log subscription filter: %s", err)
}

d.SetId(cloudwatchLogsSubscriptionFilterId(d.Get("log_group_name").(string)))
log.Printf("[DEBUG] Cloudwatch logs subscription %q created", d.Id())
return nil
}

func resourceAwsCloudwatchLogSubscriptionFilterUpdate(d *schema.ResourceData, meta interface{}) error {
Expand Down