From 26abd932f5ee28e4cea249bb15dcc333019c0b67 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Mon, 4 Mar 2019 17:45:25 -0500 Subject: [PATCH] resource/aws_cloudfront_distribution: Allow final creation and update timeout retry for AWS Go SDK retries Reference: * https://github.com/terraform-providers/terraform-provider-aws/issues/6197 When using `resource.Retry()` for handling eventual consistency, it will timebox the inner function to the configured timeout, which we generally set to a minute or two. The AWS Go SDK, when it encounters recoverable conditions such as 5XX errors or throttling errors, will automatically retry within itself up to the configured session `MaxRetries` (Terraform AWS Provider `max_retries` configuration) before returning to the calling code. For heavily utilized AWS accounts, the throttling errors will cause the outer timeout, which does not give the resource the opportunity to keep retrying outside the timebox. Here we implement this final retry by checking for timeout error from `resource.Retry()` outside the timeboxing, so the AWS Go SDK can return the proper error messaging in these situations or (hopefully) finally succeed in the case of throttling. Since this error handling condition would require extraneous amounts of resources to only potentially trigger the handling, we do not generally implement covering acceptance testing for this code, but it may be a good candidate for special Terraform AWS Provider handling within a future planned Terraform Provider linting tool. Output from acceptance testing: ``` --- PASS: TestAccAWSCloudFrontDistribution_Origin_EmptyOriginID (2.08s) --- PASS: TestAccAWSCloudFrontDistribution_Origin_EmptyDomainName (2.08s) --- PASS: TestAccAWSCloudFrontDistribution_ViewerCertificate_AcmCertificateArn (1821.71s) --- PASS: TestAccAWSCloudFrontDistribution_ViewerCertificate_AcmCertificateArn_ConflictsWithCloudFrontDefaultCertificate (1821.72s) --- PASS: TestAccAWSCloudFrontDistribution_noCustomErrorResponseConfig (2086.99s) --- PASS: TestAccAWSCloudFrontDistribution_orderedCacheBehavior (2090.63s) --- PASS: TestAccAWSCloudFrontDistribution_HTTP11Config (2092.43s) --- PASS: TestAccAWSCloudFrontDistribution_noOptionalItemsConfig (2092.72s) --- PASS: TestAccAWSCloudFrontDistribution_IsIPV6EnabledConfig (2097.43s) --- PASS: TestAccAWSCloudFrontDistribution_S3Origin (2277.83s) --- PASS: TestAccAWSCloudFrontDistribution_multiOrigin (2280.49s) --- PASS: TestAccAWSCloudFrontDistribution_customOrigin (2282.05s) --- PASS: TestAccAWSCloudFrontDistribution_S3OriginWithTags (3345.90s) ``` --- aws/resource_aws_cloudfront_distribution.go | 38 +++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/aws/resource_aws_cloudfront_distribution.go b/aws/resource_aws_cloudfront_distribution.go index 958b24eb7bb..81678a104f4 100644 --- a/aws/resource_aws_cloudfront_distribution.go +++ b/aws/resource_aws_cloudfront_distribution.go @@ -699,16 +699,25 @@ func resourceAwsCloudFrontDistributionCreate(d *schema.ResourceData, meta interf err := resource.Retry(1*time.Minute, func() *resource.RetryError { var err error resp, err = conn.CreateDistributionWithTags(params) + + // ACM and IAM certificate eventual consistency + // InvalidViewerCertificate: The specified SSL certificate doesn't exist, isn't in us-east-1 region, isn't valid, or doesn't include a valid certificate chain. + if isAWSErr(err, cloudfront.ErrCodeInvalidViewerCertificate, "") { + return resource.RetryableError(err) + } + if err != nil { - // ACM and IAM certificate eventual consistency - // InvalidViewerCertificate: The specified SSL certificate doesn't exist, isn't in us-east-1 region, isn't valid, or doesn't include a valid certificate chain. - if isAWSErr(err, cloudfront.ErrCodeInvalidViewerCertificate, "") { - return resource.RetryableError(err) - } return resource.NonRetryableError(err) } + return nil }) + + // Propagate AWS Go SDK retried error, if any + if isResourceTimeoutError(err) { + resp, err = conn.CreateDistributionWithTags(params) + } + if err != nil { return fmt.Errorf("error creating CloudFront Distribution: %s", err) } @@ -785,16 +794,25 @@ func resourceAwsCloudFrontDistributionUpdate(d *schema.ResourceData, meta interf // Handle eventual consistency issues err := resource.Retry(1*time.Minute, func() *resource.RetryError { _, err := conn.UpdateDistribution(params) + + // ACM and IAM certificate eventual consistency + // InvalidViewerCertificate: The specified SSL certificate doesn't exist, isn't in us-east-1 region, isn't valid, or doesn't include a valid certificate chain. + if isAWSErr(err, cloudfront.ErrCodeInvalidViewerCertificate, "") { + return resource.RetryableError(err) + } + if err != nil { - // ACM and IAM certificate eventual consistency - // InvalidViewerCertificate: The specified SSL certificate doesn't exist, isn't in us-east-1 region, isn't valid, or doesn't include a valid certificate chain. - if isAWSErr(err, cloudfront.ErrCodeInvalidViewerCertificate, "") { - return resource.RetryableError(err) - } return resource.NonRetryableError(err) } + return nil }) + + // Propagate AWS Go SDK retried error, if any + if isResourceTimeoutError(err) { + _, err = conn.UpdateDistribution(params) + } + if err != nil { return fmt.Errorf("error updating CloudFront Distribution (%s): %s", d.Id(), err) }