Skip to content

Commit

Permalink
service/secretsmanager: Handle read-after-create eventual consistency…
Browse files Browse the repository at this point in the history
… in resources

Reference: #16482
Reference: #16796

Output from acceptance testing in AWS Commercial:

```
--- FAIL: TestAccAwsSecretsManagerSecret_policy (40.13s) # #18461
--- PASS: TestAccAwsSecretsManagerSecret_basic (19.14s)
--- PASS: TestAccAwsSecretsManagerSecret_Description (28.90s)
--- PASS: TestAccAwsSecretsManagerSecret_KmsKeyID (36.04s)
--- PASS: TestAccAwsSecretsManagerSecret_RecoveryWindowInDays_Recreate (27.96s)
--- PASS: TestAccAwsSecretsManagerSecret_RotationLambdaARN (103.44s)
--- PASS: TestAccAwsSecretsManagerSecret_RotationRules (55.08s)
--- PASS: TestAccAwsSecretsManagerSecret_Tags (48.81s)
--- PASS: TestAccAwsSecretsManagerSecret_withNamePrefix (19.14s)

--- PASS: TestAccAwsSecretsManagerSecretPolicy_basic (46.93s)
--- PASS: TestAccAwsSecretsManagerSecretPolicy_blockPublicPolicy (49.17s)
--- PASS: TestAccAwsSecretsManagerSecretPolicy_disappears (27.04s)

--- PASS: TestAccAwsSecretsManagerSecretRotation_basic (54.74s)

--- PASS: TestAccAwsSecretsManagerSecretVersion_Base64Binary (20.22s)
--- PASS: TestAccAwsSecretsManagerSecretVersion_BasicString (19.64s)
--- PASS: TestAccAwsSecretsManagerSecretVersion_VersionStages (42.20s)
```

Output from acceptance testing in AWS GovCloud (US):

```
--- FAIL: TestAccAwsSecretsManagerSecret_policy (41.26s) # #18461
--- PASS: TestAccAwsSecretsManagerSecret_basic (25.60s)
--- PASS: TestAccAwsSecretsManagerSecret_Description (37.53s)
--- PASS: TestAccAwsSecretsManagerSecret_KmsKeyID (45.10s)
--- PASS: TestAccAwsSecretsManagerSecret_RecoveryWindowInDays_Recreate (37.67s)
--- PASS: TestAccAwsSecretsManagerSecret_RotationLambdaARN (70.43s)
--- PASS: TestAccAwsSecretsManagerSecret_RotationRules (58.87s)
--- PASS: TestAccAwsSecretsManagerSecret_Tags (64.49s)
--- PASS: TestAccAwsSecretsManagerSecret_withNamePrefix (23.75s)

--- PASS: TestAccAwsSecretsManagerSecretPolicy_basic (49.66s)
--- PASS: TestAccAwsSecretsManagerSecretPolicy_blockPublicPolicy (64.81s)
--- PASS: TestAccAwsSecretsManagerSecretPolicy_disappears (29.34s)

--- PASS: TestAccAwsSecretsManagerSecretRotation_basic (68.89s)

--- PASS: TestAccAwsSecretsManagerSecretVersion_Base64Binary (21.89s)
--- PASS: TestAccAwsSecretsManagerSecretVersion_BasicString (24.62s)
--- PASS: TestAccAwsSecretsManagerSecretVersion_VersionStages (54.57s)
```
  • Loading branch information
bflad committed Mar 29, 2021
1 parent d0fb8dd commit b96df1d
Show file tree
Hide file tree
Showing 8 changed files with 174 additions and 41 deletions.
15 changes: 15 additions & 0 deletions .changelog/18462.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
```release-note:bug
resource/aws_secretsmanager_secret: Handle read-after-create eventual consistency
```

```release-note:bug
resource/aws_secretsmanager_secret_policy: Handle read-after-create eventual consistency
```

```release-note:bug
resource/aws_secretsmanager_secret_rotation: Handle read-after-create eventual consistency
```

```release-note:bug
resource/aws_secretsmanager_secret_version: Handle read-after-create eventual consistency
```
4 changes: 2 additions & 2 deletions aws/internal/service/secretsmanager/waiter/waiter.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ import (
)

const (
// Maximum amount of time to wait for Secrets Manager deletions to propagate
DeletionPropagationTimeout = 2 * time.Minute
// Maximum amount of time to wait for Secrets Manager changes to propagate
PropagationTimeout = 2 * time.Minute
)
45 changes: 36 additions & 9 deletions aws/resource_aws_secretsmanager_secret.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/structure"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
iamwaiter "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/iam/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/secretsmanager/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsSecretsManagerSecret() *schema.Resource {
Expand Down Expand Up @@ -131,7 +133,7 @@ func resourceAwsSecretsManagerSecretCreate(d *schema.ResourceData, meta interfac

// Retry for secret recreation after deletion
var output *secretsmanager.CreateSecretOutput
err := resource.Retry(waiter.DeletionPropagationTimeout, func() *resource.RetryError {
err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error
output, err = conn.CreateSecret(input)
// Temporarily retry on these errors to support immediate secret recreation:
Expand Down Expand Up @@ -218,15 +220,40 @@ func resourceAwsSecretsManagerSecretRead(d *schema.ResourceData, meta interface{
SecretId: aws.String(d.Id()),
}

log.Printf("[DEBUG] Reading Secrets Manager Secret: %s", input)
output, err := conn.DescribeSecret(input)
if err != nil {
if isAWSErr(err, secretsmanager.ErrCodeResourceNotFoundException, "") {
log.Printf("[WARN] Secrets Manager Secret %q not found - removing from state", d.Id())
d.SetId("")
return nil
var output *secretsmanager.DescribeSecretOutput

err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

output, err = conn.DescribeSecret(input)

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}
return fmt.Errorf("error reading Secrets Manager Secret: %w", err)

return nil
})

if tfresource.TimedOut(err) {
output, err = conn.DescribeSecret(input)
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
log.Printf("[WARN] Secrets Manager Secret (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading Secrets Manager Secret (%s): %w", d.Id(), err)
}

if output == nil {
return fmt.Errorf("error reading Secrets Manager Secret (%s): empty response", d.Id())
}

d.Set("arn", output.ARN)
Expand Down
45 changes: 37 additions & 8 deletions aws/resource_aws_secretsmanager_secret_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/structure"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
iamwaiter "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/iam/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/secretsmanager/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsSecretsManagerSecretPolicy() *schema.Resource {
Expand Down Expand Up @@ -89,15 +92,41 @@ func resourceAwsSecretsManagerSecretPolicyRead(d *schema.ResourceData, meta inte
input := &secretsmanager.GetResourcePolicyInput{
SecretId: aws.String(d.Id()),
}
log.Printf("[DEBUG] Reading Secrets Manager Secret Policy: %#v", input)
res, err := conn.GetResourcePolicy(input)
if err != nil {
if isAWSErr(err, secretsmanager.ErrCodeResourceNotFoundException, "") {
log.Printf("[WARN] SecretsManager Secret Policy (%s) not found, removing from state", d.Id())
d.SetId("")
return nil

var res *secretsmanager.GetResourcePolicyOutput

err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

res, err = conn.GetResourcePolicy(input)

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}
return fmt.Errorf("error reading Secrets Manager Secret policy: %w", err)

return nil
})

if tfresource.TimedOut(err) {
res, err = conn.GetResourcePolicy(input)
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
log.Printf("[WARN] Secrets Manager Secret Policy (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading Secrets Manager Secret Policy (%s): %w", d.Id(), err)
}

if res == nil {
return fmt.Errorf("error reading Secrets Manager Secret Policy (%s): empty response", d.Id())
}

if res.ResourcePolicy != nil {
Expand Down
2 changes: 1 addition & 1 deletion aws/resource_aws_secretsmanager_secret_policy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ func testAccCheckAwsSecretsManagerSecretPolicyDestroy(s *terraform.State) error

var output *secretsmanager.DescribeSecretOutput

err := resource.Retry(waiter.DeletionPropagationTimeout, func() *resource.RetryError {
err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error
output, err = conn.DescribeSecret(secretInput)

Expand Down
44 changes: 36 additions & 8 deletions aws/resource_aws_secretsmanager_secret_rotation.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/secretsmanager/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsSecretsManagerSecretRotation() *schema.Resource {
Expand Down Expand Up @@ -99,15 +102,40 @@ func resourceAwsSecretsManagerSecretRotationRead(d *schema.ResourceData, meta in
SecretId: aws.String(d.Id()),
}

log.Printf("[DEBUG] Reading Secrets Manager Secret Rotation: %s", input)
output, err := conn.DescribeSecret(input)
if err != nil {
if isAWSErr(err, secretsmanager.ErrCodeResourceNotFoundException, "") {
log.Printf("[WARN] Secrets Manager Secret Rotation %q not found - removing from state", d.Id())
d.SetId("")
return nil
var output *secretsmanager.DescribeSecretOutput

err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

output, err = conn.DescribeSecret(input)

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
return resource.RetryableError(err)
}
return fmt.Errorf("error reading Secrets Manager Secret Rotation: %s", err)

if err != nil {
return resource.NonRetryableError(err)
}

return nil
})

if tfresource.TimedOut(err) {
output, err = conn.DescribeSecret(input)
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
log.Printf("[WARN] Secrets Manager Secret Rotation (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading Secrets Manager Secret Rotation (%s): %w", d.Id(), err)
}

if output == nil {
return fmt.Errorf("error reading Secrets Manager Secret Rotation (%s): empty response", d.Id())
}

d.Set("secret_id", d.Id())
Expand Down
2 changes: 1 addition & 1 deletion aws/resource_aws_secretsmanager_secret_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ func testAccCheckAwsSecretsManagerSecretDestroy(s *terraform.State) error {

var output *secretsmanager.DescribeSecretOutput

err := resource.Retry(waiter.DeletionPropagationTimeout, func() *resource.RetryError {
err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error
output, err = conn.DescribeSecret(input)

Expand Down
58 changes: 46 additions & 12 deletions aws/resource_aws_secretsmanager_secret_version.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/secretsmanager/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsSecretsManagerSecretVersion() *schema.Resource {
Expand Down Expand Up @@ -114,20 +118,50 @@ func resourceAwsSecretsManagerSecretVersionRead(d *schema.ResourceData, meta int
VersionId: aws.String(versionID),
}

log.Printf("[DEBUG] Reading Secrets Manager Secret Version: %s", input)
output, err := conn.GetSecretValue(input)
if err != nil {
if isAWSErr(err, secretsmanager.ErrCodeResourceNotFoundException, "") {
log.Printf("[WARN] Secrets Manager Secret Version %q not found - removing from state", d.Id())
d.SetId("")
return nil
var output *secretsmanager.GetSecretValueOutput

err = resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

output, err = conn.GetSecretValue(input)

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
return resource.RetryableError(err)
}
if isAWSErr(err, secretsmanager.ErrCodeInvalidRequestException, "You can’t perform this operation on the secret because it was deleted") {
log.Printf("[WARN] Secrets Manager Secret Version %q not found - removing from state", d.Id())
d.SetId("")
return nil

if d.IsNewResource() && tfawserr.ErrMessageContains(err, secretsmanager.ErrCodeInvalidRequestException, "You can’t perform this operation on the secret because it was deleted") {
return resource.RetryableError(err)
}
return fmt.Errorf("error reading Secrets Manager Secret Version: %s", err)

if err != nil {
return resource.NonRetryableError(err)
}

return nil
})

if tfresource.TimedOut(err) {
output, err = conn.GetSecretValue(input)
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, secretsmanager.ErrCodeResourceNotFoundException) {
log.Printf("[WARN] Secrets Manager Secret Version (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if !d.IsNewResource() && tfawserr.ErrMessageContains(err, secretsmanager.ErrCodeInvalidRequestException, "You can’t perform this operation on the secret because it was deleted") {
log.Printf("[WARN] Secrets Manager Secret Version (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading Secrets Manager Secret Version (%s): %w", d.Id(), err)
}

if output == nil {
return fmt.Errorf("error reading Secrets Manager Secret Version (%s): empty response", d.Id())
}

d.Set("secret_id", secretID)
Expand Down

0 comments on commit b96df1d

Please sign in to comment.