Skip to content

Commit

Permalink
service/appmesh: Handle read-after-create eventual consistency in res…
Browse files Browse the repository at this point in the history
…ources

Reference: #15084
Reference: #16796
Reference: https://blog.envoyproxy.io/embracing-eventual-consistency-in-soa-networking-32a5ee5d443d

Output from acceptance testing in AWS Commercial:

```
--- FAIL: TestAccAWSAppmesh_serial (1654.08s)
    --- PASS: TestAccAWSAppmesh_serial/VirtualRouter (64.89s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualRouter/basic (27.55s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualRouter/tags (37.34s)
    --- PASS: TestAccAWSAppmesh_serial/VirtualService (102.18s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualService/virtualNode (30.51s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualService/virtualRouter (27.91s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualService/tags (43.76s)
    --- PASS: TestAccAWSAppmesh_serial/GatewayRoute (180.76s)
        --- PASS: TestAccAWSAppmesh_serial/GatewayRoute/httpRoute (31.38s)
        --- PASS: TestAccAWSAppmesh_serial/GatewayRoute/http2Route (30.54s)
        --- PASS: TestAccAWSAppmesh_serial/GatewayRoute/tags (45.18s)
        --- PASS: TestAccAWSAppmesh_serial/GatewayRoute/basic (18.19s)
        --- PASS: TestAccAWSAppmesh_serial/GatewayRoute/disappears (14.95s)
        --- PASS: TestAccAWSAppmesh_serial/GatewayRoute/grpcRoute (40.53s)
    --- PASS: TestAccAWSAppmesh_serial/Mesh (73.44s)
        --- PASS: TestAccAWSAppmesh_serial/Mesh/basic (12.91s)
        --- PASS: TestAccAWSAppmesh_serial/Mesh/egressFilter (29.35s)
        --- PASS: TestAccAWSAppmesh_serial/Mesh/tags (31.18s)
    --- FAIL: TestAccAWSAppmesh_serial/Route (429.26s)
        --- PASS: TestAccAWSAppmesh_serial/Route/httpRoute (43.23s)
        --- PASS: TestAccAWSAppmesh_serial/Route/tcpRoute (42.71s)
        --- PASS: TestAccAWSAppmesh_serial/Route/tcpRouteTimeout (30.21s)
        --- PASS: TestAccAWSAppmesh_serial/Route/grpcRouteEmptyMatch (17.81s)
        --- PASS: TestAccAWSAppmesh_serial/Route/grpcRouteTimeout (31.13s)
        --- PASS: TestAccAWSAppmesh_serial/Route/http2RouteTimeout (31.08s)
        --- PASS: TestAccAWSAppmesh_serial/Route/httpRetryPolicy (32.21s)
        --- PASS: TestAccAWSAppmesh_serial/Route/routePriority (31.77s)
        --- PASS: TestAccAWSAppmesh_serial/Route/tags (44.91s)
        --- PASS: TestAccAWSAppmesh_serial/Route/grpcRoute (54.42s)
        --- PASS: TestAccAWSAppmesh_serial/Route/http2Route (30.70s)
        --- FAIL: TestAccAWSAppmesh_serial/Route/httpHeader (8.24s) # #18528
        --- PASS: TestAccAWSAppmesh_serial/Route/httpRouteTimeout (30.84s)
    --- PASS: TestAccAWSAppmesh_serial/VirtualGateway (285.66s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/listenerConnectionPool (26.22s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/listenerHealthChecks (26.25s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/listenerTls (70.19s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/basic (17.09s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/disappears (12.66s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/backendDefaultsCertificate (15.85s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/tags (38.11s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/backendDefaults (26.15s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/listenerValidation (26.61s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualGateway/logging (26.53s)
    --- PASS: TestAccAWSAppmesh_serial/VirtualNode (517.90s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/listenerOutlierDetection (26.74s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/listenerTimeout (26.37s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/tags (38.61s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/disappears (12.41s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/backendClientPolicyFile (26.83s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/cloudMapServiceDiscovery (99.43s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/basic (15.51s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/backendClientPolicyAcm (50.89s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/listenerTls (69.86s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/backendDefaults (27.28s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/backendDefaultsCertificate (15.92s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/listenerValidation (26.70s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/listenerConnectionPool (28.32s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/listenerHealthChecks (26.63s)
        --- PASS: TestAccAWSAppmesh_serial/VirtualNode/logging (26.40s)
```
  • Loading branch information
bflad committed Apr 1, 2021
1 parent bed1dec commit 5575fb9
Show file tree
Hide file tree
Showing 9 changed files with 295 additions and 38 deletions.
1 change: 0 additions & 1 deletion .semgrep.yml
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,6 @@ rules:
- aws/resource_aws_apigatewayv2_*.go
- aws/resource_aws_app_cookie_stickiness_policy.go
- aws/resource_aws_appautoscaling_*.go
- aws/resource_aws_appmesh_*.go
- aws/resource_aws_appsync_*.go
- aws/resource_aws_athena_*.go
- aws/resource_aws_autoscaling_*.go
Expand Down
10 changes: 10 additions & 0 deletions aws/internal/service/appmesh/waiter/waiter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package waiter

import (
"time"
)

const (
// Maximum amount of time to wait for Appmesh changes to propagate
PropagationTimeout = 2 * time.Minute
)
50 changes: 44 additions & 6 deletions aws/resource_aws_appmesh_gateway_route.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,14 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/appmesh"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/appmesh/finder"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/appmesh/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsAppmeshGatewayRoute() *schema.Resource {
Expand Down Expand Up @@ -320,20 +324,54 @@ func resourceAwsAppmeshGatewayRouteRead(d *schema.ResourceData, meta interface{}
conn := meta.(*AWSClient).appmeshconn
ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig

gatewayRoute, err := finder.GatewayRoute(conn, d.Get("mesh_name").(string), d.Get("virtual_gateway_name").(string), d.Get("name").(string), d.Get("mesh_owner").(string))
var gatewayRoute *appmesh.GatewayRouteData

if isAWSErr(err, appmesh.ErrCodeNotFoundException, "") {
log.Printf("[WARN] App Mesh gateway route (%s) not found, removing from state", d.Id())
err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

gatewayRoute, err = finder.GatewayRoute(conn, d.Get("mesh_name").(string), d.Get("virtual_gateway_name").(string), d.Get("name").(string), d.Get("mesh_owner").(string))

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}

return nil
})

if tfresource.TimedOut(err) {
gatewayRoute, err = finder.GatewayRoute(conn, d.Get("mesh_name").(string), d.Get("virtual_gateway_name").(string), d.Get("name").(string), d.Get("mesh_owner").(string))
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
log.Printf("[WARN] App Mesh Gateway Route (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading App Mesh gateway route: %w", err)
return fmt.Errorf("error reading App Mesh Gateway Route: %w", err)
}

if gatewayRoute == nil {
if d.IsNewResource() {
return fmt.Errorf("error reading App Mesh Gateway Route: not found after creation")
}

log.Printf("[WARN] App Mesh Gateway Route (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if gatewayRoute == nil || aws.StringValue(gatewayRoute.Status.Status) == appmesh.GatewayRouteStatusCodeDeleted {
log.Printf("[WARN] App Mesh gateway route (%s) not found, removing from state", d.Id())
if aws.StringValue(gatewayRoute.Status.Status) == appmesh.GatewayRouteStatusCodeDeleted {
if d.IsNewResource() {
return fmt.Errorf("error reading App Mesh Gateway Route: %s after creation", aws.StringValue(gatewayRoute.Status.Status))
}

log.Printf("[WARN] App Mesh Gateway Route (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}
Expand Down
45 changes: 40 additions & 5 deletions aws/resource_aws_appmesh_mesh.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/appmesh"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/appmesh/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsAppmeshMesh() *schema.Resource {
Expand Down Expand Up @@ -123,17 +127,48 @@ func resourceAwsAppmeshMeshRead(d *schema.ResourceData, meta interface{}) error
req.MeshOwner = aws.String(v.(string))
}

resp, err := conn.DescribeMesh(req)
if isAWSErr(err, appmesh.ErrCodeNotFoundException, "") {
log.Printf("[WARN] App Mesh service mesh (%s) not found, removing from state", d.Id())
var resp *appmesh.DescribeMeshOutput

err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

resp, err = conn.DescribeMesh(req)

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}

return nil
})

if tfresource.TimedOut(err) {
resp, err = conn.DescribeMesh(req)
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
log.Printf("[WARN] App Mesh Service Mesh (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading App Mesh service mesh: %s", err)
return fmt.Errorf("error reading App Mesh Service Mesh: %w", err)
}

if resp == nil || resp.Mesh == nil {
return fmt.Errorf("error reading App Mesh Service Mesh: empty response")
}

if aws.StringValue(resp.Mesh.Status.Status) == appmesh.MeshStatusCodeDeleted {
log.Printf("[WARN] App Mesh service mesh (%s) not found, removing from state", d.Id())
if d.IsNewResource() {
return fmt.Errorf("error reading App Mesh Service Mesh: %s after creation", aws.StringValue(resp.Mesh.Status.Status))
}

log.Printf("[WARN] App Mesh Service Mesh (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}
Expand Down
45 changes: 40 additions & 5 deletions aws/resource_aws_appmesh_route.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,13 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/appmesh"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/appmesh/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsAppmeshRoute() *schema.Resource {
Expand Down Expand Up @@ -730,17 +734,48 @@ func resourceAwsAppmeshRouteRead(d *schema.ResourceData, meta interface{}) error
req.MeshOwner = aws.String(v.(string))
}

resp, err := conn.DescribeRoute(req)
if isAWSErr(err, appmesh.ErrCodeNotFoundException, "") {
log.Printf("[WARN] App Mesh route (%s) not found, removing from state", d.Id())
var resp *appmesh.DescribeRouteOutput

err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

resp, err = conn.DescribeRoute(req)

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}

return nil
})

if tfresource.TimedOut(err) {
resp, err = conn.DescribeRoute(req)
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
log.Printf("[WARN] App Mesh Route (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading App Mesh route: %s", err)
return fmt.Errorf("error reading App Mesh Route: %w", err)
}

if resp == nil || resp.Route == nil {
return fmt.Errorf("error reading App Mesh Route: empty response")
}

if aws.StringValue(resp.Route.Status.Status) == appmesh.RouteStatusCodeDeleted {
log.Printf("[WARN] App Mesh route (%s) not found, removing from state", d.Id())
if d.IsNewResource() {
return fmt.Errorf("error reading App Mesh Route: %s after creation", aws.StringValue(resp.Route.Status.Status))
}

log.Printf("[WARN] App Mesh Route (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}
Expand Down
50 changes: 44 additions & 6 deletions aws/resource_aws_appmesh_virtual_gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,14 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/appmesh"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/appmesh/finder"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/appmesh/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsAppmeshVirtualGateway() *schema.Resource {
Expand Down Expand Up @@ -706,20 +710,54 @@ func resourceAwsAppmeshVirtualGatewayRead(d *schema.ResourceData, meta interface
conn := meta.(*AWSClient).appmeshconn
ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig

virtualGateway, err := finder.VirtualGateway(conn, d.Get("mesh_name").(string), d.Get("name").(string), d.Get("mesh_owner").(string))
var virtualGateway *appmesh.VirtualGatewayData

if isAWSErr(err, appmesh.ErrCodeNotFoundException, "") {
log.Printf("[WARN] App Mesh virtual gateway (%s) not found, removing from state", d.Id())
err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

virtualGateway, err = finder.VirtualGateway(conn, d.Get("mesh_name").(string), d.Get("name").(string), d.Get("mesh_owner").(string))

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}

return nil
})

if tfresource.TimedOut(err) {
virtualGateway, err = finder.VirtualGateway(conn, d.Get("mesh_name").(string), d.Get("name").(string), d.Get("mesh_owner").(string))
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
log.Printf("[WARN] App Mesh Virtual Gateway (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading App Mesh virtual gateway (%s): %w", d.Id(), err)
return fmt.Errorf("error reading App Mesh Virtual Gateway: %w", err)
}

if virtualGateway == nil {
if d.IsNewResource() {
return fmt.Errorf("error reading App Mesh Virtual Gateway: not found after creation")
}

log.Printf("[WARN] App Mesh Virtual Gateway (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if virtualGateway == nil || aws.StringValue(virtualGateway.Status.Status) == appmesh.VirtualGatewayStatusCodeDeleted {
log.Printf("[WARN] App Mesh virtual gateway (%s) not found, removing from state", d.Id())
if aws.StringValue(virtualGateway.Status.Status) == appmesh.VirtualGatewayStatusCodeDeleted {
if d.IsNewResource() {
return fmt.Errorf("error reading App Mesh Virtual Gateway: %s after creation", aws.StringValue(virtualGateway.Status.Status))
}

log.Printf("[WARN] App Mesh Virtual Gateway (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}
Expand Down
42 changes: 37 additions & 5 deletions aws/resource_aws_appmesh_virtual_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/appmesh"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/appmesh/waiter"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
)

func resourceAwsAppmeshVirtualNode() *schema.Resource {
Expand Down Expand Up @@ -1086,20 +1090,48 @@ func resourceAwsAppmeshVirtualNodeRead(d *schema.ResourceData, meta interface{})
req.MeshOwner = aws.String(v.(string))
}

resp, err := conn.DescribeVirtualNode(req)
var resp *appmesh.DescribeVirtualNodeOutput

if isAWSErr(err, appmesh.ErrCodeNotFoundException, "") {
log.Printf("[WARN] App Mesh virtual node (%s) not found, removing from state", d.Id())
err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
var err error

resp, err = conn.DescribeVirtualNode(req)

if d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}

return nil
})

if tfresource.TimedOut(err) {
resp, err = conn.DescribeVirtualNode(req)
}

if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, appmesh.ErrCodeNotFoundException) {
log.Printf("[WARN] App Mesh Virtual Node (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

if err != nil {
return fmt.Errorf("error reading App Mesh virtual node (%s): %w", d.Id(), err)
return fmt.Errorf("error reading App Mesh Virtual Node: %w", err)
}

if resp == nil || resp.VirtualNode == nil {
return fmt.Errorf("error reading App Mesh Virtual Node: empty response")
}

if aws.StringValue(resp.VirtualNode.Status.Status) == appmesh.VirtualNodeStatusCodeDeleted {
log.Printf("[WARN] App Mesh virtual node (%s) not found, removing from state", d.Id())
if d.IsNewResource() {
return fmt.Errorf("error reading App Mesh Virtual Node: %s after creation", aws.StringValue(resp.VirtualNode.Status.Status))
}

log.Printf("[WARN] App Mesh Virtual Node (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}
Expand Down
Loading

0 comments on commit 5575fb9

Please sign in to comment.