Merge pull request #362 from hashicorp/b-cut-expired-creds-retry

Avoid retries on expired credentials
hashicorp · Feb 22, 2023 · 1937c18 · 1937c18
2 parents 87c729d + c4d0ab0
commit 1937c18
Show file tree

Hide file tree

Showing 6 changed files with 432 additions and 4 deletions.
diff --git a/aws_config.go b/aws_config.go
@@ -96,7 +96,7 @@ func GetAwsConfig(ctx context.Context, c *Config) (context.Context, aws.Config,
 
 	if !c.SkipCredsValidation {
 		if _, _, err := getAccountIDAndPartitionFromSTSGetCallerIdentity(baseCtx, stsClient(baseCtx, awsConfig, c)); err != nil {
-			return ctx, awsConfig, fmt.Errorf("error validating provider credentials: %w", err)
+			return ctx, awsConfig, fmt.Errorf("validating provider credentials: %w", err)
 		}
 	}
 

diff --git a/aws_config_test.go b/aws_config_test.go
@@ -22,6 +22,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"github.com/aws/smithy-go"
 	"github.com/aws/smithy-go/middleware"
 	smithyhttp "github.com/aws/smithy-go/transport/http"
 	"github.com/google/go-cmp/cmp"
@@ -94,6 +95,52 @@ func TestGetAwsConfig(t *testing.T) {
 				servicemocks.MockStsGetCallerIdentityValidEndpoint,
 			},
 		},
+		{
+			Config: &Config{
+				AccessKey:  servicemocks.MockStaticAccessKey,
+				Region:     "us-east-1",
+				SecretKey:  servicemocks.MockStaticSecretKey,
+				MaxRetries: 100,
+			},
+			Description:    "ExpiredToken",
+			ExpectedRegion: "us-east-1",
+			ExpectedError: func(err error) bool {
+				return strings.Contains(err.Error(), "ExpiredToken")
+			},
+			MockStsEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredToken,
+			},
+		},
+		{
+			Config: &Config{
+				AccessKey: servicemocks.MockStaticAccessKey,
+				Region:    "us-east-1",
+				SecretKey: servicemocks.MockStaticSecretKey,
+			},
+			Description:    "ExpiredTokenException",
+			ExpectedRegion: "us-east-1",
+			ExpectedError: func(err error) bool {
+				return strings.Contains(err.Error(), "ExpiredTokenException")
+			},
+			MockStsEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredTokenException,
+			},
+		},
+		{
+			Config: &Config{
+				AccessKey: servicemocks.MockStaticAccessKey,
+				Region:    "us-east-1",
+				SecretKey: servicemocks.MockStaticSecretKey,
+			},
+			Description:    "RequestExpired",
+			ExpectedRegion: "us-east-1",
+			ExpectedError: func(err error) bool {
+				return strings.Contains(err.Error(), "RequestExpired")
+			},
+			MockStsEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityInvalidBodyRequestExpired,
+			},
+		},
 		{
 			Config: &Config{
 				AccessKey: servicemocks.MockStaticAccessKey,
@@ -3043,6 +3090,76 @@ func TestRetryHandlers(t *testing.T) {
 				return results
 			}(),
 		},
+		"no retries for ExpiredToken": {
+			NextHandler: func() middleware.FinalizeHandler {
+				num := 0
+				reqsErrs := make([]error, 2)
+				for i := 0; i < 2; i++ {
+					reqsErrs[i] = &smithy.OperationError{
+						ServiceID:     "STS",
+						OperationName: "GetCallerIdentity",
+						Err: &smithyhttp.ResponseError{
+							Response: &smithyhttp.Response{
+								Response: &http.Response{
+									StatusCode: 403,
+								},
+							},
+							Err: &smithy.GenericAPIError{
+								Code:    "ExpiredToken",
+								Message: "The security token included in the request is expired",
+							},
+						},
+					}
+				}
+				return middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+					if num >= len(reqsErrs) {
+						err = fmt.Errorf("more requests than expected")
+					} else {
+						err = reqsErrs[num]
+						num++
+					}
+					return out, metadata, err
+				})
+			},
+			Err: &smithy.OperationError{
+				ServiceID:     "STS",
+				OperationName: "GetCallerIdentity",
+				Err: &smithyhttp.ResponseError{
+					Response: &smithyhttp.Response{
+						Response: &http.Response{
+							StatusCode: 403,
+						},
+					},
+					Err: &smithy.GenericAPIError{
+						Code:    "ExpiredToken",
+						Message: "The security token included in the request is expired",
+					},
+				},
+			},
+			ExpectResults: func() retry.AttemptResults {
+				results := retry.AttemptResults{
+					Results: make([]retry.AttemptResult, 1),
+				}
+				results.Results[0] = retry.AttemptResult{
+					Err: &smithy.OperationError{
+						ServiceID:     "STS",
+						OperationName: "GetCallerIdentity",
+						Err: &smithyhttp.ResponseError{
+							Response: &smithyhttp.Response{
+								Response: &http.Response{
+									StatusCode: 403,
+								},
+							},
+							Err: &smithy.GenericAPIError{
+								Code:    "ExpiredToken",
+								Message: "The security token included in the request is expired",
+							},
+						},
+					},
+				}
+				return results
+			}(),
+		},
 		"stops at maxRetries for other network errors": {
 			NextHandler: func() middleware.FinalizeHandler {
 				num := 0

diff --git a/awsauth_test.go b/awsauth_test.go
@@ -320,6 +320,48 @@ func TestGetAccountIDAndPartitionFromSTSGetCallerIdentity(t *testing.T) {
 			},
 			ErrCount: 1,
 		},
+		{
+			Description: "sts:GetCallerIdentity ExpiredToken with invalid JSON response",
+			MockEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredToken,
+			},
+			ErrCount: 1,
+		},
+		{
+			Description: "sts:GetCallerIdentity ExpiredToken with valid JSON response",
+			MockEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityValidBodyExpiredToken,
+			},
+			ErrCount: 1,
+		},
+		{
+			Description: "sts:GetCallerIdentity ExpiredTokenException with invalid JSON response",
+			MockEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredTokenException,
+			},
+			ErrCount: 1,
+		},
+		{
+			Description: "sts:GetCallerIdentity ExpiredTokenException with valid JSON response",
+			MockEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityValidBodyExpiredTokenException,
+			},
+			ErrCount: 1,
+		},
+		{
+			Description: "sts:GetCallerIdentity RequestExpired with invalid JSON response",
+			MockEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityInvalidBodyRequestExpired,
+			},
+			ErrCount: 1,
+		},
+		{
+			Description: "sts:GetCallerIdentity RequestExpired with valid JSON response",
+			MockEndpoints: []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityValidBodyRequestExpired,
+			},
+			ErrCount: 1,
+		},
 		{
 			Description: "sts:GetCallerIdentity success",
 			MockEndpoints: []*servicemocks.MockEndpoint{

diff --git a/servicemocks/mock.go b/servicemocks/mock.go
@@ -113,6 +113,72 @@ const (
   <Message>User: arn:aws:iam::123456789012:user/Bob is not authorized to perform: sts:GetCallerIdentity</Message>
 </Error>
 <RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
+</ErrorResponse>`
+	// MockStsGetCallerIdentityValidResponseBodyExpiredToken uses code "ExpiredToken", seemingly the most common
+	// code. Errors usually have an invalid body but this may be fixed at some point.
+	MockStsGetCallerIdentityValidResponseBodyExpiredToken = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+<Error>
+  <Type>Sender</Type>
+  <Code>ExpiredToken</Code>
+  <Message>The security token included in the request is expired</Message>
+</Error>
+<ResponseMetadata>
+  <RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
+</ResponseMetadata>
+</ErrorResponse>`
+	// MockStsGetCallerIdentityInvalidResponseBodyExpiredToken uses code "ExpiredToken", seemingly the most common
+	// code. Errors usually have an invalid body.
+	MockStsGetCallerIdentityInvalidResponseBodyExpiredToken = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+<Error>
+  <Type>Sender</Type>
+  <Code>ExpiredToken</Code>
+  <Message>The security token included in the request is expired</Message>
+</Error>
+<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
+</ErrorResponse>`
+	// MockStsGetCallerIdentityValidResponseBodyExpiredTokenException uses code "ExpiredTokenException", a more rare code
+	// but used at least by Fargate. Errors usually have an invalid body but this may change.
+	MockStsGetCallerIdentityValidResponseBodyExpiredTokenException = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+<Error>
+  <Type>Sender</Type>
+  <Code>ExpiredTokenException</Code>
+  <Message>The security token included in the request is expired</Message>
+</Error>
+<ResponseMetadata>
+  <RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
+</ResponseMetadata>
+</ErrorResponse>`
+	// MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException uses code "ExpiredTokenException", a more rare code
+	// but used at least by Fargate. Errors usually have an invalid body but this may change.
+	MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+<Error>
+  <Type>Sender</Type>
+  <Code>ExpiredTokenException</Code>
+  <Message>The security token included in the request is expired</Message>
+</Error>
+<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
+</ErrorResponse>`
+	// MockStsGetCallerIdentityValidResponseBodyRequestExpired uses code "RequestExpired", a code only used in EC2.
+	// Errors usually have an invalid body but this may change.
+	MockStsGetCallerIdentityValidResponseBodyRequestExpired = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+<Error>
+  <Type>Sender</Type>
+  <Code>RequestExpired</Code>
+  <Message>The security token included in the request is expired</Message>
+</Error>
+<ResponseMetadata>
+  <RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
+</ResponseMetadata>
+</ErrorResponse>`
+	// MockStsGetCallerIdentityInvalidResponseBodyRequestExpired uses code "RequestExpired", a code only used in EC2.
+	// Errors usually have an invalid body but this may change.
+	MockStsGetCallerIdentityInvalidResponseBodyRequestExpired = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+<Error>
+  <Type>Sender</Type>
+  <Code>RequestExpired</Code>
+  <Message>The security token included in the request is expired</Message>
+</Error>
+<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
 </ErrorResponse>`
 	MockStsGetCallerIdentityPartition         = `aws`
 	MockStsGetCallerIdentityValidResponseBody = `<GetCallerIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
@@ -211,6 +277,96 @@ var (
 			StatusCode:  http.StatusForbidden,
 		},
 	}
+	MockStsGetCallerIdentityInvalidBodyExpiredToken = &MockEndpoint{
+		Request: &MockRequest{
+			Body: url.Values{
+				"Action":  []string{"GetCallerIdentity"},
+				"Version": []string{"2011-06-15"},
+			}.Encode(),
+			Method: http.MethodPost,
+			Uri:    "/",
+		},
+		Response: &MockResponse{
+			Body:        MockStsGetCallerIdentityInvalidResponseBodyExpiredToken,
+			ContentType: "text/xml",
+			StatusCode:  http.StatusForbidden,
+		},
+	}
+	MockStsGetCallerIdentityValidBodyExpiredToken = &MockEndpoint{
+		Request: &MockRequest{
+			Body: url.Values{
+				"Action":  []string{"GetCallerIdentity"},
+				"Version": []string{"2011-06-15"},
+			}.Encode(),
+			Method: http.MethodPost,
+			Uri:    "/",
+		},
+		Response: &MockResponse{
+			Body:        MockStsGetCallerIdentityValidResponseBodyExpiredToken,
+			ContentType: "text/xml",
+			StatusCode:  http.StatusForbidden,
+		},
+	}
+	MockStsGetCallerIdentityInvalidBodyExpiredTokenException = &MockEndpoint{
+		Request: &MockRequest{
+			Body: url.Values{
+				"Action":  []string{"GetCallerIdentity"},
+				"Version": []string{"2011-06-15"},
+			}.Encode(),
+			Method: http.MethodPost,
+			Uri:    "/",
+		},
+		Response: &MockResponse{
+			Body:        MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException,
+			ContentType: "text/xml",
+			StatusCode:  http.StatusForbidden,
+		},
+	}
+	MockStsGetCallerIdentityValidBodyExpiredTokenException = &MockEndpoint{
+		Request: &MockRequest{
+			Body: url.Values{
+				"Action":  []string{"GetCallerIdentity"},
+				"Version": []string{"2011-06-15"},
+			}.Encode(),
+			Method: http.MethodPost,
+			Uri:    "/",
+		},
+		Response: &MockResponse{
+			Body:        MockStsGetCallerIdentityValidResponseBodyExpiredTokenException,
+			ContentType: "text/xml",
+			StatusCode:  http.StatusForbidden,
+		},
+	}
+	MockStsGetCallerIdentityInvalidBodyRequestExpired = &MockEndpoint{
+		Request: &MockRequest{
+			Body: url.Values{
+				"Action":  []string{"GetCallerIdentity"},
+				"Version": []string{"2011-06-15"},
+			}.Encode(),
+			Method: http.MethodPost,
+			Uri:    "/",
+		},
+		Response: &MockResponse{
+			Body:        MockStsGetCallerIdentityInvalidResponseBodyRequestExpired,
+			ContentType: "text/xml",
+			StatusCode:  http.StatusForbidden,
+		},
+	}
+	MockStsGetCallerIdentityValidBodyRequestExpired = &MockEndpoint{
+		Request: &MockRequest{
+			Body: url.Values{
+				"Action":  []string{"GetCallerIdentity"},
+				"Version": []string{"2011-06-15"},
+			}.Encode(),
+			Method: http.MethodPost,
+			Uri:    "/",
+		},
+		Response: &MockResponse{
+			Body:        MockStsGetCallerIdentityValidResponseBodyRequestExpired,
+			ContentType: "text/xml",
+			StatusCode:  http.StatusForbidden,
+		},
+	}
 	MockStsGetCallerIdentityValidEndpoint = &MockEndpoint{
 		Request: &MockRequest{
 			Body: url.Values{

diff --git a/v2/awsv1shim/session.go b/v2/awsv1shim/session.go
@@ -129,9 +129,17 @@ func GetSession(ctx context.Context, awsC *awsv2.Config, c *awsbase.Config) (*se
 	sess.Handlers.Retry.PushBack(func(r *request.Request) {
 		logger := logging.RetrieveLogger(r.Context())
 
+		if r.IsErrorExpired() {
+			logger.Warn(ctx, "Disabling retries after next request due to expired credentials", map[string]any{
+				"error": r.Error,
+			})
+			r.Retryable = aws.Bool(false)
+		}
+
 		if r.RetryCount < constants.MaxNetworkRetryCount {
 			return
 		}
+
 		// RequestError: send request failed
 		// caused by: Post https://FQDN/: dial tcp: lookup FQDN: no such host
 		if tfawserr.ErrMessageAndOrigErrContain(r.Error, request.ErrCodeRequestError, "send request failed", "no such host") {