provider: AWS SDK for Go v1-compatible Backoff mechanism

.changelog/37404.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+resource/provider: Change the AWS SDK for Go v2 API client [`BackoffDelayer`](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2@v1.26.1/aws/retry#BackoffDelayer) to maintain behavioral compatibility with AWS SDK for Go v1
+```

internal/conns/backoff_delayer.go

@@ -0,0 +1,86 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package conns
+
+import (
+	"math"
+	"math/rand"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
+)
+
+// AWS SDK for Go v1 compatible Backoff.
+// See https://github.com/aws/aws-sdk-go/blob/e7dfa8a81550571e247af1ed63a698f9f43a4d51/aws/client/default_retryer.go#L78.
+
+type v1CompatibleBackoff struct {
+	maxRetryDelay time.Duration
+}
+
+// AWS SDK for Go v1 compatible Backoff.
+// See https://github.com/aws/aws-sdk-go/blob/e7dfa8a81550571e247af1ed63a698f9f43a4d51/aws/client/default_retryer.go#L78.
+func (c *v1CompatibleBackoff) BackoffDelay(attempt int, err error) (time.Duration, error) {
+	const (
+		defaultMinRetryDelay    = 30 * time.Millisecond
+		defaultMinThrottleDelay = 500 * time.Millisecond
+	)
+	minDelay := defaultMinRetryDelay
+
+	if retry.IsErrorThrottles(retry.DefaultThrottles).IsErrorThrottle(err).Bool() {
+		minDelay = defaultMinThrottleDelay
+	}
+
+	maxDelay := c.maxRetryDelay
+	var delay time.Duration
+
+	// Logic to cap the retry count based on the minDelay provided.
+	actualRetryCount := int(math.Log2(float64(minDelay))) + 1
+	if actualRetryCount < 63-attempt {
+		delay = time.Duration(1<<uint64(attempt)) * getJitterDelay(minDelay)
+		if delay > maxDelay {
+			delay = getJitterDelay(maxDelay / 2) //nolint:mnd // copied verbatim
+		}
+	} else {
+		delay = getJitterDelay(maxDelay / 2) //nolint:mnd // copied verbatim
+	}
+
+	return delay, nil
+}
+
+func getJitterDelay(duration time.Duration) time.Duration {
+	return time.Duration(seededRand.Int63n(int64(duration)) + int64(duration))
+}
+
+// lockedSource is a thread-safe implementation of rand.Source.
+type lockedSource struct {
+	src rand.Source
+}
+
+func (r *lockedSource) Int63() (n int64) {
+	r.lock()
+	n = r.src.Int63()
+	r.unlock()
+	return
+}
+
+func (r *lockedSource) Seed(seed int64) {
+	r.lock()
+	r.src.Seed(seed)
+	r.unlock()
+}
+
+func (r *lockedSource) key() string {
+	return "backoff-rand-source"
+}
+
+func (r *lockedSource) lock() {
+	GlobalMutexKV.Lock(r.key())
+}
+
+func (r *lockedSource) unlock() {
+	GlobalMutexKV.Unlock(r.key())
+}
+
+// seededRand is a new RNG using a thread safe implementation of rand.Source.
+var seededRand = rand.New(&lockedSource{src: rand.NewSource(time.Now().UnixNano())})

internal/conns/backoff_delayer_test.go

@@ -0,0 +1,52 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package conns
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	awsclient "github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+	smithy "github.com/aws/smithy-go"
+)
+
+func TestBackoffDelayer(t *testing.T) {
+	t.Parallel()
+
+	maxBackoff := 300 * time.Second
+	maxRetries := 25
+	v2 := retry.NewExponentialJitterBackoff(maxBackoff)
+	v1 := awsclient.DefaultRetryer{
+		NumMaxRetries:    maxRetries,
+		MaxRetryDelay:    maxBackoff,
+		MaxThrottleDelay: maxBackoff,
+	}
+	v1compat := &v1CompatibleBackoff{
+		maxRetryDelay: maxBackoff,
+	}
+
+	err1 := awserr.New("ThrottlingException", "Rate exceeded", nil)
+	err2 := &smithy.GenericAPIError{
+		Code:    "ThrottlingException",
+		Message: "Rate exceeded",
+	}
+	req := request.Request{
+		Error: err1,
+		HTTPResponse: &http.Response{
+			StatusCode: http.StatusBadRequest,
+		},
+	}
+	for i := 0; i < maxRetries; i++ {
+		d1 := v1.RetryRules(&req)
+		req.RetryCount++
+		d2, _ := v2.BackoffDelay(i, err2)
+		d1compat, _ := v1compat.BackoffDelay(i, err2)
+
+		t.Logf("%d v1: %s, v2: %s v1compat: %s\n", i, d1.String(), d2.String(), d1compat.String())
+	}
+}

internal/conns/config.go

@@ -70,6 +70,9 @@ func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWS
 
 	ctx, logger := logging.NewTfLogger(ctx)
 
+	const (
+		maxBackoff = 300 * time.Second // AWS SDK for Go v1 DefaultRetryerMaxRetryDelay: https://github.com/aws/aws-sdk-go/blob/9f6e3bb9f523aef97fa1cd5c5f8ba8ecf212e44e/aws/client/default_retryer.go#L48-L49.
+	)
 	awsbaseConfig := awsbase.Config{
 		AccessKey:         c.AccessKey,
 		AllowedAccountIds: c.AllowedAccountIds,
@@ -81,6 +84,7 @@ func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWS
 			},
 		},
 		AssumeRoleWithWebIdentity:      c.AssumeRoleWithWebIdentity,
+		Backoff:                        &v1CompatibleBackoff{maxRetryDelay: maxBackoff},
 		CallerDocumentationURL:         "https://registry.terraform.io/providers/hashicorp/aws",
 		CallerName:                     "Terraform AWS Provider",
 		EC2MetadataServiceEnableState:  c.EC2MetadataServiceEnableState,
@@ -92,7 +96,7 @@ func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWS
 		HTTPSProxy:                     c.HTTPSProxy,
 		HTTPProxyMode:                  awsbase.HTTPProxyModeLegacy,
 		Logger:                         logger,
-		MaxBackoff:                     300 * time.Second, // AWS SDK for Go v1 DefaultRetryerMaxRetryDelay: https://github.com/aws/aws-sdk-go/blob/9f6e3bb9f523aef97fa1cd5c5f8ba8ecf212e44e/aws/client/default_retryer.go#L48-L49.
+		MaxBackoff:                     maxBackoff,
 		MaxRetries:                     c.MaxRetries,
 		NoProxy:                        c.NoProxy,
 		Profile:                        c.Profile,