Set up retries for assume role chain calls

pkg/awsUtil/sts.go

@@ -5,13 +5,15 @@ import (
 	"errors"
 	"fmt"
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/aws/aws-sdk-go-v2/service/sts/types"
 	"github.com/openshift/backplane-cli/pkg/utils"
 	"net/http"
 	"net/url"
+	"time"
 )
 
 func StsClientWithProxy(proxyUrl string) (*sts.Client, error) {
@@ -107,6 +109,15 @@ func AssumeRoleSequence(roleSessionName string, seedClient STSRoleAssumer, roleA
 						},
 					},
 				}),
+				config.WithRetryer(func() aws.Retryer {
+					return retry.NewStandard(func(options *retry.StandardOptions) {
+						options.Retryables = append(options.Retryables, retry.RetryableHTTPStatusCode{
+							Codes: map[int]struct{}{401: {}, 403: {}, 404: {}}, // Handle IAM eventual consistency because backplane api modifies trust policy
+						})
+						options.MaxAttempts = 5
+						options.MaxBackoff = 20 * time.Second
+					})
+				}),
 				config.WithRegion("us-east-1"), // We don't care about region here, but the API still wants to see one set
 			)
 			if err != nil {