From e709f3a4c19dc9db4d380bb1107a96fbe4011c8f Mon Sep 17 00:00:00 2001
From: Dirk Avery <dirk.avery@gmail.com>
Date: Thu, 18 Apr 2024 16:07:18 -0400
Subject: [PATCH 1/3] cloudformation/stack_set: Add retry on create

---
 internal/service/cloudformation/stack_set.go | 52 +++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/internal/service/cloudformation/stack_set.go b/internal/service/cloudformation/stack_set.go
index 38b68396d32..a8c9c86ec8f 100644
--- a/internal/service/cloudformation/stack_set.go
+++ b/internal/service/cloudformation/stack_set.go
@@ -266,7 +266,57 @@ func resourceStackSetCreate(ctx context.Context, d *schema.ResourceData, meta in
 		input.TemplateURL = aws.String(v.(string))
 	}
 
-	_, err := conn.CreateStackSetWithContext(ctx, input)
+	_, err := tfresource.RetryWhen(ctx, propagationTimeout,
+		func() (interface{}, error) {
+			output, err := conn.CreateStackSetWithContext(ctx, input)
+			if err != nil {
+				return nil, err
+			}
+
+			operation, err := WaitStackSetCreated(ctx, conn, name, d.Get("call_as").(string), d.Timeout(schema.TimeoutCreate))
+			if err != nil {
+				return nil, fmt.Errorf("waiting for completion (%s): %w", aws.StringValue(output.StackSetId), err)
+			}
+			return operation, nil
+		},
+		func(err error) (bool, error) {
+			if err == nil {
+				return false, nil
+			}
+
+			message := err.Error()
+
+			// IAM eventual consistency
+			if strings.Contains(message, "AccountGate check failed") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			// User: XXX is not authorized to perform: cloudformation:CreateStack on resource: YYY
+			if strings.Contains(message, "is not authorized") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			// XXX role has insufficient YYY permissions
+			if strings.Contains(message, "role has insufficient") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			// Account XXX should have YYY role with trust relationship to Role ZZZ
+			if strings.Contains(message, "role with trust relationship") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			if strings.Contains(message, "The security token included in the request is invalid") {
+				return true, err
+			}
+
+			return false, err
+		},
+	)
 
 	if err != nil {
 		return sdkdiag.AppendErrorf(diags, "creating CloudFormation StackSet (%s): %s", name, err)

From 161e9c4fd7ae876024ab8463f9d3b1e79ba73048 Mon Sep 17 00:00:00 2001
From: Dirk Avery <dirk.avery@gmail.com>
Date: Thu, 18 Apr 2024 16:13:21 -0400
Subject: [PATCH 2/3] Add changelog

---
 .changelog/36982.txt | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 .changelog/36982.txt

diff --git a/.changelog/36982.txt b/.changelog/36982.txt
new file mode 100644
index 00000000000..b47770dfe42
--- /dev/null
+++ b/.changelog/36982.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+resource/aws_cloudformation_stack_set: Add retry when creating to potentially help with eventual consistency problems
+```
\ No newline at end of file

From a41a2d5f03d51102c3bc377610e385f8b3937cf4 Mon Sep 17 00:00:00 2001
From: Dirk Avery <dirk.avery@gmail.com>
Date: Thu, 18 Apr 2024 16:13:51 -0400
Subject: [PATCH 3/3] cloudformation/stack_set: Status and waiter

---
 internal/service/cloudformation/status.go | 16 ++++++++++++++++
 internal/service/cloudformation/wait.go   | 22 ++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/internal/service/cloudformation/status.go b/internal/service/cloudformation/status.go
index 0a3c457c486..7814a1e24b0 100644
--- a/internal/service/cloudformation/status.go
+++ b/internal/service/cloudformation/status.go
@@ -28,6 +28,22 @@ func StatusChangeSet(ctx context.Context, conn *cloudformation.CloudFormation, s
 	}
 }
 
+func StatusStackSet(ctx context.Context, conn *cloudformation.CloudFormation, name, callAs string) retry.StateRefreshFunc {
+	return func() (interface{}, string, error) {
+		output, err := FindStackSetByName(ctx, conn, name, callAs)
+
+		if tfresource.NotFound(err) {
+			return nil, "", nil
+		}
+
+		if err != nil {
+			return nil, "", err
+		}
+
+		return output, aws.StringValue(output.Status), nil
+	}
+}
+
 func StatusStackSetOperation(ctx context.Context, conn *cloudformation.CloudFormation, stackSetName, operationID, callAs string) retry.StateRefreshFunc {
 	return func() (interface{}, string, error) {
 		output, err := FindStackSetOperationByStackSetNameAndOperationID(ctx, conn, stackSetName, operationID, callAs)
diff --git a/internal/service/cloudformation/wait.go b/internal/service/cloudformation/wait.go
index 813420f536a..97228194ac5 100644
--- a/internal/service/cloudformation/wait.go
+++ b/internal/service/cloudformation/wait.go
@@ -40,6 +40,28 @@ func WaitChangeSetCreated(ctx context.Context, conn *cloudformation.CloudFormati
 	return nil, err
 }
 
+func WaitStackSetCreated(ctx context.Context, conn *cloudformation.CloudFormation, name, callAs string, timeout time.Duration) (*cloudformation.StackSet, error) {
+	stateConf := retry.StateChangeConf{
+		Pending: []string{},
+		Target:  []string{cloudformation.StackSetStatusActive},
+		Timeout: ChangeSetCreatedTimeout,
+		Refresh: StatusStackSet(ctx, conn, name, callAs),
+		Delay:   15 * time.Second,
+	}
+
+	outputRaw, err := stateConf.WaitForStateContext(ctx)
+
+	if output, ok := outputRaw.(*cloudformation.StackSet); ok {
+		if status := aws.StringValue(output.Status); status == cloudformation.ChangeSetStatusFailed {
+			tfresource.SetLastError(err, fmt.Errorf("describing CloudFormation Stack Set (%s) results: %w", name, err))
+		}
+
+		return output, err
+	}
+
+	return nil, err
+}
+
 const (
 	stackSetOperationDelay = 5 * time.Second
 )