Extend AWS CloudFormation commands

Adds support for enabling TMC integration using
ENABLE_TMC_CLOUD_PROVIDER_PERMISSIONS=true

as well as adding the generate-cloudformation-template
subcommand to `management-cluster permissions aws`, which
will allow users to apply CloudFormation by themselves,
or convert the template into IAM policies or Terraform.

Signed-off-by: Naadir Jeewa <jeewan@vmware.com>

cmd/cli/plugin/managementcluster/permissions_aws_generate.go

@@ -0,0 +1,36 @@
+// Copyright 2021 VMware, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var generateAWSCloudFormationTemplateCmd = &cobra.Command{
+	Use:   "generate-cloudformation-template",
+	Short: "Generate AWS CloudFormation Template",
+	Long:  `Generate AWS CloudFormation Template`,
+	RunE:  generateCloudFormationTemplate,
+}
+
+func init() {
+	generateAWSCloudFormationTemplateCmd.Flags().StringVarP(&setAWSPermissionsOps.clusterConfigFile, "file", "f", "", "Optional, configuration file from which to read the aws credentials. Falls back to using the default AWS credentials chain if not provided.")
+	awsPermissionsCmd.AddCommand(generateAWSCloudFormationTemplateCmd)
+}
+
+func generateCloudFormationTemplate(cmd *cobra.Command, args []string) error {
+	forceUpdateTKGCompatibilityImage := false
+	tkgctlClient, err := newTKGCtlClient(forceUpdateTKGCompatibilityImage)
+	if err != nil {
+		return err
+	}
+	template, err := tkgctlClient.GenerateAWSCloudFormationTemplate(setAWSPermissionsOps.clusterConfigFile)
+	if err != nil {
+		return err
+	}
+	fmt.Println(template)
+	return nil
+}

cmd/cli/plugin/managementcluster/permissions_aws_set.go

@@ -14,11 +14,11 @@ var setAWSPermissionsCmd = &cobra.Command{
 	RunE:  setAWSPermissions,
 }
 
-type setAWSPermissionsOptions struct {
+type awsPermissionsOptions struct {
 	clusterConfigFile string
 }
 
-var setAWSPermissionsOps setAWSPermissionsOptions
+var setAWSPermissionsOps awsPermissionsOptions
 
 func init() {
 	setAWSPermissionsCmd.Flags().StringVarP(&setAWSPermissionsOps.clusterConfigFile, "file", "f", "", "Optional, configuration file from which to read the aws credentials. Falls back to using the default AWS credentials chain if not provided.")

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/aunum/log v0.0.0-20200821225356-38d2e2c8b489
 	github.com/avinetworks/sdk v0.0.0-20201123134013-c157ef55b6f7
 	github.com/aws/aws-sdk-go v1.40.56
+	github.com/awslabs/goformation/v4 v4.19.5
 	github.com/briandowns/spinner v1.16.0
 	github.com/cppforlife/go-cli-ui v0.0.0-20200716203538-1e47f820817f
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
@@ -70,6 +71,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rs/xid v1.2.1
 	github.com/satori/go.uuid v1.2.0
+	github.com/sergi/go-diff v1.2.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/spf13/afero v1.6.0
 	github.com/spf13/cobra v1.2.1

pkg/v1/providers/config_default.yaml

@@ -414,6 +414,9 @@ TKG_PROXY_CA_CERT: ""
 #! IP Family setting
 TKG_IP_FAMILY:
 
+#! Configure cloud provider permissions for TMC enablement. Only affects AWS at present.
+ENABLE_TMC_CLOUD_PROVIDER_PERMISSIONS: false
+
 #! Node Nameservers (currently only supports the vSphere provider)
 CONTROL_PLANE_NODE_NAMESERVERS:
 WORKER_NODE_NAMESERVERS:

pkg/v1/tkg/aws/client.go

@@ -15,14 +15,17 @@ import (
 	utilpointer "k8s.io/utils/pointer"
 	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cloudformation/bootstrap"
 	cloudformation "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cloudformation/service"
+	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/configreader"
 	awscreds "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/credentials"
+	iamv1 "sigs.k8s.io/cluster-api-provider-aws/iam/api/v1beta1"
 
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/web/server/models"
 )
 
 const (
 	defaultControlPlaneMinMemoryInGB = 8
 	defaultControlPlaneMinCPU        = 2
+	tanzuMissionControlPoliciesSID   = "tmccloudvmwarecom"
 )
 
 type client struct {
@@ -250,18 +253,80 @@ func (c *client) ListSubnets(vpcID string) ([]*models.AWSSubnet, error) {
 }
 
 func (c *client) CreateCloudFormationStack() error {
-	template := bootstrap.NewTemplate()
-	setDefaultCloudFormationTemplateValue(&template)
-	cfnSvc := cloudformation.NewService(cfn.New(c.session))
-	err := cfnSvc.ReconcileBootstrapStack(template.Spec.StackName, *template.RenderCloudFormation())
+	template, err := c.GenerateBootstrapTemplate(GenerateBootstrapTemplateInput{})
 	if err != nil {
 		return err
 	}
+	return c.CreateCloudFormationStackWithTemplate(template)
+}
 
+func (c *client) CreateCloudFormationStackWithTemplate(template *bootstrap.Template) error {
+	cfnSvc := cloudformation.NewService(cfn.New(c.session))
+	if err := cfnSvc.ReconcileBootstrapStack(template.Spec.StackName, *template.RenderCloudFormation()); err != nil {
+		return err
+	}
 	return cfnSvc.ShowStackResources(template.Spec.StackName)
 }
 
-func setDefaultCloudFormationTemplateValue(t *bootstrap.Template) {
+// GenerateBootstrapTemplateInput is the input to the GenerateBootstrapTemplate func
+type GenerateBootstrapTemplateInput struct {
+	// BootstrapConfigFile is the path to a CAPA bootstrapv1 configuration file that can be used
+	// to customize IAM policies
+	BootstrapConfigFile string
+	// EnableTanzuMissionControlPermissions if true will add IAM permissions for use by Tanzu Mission Control
+	// to all nodes
+	EnableTanzuMissionControlPermissions bool
+}
+
+// GenerateBootstrapTemplate generates a wrapped CAPA bootstrapv1 configuration specification that controls
+// the generation of CloudFormation stacks
+func (c *client) GenerateBootstrapTemplate(i GenerateBootstrapTemplateInput) (*bootstrap.Template, error) {
+	template := bootstrap.NewTemplate()
+	if i.BootstrapConfigFile != "" {
+		spec, err := configreader.LoadConfigFile(i.BootstrapConfigFile)
+		if err != nil {
+			return nil, err
+		}
+		template.Spec = &spec.Spec
+	}
+	setDefaultsBootstrapTemplate(&template)
+	if i.EnableTanzuMissionControlPermissions {
+		ensureTanzuMissionControlPermissions(&template)
+	}
+	return &template, nil
+}
+
+func ensureTanzuMissionControlPermissions(t *bootstrap.Template) {
+	t.Spec.Nodes.ExtraStatements = ensureTanzuMissionControlPermissionsForRole(t.Spec.Nodes.ExtraStatements)
+	t.Spec.ControlPlane.ExtraStatements = ensureTanzuMissionControlPermissionsForRole(t.Spec.ControlPlane.ExtraStatements)
+}
+
+func ensureTanzuMissionControlPermissionsForRole(statements []iamv1.StatementEntry) []iamv1.StatementEntry {
+	tmcStatementEntry := iamv1.StatementEntry{
+		Sid:    tanzuMissionControlPoliciesSID,
+		Effect: iamv1.EffectAllow,
+		Action: iamv1.Actions{
+			"ec2:DescribeKeyPairs",
+			"elasticloadbalancing:DescribeLoadBalancers",
+			"servicequotas:ListServiceQuotas",
+			"iam:GetPolicy",
+			"iam:ListAttachedRolePoliices",
+			"iam:GetPolicyVersion",
+			"iam:ListRoleTags",
+		},
+		Resource: iamv1.Resources{iamv1.Any},
+	}
+	for i, statementEntry := range statements {
+		if statementEntry.Sid == tanzuMissionControlPoliciesSID {
+			statements[i] = tmcStatementEntry
+			return statements
+		}
+	}
+	statements = append(statements, tmcStatementEntry)
+	return statements
+}
+
+func setDefaultsBootstrapTemplate(t *bootstrap.Template) {
 	t.Spec.NameSuffix = utilpointer.StringPtr(DefaultCloudFormationNameSuffix)
 	t.Spec.StackName = DefaultCloudFormationStackName
 	t.Spec.BootstrapUser.UserName = DefaultCloudFormationBootstrapUserName

pkg/v1/tkg/aws/client_test.go

@@ -4,11 +4,18 @@
 package aws_test
 
 import (
+	"fmt"
+	"os"
+	"path"
 	"testing"
 
+	"github.com/awslabs/goformation/v4/cloudformation"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sergi/go-diff/diffmatchpatch"
+	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cloudformation/bootstrap"
 	awscreds "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/credentials"
+	"sigs.k8s.io/yaml"
 
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/aws"
 )
@@ -36,15 +43,17 @@ var _ = Describe("Unit tests for aws client", func() {
 		awsClient       aws.Client
 	)
 
+	JustBeforeEach(func() {
+		awsCreds := awscreds.AWSCredentials{
+			AccessKeyID:     accessKeyID,
+			SecretAccessKey: secretAccessKey,
+			Region:          region,
+		}
+		awsClient, err = aws.New(awsCreds)
+	})
 	Describe("Encode aws credentials", func() {
 		var encodedCreds string
 		JustBeforeEach(func() {
-			awsCreds := awscreds.AWSCredentials{
-				AccessKeyID:     accessKeyID,
-				SecretAccessKey: secretAccessKey,
-				Region:          region,
-			}
-			awsClient, err = aws.New(awsCreds)
 			Expect(err).ToNot(HaveOccurred())
 			encodedCreds, err = awsClient.EncodeCredentials()
 		})
@@ -73,4 +82,42 @@ var _ = Describe("Unit tests for aws client", func() {
 			})
 		})
 	})
+
+	Describe("Generating Cloudformation", func() {
+		awsClient, err = aws.New(awscreds.AWSCredentials{
+			AccessKeyID:     fakeRegion,
+			SecretAccessKey: fakeAccessKeyID,
+			Region:          fakeSecretAccessKey,
+		})
+		Expect(err).ToNot(HaveOccurred())
+	})
+	Context("Defaults", func() {
+		template, err := awsClient.GenerateBootstrapTemplate(aws.GenerateBootstrapTemplateInput{})
+		Expect(err).NotTo(HaveOccurred())
+		testBootstrapTemplate("default", template)
+	})
+	Context("With TMC Permissions", func() {
+		template, err := awsClient.GenerateBootstrapTemplate(aws.GenerateBootstrapTemplateInput{
+			EnableTanzuMissionControlPermissions: true,
+		})
+		Expect(err).NotTo(HaveOccurred())
+		testBootstrapTemplate("with-tmc", template)
+	})
 })
+
+func testBootstrapTemplate(name string, template *bootstrap.Template) {
+	defer GinkgoRecover()
+	cfn := cloudformation.Template{}
+	data, err := os.ReadFile(path.Join("fixtures", "cloudformation-"+name+".yaml"))
+	Expect(err).ToNot(HaveOccurred())
+	err = yaml.Unmarshal(data, cfn)
+	Expect(err).ToNot(HaveOccurred())
+	tData, err := template.RenderCloudFormation().YAML()
+	Expect(err).ToNot(HaveOccurred())
+	err = os.WriteFile("/tmp/tmp1", tData, 0600)
+	Expect(err).ToNot(HaveOccurred())
+	dmp := diffmatchpatch.New()
+	diffs := dmp.DiffMain(string(tData), string(data), false)
+	out := dmp.DiffPrettyText(diffs)
+	Expect(string(tData)).To(Equal(string(data)), fmt.Sprintf("Differing output (%s):\n%s", name, out))
+}