Merge pull request #3 from envato/andrewjhumphrey-skip-tagged-orig-pr73

Skip resources tagged with aws:cloudformation:stack-name

iamy.go

@@ -39,15 +39,21 @@ type Ui struct {
 	Exit         func(code int)
 }
 
+// CFN automatically tags resources with this and other tags:
+// https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html
+const cloudformationStackNameTag = "aws:cloudformation:stack-name"
+
 func main() {
 	var (
-		debug     = kingpin.Flag("debug", "Show debugging output").Bool()
-		pull      = kingpin.Command("pull", "Syncs IAM users, groups and policies from the active AWS account to files")
-		pullDir   = pull.Flag("dir", "The directory to dump yaml files to").Default(defaultDir).Short('d').String()
-		canDelete = pull.Flag("delete", "Delete extraneous files from destination dir").Bool()
-		lookupCfn = pull.Flag("accurate-cfn", "Fetch all known resource names from cloudformation to get exact filtering").Bool()
-		push      = kingpin.Command("push", "Syncs IAM users, groups and policies from files to the active AWS account")
-		pushDir   = push.Flag("dir", "The directory to load yaml files from").Default(defaultDir).Short('d').ExistingDir()
+		debug         = kingpin.Flag("debug", "Show debugging output").Bool()
+		pull          = kingpin.Command("pull", "Syncs IAM users, groups and policies from the active AWS account to files")
+		pullDir       = pull.Flag("dir", "The directory to dump yaml files to").Default(defaultDir).Short('d').String()
+		canDelete     = pull.Flag("delete", "Delete extraneous files from destination dir").Bool()
+		lookupCfn     = pull.Flag("accurate-cfn", "Fetch all known resource names from cloudformation to get exact filtering").Bool()
+		skipCfnTagged = pull.Flag("skip-cfn-tagged", fmt.Sprintf("Shorthand for --skip-tagged %s", cloudformationStackNameTag)).Bool()
+		skipTagged    = pull.Flag("skip-tagged", "Skips IAM entities (or buckets associated with bucket policies) tagged with a given tag").Strings()
+		push          = kingpin.Command("push", "Syncs IAM users, groups and policies from files to the active AWS account")
+		pushDir       = push.Flag("dir", "The directory to load yaml files from").Default(defaultDir).Short('d').ExistingDir()
 	)
 	dryRun = kingpin.Flag("dry-run", "Show what would happen, but don't prompt to do it").Bool()
 
@@ -73,18 +79,24 @@ func main() {
 	}
 
 	performVersionChecks()
+	if *skipCfnTagged {
+		*skipTagged = append(*skipTagged, cloudformationStackNameTag)
+	}
 
 	switch cmd {
 	case push.FullCommand():
 		PushCommand(ui, PushCommandInput{
-			Dir: *pushDir,
+			Dir:                  *pushDir,
+			HeuristicCfnMatching: !*lookupCfn,
+			SkipTagged:           *skipTagged,
 		})
 
 	case pull.FullCommand():
 		PullCommand(ui, PullCommandInput{
 			Dir:                  *pullDir,
 			CanDelete:            *canDelete,
 			HeuristicCfnMatching: !*lookupCfn,
+			SkipTagged:           *skipTagged,
 		})
 	}
 }

iamy/aws.go

@@ -17,6 +17,7 @@ type AwsFetcher struct {
 	// when pushing to AWS
 	SkipFetchingPolicyAndRoleDescriptions bool
 	HeuristicCfnMatching                  bool
+	SkipTagged                            []string
 
 	Debug *log.Logger
 
@@ -99,7 +100,7 @@ func (a *AwsFetcher) fetchS3Data() error {
 		if b.policyJson == "" {
 			continue
 		}
-		if ok, err := a.isSkippableManagedResource(CfnS3Bucket, b.name); ok {
+		if ok, err := a.isSkippableManagedResource(CfnS3Bucket, b.name, b.tags); ok {
 			log.Printf(err)
 			continue
 		}
@@ -209,7 +210,7 @@ func (a *AwsFetcher) marshalRoleDescriptionAsync(roleName string, target *string
 
 func (a *AwsFetcher) populateInstanceProfileData(resp *iam.ListInstanceProfilesOutput) error {
 	for _, profileResp := range resp.InstanceProfiles {
-		if ok, err := a.isSkippableManagedResource(CfnInstanceProfile, *profileResp.InstanceProfileName); ok {
+		if ok, err := a.isSkippableManagedResource(CfnInstanceProfile, *profileResp.InstanceProfileName, map[string]string{}); ok {
 			log.Printf(err)
 			continue
 		}
@@ -229,7 +230,12 @@ func (a *AwsFetcher) populateInstanceProfileData(resp *iam.ListInstanceProfilesO
 
 func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOutput) error {
 	for _, userResp := range resp.UserDetailList {
-		if ok, err := a.isSkippableManagedResource(CfnIamUser, *userResp.UserName); ok {
+		tags := make(map[string]string)
+		for _, tag := range userResp.Tags {
+			tags[*tag.Key] = *tag.Value
+		}
+
+		if ok, err := a.isSkippableManagedResource(CfnIamUser, *userResp.UserName, tags); ok {
 			log.Printf(err)
 			continue
 		}
@@ -239,7 +245,6 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut
 				Name: *userResp.UserName,
 				Path: *userResp.Path,
 			},
-			Tags: make(map[string]string),
 		}
 
 		for _, g := range userResp.GroupList {
@@ -251,15 +256,13 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut
 		if err := a.populateInlinePolicies(userResp.UserPolicyList, &user.InlinePolicies); err != nil {
 			return err
 		}
-		for _, t := range userResp.Tags {
-			user.Tags[*t.Key] = *t.Value
-		}
+		user.Tags = tags
 
 		a.data.Users = append(a.data.Users, &user)
 	}
 
 	for _, groupResp := range resp.GroupDetailList {
-		if ok, err := a.isSkippableManagedResource(CfnIamGroup, *groupResp.GroupName); ok {
+		if ok, err := a.isSkippableManagedResource(CfnIamGroup, *groupResp.GroupName, map[string]string{}); ok {
 			log.Printf(err)
 			continue
 		}
@@ -280,7 +283,12 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut
 	}
 
 	for _, roleResp := range resp.RoleDetailList {
-		if ok, err := a.isSkippableManagedResource(CfnIamRole, *roleResp.RoleName); ok {
+		tags := make(map[string]string)
+		for _, tag := range roleResp.Tags {
+			tags[*tag.Key] = *tag.Value
+		}
+
+		if ok, err := a.isSkippableManagedResource(CfnIamRole, *roleResp.RoleName, tags); ok {
 			log.Printf(err)
 			continue
 		}
@@ -310,7 +318,7 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut
 	}
 
 	for _, policyResp := range resp.Policies {
-		if ok, err := a.isSkippableManagedResource(CfnIamPolicy, *policyResp.PolicyName); ok {
+		if ok, err := a.isSkippableManagedResource(CfnIamPolicy, *policyResp.PolicyName, map[string]string{}); ok {
 			log.Printf(err)
 			continue
 		}
@@ -403,7 +411,16 @@ func (a *AwsFetcher) getAccount() (*Account, error) {
 //
 // Returns a boolean of whether it can be skipped and a string of the
 // reasoning why it was skipped.
-func (a *AwsFetcher) isSkippableManagedResource(cfnType CfnResourceType, resourceIdentifier string) (bool, string) {
+
+func (a *AwsFetcher) isSkippableManagedResource(cfnType CfnResourceType, resourceIdentifier string, tags map[string]string) (bool, string) {
+	if len(a.SkipTagged) > 0 {
+		for _, tag := range a.SkipTagged {
+			if stackName, ok := tags[tag]; ok {
+				return true, fmt.Sprintf("Skipping resource %s tagged with %s in stack %s", resourceIdentifier, tag, stackName)
+			}
+		}
+	}
+
 	if a.cfn.IsManagedResource(cfnType, resourceIdentifier) {
 		return true, fmt.Sprintf("CloudFormation generated resource %s", resourceIdentifier)
 	}

iamy/aws_test.go

@@ -2,8 +2,12 @@ package iamy
 
 import (
 	"testing"
+
+	"github.com/aws/aws-sdk-go/service/iam"
 )
 
+const cloudformationStackNameTag = "aws:cloudformation:stack-name"
+
 func TestIsSkippableManagedResource(t *testing.T) {
 	skippables := []string{
 		"myalias-123/iam/role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot.yaml",
@@ -22,7 +26,7 @@ func TestIsSkippableManagedResource(t *testing.T) {
 	for _, name := range skippables {
 		t.Run(name, func(t *testing.T) {
 
-			skipped, err := f.isSkippableManagedResource(CfnIamRole, name)
+			skipped, err := f.isSkippableManagedResource(CfnIamRole, name, map[string]string{})
 			if skipped == false {
 				t.Errorf("expected %s to be skipped but got false", name)
 			}
@@ -36,7 +40,7 @@ func TestIsSkippableManagedResource(t *testing.T) {
 	for _, name := range nonSkippables {
 		t.Run(name, func(t *testing.T) {
 
-			skipped, err := f.isSkippableManagedResource(CfnIamRole, name)
+			skipped, err := f.isSkippableManagedResource(CfnIamRole, name, map[string]string{})
 			if skipped == true {
 				t.Errorf("expected %s to not be skipped but got true", name)
 			}
@@ -47,3 +51,128 @@ func TestIsSkippableManagedResource(t *testing.T) {
 		})
 	}
 }
+
+func TestSkippableS3TaggedResources(t *testing.T) {
+	f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}}
+	skippableTags := map[string]string{cloudformationStackNameTag: "my-stack"}
+
+	skipped, err := f.isSkippableManagedResource(CfnS3Bucket, "my-bucket", skippableTags)
+	if err == "" {
+		t.Errorf("expected an error message but it was empty")
+	}
+	if skipped == false {
+		t.Errorf("expected resource to be skipped but got false")
+	}
+}
+
+func TestSkippableS3TaggedResources_WithNoSkipTags(t *testing.T) {
+	f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{}}
+	skippableTags := map[string]string{cloudformationStackNameTag: "my-stack"}
+
+	skipped, err := f.isSkippableManagedResource(CfnS3Bucket, "my-bucket", skippableTags)
+	if err != "" {
+		t.Errorf("expected no error message but it was " + err)
+	}
+	if skipped == true {
+		t.Errorf("expected resource to not be skipped but got true")
+	}
+}
+
+func TestNonSkippableTaggedResources(t *testing.T) {
+	f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}}
+	nonSkippableTags := map[string]string{"Name": "blah"}
+
+	skipped, err := f.isSkippableManagedResource(CfnS3Bucket, "my-bucket", nonSkippableTags)
+	if err != "" {
+		t.Errorf("expected no error message but got: %s", err)
+	}
+	if skipped == true {
+		t.Errorf("expected resource to not be skipped but got true")
+	}
+}
+
+func TestSkippableIAMUserResource(t *testing.T) {
+	f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}}
+	key := cloudformationStackNameTag
+	val := "my-stack"
+	userName := "my-user"
+	path := "/"
+	userList := []*iam.UserDetail{
+		{Tags: []*iam.Tag{{Key: &key, Value: &val}}, UserName: &userName, Path: &path},
+	}
+
+	resp := iam.GetAccountAuthorizationDetailsOutput{UserDetailList: userList}
+	f.populateIamData(&resp)
+	for _, user := range f.data.Users {
+		if user.Name == userName {
+			t.Error("Expected to skip user with CFN tags")
+		}
+	}
+}
+
+func TestSkippableIAMUserResource_WithNoSkipTags(t *testing.T) {
+	f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{}}
+	key := cloudformationStackNameTag
+	val := "my-stack"
+	userName := "my-user"
+	path := "/"
+	userList := []*iam.UserDetail{
+		{Tags: []*iam.Tag{{Key: &key, Value: &val}}, UserName: &userName, Path: &path},
+	}
+
+	resp := iam.GetAccountAuthorizationDetailsOutput{UserDetailList: userList}
+	f.populateIamData(&resp)
+	foundUser := false
+	for _, user := range f.data.Users {
+		if user.Name == userName {
+			foundUser = true
+		}
+	}
+
+	if !foundUser {
+		t.Error("Expected to not skip user with CFN tags when SkipTagged: []string{}")
+	}
+}
+
+func TestSkippableIAMRoleResource(t *testing.T) {
+	f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}}
+	key := cloudformationStackNameTag
+	val := "my-stack"
+	roleName := "my-role"
+	path := "/"
+	roleList := []*iam.RoleDetail{
+		{Tags: []*iam.Tag{{Key: &key, Value: &val}}, RoleName: &roleName, Path: &path},
+	}
+
+	resp := iam.GetAccountAuthorizationDetailsOutput{RoleDetailList: roleList}
+	f.populateIamData(&resp)
+	for _, role := range f.data.Roles {
+		if role.Name == roleName {
+			t.Error("Expected to skip role with CFN tags")
+		}
+	}
+}
+
+func TestSkippableIAMRoleResource_WithNoSkipTags(t *testing.T) {
+	f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{}, SkipFetchingPolicyAndRoleDescriptions: true}
+	key := cloudformationStackNameTag
+	val := "my-stack"
+	roleName := "my-role"
+	path := "/"
+	str := "{}"
+	roleList := []*iam.RoleDetail{
+		{Tags: []*iam.Tag{{Key: &key, Value: &val}}, RoleName: &roleName, Path: &path, AssumeRolePolicyDocument: &str},
+	}
+
+	resp := iam.GetAccountAuthorizationDetailsOutput{RoleDetailList: roleList}
+	f.populateIamData(&resp)
+	foundRole := false
+	for _, role := range f.data.Roles {
+		if role.Name == roleName {
+			foundRole = true
+		}
+	}
+	if !foundRole {
+		t.Error("Expected to not skip role with CFN tags and SkipTagged: []string{}")
+	}
+}

iamy/iam.go

@@ -27,7 +27,7 @@ func (c *iamClient) getPolicyDescription(arn string) (string, error) {
 
 func (c *iamClient) getRoleDescription(name string) (string, error) {
 	resp, err := c.GetRole(&iam.GetRoleInput{RoleName: &name})
-	if err == nil && resp.Role.Description != nil {
+	if err == nil && resp.Role != nil && resp.Role.Description != nil {
 		return *resp.Role.Description, nil
 	}
 	return "", err