diff --git a/iamy.go b/iamy.go index 722f508..4804e0b 100644 --- a/iamy.go +++ b/iamy.go @@ -39,15 +39,21 @@ type Ui struct { Exit func(code int) } +// CFN automatically tags resources with this and other tags: +// https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html +const cloudformationStackNameTag = "aws:cloudformation:stack-name" + func main() { var ( - debug = kingpin.Flag("debug", "Show debugging output").Bool() - pull = kingpin.Command("pull", "Syncs IAM users, groups and policies from the active AWS account to files") - pullDir = pull.Flag("dir", "The directory to dump yaml files to").Default(defaultDir).Short('d').String() - canDelete = pull.Flag("delete", "Delete extraneous files from destination dir").Bool() - lookupCfn = pull.Flag("accurate-cfn", "Fetch all known resource names from cloudformation to get exact filtering").Bool() - push = kingpin.Command("push", "Syncs IAM users, groups and policies from files to the active AWS account") - pushDir = push.Flag("dir", "The directory to load yaml files from").Default(defaultDir).Short('d').ExistingDir() + debug = kingpin.Flag("debug", "Show debugging output").Bool() + pull = kingpin.Command("pull", "Syncs IAM users, groups and policies from the active AWS account to files") + pullDir = pull.Flag("dir", "The directory to dump yaml files to").Default(defaultDir).Short('d').String() + canDelete = pull.Flag("delete", "Delete extraneous files from destination dir").Bool() + lookupCfn = pull.Flag("accurate-cfn", "Fetch all known resource names from cloudformation to get exact filtering").Bool() + skipCfnTagged = pull.Flag("skip-cfn-tagged", fmt.Sprintf("Shorthand for --skip-tagged %s", cloudformationStackNameTag)).Bool() + skipTagged = pull.Flag("skip-tagged", "Skips IAM entities (or buckets associated with bucket policies) tagged with a given tag").Strings() + push = kingpin.Command("push", "Syncs IAM users, groups and policies from files to the active AWS account") + pushDir = push.Flag("dir", "The directory to load yaml files from").Default(defaultDir).Short('d').ExistingDir() ) dryRun = kingpin.Flag("dry-run", "Show what would happen, but don't prompt to do it").Bool() @@ -73,11 +79,16 @@ func main() { } performVersionChecks() + if *skipCfnTagged { + *skipTagged = append(*skipTagged, cloudformationStackNameTag) + } switch cmd { case push.FullCommand(): PushCommand(ui, PushCommandInput{ - Dir: *pushDir, + Dir: *pushDir, + HeuristicCfnMatching: !*lookupCfn, + SkipTagged: *skipTagged, }) case pull.FullCommand(): @@ -85,6 +96,7 @@ func main() { Dir: *pullDir, CanDelete: *canDelete, HeuristicCfnMatching: !*lookupCfn, + SkipTagged: *skipTagged, }) } } diff --git a/iamy/aws.go b/iamy/aws.go index 3e0c6a9..84d46c2 100644 --- a/iamy/aws.go +++ b/iamy/aws.go @@ -17,6 +17,7 @@ type AwsFetcher struct { // when pushing to AWS SkipFetchingPolicyAndRoleDescriptions bool HeuristicCfnMatching bool + SkipTagged []string Debug *log.Logger @@ -99,7 +100,7 @@ func (a *AwsFetcher) fetchS3Data() error { if b.policyJson == "" { continue } - if ok, err := a.isSkippableManagedResource(CfnS3Bucket, b.name); ok { + if ok, err := a.isSkippableManagedResource(CfnS3Bucket, b.name, b.tags); ok { log.Printf(err) continue } @@ -209,7 +210,7 @@ func (a *AwsFetcher) marshalRoleDescriptionAsync(roleName string, target *string func (a *AwsFetcher) populateInstanceProfileData(resp *iam.ListInstanceProfilesOutput) error { for _, profileResp := range resp.InstanceProfiles { - if ok, err := a.isSkippableManagedResource(CfnInstanceProfile, *profileResp.InstanceProfileName); ok { + if ok, err := a.isSkippableManagedResource(CfnInstanceProfile, *profileResp.InstanceProfileName, map[string]string{}); ok { log.Printf(err) continue } @@ -229,7 +230,12 @@ func (a *AwsFetcher) populateInstanceProfileData(resp *iam.ListInstanceProfilesO func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOutput) error { for _, userResp := range resp.UserDetailList { - if ok, err := a.isSkippableManagedResource(CfnIamUser, *userResp.UserName); ok { + tags := make(map[string]string) + for _, tag := range userResp.Tags { + tags[*tag.Key] = *tag.Value + } + + if ok, err := a.isSkippableManagedResource(CfnIamUser, *userResp.UserName, tags); ok { log.Printf(err) continue } @@ -239,7 +245,6 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut Name: *userResp.UserName, Path: *userResp.Path, }, - Tags: make(map[string]string), } for _, g := range userResp.GroupList { @@ -251,15 +256,13 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut if err := a.populateInlinePolicies(userResp.UserPolicyList, &user.InlinePolicies); err != nil { return err } - for _, t := range userResp.Tags { - user.Tags[*t.Key] = *t.Value - } + user.Tags = tags a.data.Users = append(a.data.Users, &user) } for _, groupResp := range resp.GroupDetailList { - if ok, err := a.isSkippableManagedResource(CfnIamGroup, *groupResp.GroupName); ok { + if ok, err := a.isSkippableManagedResource(CfnIamGroup, *groupResp.GroupName, map[string]string{}); ok { log.Printf(err) continue } @@ -280,7 +283,12 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut } for _, roleResp := range resp.RoleDetailList { - if ok, err := a.isSkippableManagedResource(CfnIamRole, *roleResp.RoleName); ok { + tags := make(map[string]string) + for _, tag := range roleResp.Tags { + tags[*tag.Key] = *tag.Value + } + + if ok, err := a.isSkippableManagedResource(CfnIamRole, *roleResp.RoleName, tags); ok { log.Printf(err) continue } @@ -310,7 +318,7 @@ func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOut } for _, policyResp := range resp.Policies { - if ok, err := a.isSkippableManagedResource(CfnIamPolicy, *policyResp.PolicyName); ok { + if ok, err := a.isSkippableManagedResource(CfnIamPolicy, *policyResp.PolicyName, map[string]string{}); ok { log.Printf(err) continue } @@ -403,7 +411,16 @@ func (a *AwsFetcher) getAccount() (*Account, error) { // // Returns a boolean of whether it can be skipped and a string of the // reasoning why it was skipped. -func (a *AwsFetcher) isSkippableManagedResource(cfnType CfnResourceType, resourceIdentifier string) (bool, string) { + +func (a *AwsFetcher) isSkippableManagedResource(cfnType CfnResourceType, resourceIdentifier string, tags map[string]string) (bool, string) { + if len(a.SkipTagged) > 0 { + for _, tag := range a.SkipTagged { + if stackName, ok := tags[tag]; ok { + return true, fmt.Sprintf("Skipping resource %s tagged with %s in stack %s", resourceIdentifier, tag, stackName) + } + } + } + if a.cfn.IsManagedResource(cfnType, resourceIdentifier) { return true, fmt.Sprintf("CloudFormation generated resource %s", resourceIdentifier) } diff --git a/iamy/aws_test.go b/iamy/aws_test.go index d103162..8aa8a0d 100644 --- a/iamy/aws_test.go +++ b/iamy/aws_test.go @@ -2,8 +2,12 @@ package iamy import ( "testing" + + "github.com/aws/aws-sdk-go/service/iam" ) +const cloudformationStackNameTag = "aws:cloudformation:stack-name" + func TestIsSkippableManagedResource(t *testing.T) { skippables := []string{ "myalias-123/iam/role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot.yaml", @@ -22,7 +26,7 @@ func TestIsSkippableManagedResource(t *testing.T) { for _, name := range skippables { t.Run(name, func(t *testing.T) { - skipped, err := f.isSkippableManagedResource(CfnIamRole, name) + skipped, err := f.isSkippableManagedResource(CfnIamRole, name, map[string]string{}) if skipped == false { t.Errorf("expected %s to be skipped but got false", name) } @@ -36,7 +40,7 @@ func TestIsSkippableManagedResource(t *testing.T) { for _, name := range nonSkippables { t.Run(name, func(t *testing.T) { - skipped, err := f.isSkippableManagedResource(CfnIamRole, name) + skipped, err := f.isSkippableManagedResource(CfnIamRole, name, map[string]string{}) if skipped == true { t.Errorf("expected %s to not be skipped but got true", name) } @@ -47,3 +51,128 @@ func TestIsSkippableManagedResource(t *testing.T) { }) } } + +func TestSkippableS3TaggedResources(t *testing.T) { + f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}} + skippableTags := map[string]string{cloudformationStackNameTag: "my-stack"} + + skipped, err := f.isSkippableManagedResource(CfnS3Bucket, "my-bucket", skippableTags) + if err == "" { + t.Errorf("expected an error message but it was empty") + } + if skipped == false { + t.Errorf("expected resource to be skipped but got false") + } +} + +func TestSkippableS3TaggedResources_WithNoSkipTags(t *testing.T) { + f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{}} + skippableTags := map[string]string{cloudformationStackNameTag: "my-stack"} + + skipped, err := f.isSkippableManagedResource(CfnS3Bucket, "my-bucket", skippableTags) + if err != "" { + t.Errorf("expected no error message but it was " + err) + } + if skipped == true { + t.Errorf("expected resource to not be skipped but got true") + } +} + +func TestNonSkippableTaggedResources(t *testing.T) { + f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}} + nonSkippableTags := map[string]string{"Name": "blah"} + + skipped, err := f.isSkippableManagedResource(CfnS3Bucket, "my-bucket", nonSkippableTags) + if err != "" { + t.Errorf("expected no error message but got: %s", err) + } + if skipped == true { + t.Errorf("expected resource to not be skipped but got true") + } +} + +func TestSkippableIAMUserResource(t *testing.T) { + f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}} + key := cloudformationStackNameTag + val := "my-stack" + userName := "my-user" + path := "/" + userList := []*iam.UserDetail{ + {Tags: []*iam.Tag{{Key: &key, Value: &val}}, UserName: &userName, Path: &path}, + } + + resp := iam.GetAccountAuthorizationDetailsOutput{UserDetailList: userList} + f.populateIamData(&resp) + for _, user := range f.data.Users { + if user.Name == userName { + t.Error("Expected to skip user with CFN tags") + } + } +} + +func TestSkippableIAMUserResource_WithNoSkipTags(t *testing.T) { + f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{}} + key := cloudformationStackNameTag + val := "my-stack" + userName := "my-user" + path := "/" + userList := []*iam.UserDetail{ + {Tags: []*iam.Tag{{Key: &key, Value: &val}}, UserName: &userName, Path: &path}, + } + + resp := iam.GetAccountAuthorizationDetailsOutput{UserDetailList: userList} + f.populateIamData(&resp) + foundUser := false + for _, user := range f.data.Users { + if user.Name == userName { + foundUser = true + } + } + + if !foundUser { + t.Error("Expected to not skip user with CFN tags when SkipTagged: []string{}") + } +} + +func TestSkippableIAMRoleResource(t *testing.T) { + f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{cloudformationStackNameTag}} + key := cloudformationStackNameTag + val := "my-stack" + roleName := "my-role" + path := "/" + roleList := []*iam.RoleDetail{ + {Tags: []*iam.Tag{{Key: &key, Value: &val}}, RoleName: &roleName, Path: &path}, + } + + resp := iam.GetAccountAuthorizationDetailsOutput{RoleDetailList: roleList} + f.populateIamData(&resp) + for _, role := range f.data.Roles { + if role.Name == roleName { + t.Error("Expected to skip role with CFN tags") + } + } +} + +func TestSkippableIAMRoleResource_WithNoSkipTags(t *testing.T) { + f := AwsFetcher{cfn: &cfnClient{}, SkipTagged: []string{}, SkipFetchingPolicyAndRoleDescriptions: true} + key := cloudformationStackNameTag + val := "my-stack" + roleName := "my-role" + path := "/" + str := "{}" + roleList := []*iam.RoleDetail{ + {Tags: []*iam.Tag{{Key: &key, Value: &val}}, RoleName: &roleName, Path: &path, AssumeRolePolicyDocument: &str}, + } + + resp := iam.GetAccountAuthorizationDetailsOutput{RoleDetailList: roleList} + f.populateIamData(&resp) + foundRole := false + for _, role := range f.data.Roles { + if role.Name == roleName { + foundRole = true + } + } + if !foundRole { + t.Error("Expected to not skip role with CFN tags and SkipTagged: []string{}") + } +} diff --git a/iamy/iam.go b/iamy/iam.go index d6489ee..17f869c 100644 --- a/iamy/iam.go +++ b/iamy/iam.go @@ -27,7 +27,7 @@ func (c *iamClient) getPolicyDescription(arn string) (string, error) { func (c *iamClient) getRoleDescription(name string) (string, error) { resp, err := c.GetRole(&iam.GetRoleInput{RoleName: &name}) - if err == nil && resp.Role.Description != nil { + if err == nil && resp.Role != nil && resp.Role.Description != nil { return *resp.Role.Description, nil } return "", err diff --git a/iamy/s3.go b/iamy/s3.go index 5d846d6..b7c026d 100644 --- a/iamy/s3.go +++ b/iamy/s3.go @@ -13,6 +13,7 @@ import ( ) const NoSuchBucketPolicyErrCode = "NoSuchBucketPolicy" +const NoSuchTagSetErrCode = "NoSuchTagSet" func newRegionClientMap(s *session.Session) *regionClientMap { return ®ionClientMap{ @@ -62,6 +63,7 @@ type bucket struct { name string policyJson string exists bool + tags map[string]string } func (c *s3Client) withRegion(region string) s3iface.S3API { @@ -86,6 +88,13 @@ func (c *s3Client) populateBucket(b *bucket) error { } region := s3.NormalizeBucketLocation(normaliseString(r.LocationConstraint)) + + tags, err := c.fetchTags(b.name, region) + if err != nil { + return err + } + b.tags = tags + b.policyJson, err = c.GetBucketPolicyDoc(b.name, region) return err @@ -103,6 +112,7 @@ func (c *s3Client) listAllBuckets() ([]*bucket, error) { for _, rb := range bucketListResp.Buckets { b := bucket{name: *rb.Name} + b.exists = true buckets = append(buckets, &b) wg.Add(1) @@ -115,8 +125,6 @@ func (c *s3Client) listAllBuckets() ([]*bucket, error) { oneOfTheErrorsDuringPopulation = errors.New(fmt.Sprintf("Error while getting details for S3 bucket %s: %s", b.name, err)) } } - } else { - b.exists = true } }() } @@ -153,3 +161,23 @@ func (c *s3Client) GetBucketPolicyDoc(name, region string) (string, error) { return *resp.Policy, nil } + +func (c *s3Client) fetchTags(name, region string) (map[string]string, error) { + tags := make(map[string]string) + clientForRegion := c.withRegion(region) + tagsResponse, err := clientForRegion.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: aws.String(name)}) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == NoSuchTagSetErrCode { + return tags, nil + } + } + return tags, err + } + for _, tag := range tagsResponse.TagSet { + if tag != nil { + tags[*tag.Key] = *tag.Value + } + } + return tags, nil +} diff --git a/pull.go b/pull.go index bb57eee..8f5eeed 100644 --- a/pull.go +++ b/pull.go @@ -10,10 +10,11 @@ type PullCommandInput struct { Dir string CanDelete bool HeuristicCfnMatching bool + SkipTagged []string } func PullCommand(ui Ui, input PullCommandInput) { - aws := iamy.AwsFetcher{Debug: ui.Debug, HeuristicCfnMatching: input.HeuristicCfnMatching} + aws := iamy.AwsFetcher{Debug: ui.Debug, HeuristicCfnMatching: input.HeuristicCfnMatching, SkipTagged: input.SkipTagged} data, err := aws.Fetch() if err != nil { ui.Error.Fatal(fmt.Printf("%s", err)) diff --git a/push.go b/push.go index cd68ab5..169a956 100644 --- a/push.go +++ b/push.go @@ -12,7 +12,9 @@ import ( ) type PushCommandInput struct { - Dir string + Dir string + HeuristicCfnMatching bool + SkipTagged []string } func PushCommand(ui Ui, input PushCommandInput) { @@ -22,6 +24,8 @@ func PushCommand(ui Ui, input PushCommandInput) { aws := iamy.AwsFetcher{ SkipFetchingPolicyAndRoleDescriptions: true, Debug: ui.Debug, + HeuristicCfnMatching: input.HeuristicCfnMatching, + SkipTagged: input.SkipTagged, } allDataFromYaml, err := yaml.Load()