Skip to content

Commit

Permalink
Fetch only the S3 buckets and related objects of the current region. C…
Browse files Browse the repository at this point in the history 
…loses #44.
  • Loading branch information
@fxaguessy
fxaguessy committed Feb 20, 2017
1 parent ab6ba32 commit 5bfa794
Show file tree
Hide file tree
Showing 8 changed files with 324 additions and 164 deletions.
6 changes: 5 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,8 @@

### Features

- [#33](https://github.com/wallix/awless/issues/33): Ability to set AWS profile using `aws.profile` config key
- [#33](https://github.com/wallix/awless/issues/33): Ability to set AWS profile using `aws.profile` config key

### Bugfixes

- [#44](https://github.com/wallix/awless/issues/44): Fetch only the S3 buckets and related objects of the current region.
125 changes: 78 additions & 47 deletions cloud/aws/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package aws

import (
"fmt"
"regexp"
"sort"
"sync"
Expand Down Expand Up @@ -59,6 +60,12 @@ type Security interface {
GetAccountId() (string, error)
}

type oncer struct {
sync.Once
result interface{}
err error
}

type security struct {
stsiface.STSAPI
}
Expand Down Expand Up @@ -149,33 +156,34 @@ func (s *Access) fetch_all_user_graph() (*graph.Graph, []*iam.UserDetail, error)
return g, userDetails, nil
}

func (s *Storage) fetch_all_storageobject_graph() (*graph.Graph, []*s3.Object, error) {
func (s *Storage) fetch_all_bucket_graph() (*graph.Graph, []*s3.Bucket, error) {
g := graph.NewGraph()
var cloudResources []*s3.Object
var wg sync.WaitGroup
errc := make(chan error)
var buckets []*s3.Bucket
bucketM := &sync.Mutex{}

s.foreach_bucket(func(b *s3.Bucket) error {
wg.Add(1)
go func(bucket *s3.Bucket) {
defer wg.Done()
errc <- s.fetchObjectsForBucket(bucket, g)
}(b)
err := s.foreach_bucket_parallel(func(b *s3.Bucket) error {
bucketM.Lock()
buckets = append(buckets, b)
bucketM.Unlock()
res, err := newResource(b)
g.AddResource(res)
if err != nil {
return fmt.Errorf("build resource for bucket `%s`: %s", awssdk.StringValue(b.Name), err)
}
return nil
})
return g, buckets, err
}

go func() {
wg.Wait()
close(errc)
}()
func (s *Storage) fetch_all_storageobject_graph() (*graph.Graph, []*s3.Object, error) {
g := graph.NewGraph()
var cloudResources []*s3.Object

for err := range errc {
if err != nil {
return g, cloudResources, err
}
}
err := s.foreach_bucket_parallel(func(b *s3.Bucket) error {
return s.fetchObjectsForBucket(b, g)
})

return g, cloudResources, nil
return g, cloudResources, err
}

func (s *Storage) fetchObjectsForBucket(bucket *s3.Bucket, g *graph.Graph) error {
Expand All @@ -201,60 +209,83 @@ func (s *Storage) fetchObjectsForBucket(bucket *s3.Bucket, g *graph.Graph) error
return nil
}

func (s *Storage) foreach_bucket(f func(b *s3.Bucket) error) error {
out, err := s.fetch_all_bucket()
func (s *Storage) getBucketsPerRegion() ([]*s3.Bucket, error) {
var buckets []*s3.Bucket
out, err := s.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
return err
return buckets, err
}

for _, output := range out.(*s3.ListBucketsOutput).Buckets {
err := f(output)
if err != nil {
return err
}
}
bucketc := make(chan *s3.Bucket)
errc := make(chan error)

return nil
}
var wg sync.WaitGroup

for _, bucket := range out.Buckets {
wg.Add(1)
go func(b *s3.Bucket) {
defer wg.Done()
loc, err := s.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: b.Name})
if err != nil {
errc <- err
return
}
if awssdk.StringValue(loc.LocationConstraint) == s.region {
bucketc <- b
}
}(bucket)
}
go func() {
wg.Wait()
close(bucketc)
}()

func (s *Storage) fetch_all_bucket() (interface{}, error) {
return s.ListBuckets(&s3.ListBucketsInput{})
for {
select {
case err := <-errc:
if err != nil {
return buckets, err
}
case b, ok := <-bucketc:
if !ok {
return buckets, nil
}
buckets = append(buckets, b)
}
}
}

func (s *Storage) fetch_all_bucket_graph() (*graph.Graph, []*s3.Bucket, error) {
g := graph.NewGraph()
out, err := s.fetch_all_bucket()
if err != nil {
return nil, nil, err
func (s *Storage) foreach_bucket_parallel(f func(b *s3.Bucket) error) error {
s.once.Do(func() {
s.once.result, s.once.err = s.getBucketsPerRegion()
})
if s.once.err != nil {
return s.once.err
}
var buckets []*s3.Bucket
buckets := s.once.result.([]*s3.Bucket)

errc := make(chan error)
var wg sync.WaitGroup

for _, output := range out.(*s3.ListBucketsOutput).Buckets {
buckets = append(buckets, output)
for _, output := range buckets {
wg.Add(1)
go func(b *s3.Bucket) {
defer wg.Done()
res, err := newResource(b)
if err != nil {
if err := f(b); err != nil {
errc <- err
}
g.AddResource(res)
}(output)
}

go func() {
wg.Wait()
close(errc)
}()

for err := range errc {
if err != nil {
return g, nil, nil
return err
}
}

return g, buckets, nil
return nil
}
178 changes: 65 additions & 113 deletions cloud/aws/api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,8 @@ import (

awssdk "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/iam/iamiface"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/wallix/awless/graph"
)

func TestRegionsValid(t *testing.T) {
Expand Down Expand Up @@ -249,6 +245,71 @@ func TestBuildInfraRdfGraph(t *testing.T) {
}
}

func TestBuildStorageRdfGraph(t *testing.T) {
buckets := map[string][]*s3.Bucket{
"us-west-1": {
{Name: awssdk.String("bucket_us_1")},
{Name: awssdk.String("bucket_us_2")},
{Name: awssdk.String("bucket_us_3")},
},
"eu-west-1": {
{Name: awssdk.String("bucket_eu_1")},
{Name: awssdk.String("bucket_eu_2")},
},
}
objects := map[string][]*s3.Object{
"bucket_us_1": {
{Key: awssdk.String("obj_1")},
{Key: awssdk.String("obj_2")},
},
"bucket_us_2": {},
"bucket_us_3": {
{Key: awssdk.String("obj_3")},
},
"bucket_eu_1": {
{Key: awssdk.String("obj_4")},
},
"bucket_eu_2": {
{Key: awssdk.String("obj_5")},
{Key: awssdk.String("obj_6")},
},
}
bucketsACL := map[string][]*s3.Grant{
"bucket_us_1": {
{Permission: awssdk.String("Read"), Grantee: &s3.Grantee{ID: awssdk.String("usr_1")}},
},
"bucket_us_3": {
{Permission: awssdk.String("Write"), Grantee: &s3.Grantee{URI: awssdk.String("usr_2")}},
},
"bucket_eu_1": {
{Permission: awssdk.String("Write"), Grantee: &s3.Grantee{URI: awssdk.String("usr_2")}},
},
"bucket_eu_2": {
{Permission: awssdk.String("Write"), Grantee: &s3.Grantee{URI: awssdk.String("usr_1")}},
},
}

mocks3 := &mockS3{bucketsPerRegion: buckets, objectsPerBucket: objects, bucketsACL: bucketsACL}
StorageService = mocks3
storage := Storage{S3API: mocks3, region: "eu-west-1"}

g, err := storage.FetchResources()
if err != nil {
t.Fatal(err)
}

result := g.MustMarshal()

expectContent, err := ioutil.ReadFile(filepath.Join("testdata", "storage.rdf"))
if err != nil {
t.Fatal(err)
}

if got, want := result, string(expectContent); got != want {
t.Fatalf("got\n[%s]\n\nwant\n[%s]", got, want)
}
}

func TestBuildEmptyRdfGraphWhenNoData(t *testing.T) {
expect := `/region<eu-west-1> "has_type"@[] "/region"^^type:text`
access := Access{IAMAPI: &mockIam{}, region: "eu-west-1"}
Expand Down Expand Up @@ -276,115 +337,6 @@ func TestBuildEmptyRdfGraphWhenNoData(t *testing.T) {
}
}

type mockEc2 struct {
ec2iface.EC2API
vpcs []*ec2.Vpc
subnets []*ec2.Subnet
instances []*ec2.Instance
securityGroups []*ec2.SecurityGroup
keyPairs []*ec2.KeyPairInfo
internetGateways []*ec2.InternetGateway
routeTables []*ec2.RouteTable
}

func (m *mockEc2) DescribeVpcs(input *ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error) {
return &ec2.DescribeVpcsOutput{Vpcs: m.vpcs}, nil
}

func (m *mockEc2) DescribeSubnets(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) {
return &ec2.DescribeSubnetsOutput{Subnets: m.subnets}, nil
}

func (m *mockEc2) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
return &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: m.instances}}}, nil
}

func (m *mockEc2) DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) {
return &ec2.DescribeSecurityGroupsOutput{SecurityGroups: m.securityGroups}, nil
}

func (m *mockEc2) DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) {
return &ec2.DescribeKeyPairsOutput{KeyPairs: m.keyPairs}, nil
}

func (m *mockEc2) DescribeInternetGateways(input *ec2.DescribeInternetGatewaysInput) (*ec2.DescribeInternetGatewaysOutput, error) {
return &ec2.DescribeInternetGatewaysOutput{InternetGateways: m.internetGateways}, nil
}

func (m *mockEc2) DescribeRouteTables(input *ec2.DescribeRouteTablesInput) (*ec2.DescribeRouteTablesOutput, error) {
return &ec2.DescribeRouteTablesOutput{RouteTables: m.routeTables}, nil
}

// Not tested
func (m *mockEc2) DescribeVolumes(input *ec2.DescribeVolumesInput) (*ec2.DescribeVolumesOutput, error) {
return &ec2.DescribeVolumesOutput{}, nil
}

type mockIam struct {
iamiface.IAMAPI
groups []*iam.GroupDetail
managedPolicies []*iam.ManagedPolicyDetail
roles []*iam.RoleDetail
users []*iam.User
usersDetails []*iam.UserDetail
}

func (m *mockIam) ListUsers(input *iam.ListUsersInput) (*iam.ListUsersOutput, error) {
return &iam.ListUsersOutput{Users: m.users}, nil
}

func (m *mockIam) ListPolicies(input *iam.ListPoliciesInput) (*iam.ListPoliciesOutput, error) {
var policies []*iam.Policy
for _, p := range m.managedPolicies {
policy := &iam.Policy{PolicyId: p.PolicyId, PolicyName: p.PolicyName}
policies = append(policies, policy)
}
return &iam.ListPoliciesOutput{Policies: policies}, nil
}

func (m *mockIam) GetAccountAuthorizationDetails(input *iam.GetAccountAuthorizationDetailsInput) (*iam.GetAccountAuthorizationDetailsOutput, error) {
return &iam.GetAccountAuthorizationDetailsOutput{GroupDetailList: m.groups, Policies: m.managedPolicies, RoleDetailList: m.roles, UserDetailList: m.usersDetails}, nil
}

func stringInSlice(s string, slice []string) bool {
for _, v := range slice {
if v == s {
return true
}
}
return false
}

type mockS3 struct {
s3iface.S3API
bucketsACL map[string][]*s3.Grant
}

func (m *mockS3) GetBucketAcl(input *s3.GetBucketAclInput) (*s3.GetBucketAclOutput, error) {
return &s3.GetBucketAclOutput{Grants: m.bucketsACL[awssdk.StringValue(input.Bucket)]}, nil
}
func (m *mockS3) Name() string {
return ""
}
func (m *mockS3) Provider() string {
return ""
}
func (m *mockS3) ProviderAPI() string {
return ""
}
func (m *mockS3) ProviderRunnableAPI() interface{} {
return m
}
func (m *mockS3) ResourceTypes() []string {
return []string{}
}
func (m *mockS3) FetchResources() (*graph.Graph, error) {
return nil, nil
}
func (m *mockS3) FetchByType(t string) (*graph.Graph, error) {
return nil, nil
}

func diffText(actual, expected string) error {
actuals := strings.Split(actual, "\n")
expecteds := strings.Split(expected, "\n")
Expand Down
Loading

0 comments on commit 5bfa794

Please sign in to comment.