Skip to content

Commit

Permalink
Add options for adding "fields" for AWS S3 (#49)
Browse files Browse the repository at this point in the history 
* Adding S3 fields
  • Loading branch information
@shahariaazam
shahariaazam authored Aug 28, 2023
1 parent abe9bdf commit 07e3797
Show file tree
Hide file tree
Showing 12 changed files with 188 additions and 121 deletions.
13 changes: 13 additions & 0 deletions .codecov.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,19 @@ coverage:
range: 80..100
round: down
precision: 2
status:
project: # measuring the overall project coverage
default: # context, you can create multiple ones with custom titles
enabled: yes # must be yes|true to enable this status
target: 90 # specify the target coverage for each commit status
# option: "auto" (must increase from parent commit or pull request base)
# option: "X%" a static target percentage to hit
threshold: null # allowed to drop X% and still result in a "success" commit status
if_not_found: success # if parent is not found report status as success, error, or failure
if_ci_failed: error # if ci fails report status as success, error, or failure
patch:
default:
enabled: no
ignore:
- ".github/**"
- "ci/**"
Expand Down
11 changes: 0 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,17 +307,6 @@ source:
account_id: "xxx"
```
#### Available metadata for AWS S3 Source
| Meta Key | Description |
|---------------------|-----------------------------------------------------------------------------------------------------|
| AWS-S3-Bucket-Name | Name of the bucket name |
| Scanner-Label | Name of the source configured in config.yaml file |
| AWS-S3-Region | Region name of AWS S3 source |
| AWS-ARN | AWS ARN for the bucket |
| AWS-S3-Tag-{tagKey} | Every tag key associated with the source will be added as metadata. {tagKey} is the tag key from S3 |
### :octocat: GitHub Repository
It will fetch all GitHub repositories with metadata
Expand Down
8 changes: 8 additions & 0 deletions cmd/testdata/valid_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,14 @@ source:
- region
- arn
- tags
aws_s3_one:
type: aws_s3
configuration: *aws_conf
fields:
- bucket_name
- region
- arn
- tags
relations:
criteria:
- name: "file-system-rule1"
Expand Down
6 changes: 6 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7
github.com/aws/aws-sdk-go-v2/service/rds v1.51.0
github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.7
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
github.com/google/go-github/v50 v50.2.0
github.com/google/uuid v1.3.0
github.com/kyokomi/emoji v2.2.4+incompatible
Expand All @@ -26,11 +27,16 @@ require (

require (
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect
Expand Down
12 changes: 12 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ github.com/aws/aws-sdk-go v1.44.234/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8
github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM=
github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw=
github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY=
github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c=
Expand All @@ -21,17 +23,27 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45lt
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1 h1:xn5CI639mnWvdiweqoRx/H221Ia9Asx9XxfIRhe0MPo=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1/go.mod h1:ZZLfkd1Y7fjXujjMg1CFqNmaTl314eCbShlHQO7VTWo=
github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7 h1:oQ1Esut3iaL2Dydt2RBd9gbuUevToXpdTI+Uh1xXryI=
github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7/go.mod h1:RHhgOMnMIkgB4TmxQat9obSnZ6fF1fuA27+itZKUi1o=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8=
github.com/aws/aws-sdk-go-v2/service/rds v1.51.0 h1:9yschHJVfDwU1aXriWZOUzX4/vrv0L2sq8nVHRH97uU=
github.com/aws/aws-sdk-go-v2/service/rds v1.51.0/go.mod h1:UNv1vk1fU1NJefzteykVpVLA88w4WxB05g3vp2kQhYM=
github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.7 h1:68kjp2WO8gv2tqBxVmffdmMUGuk7SXEinCwrPSPcXXo=
github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.7/go.mod h1:0xtogtnBtKXuOOweAUxTrxuBZhYTFX3KjQnrSXQr6lM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4=
Expand Down
19 changes: 18 additions & 1 deletion pkg/config/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,9 +279,26 @@
},
"configuration": {
"$ref": "#/definitions/source.aws_common.configuration"
},
"fields": {
"$ref": "#/definitions/source.aws_s3.fields"
}
},
"required": ["type", "configuration"]
"required": ["type", "configuration", "fields"]
},
"source.aws_s3.fields": {
"type": "array",
"items": {
"type": "string",
"enum": [
"bucket_name",
"region",
"arn",
"tags"
]
},
"uniqueItems": true,
"additionalItems": false
},
"source.aws_rds": {
"type": "object",
Expand Down
5 changes: 5 additions & 0 deletions pkg/config/schema_validator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ source:
session_token: "xxxx"
region: "x"
account_id: "xxx"
fields:
- bucket_name
- region
- arn
- tags
aws_rds_one:
type: aws_rds
configuration: *aws_conf
Expand Down
5 changes: 5 additions & 0 deletions pkg/config/testdata/valid_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ source:
session_token: "xxxx"
region: "x"
account_id: "xxx"
fields:
- bucket_name
- region
- arn
- tags
aws_rds_one:
type: aws_rds
configuration: *aws_conf
Expand Down
95 changes: 53 additions & 42 deletions pkg/scanner/aws_s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,92 +2,103 @@
package scanner

import (
"context"
"fmt"

"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/shaharia-lab/teredix/pkg"
"github.com/shaharia-lab/teredix/pkg/resource"
"github.com/shaharia-lab/teredix/pkg/util"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go-v2/aws"

"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3"
)

const (
s3fieldBucketName = "bucketName"
s3fieldRegion = "region"
s3fieldARN = "arn"
s3fieldTags = "tags"

s3ARNFormat = "arn:aws:s3:::%s"
)

// AWSS3Client build aws client
type AWSS3Client interface {
ListBuckets(listBucketInput *s3.ListBucketsInput) (*s3.ListBucketsOutput, error)
GetBucketTagging(bucketTaggingInput *s3.GetBucketTaggingInput) (*s3.GetBucketTaggingOutput, error)
ListBuckets(ctx context.Context, params *s3.ListBucketsInput, optFns ...func(*s3.Options)) (*s3.ListBucketsOutput, error)
GetBucketTagging(ctx context.Context, params *s3.GetBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.GetBucketTaggingOutput, error)
}

// AWSS3 AWS S3 source
type AWSS3 struct {
SourceName string
S3Client AWSS3Client
Region string
Fields []string
}

// NewAWSS3 construct AWS S3 source
func NewAWSS3(sourceName string, region string, s3Client AWSS3Client) *AWSS3 {
func NewAWSS3(sourceName string, region string, s3Client AWSS3Client, fields []string) *AWSS3 {
return &AWSS3{
SourceName: sourceName,
S3Client: s3Client,
Region: region,
Fields: fields,
}
}

// Scan discover resource and send to resource channel
func (a *AWSS3) Scan(resourceChannel chan resource.Resource) error {
result, err := a.S3Client.ListBuckets(nil)
// List all S3 buckets
output, err := a.S3Client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
if err != nil {
return fmt.Errorf("failed to list buckets. error: %w", err)
return fmt.Errorf("unable to list buckets: %w", err)
}

for _, bucket := range result.Buckets {
resourceChannel <- a.mapToResource(bucket)
for _, bucket := range output.Buckets {
resourceChannel <- resource.Resource{
Kind: pkg.ResourceKindAWSS3,
UUID: util.GenerateUUID(),
Name: aws.ToString(bucket.Name),
ExternalID: aws.ToString(bucket.Name),
RelatedWith: nil,
MetaData: a.getMetaData(bucket),
}
}

return nil
}

func (a *AWSS3) mapToResource(bucket *s3.Bucket) resource.Resource {
res := resource.Resource{
Kind: pkg.ResourceKindAWSS3,
UUID: util.GenerateUUID(),
Name: aws.StringValue(bucket.Name),
ExternalID: aws.StringValue(bucket.Name),
RelatedWith: nil,
MetaData: []resource.MetaData{
{
Key: "AWS-S3-Bucket-Name",
Value: aws.StringValue(bucket.Name),
},
{
Key: pkg.MetaKeyScannerLabel,
Value: a.SourceName,
},
{
Key: "AWS-S3-Region",
Value: a.Region,
},
{
Key: "AWS-ARN",
Value: fmt.Sprintf("arn:aws:s3:::%s", aws.StringValue(bucket.Name)),
},
func (a *AWSS3) getMetaData(bucket types.Bucket) []resource.MetaData {
mappings := map[string]func() string{
s3fieldBucketName: func() string { return aws.ToString(bucket.Name) },
s3fieldARN: func() string {
return fmt.Sprintf(s3ARNFormat, aws.ToString(bucket.Name))
},
s3fieldRegion: func() string { return a.Region },
}

bucketName := aws.StringValue(bucket.Name)
getTags := func() []ResourceTag {
var tt []ResourceTag

tagResult, _ := a.S3Client.GetBucketTagging(&s3.GetBucketTaggingInput{
Bucket: aws.String(bucketName),
})
if util.IsFieldExistsInConfig(s3fieldTags, a.Fields) == false {
return tt
}

for _, tag := range tagResult.TagSet {
res.MetaData = append(res.MetaData, resource.MetaData{
Key: fmt.Sprintf("AWS-S3-Tag-%s", aws.StringValue(tag.Key)),
Value: aws.StringValue(tag.Value),
tagResult, _ := a.S3Client.GetBucketTagging(context.TODO(), &s3.GetBucketTaggingInput{
Bucket: aws.String(aws.ToString(bucket.Name)),
})

for _, tag := range tagResult.TagSet {
tt = append(tt, ResourceTag{
Key: aws.ToString(tag.Key),
Value: aws.ToString(tag.Value),
})
}

return tt
}

return res
return NewFieldMapper(mappings, getTags, a.Fields).getResourceMetaData()
}
Loading

0 comments on commit 07e3797

Please sign in to comment.