Add options for adding "fields" for AWS S3

.codecov.yml

@@ -2,6 +2,19 @@ coverage:
   range: 80..100
   round: down
   precision: 2
+  status:
+    project: # measuring the overall project coverage
+      default: # context, you can create multiple ones with custom titles
+        enabled: yes           # must be yes|true to enable this status
+        target: 90             # specify the target coverage for each commit status
+        #   option: "auto" (must increase from parent commit or pull request base)
+        #   option: "X%" a static target percentage to hit
+        threshold: null        # allowed to drop X% and still result in a "success" commit status
+        if_not_found: success  # if parent is not found report status as success, error, or failure
+        if_ci_failed: error    # if ci fails report status as success, error, or failure
+    patch:
+      default:
+        enabled: no
 ignore:
   - ".github/**"
   - "ci/**"

README.md

@@ -307,17 +307,6 @@ source:
       account_id: "xxx"
 ```
 
-#### Available metadata for AWS S3 Source
-
-| Meta Key            | Description                                                                                         |
-|---------------------|-----------------------------------------------------------------------------------------------------|
-| AWS-S3-Bucket-Name  | Name of the bucket name                                                                             |
-| Scanner-Label       | Name of the source configured in config.yaml file                                                   |
-| AWS-S3-Region       | Region name of AWS S3 source                                                                        |
-| AWS-ARN             | AWS ARN for the bucket                                                                              |
-| AWS-S3-Tag-{tagKey} | Every tag key associated with the source will be added as metadata. {tagKey} is the tag key from S3 |
-
-
 ### :octocat: GitHub Repository
 
 It will fetch all GitHub repositories with metadata

cmd/testdata/valid_config.yaml

@@ -88,6 +88,14 @@ source:
       - region
       - arn
       - tags
+  aws_s3_one:
+    type: aws_s3
+    configuration: *aws_conf
+    fields:
+      - bucket_name
+      - region
+      - arn
+      - tags
 relations:
   criteria:
     - name: "file-system-rule1"

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7
 	github.com/aws/aws-sdk-go-v2/service/rds v1.51.0
 	github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.7
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
 	github.com/google/go-github/v50 v50.2.0
 	github.com/google/uuid v1.3.0
 	github.com/kyokomi/emoji v2.2.4+incompatible
@@ -26,11 +27,16 @@ require (
 
 require (
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect

go.sum

@@ -7,6 +7,8 @@ github.com/aws/aws-sdk-go v1.44.234/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8
 github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
 github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM=
 github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw=
 github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c=
@@ -21,17 +23,27 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45lt
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1 h1:xn5CI639mnWvdiweqoRx/H221Ia9Asx9XxfIRhe0MPo=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1/go.mod h1:ZZLfkd1Y7fjXujjMg1CFqNmaTl314eCbShlHQO7VTWo=
 github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7 h1:oQ1Esut3iaL2Dydt2RBd9gbuUevToXpdTI+Uh1xXryI=
 github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7/go.mod h1:RHhgOMnMIkgB4TmxQat9obSnZ6fF1fuA27+itZKUi1o=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8=
 github.com/aws/aws-sdk-go-v2/service/rds v1.51.0 h1:9yschHJVfDwU1aXriWZOUzX4/vrv0L2sq8nVHRH97uU=
 github.com/aws/aws-sdk-go-v2/service/rds v1.51.0/go.mod h1:UNv1vk1fU1NJefzteykVpVLA88w4WxB05g3vp2kQhYM=
 github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.7 h1:68kjp2WO8gv2tqBxVmffdmMUGuk7SXEinCwrPSPcXXo=
 github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.7/go.mod h1:0xtogtnBtKXuOOweAUxTrxuBZhYTFX3KjQnrSXQr6lM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4=

pkg/config/schema.json

@@ -279,9 +279,26 @@
         },
         "configuration": {
           "$ref": "#/definitions/source.aws_common.configuration"
+        },
+        "fields": {
+          "$ref": "#/definitions/source.aws_s3.fields"
         }
       },
-      "required": ["type", "configuration"]
+      "required": ["type", "configuration", "fields"]
+    },
+    "source.aws_s3.fields": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": [
+          "bucket_name",
+          "region",
+          "arn",
+          "tags"
+        ]
+      },
+      "uniqueItems": true,
+      "additionalItems": false
     },
     "source.aws_rds": {
       "type": "object",

pkg/config/schema_validator_test.go

@@ -72,6 +72,11 @@ source:
       session_token: "xxxx"
       region: "x"
       account_id: "xxx"
+    fields:
+      - bucket_name
+      - region
+      - arn
+      - tags
   aws_rds_one:
     type: aws_rds
     configuration: *aws_conf

pkg/config/testdata/valid_config.yaml

@@ -52,6 +52,11 @@ source:
       session_token: "xxxx"
       region: "x"
       account_id: "xxx"
+    fields:
+      - bucket_name
+      - region
+      - arn
+      - tags
   aws_rds_one:
     type: aws_rds
     configuration: *aws_conf

pkg/scanner/aws_s3.go

@@ -2,92 +2,103 @@
 package scanner
 
 import (
+	"context"
 	"fmt"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/shaharia-lab/teredix/pkg"
 	"github.com/shaharia-lab/teredix/pkg/resource"
 	"github.com/shaharia-lab/teredix/pkg/util"
 
-	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go-v2/aws"
 
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+const (
+	s3fieldBucketName = "bucketName"
+	s3fieldRegion     = "region"
+	s3fieldARN        = "arn"
+	s3fieldTags       = "tags"
+
+	s3ARNFormat = "arn:aws:s3:::%s"
 )
 
 // AWSS3Client build aws client
 type AWSS3Client interface {
-	ListBuckets(listBucketInput *s3.ListBucketsInput) (*s3.ListBucketsOutput, error)
-	GetBucketTagging(bucketTaggingInput *s3.GetBucketTaggingInput) (*s3.GetBucketTaggingOutput, error)
+	ListBuckets(ctx context.Context, params *s3.ListBucketsInput, optFns ...func(*s3.Options)) (*s3.ListBucketsOutput, error)
+	GetBucketTagging(ctx context.Context, params *s3.GetBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.GetBucketTaggingOutput, error)
 }
 
 // AWSS3 AWS S3 source
 type AWSS3 struct {
 	SourceName string
 	S3Client   AWSS3Client
 	Region     string
+	Fields     []string
 }
 
 // NewAWSS3 construct AWS S3 source
-func NewAWSS3(sourceName string, region string, s3Client AWSS3Client) *AWSS3 {
+func NewAWSS3(sourceName string, region string, s3Client AWSS3Client, fields []string) *AWSS3 {
 	return &AWSS3{
 		SourceName: sourceName,
 		S3Client:   s3Client,
 		Region:     region,
+		Fields:     fields,
 	}
 }
 
 // Scan discover resource and send to resource channel
 func (a *AWSS3) Scan(resourceChannel chan resource.Resource) error {
-	result, err := a.S3Client.ListBuckets(nil)
+	// List all S3 buckets
+	output, err := a.S3Client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
 	if err != nil {
-		return fmt.Errorf("failed to list buckets. error: %w", err)
+		return fmt.Errorf("unable to list buckets: %w", err)
 	}
 
-	for _, bucket := range result.Buckets {
-		resourceChannel <- a.mapToResource(bucket)
+	for _, bucket := range output.Buckets {
+		resourceChannel <- resource.Resource{
+			Kind:        pkg.ResourceKindAWSS3,
+			UUID:        util.GenerateUUID(),
+			Name:        aws.ToString(bucket.Name),
+			ExternalID:  aws.ToString(bucket.Name),
+			RelatedWith: nil,
+			MetaData:    a.getMetaData(bucket),
+		}
 	}
 
 	return nil
 }
 
-func (a *AWSS3) mapToResource(bucket *s3.Bucket) resource.Resource {
-	res := resource.Resource{
-		Kind:        pkg.ResourceKindAWSS3,
-		UUID:        util.GenerateUUID(),
-		Name:        aws.StringValue(bucket.Name),
-		ExternalID:  aws.StringValue(bucket.Name),
-		RelatedWith: nil,
-		MetaData: []resource.MetaData{
-			{
-				Key:   "AWS-S3-Bucket-Name",
-				Value: aws.StringValue(bucket.Name),
-			},
-			{
-				Key:   pkg.MetaKeyScannerLabel,
-				Value: a.SourceName,
-			},
-			{
-				Key:   "AWS-S3-Region",
-				Value: a.Region,
-			},
-			{
-				Key:   "AWS-ARN",
-				Value: fmt.Sprintf("arn:aws:s3:::%s", aws.StringValue(bucket.Name)),
-			},
+func (a *AWSS3) getMetaData(bucket types.Bucket) []resource.MetaData {
+	mappings := map[string]func() string{
+		s3fieldBucketName: func() string { return aws.ToString(bucket.Name) },
+		s3fieldARN: func() string {
+			return fmt.Sprintf(s3ARNFormat, aws.ToString(bucket.Name))
 		},
+		s3fieldRegion: func() string { return a.Region },
 	}
 
-	bucketName := aws.StringValue(bucket.Name)
+	getTags := func() []ResourceTag {
+		var tt []ResourceTag
 
-	tagResult, _ := a.S3Client.GetBucketTagging(&s3.GetBucketTaggingInput{
-		Bucket: aws.String(bucketName),
-	})
+		if util.IsFieldExistsInConfig(s3fieldTags, a.Fields) == false {
+			return tt
+		}
 
-	for _, tag := range tagResult.TagSet {
-		res.MetaData = append(res.MetaData, resource.MetaData{
-			Key:   fmt.Sprintf("AWS-S3-Tag-%s", aws.StringValue(tag.Key)),
-			Value: aws.StringValue(tag.Value),
+		tagResult, _ := a.S3Client.GetBucketTagging(context.TODO(), &s3.GetBucketTaggingInput{
+			Bucket: aws.String(aws.ToString(bucket.Name)),
 		})
+
+		for _, tag := range tagResult.TagSet {
+			tt = append(tt, ResourceTag{
+				Key:   aws.ToString(tag.Key),
+				Value: aws.ToString(tag.Value),
+			})
+		}
+
+		return tt
 	}
 
-	return res
+	return NewFieldMapper(mappings, getTags, a.Fields).getResourceMetaData()
 }