Added fields option for AWS ECR (#45)

- Added fields option for AWS ECR
- Re-factored field mapper for sources to reduce code duplication

pkg/cmd/testdata/valid_config.yaml

@@ -55,7 +55,7 @@ source:
       - topics
   aws_ec2_one:
     type: aws_ec2
-    configuration:
+    configuration: &aws_conf
       access_key: "xxxx"
       secret_key: "xxxx"
       session_token: "xxxx"
@@ -71,12 +71,15 @@ source:
       - instance_state
       - vpc_id
       - tags
-  #two:
-  #  type: kubernetes
-  #  configuration:
-  #    kube_config_file_path: "another_path"
-  #  depends_on:
-  #    - "one"
+  aws_ecr_one:
+    type: aws_ecr
+    configuration: *aws_conf
+    fields:
+      - repository_name
+      - repository_uri
+      - registry_id
+      - arn
+      - tags
 relations:
   criteria:
     - name: "file-system-rule1"

pkg/config/schema.json

@@ -235,7 +235,7 @@
           "$ref": "#/definitions/source.github_repository.fields"
         }
       },
-      "required": ["type", "configuration"]
+      "required": ["type", "configuration", "fields"]
     },
     "source.github_repository.configuration": {
       "type": "object",
@@ -310,7 +310,26 @@
           "$ref": "#/definitions/source.aws_ec2.fields"
         }
       },
-      "required": ["type", "configuration"]
+      "required": ["type", "configuration", "fields"]
+    },
+    "source.aws_ec2.fields": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": [
+          "instance_id",
+          "image_id",
+          "private_dns_name",
+          "instance_type",
+          "architecture",
+          "instance_lifecycle",
+          "instance_state",
+          "vpc_id",
+          "tags"
+        ]
+      },
+      "uniqueItems": true,
+      "additionalItems": false
     },
     "source.aws_ecr": {
       "type": "object",
@@ -321,10 +340,28 @@
         },
         "configuration": {
           "$ref": "#/definitions/source.aws_common.configuration"
+        },
+        "fields": {
+          "$ref": "#/definitions/source.aws_ecr.fields"
         }
       },
       "required": ["type", "configuration"]
     },
+    "source.aws_ecr.fields": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": [
+          "repository_name",
+          "arn",
+          "registry_id",
+          "repository_uri",
+          "tags"
+        ]
+      },
+      "uniqueItems": true,
+      "additionalItems": false
+    },
     "source.aws_common.configuration": {
       "type": "object",
       "properties": {
@@ -351,25 +388,6 @@
       },
       "required": ["access_key", "secret_key", "session_token", "region", "account_id"]
     },
-    "source.aws_ec2.fields": {
-      "type": "array",
-      "items": {
-        "type": "string",
-        "enum": [
-          "instance_id",
-          "image_id",
-          "private_dns_name",
-          "instance_type",
-          "architecture",
-          "instance_lifecycle",
-          "instance_state",
-          "vpc_id",
-          "tags"
-        ]
-      },
-      "uniqueItems": true,
-      "additionalItems": false
-    },
     "storage.postgresql": {
       "type": "object",
       "properties": {

pkg/config/schema_validator_test.go

@@ -96,6 +96,12 @@ source:
   aws_ecr_example:
     type: aws_ecr
     configuration: *aws_conf
+    fields:
+      - repository_name
+      - repository_uri
+      - registry_id
+      - arn
+      - tags
 relations:
   criteria:
     - name: "file-system-rule1"

pkg/config/testdata/valid_config.yaml

@@ -71,6 +71,12 @@ source:
   aws_ecr_example:
     type: aws_ecr
     configuration: *aws_conf
+    fields:
+      - repository_name
+      - repository_uri
+      - registry_id
+      - arn
+      - tags
 relations:
   criteria:
     - name: "file-system-rule1"

pkg/source/scanner/aws_ec2.go

@@ -3,12 +3,9 @@ package scanner
 
 import (
 	"context"
-	"fmt"
-	"strings"
 
 	"github.com/shaharia-lab/teredix/pkg"
 	"github.com/shaharia-lab/teredix/pkg/resource"
-	"github.com/shaharia-lab/teredix/pkg/util"
 
 	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
 
@@ -69,7 +66,13 @@ func (a *AWSEC2) Scan(resourceChannel chan resource.Resource) error {
 		// Loop through instances and their tags
 		for _, reservation := range resp.Reservations {
 			for _, instance := range reservation.Instances {
-				resourceChannel <- a.mapToResource(instance)
+				resourceChannel <- resource.Resource{
+					Name:       *instance.InstanceId,
+					Kind:       pkg.ResourceKindAWSEC2,
+					UUID:       *instance.InstanceId,
+					ExternalID: *instance.InstanceId,
+					MetaData:   a.getMetaData(instance),
+				}
 			}
 		}
 
@@ -83,6 +86,22 @@ func (a *AWSEC2) Scan(resourceChannel chan resource.Resource) error {
 	return nil
 }
 
+func (a *AWSEC2) getMetaData(instance types.Instance) []resource.MetaData {
+	mappings := map[string]func() string{
+		fieldInstanceID:        func() string { return safeDereference(instance.InstanceId) },
+		fieldImageID:           func() string { return safeDereference(instance.ImageId) },
+		fieldPrivateDNSName:    func() string { return safeDereference(instance.PrivateDnsName) },
+		fieldInstanceType:      func() string { return stringValueOrDefault(string(instance.InstanceType)) },
+		fieldArchitecture:      func() string { return stringValueOrDefault(string(instance.Architecture)) },
+		fieldInstanceLifecycle: func() string { return stringValueOrDefault(string(instance.InstanceLifecycle)) },
+		fieldInstanceState:     func() string { return stringValueOrDefault(string(instance.State.Name)) },
+		fieldVpcID:             func() string { return safeDereference(instance.VpcId) },
+	}
+	return NewFieldMapper(mappings, func() []types.Tag {
+		return instance.Tags
+	}, a.Fields).getResourceMetaData()
+}
+
 func (a *AWSEC2) makeAPICallToAWS(nextToken string) (*ec2.DescribeInstancesOutput, error) {
 	// Describe instances for current page
 	params := &ec2.DescribeInstancesInput{
@@ -112,67 +131,3 @@ func (a *AWSEC2) makeAPICallToAWS(nextToken string) (*ec2.DescribeInstancesOutpu
 	}
 	return resp, nil
 }
-
-func (a *AWSEC2) mapToResource(instance types.Instance) resource.Resource {
-	var repoMeta []resource.MetaData
-	for _, mapper := range a.fieldMapper(instance) {
-		if util.IsFieldExistsInConfig(mapper.field, a.Fields) || strings.Contains(mapper.field, "tag_") {
-			val := mapper.value()
-			if val != "" {
-				repoMeta = append(repoMeta, resource.MetaData{Key: mapper.field, Value: val})
-			}
-		}
-	}
-
-	return resource.Resource{
-		Name:       *instance.InstanceId,
-		Kind:       pkg.ResourceKindAWSEC2,
-		UUID:       *instance.InstanceId,
-		ExternalID: *instance.InstanceId,
-		MetaData:   repoMeta,
-	}
-}
-
-func safeDereference(s *string) string {
-	if s != nil {
-		return *s
-	}
-	return ""
-}
-
-func stringValueOrDefault(s string) string {
-	if s != "" {
-		return s
-	}
-	return ""
-}
-
-func (a *AWSEC2) fieldMapper(instance types.Instance) []MetaDataMapper {
-	var fieldMapper []MetaDataMapper
-
-	mappings := map[string]func() string{
-		fieldInstanceID:        func() string { return safeDereference(instance.InstanceId) },
-		fieldImageID:           func() string { return safeDereference(instance.ImageId) },
-		fieldPrivateDNSName:    func() string { return safeDereference(instance.PrivateDnsName) },
-		fieldInstanceType:      func() string { return stringValueOrDefault(string(instance.InstanceType)) },
-		fieldArchitecture:      func() string { return stringValueOrDefault(string(instance.Architecture)) },
-		fieldInstanceLifecycle: func() string { return stringValueOrDefault(string(instance.InstanceLifecycle)) },
-		fieldInstanceState:     func() string { return stringValueOrDefault(string(instance.State.Name)) },
-		fieldVpcID:             func() string { return safeDereference(instance.VpcId) },
-	}
-
-	for field, fn := range mappings {
-		fieldMapper = append(fieldMapper, MetaDataMapper{field: field, value: fn})
-	}
-
-	if util.IsFieldExistsInConfig(fieldTags, a.Fields) {
-		for _, tag := range instance.Tags {
-			fieldMapper = append(fieldMapper, MetaDataMapper{
-				field: fmt.Sprintf("tag_%s", safeDereference(tag.Key)),
-				value: func() string { return safeDereference(tag.Value) },
-			})
-		}
-	}
-
-	return fieldMapper
-}

pkg/source/scanner/aws_ecr.go

@@ -3,8 +3,9 @@ package scanner
 
 import (
 	"context"
-	"fmt"
 
+	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	ecrTypes "github.com/aws/aws-sdk-go-v2/service/ecr/types"
 	"github.com/shaharia-lab/teredix/pkg"
 	"github.com/shaharia-lab/teredix/pkg/resource"
 	"github.com/shaharia-lab/teredix/pkg/util"
@@ -13,6 +14,14 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/ecr"
 )
 
+const (
+	ecrFieldRepositoryName = "repositoryName"
+	ecrFieldArn            = "repositoryArn"
+	ecrFieldRegistryID     = "registryID"
+	ecrFieldRepositoryURI  = "repositoryURI"
+	ecrFieldTags           = "tags"
+)
+
 // EcrClient build aws client
 type EcrClient interface {
 	DescribeRepositories(context.Context, *ecr.DescribeRepositoriesInput, ...func(options *ecr.Options)) (*ecr.DescribeRepositoriesOutput, error)
@@ -27,16 +36,18 @@ type AWSECR struct {
 	Region                 string
 	AccountID              string
 	ResourceTaggingService util.ResourceTaggingServiceClient
+	Fields                 []string
 }
 
 // NewAWSECR construct AWS ECR source
-func NewAWSECR(sourceName string, region string, accountID string, ecrClient EcrClient, resourceTaggingService util.ResourceTaggingServiceClient) *AWSECR {
+func NewAWSECR(sourceName string, region string, accountID string, ecrClient EcrClient, resourceTaggingService util.ResourceTaggingServiceClient, fields []string) *AWSECR {
 	return &AWSECR{
 		SourceName:             sourceName,
 		ECRClient:              ecrClient,
 		Region:                 region,
 		AccountID:              accountID,
 		ResourceTaggingService: resourceTaggingService,
+		Fields:                 fields,
 	}
 }
 
@@ -68,40 +79,7 @@ func (a *AWSECR) Scan(resourceChannel chan resource.Resource) error {
 				Kind:       pkg.ResourceKindAWSECR,
 				UUID:       util.GenerateUUID(),
 				ExternalID: *repository.RepositoryArn,
-				MetaData: []resource.MetaData{
-					{
-						Key:   "AWS-ECR-Repository-Name",
-						Value: *repository.RepositoryName,
-					},
-					{
-						Key:   "AWS-ECR-Repository-Arn",
-						Value: *repository.RepositoryArn,
-					},
-					{
-						Key:   "AWS-ECR-Registry-Id",
-						Value: *repository.RegistryId,
-					},
-					{
-						Key:   "AWS-ECR-Repository-URI",
-						Value: *repository.RepositoryUri,
-					},
-					{
-						Key:   pkg.MetaKeyScannerLabel,
-						Value: a.SourceName,
-					},
-				},
-			}
-
-			tags, err := util.GetAWSResourceTagByARN(context.Background(), a.ResourceTaggingService, *repository.RepositoryArn)
-			if err != nil {
-				return err
-			}
-
-			for tagKey, tagValue := range tags {
-				res.MetaData = append(res.MetaData, resource.MetaData{
-					Key:   fmt.Sprintf("AWS-ECR-%s", tagKey),
-					Value: tagValue,
-				})
+				MetaData:   a.getMetaData(repository),
 			}
 
 			resourceChannel <- res
@@ -117,3 +95,31 @@ func (a *AWSECR) Scan(resourceChannel chan resource.Resource) error {
 
 	return nil
 }
+
+func (a *AWSECR) getMetaData(repository ecrTypes.Repository) []resource.MetaData {
+	mappings := map[string]func() string{
+		ecrFieldRepositoryName: func() string { return stringValueOrDefault(*repository.RepositoryName) },
+		ecrFieldArn:            func() string { return stringValueOrDefault(*repository.RepositoryArn) },
+		ecrFieldRegistryID:     func() string { return stringValueOrDefault(*repository.RegistryId) },
+		ecrFieldRepositoryURI:  func() string { return stringValueOrDefault(*repository.RepositoryUri) },
+	}
+
+	getTags := func() []types.Tag {
+		tags, err := util.GetAWSResourceTagByARN(context.Background(), a.ResourceTaggingService, *repository.RepositoryArn)
+		if err != nil {
+			return []types.Tag{}
+		}
+
+		var tt []types.Tag
+		for key, val := range tags {
+			tt = append(tt, types.Tag{
+				Key:   aws.String(key),
+				Value: aws.String(val),
+			})
+		}
+
+		return tt
+	}
+
+	return NewFieldMapper(mappings, getTags, a.Fields).getResourceMetaData()
+}