Maha/paramgen (#178)

* use paramgen and util.parseConfig * fix copyright year linter * generated files * resolve conflicts * address reviews * fix go.mod
ConduitIO · Jan 10, 2023 · 4720dbf · 4720dbf
1 parent 1258484
commit 4720dbf
Show file tree

Hide file tree

Showing 12 changed files with 198 additions and 439 deletions.
diff --git a/config/config.go b/config/config.go
@@ -14,8 +14,6 @@
 
 package config
 
-import "fmt"
-
 const (
 	// ConfigKeyAWSAccessKeyID is the config name for AWS access secret key
 	ConfigKeyAWSAccessKeyID = "aws.accessKeyId"
@@ -35,52 +33,14 @@ const (
 
 // Config represents configuration needed for S3
 type Config struct {
-	AWSAccessKeyID     string
-	AWSSecretAccessKey string
-	AWSRegion          string
-	AWSBucket          string
-	Prefix             string
-}
-
-// Parse attempts to parse plugins.Config into a Config struct
-func Parse(cfg map[string]string) (Config, error) {
-	accessKeyID, ok := cfg[ConfigKeyAWSAccessKeyID]
-
-	if !ok {
-		return Config{}, requiredConfigErr(ConfigKeyAWSAccessKeyID)
-	}
-
-	secretAccessKey, ok := cfg[ConfigKeyAWSSecretAccessKey]
-
-	if !ok {
-		return Config{}, requiredConfigErr(ConfigKeyAWSSecretAccessKey)
-	}
-
-	region, ok := cfg[ConfigKeyAWSRegion]
-
-	if !ok {
-		return Config{}, requiredConfigErr(ConfigKeyAWSRegion)
-	}
-
-	bucket, ok := cfg[ConfigKeyAWSBucket]
-
-	if !ok {
-		return Config{}, requiredConfigErr(ConfigKeyAWSBucket)
-	}
-
-	prefix := cfg[ConfigKeyPrefix]
-
-	config := Config{
-		AWSAccessKeyID:     accessKeyID,
-		AWSSecretAccessKey: secretAccessKey,
-		AWSRegion:          region,
-		AWSBucket:          bucket,
-		Prefix:             prefix,
-	}
-
-	return config, nil
-}
-
-func requiredConfigErr(name string) error {
-	return fmt.Errorf("%q config value must be set", name)
+	// AWS access key id.
+	AWSAccessKeyID string `json:"aws.accessKeyId" validate:"required"`
+	// AWS secret access key.
+	AWSSecretAccessKey string `json:"aws.secretAccessKey" validate:"required"`
+	// the AWS S3 bucket region
+	AWSRegion string `json:"aws.region" validate:"required"`
+	// the AWS S3 bucket name.
+	AWSBucket string `json:"aws.bucket" validate:"required"`
+	// the S3 key prefix.
+	Prefix string
 }
diff --git a/config/config_test.go b/config/config_test.go
@@ -16,6 +16,9 @@ package config
 
 import (
 	"testing"
+
+	sdk "github.com/conduitio/conduit-connector-sdk"
+	"github.com/matryer/is"
 )
 
 var exampleConfig = map[string]string{
@@ -26,156 +29,17 @@ var exampleConfig = map[string]string{
 	"prefix":              "conduit-",
 }
 
-func configWith(pairs ...string) map[string]string {
-	cfg := make(map[string]string)
-
-	for key, value := range exampleConfig {
-		cfg[key] = value
-	}
-
-	for i := 0; i < len(pairs); i += 2 {
-		key := pairs[i]
-		value := pairs[i+1]
-		cfg[key] = value
-	}
-
-	return cfg
-}
-
-func configWithout(keys ...string) map[string]string {
-	cfg := make(map[string]string)
-
-	for key, value := range exampleConfig {
-		cfg[key] = value
-	}
-
-	for _, key := range keys {
-		delete(cfg, key)
-	}
-
-	return cfg
-}
-
-func TestAWSAccessKeyID(t *testing.T) {
-	t.Run("Successful", func(t *testing.T) {
-		c, err := Parse(configWith("aws.accessKeyId", "some-value"))
-
-		if err != nil {
-			t.Fatalf("expected no error, got %v", err)
-		}
-
-		if c.AWSAccessKeyID != "some-value" {
-			t.Fatalf("expected AWSAccessKeyID to be %q, got %q", "some-value", c.AWSAccessKeyID)
-		}
-	})
-
-	t.Run("Missing", func(t *testing.T) {
-		_, err := Parse(configWithout("aws.accessKeyId"))
-
-		if err == nil {
-			t.Fatal("expected error, got nothing")
-		}
-
-		expectedErrMsg := `"aws.accessKeyId" config value must be set`
-
-		if err.Error() != expectedErrMsg {
-			t.Fatalf("expected error msg to be %q, got %q", expectedErrMsg, err.Error())
-		}
-	})
-}
-
-func TestAWSSecretAccessKey(t *testing.T) {
-	t.Run("Successful", func(t *testing.T) {
-		c, err := Parse(configWith("aws.secretAccessKey", "some-value"))
-
-		if err != nil {
-			t.Fatalf("expected no error, got %v", err)
-		}
-
-		if c.AWSSecretAccessKey != "some-value" {
-			t.Fatalf("expected AWSSecretAccessKey to be %q, got %q", "some-value", c.AWSSecretAccessKey)
-		}
-	})
-
-	t.Run("Missing", func(t *testing.T) {
-		_, err := Parse(configWithout("aws.secretAccessKey"))
-
-		if err == nil {
-			t.Fatal("expected error, got nothing")
-		}
-
-		expectedErrMsg := `"aws.secretAccessKey" config value must be set`
-
-		if err.Error() != expectedErrMsg {
-			t.Fatalf("expected error msg to be %q, got %q", expectedErrMsg, err.Error())
-		}
-	})
-}
-
-func TestAWSRegion(t *testing.T) {
-	t.Run("Successful", func(t *testing.T) {
-		c, err := Parse(configWith("aws.region", "us-west-2"))
-
-		if err != nil {
-			t.Fatalf("expected no error, got %v", err)
-		}
-
-		if c.AWSRegion != "us-west-2" {
-			t.Fatalf("expected AWSRegion to be %q, got %q", "us-west-2", c.AWSRegion)
-		}
-	})
-
-	t.Run("Missing", func(t *testing.T) {
-		_, err := Parse(configWithout("aws.region"))
-
-		if err == nil {
-			t.Fatal("expected error, got nothing")
-		}
-
-		expectedErrMsg := `"aws.region" config value must be set`
-
-		if err.Error() != expectedErrMsg {
-			t.Fatalf("expected error msg to be %q, got %q", expectedErrMsg, err.Error())
-		}
-	})
-}
-
-func TestAWSBucket(t *testing.T) {
-	t.Run("Successful", func(t *testing.T) {
-		c, err := Parse(configWith("aws.bucket", "foobar"))
-
-		if err != nil {
-			t.Fatalf("expected no error, got %v", err)
-		}
-
-		if c.AWSBucket != "foobar" {
-			t.Fatalf("expected AWSBucket to be %q, got %q", "foobar", c.AWSBucket)
-		}
-	})
-
-	t.Run("Missing", func(t *testing.T) {
-		_, err := Parse(configWithout("aws.bucket"))
-
-		if err == nil {
-			t.Fatal("expected error, got nothing")
-		}
-
-		expectedErrMsg := `"aws.bucket" config value must be set`
-
-		if err.Error() != expectedErrMsg {
-			t.Fatalf("expected error msg to be %q, got %q", expectedErrMsg, err.Error())
-		}
-	})
-}
-
-func TestPrefix(t *testing.T) {
-	c, err := Parse(configWith("prefix", "some/value"))
-
-	if err != nil {
-		t.Fatalf("expected no error, got %v", err)
-	}
-
-	if c.Prefix != "some/value" {
-		t.Fatalf("expected Prefix to be %q, got %q", "some/value", c.Prefix)
+func TestParseConfig(t *testing.T) {
+	is := is.New(t)
+	var got Config
+	err := sdk.Util.ParseConfig(exampleConfig, &got)
+	want := Config{
+		AWSAccessKeyID:     "access-key-123",
+		AWSSecretAccessKey: "secret-key-321",
+		AWSRegion:          "us-west-2",
+		AWSBucket:          "foobucket",
+		Prefix:             "conduit-",
 	}
+	is.NoErr(err)
+	is.Equal(want, got)
 }
diff --git a/destination/config.go b/destination/config.go
@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:generate paramgen -output=paramgen_dest.go Config
+
 package destination
 
 import (
-	"fmt"
-	"strings"
-
 	"github.com/conduitio/conduit-connector-s3/config"
 	"github.com/conduitio/conduit-connector-s3/destination/format"
 )
@@ -30,48 +29,6 @@ const (
 // Config represents S3 configuration with Destination specific configurations
 type Config struct {
 	config.Config
-	Format format.Format
-}
-
-// Parse attempts to parse plugins.Config into a Config struct that Destination could
-// utilize
-func Parse(cfg map[string]string) (Config, error) {
-	// first parse common fields
-	common, err := config.Parse(cfg)
-	if err != nil {
-		return Config{}, err
-	}
-
-	formatString, ok := cfg[ConfigKeyFormat]
-
-	if !ok {
-		return Config{}, requiredConfigErr(ConfigKeyFormat)
-	}
-
-	formatValue, err := format.Parse(formatString)
-
-	if err != nil {
-		var allFormats []string
-
-		for _, f := range format.All {
-			allFormats = append(allFormats, string(f))
-		}
-
-		return Config{}, fmt.Errorf(
-			"%q config value should be one of (%s)",
-			ConfigKeyFormat,
-			strings.Join(allFormats, ", "),
-		)
-	}
-
-	destinationConfig := Config{
-		Config: common,
-		Format: formatValue,
-	}
-
-	return destinationConfig, nil
-}
-
-func requiredConfigErr(name string) error {
-	return fmt.Errorf("%q config value must be set", name)
+	// the destination format, either "json" or "parquet".
+	Format format.Format `validate:"required,inclusion=parquet|json"`
 }