Allow config overrides with merging and multiple upload variants

aws/uploader.go

@@ -501,7 +501,7 @@ func (u *Uploader) tagImageAndSnapshot(ctx context.Context, amiID, region string
 }
 
 func (u *Uploader) publishImage(ctx context.Context, amiID, region string) error {
-	if !u.config.AWS.Publish {
+	if !u.config.AWS.Publish.UnwrapOr(false) {
 		return nil
 	}
 

cmd.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -23,7 +24,8 @@ import (
 )
 
 const (
-	configName = "uplosi.toml"
+	configName = "uplosi.conf"
+	configDir  = "uplosi.conf.d"
 )
 
 var (
@@ -33,12 +35,12 @@ var (
 
 func newCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:              "uplosi <provider> <image>",
+		Use:              "uplosi <image>",
 		Short:            "uplosi is a tool for uploading images to a cloud provider",
 		PersistentPreRun: preRunRoot,
 		RunE:             run,
 		Version:          version,
-		Args:             cobra.MatchAll(cobra.ExactArgs(2), isCSP(0)),
+		Args:             cobra.ExactArgs(1),
 	}
 	cmd.SetOut(os.Stdout)
 	cmd.InitDefaultVersionFlag()
@@ -47,110 +49,177 @@ func newCmd() *cobra.Command {
 	)
 
 	cmd.Flags().BoolP("increment-version", "i", false, "increment version number in config after upload")
+	cmd.Flags().StringSlice("enable-variant-glob", []string{"*"}, "list of variant name globs to enable")
+	cmd.Flags().StringSlice("disable-variant-glob", nil, "list of variant name globs to disable")
 
 	return cmd
 }
 
 func run(cmd *cobra.Command, args []string) error {
 	logger := log.New(cmd.OutOrStderr(), "", log.LstdFlags)
-	provider := args[0]
-	imagePath := args[1]
+	imagePath := args[0]
 
 	flags, err := parseUploadFlags(cmd)
 	if err != nil {
 		return fmt.Errorf("parsing flags: %w", err)
 	}
 
-	var config uploader.Config
-	if err := readTOMLFile(configName, &config); err != nil {
-		return fmt.Errorf("reading config: %w", err)
+	config, err := parseConfigFiles()
+	if err != nil {
+		return fmt.Errorf("parsing config files: %w", err)
+	}
+
+	versionFiles := map[string][]byte{}
+	versionFileLookup := func(name string) ([]byte, error) {
+		if _, ok := versionFiles[name]; !ok {
+			ver, err := os.ReadFile(name)
+			if err != nil {
+				return nil, fmt.Errorf("reading version file: %w", err)
+			}
+			versionFiles[name] = ver
+		}
+		return versionFiles[name], nil
+	}
+
+	allRefs := []string{}
+	err = config.ForEach(
+		func(name string, cfg uploader.Config) error {
+			refs, err := uploadVariant(cmd.Context(), imagePath, name, cfg, logger)
+			if err != nil {
+				return err
+			}
+			allRefs = append(allRefs, refs...)
+			return nil
+		},
+		versionFileLookup,
+		func(name string) bool {
+			return filterGlobAny(flags.enableVariantGlobs, name)
+		},
+		func(name string) bool {
+			return !filterGlobAny(flags.disableVariantGlobs, name)
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("uploading variants: %w", err)
+	}
+
+	for _, ref := range allRefs {
+		fmt.Println(ref)
 	}
 
+	// if flags.incrementVersion {
+	// 	newVer, err := incrementSemver(config.ImageVersion)
+	// 	if err != nil {
+	// 		return fmt.Errorf("incrementing semver: %w", err)
+	// 	}
+	// 	config.ImageVersion = newVer
+	// 	if err := writeTOMLFile(configName, config); err != nil {
+	// 		return fmt.Errorf("writing config: %w", err)
+	// 	}
+	// }
+
+	return nil
+}
+
+func uploadVariant(ctx context.Context, imagePath, variant string, config uploader.Config, logger *log.Logger) ([]string, error) {
 	var prepper Prepper
 	var upload Uploader
+	var err error
 
-	switch strings.ToLower(provider) {
+	if len(variant) > 0 {
+		log.Println("Uploading variant", variant)
+	}
+
+	switch strings.ToLower(config.Provider) {
 	case "aws":
 		prepper = &aws.Prepper{}
 		upload, err = aws.NewUploader(config, logger)
 		if err != nil {
-			return fmt.Errorf("creating aws uploader: %w", err)
+			return nil, fmt.Errorf("creating aws uploader: %w", err)
 		}
 	case "azure":
 		prepper = &azure.Prepper{}
 		upload, err = azure.NewUploader(config, logger)
 		if err != nil {
-			return fmt.Errorf("creating azure uploader: %w", err)
+			return nil, fmt.Errorf("creating azure uploader: %w", err)
 		}
 	case "gcp":
 		prepper = &gcp.Prepper{}
 		upload, err = gcp.NewUploader(config, logger)
 		if err != nil {
-			return fmt.Errorf("creating gcp uploader: %w", err)
+			return nil, fmt.Errorf("creating gcp uploader: %w", err)
 		}
+	default:
+		return nil, fmt.Errorf("unknown provider: %s", config.Provider)
 	}
 
 	tmpDir, err := os.MkdirTemp("", "uplosi-")
 	if err != nil {
-		return fmt.Errorf("creating temp dir: %w", err)
+		return nil, fmt.Errorf("creating temp dir: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
 
-	imagePath, err = prepper.Prepare(cmd.Context(), imagePath, tmpDir)
+	imagePath, err = prepper.Prepare(ctx, imagePath, tmpDir)
 	if err != nil {
-		return fmt.Errorf("preparing image: %w", err)
+		return nil, fmt.Errorf("preparing image: %w", err)
 	}
 	image, err := os.Open(imagePath)
 	if err != nil {
-		return fmt.Errorf("opening image: %w", err)
+		return nil, fmt.Errorf("opening image: %w", err)
 	}
 	defer image.Close()
 	imageFi, err := image.Stat()
 	if err != nil {
-		return fmt.Errorf("getting image stats: %w", err)
+		return nil, fmt.Errorf("getting image stats: %w", err)
 	}
 
 	req := &uploader.Request{
 		Image: image,
 		Size:  imageFi.Size(),
 	}
-	refs, err := upload.Upload(cmd.Context(), req)
+	refs, err := upload.Upload(ctx, req)
 	if err != nil {
-		return fmt.Errorf("uploading image: %w", err)
-	}
-
-	for _, ref := range refs {
-		fmt.Println(ref)
-	}
-
-	if flags.incrementVersion {
-		newVer, err := incrementSemver(config.ImageVersion)
-		if err != nil {
-			return fmt.Errorf("incrementing semver: %w", err)
-		}
-		config.ImageVersion = newVer
-		if err := writeTOMLFile(configName, config); err != nil {
-			return fmt.Errorf("writing config: %w", err)
-		}
+		return nil, fmt.Errorf("uploading image: %w", err)
 	}
 
-	return nil
+	return refs, nil
 }
 
 type uploadFlags struct {
-	incrementVersion bool
+	incrementVersion    bool
+	enableVariantGlobs  []string
+	disableVariantGlobs []string
 }
 
 func parseUploadFlags(cmd *cobra.Command) (*uploadFlags, error) {
 	incrementVersion, err := cmd.Flags().GetBool("increment-version")
 	if err != nil {
 		return nil, fmt.Errorf("getting increment-version flag: %w", err)
 	}
+	enableVariantGlobs, err := cmd.Flags().GetStringSlice("enable-variant-glob")
+	if err != nil {
+		return nil, fmt.Errorf("getting enable-variant-glob flag: %w", err)
+	}
+	disableVariantGlobs, err := cmd.Flags().GetStringSlice("disable-variant-glob")
+	if err != nil {
+		return nil, fmt.Errorf("getting disable-variant-glob flag: %w", err)
+	}
 	return &uploadFlags{
-		incrementVersion: incrementVersion,
+		incrementVersion:    incrementVersion,
+		enableVariantGlobs:  enableVariantGlobs,
+		disableVariantGlobs: disableVariantGlobs,
 	}, nil
 }
 
+func filterGlobAny(globs []string, name string) bool {
+	for _, glob := range globs {
+		if ok, _ := filepath.Match(glob, name); ok {
+			return true
+		}
+	}
+	return false
+}
+
 func readTOMLFile(path string, data any) error {
 	configFile, err := os.OpenFile(path, os.O_RDONLY, os.ModeAppend)
 	if err != nil {
@@ -179,17 +248,6 @@ func supportedCSPs() []string {
 	return []string{"aws", "azure", "gcp"}
 }
 
-func isCSP(position int) func(*cobra.Command, []string) error {
-	return func(cmd *cobra.Command, args []string) error {
-		for _, csp := range supportedCSPs() {
-			if args[position] == csp {
-				return nil
-			}
-		}
-		return fmt.Errorf("unsupported cloud service provider: %s", args[position])
-	}
-}
-
 type Prepper interface {
 	Prepare(ctx context.Context, imagePath, tmpDir string) (string, error)
 }
@@ -198,6 +256,37 @@ type Uploader interface {
 	Upload(ctx context.Context, req *uploader.Request) (refs []string, retErr error)
 }
 
+func parseConfigFiles() (*uploader.ConfigFile, error) {
+	var config uploader.ConfigFile
+	if err := readTOMLFile(configName, &config); err != nil {
+		return nil, fmt.Errorf("reading config: %w", err)
+	}
+
+	dirEntries, err := os.ReadDir(configDir)
+	if os.IsNotExist(err) {
+		return &config, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading config dir: %w", err)
+	}
+	for _, dirEntry := range dirEntries {
+		var cfgOverlay uploader.ConfigFile
+		if dirEntry.IsDir() {
+			continue
+		}
+		if filepath.Ext(dirEntry.Name()) != ".conf" {
+			continue
+		}
+		if err := readTOMLFile(configName, &cfgOverlay); err != nil {
+			return nil, fmt.Errorf("reading config: %w", err)
+		}
+		if err := config.Merge(cfgOverlay); err != nil {
+			return nil, fmt.Errorf("merging config: %w", err)
+		}
+	}
+	return &config, nil
+}
+
 func canonicalSemver(version string) error {
 	ver := "v" + version
 	if !semver.IsValid(ver) {

go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	cloud.google.com/go/compute v1.23.0
 	cloud.google.com/go/storage v1.33.0
+	dario.cat/mergo v1.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.2.0
@@ -33,6 +34,7 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
@@ -43,6 +45,7 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/grpc v1.56.2 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 )
 
 require (

go.sum

@@ -10,6 +10,8 @@ cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
 cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
 cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
 cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.2 h1:t5+QXLCK9SVi0PPdaY0PrFvYUo24KwA0QwxnaHRSVd4=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.2/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
@@ -145,6 +147,11 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
@@ -284,8 +291,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=