⭐️ remote builtin providers

.vscode/launch.json

@@ -197,6 +197,17 @@
       "args": [
         "shell", "ssh", "user@18.215.249.49",
       ],
+    },
+    {
+      "name": "Configure Built-in Providers",
+      "type": "go",
+      "request": "launch",
+      "program": "${workspaceRoot}/providers-sdk/v1/util/configure",
+      "cwd": "${workspaceRoot}",
+      "args": [
+        "-f", "${workspaceRoot}/providers.yaml", 
+        "-o", "${workspaceRoot}/providers/builtin_dev.go"
+      ]
     }
   ]
 }

Makefile

@@ -169,7 +169,7 @@ providers/proto:
 
 .PHONY: providers/config
 providers/config:
-	go run ./providers-sdk/v1/util/configure/configure.go -f providers.yaml -o providers/builtin_dev.go
+	go run ./providers-sdk/v1/util/configure -f providers.yaml -o providers/builtin_dev.go
 
 .PHONY: providers/defaults
 providers/defaults:

providers-sdk/v1/util/configure/configure.go

@@ -23,10 +23,6 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-type ProvidersConf struct {
-	Builtin []string `json:"builtin"`
-}
-
 func init() {
 	rootCmd.Flags().StringP("file", "f", "providers.yaml", "config file for providers")
 	rootCmd.Flags().StringP("output", "o", "providers/builtin_dev.go", "output go-file for builtin dev providers")
@@ -77,18 +73,18 @@ var rootCmd = &cobra.Command{
 		if err = os.WriteFile(outPath, builtinGo, 0o644); err != nil {
 			log.Fatal().Err(err).Str("path", outPath).Msg("failed to write output")
 		}
-		log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(1/3) configured builtin providers")
+		log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(1/3) configured builtin providers")
 
 		buildProviders(conf.Builtin)
-		log.Info().Strs("providers", conf.Builtin).Msg("(2/3) built providers")
+		log.Info().Strs("providers", conf.Providers()).Msg("(2/3) built providers")
 
 		rewireDependencies(conf.Builtin)
-		log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(3/3) rewired dependencies/files")
+		log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(3/3) rewired dependencies/files")
 	},
 }
 
 var editProvidersCmd = &cobra.Command{
-	Use:   "edit-providers NAME1 NAME2 [-f config]",
+	Use:   "edit-providers NAME1 NAME2:REMOTE:GOPACKAGE [-f config]",
 	Short: "adds a provider to the config",
 	Run: func(cmd *cobra.Command, args []string) {
 		confPath, err := cmd.Flags().GetString("file")
@@ -108,15 +104,29 @@ var editProvidersCmd = &cobra.Command{
 		}
 
 		if len(conf.Builtin) > 0 {
-			log.Warn().Strs("providers", conf.Builtin).Msg("overwrite existing providers in config")
+			log.Warn().Strs("providers", conf.Providers()).Msg("overwrite existing providers in config")
 		}
 
 		// set new providers
-		conf.Builtin = args
-		slices.Sort(conf.Builtin)
+		for _, arg := range args {
+			if !strings.Contains(arg, ":") {
+				conf.Builtin = append(conf.Builtin, Builtin{Name: arg})
+				continue
+			}
+
+			parts := strings.Split(arg, ":")
+			if len(parts) != 3 {
+				log.Fatal().Str("provider", arg).Msg("invalid provider format, must be NAME:REMOTE:GOPACKAGE")
+			}
+
+			conf.Builtin = append(conf.Builtin, Builtin{Name: parts[0], Remote: parts[1], GoPackage: parts[2]})
+		}
+		slices.SortFunc(conf.Builtin, func(a, b Builtin) int {
+			return strings.Compare(a.Name, b.Name)
+		})
 		conf.Builtin = slices.Compact(conf.Builtin)
 
-		log.Info().Strs("providers", conf.Builtin).Msg("configured providers")
+		log.Info().Strs("providers", conf.Providers()).Msg("configured providers")
 
 		raw, err = yaml.Marshal(conf)
 		if err != nil {
@@ -137,13 +147,15 @@ func genBuiltinGo(conf ProvidersConf) ([]byte, error) {
 
 	for _, provider := range conf.Builtin {
 		// imports cannot contain dashes
-		trimProvider := strings.Replace(provider, "-", "", -1)
-		imports += fmt.Sprintf("\t%sconf \"go.mondoo.com/cnquery/v11/providers/%s/config\"\n", trimProvider, provider)
-		imports += fmt.Sprintf("\t%s \"go.mondoo.com/cnquery/v11/providers/%s/provider\"\n", trimProvider, provider)
+		trimProvider := strings.Replace(provider.Name, "-", "", -1)
+		imports += fmt.Sprintf("\t%sconf \"%s/config\"\n", trimProvider, provider.GoPackage)
+		imports += fmt.Sprintf("\t%s \"%s/provider\"\n", trimProvider, provider.GoPackage)
+
 		infos += fmt.Sprintf(
 			"//go:embed %s.resources.json\n"+
 				"var %sInfo []byte\n",
-			provider, trimProvider)
+			provider.Name, trimProvider)
+
 		configs += fmt.Sprintf(`
 	builtinProviders[%sconf.Config.ID] = &builtinProvider{
 		Runtime: &RunningProvider{
@@ -155,7 +167,7 @@ func genBuiltinGo(conf ProvidersConf) ([]byte, error) {
 		},
 		Config: &%sconf.Config,
 	}
-`, trimProvider, trimProvider, trimProvider, trimProvider, provider, trimProvider, trimProvider)
+`, trimProvider, trimProvider, trimProvider, trimProvider, provider.Name, trimProvider, trimProvider)
 	}
 
 	res := fmt.Sprintf(template, imports, infos, configs)
@@ -166,7 +178,7 @@ const template = `// Copyright (c) Mondoo, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 //
 // This file is auto-generated by 'make providers/config'
-// and configured via 'providers.yaml'
+// and configured via 'providers.yaml'; DO NOT EDIT.
 
 package providers
 
@@ -196,26 +208,31 @@ func init() {
 }
 `
 
-func buildProviders(providers []string) {
+func buildProviders(providers []Builtin) {
 	for i, provider := range providers {
-		cmd := exec.Command("make", "providers/build/"+provider)
+		cmd := exec.Command("make", "providers/build/"+provider.Name)
+		if provider.Remote != "" {
+			cmd = exec.Command("make", "provider/generate")
+			cmd.Dir = provider.Remote
+		}
+
 		var out bytes.Buffer
 		cmd.Stdout = &out
 		cmd.Stderr = &out
-		log.Debug().Str("provider", provider).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers)))
+		log.Debug().Str("provider", provider.Name).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers)))
 		if err := cmd.Run(); err != nil {
 			fmt.Println(out.String())
-			log.Error().Err(err).Str("provider", provider).Msg("failed to build provider")
+			log.Error().Err(err).Str("provider", provider.Name).Msg("failed to build provider")
 		}
 
 		// inefficient copy...
-		src := "providers/" + provider + "/resources/" + provider + ".resources.json"
+		src := provider.Resource()
 		raw, err := os.ReadFile(src)
 		if err != nil {
 			log.Fatal().Err(err).Str("src", src).Msg("failed to read resources json")
 		}
 
-		dst := "providers/" + provider + ".resources.json"
+		dst := provider.Dist()
 		err = os.WriteFile(dst, raw, 0o644)
 		if err != nil {
 			log.Fatal().Err(err).Str("dst", dst).Msg("failed to write resources json")
@@ -228,7 +245,7 @@ var (
 	reBuiltinDep     = regexp.MustCompile(`go.mondoo.com/cnquery/v11/providers/.*`)
 )
 
-func rewireDependencies(providers []string) {
+func rewireDependencies(providers []Builtin) {
 	raw, err := os.ReadFile("go.mod")
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to read go.mod")
@@ -241,28 +258,33 @@ func rewireDependencies(providers []string) {
 	deps := ""
 	replace := ""
 	for _, provider := range providers {
-		_, err := os.Stat("providers/" + provider + "/go.mod")
+		goModPath := provider.GoMod()
+		_, err := os.Stat(goModPath)
 		if err != nil {
 			if errors.Is(err, os.ErrNotExist) {
-				log.Info().Str("provider", provider).Msg("skipping provider without go.mod")
+				log.Info().Str("provider", provider.Name).Msg("skipping provider without go.mod")
 				continue
 			} else {
-				log.Fatal().Err(err).Str("provider", provider).Msg("failed to stat provider go.mod")
+				log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to stat provider go.mod")
 			}
 		}
 
-		goModContent, err := os.ReadFile("providers/" + provider + "/go.mod")
+		goModContent, err := os.ReadFile(goModPath)
 		if err != nil {
-			log.Fatal().Err(err).Str("provider", provider).Msg("failed to read provider go.mod")
+			log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to read provider go.mod")
 		}
-		goMod, err := modfile.Parse(fmt.Sprintf("%s/go.mod", provider), goModContent, nil)
+		goMod, err := modfile.Parse(goModPath, goModContent, nil)
 		if err != nil {
-			log.Fatal().Err(err).Str("provider", provider).Msg("failed to parse provider go.mod")
+			log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to parse provider go.mod")
 		}
 
 		// we don't care about the specific version for dev
-		deps += "\n\tgo.mondoo.com/cnquery/v11/providers/" + provider + " v0.0.0"
-		replace += "\nreplace go.mondoo.com/cnquery/v11/providers/" + provider + " => ./providers/" + provider
+		deps += "\n\t" + provider.GoPackage + " v0.0.0"
+		if provider.Remote != "" {
+			replace += "\nreplace " + provider.GoPackage + " => " + provider.Remote
+		} else {
+			replace += "\nreplace " + provider.GoPackage + " => ./providers/" + provider.Name
+		}
 		// if the provider has any specific pinned replacements, we also add those to allow compiling
 		for _, r := range goMod.Replace {
 			// special case: we don't want to pull in provider's 'replace go.mondoo.com/cnquery/v11 => ../..' in.

providers-sdk/v1/util/configure/provider_conf.go

@@ -0,0 +1,102 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+)
+
+type ProvidersConf struct {
+	Builtin []Builtin `json:"builtin"`
+
+	providers []string // Providers names
+}
+
+func (c ProvidersConf) Providers() []string {
+	if len(c.providers) == 0 {
+		for _, b := range c.Builtin {
+			c.providers = append(c.providers, b.Name)
+		}
+	}
+	return c.providers
+}
+
+type Builtin struct {
+	Name      string
+	Remote    string
+	GoPackage string
+}
+
+func (b Builtin) Cwd() string {
+	if b.Remote != "" {
+		return b.Remote
+	}
+
+	return ""
+}
+
+func (b Builtin) Resource() string {
+	if b.Remote != "" {
+		return fmt.Sprintf("%s/resources/%s.resources.json", b.Remote, b.Name)
+	}
+	return "providers/" + b.Name + "/resources/" + b.Name + ".resources.json"
+}
+
+func (b Builtin) Dist() string {
+	return "providers/" + b.Name + ".resources.json"
+}
+
+func (b Builtin) GoMod() string {
+	if b.Remote != "" {
+		return b.Remote + "/go.mod"
+	}
+	return "providers/" + b.Name + "/go.mod"
+}
+
+func (b *Builtin) UnmarshalJSON(data []byte) error {
+	var name string
+	if err := json.Unmarshal(data, &name); err == nil {
+		b.Name = name
+		b.GoPackage = "go.mondoo.com/cnquery/v11/providers/" + name
+
+		return nil
+	}
+
+	var raw struct {
+		Name      string `json:"name"`
+		Remote    string `json:"remote"`
+		GoPackage string `json:"goPackage"`
+	}
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+
+	b.Name = raw.Name
+	b.Remote = raw.Remote
+	if strings.HasPrefix(b.Remote, "~/") {
+		b.Remote = os.ExpandEnv(strings.Replace(b.Remote, "~/", "$HOME/", 1))
+	}
+
+	b.GoPackage = raw.GoPackage
+
+	return nil
+}
+
+func (b *Builtin) MarshalJSON() ([]byte, error) {
+	if b.Remote == "" && b.GoPackage == "" {
+		return json.Marshal(b.Name)
+	}
+
+	return json.Marshal(struct {
+		Name      string `json:"name"`
+		Remote    string `json:"remote"`
+		GoPackage string `json:"goPackage"`
+	}{
+		Name:      b.Name,
+		Remote:    b.Remote,
+		GoPackage: b.GoPackage,
+	})
+}

providers.yaml

@@ -12,4 +12,11 @@
 # In this case it will be pulled from your local repo
 # instead of using the pre-installed provider in 
 # your OS (like ~/.mondoo/providers/os).
+#
+# If you want to test a provider that is not part of
+# the cnquery repo, you can add it here as well.
+# Example:
+# - name: my-awesome-provider
+#   remote: ~/path/to/my-awesome-provider
+#   goPackage: github.com/me/my-awesome-provider
 builtin: []

providers/builtin_dev.go

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: BUSL-1.1
 //
 // This file is auto-generated by 'make providers/config'
-// and configured via 'providers.yaml'
+// and configured via 'providers.yaml'; DO NOT EDIT.
 
 package providers