Version-aware plugin catalog (#16688)

Adds support for using semantic version information when registering
and managing plugins. New `detailed` field in the response data for listing
plugins and new `version` field in the response data for reading a
single plugin.

changelog/16688.txt

@@ -0,0 +1,9 @@
+```release-note:change
+plugins: `GET /sys/plugins/catalog` endpoint now returns an additional `detailed` field in the response data with a list of additional plugin metadata.
+```
+```release-note:change
+plugins: `GET /sys/plugins/catalog/:type/:name` endpoint now returns an additional `version` field in the response data.
+```
+```release-note:improvement
+plugins: Plugin catalog supports registering and managing plugins with semantic version information.
+```

sdk/helper/pluginutil/run_config.go

@@ -16,6 +16,7 @@ import (
 type PluginClientConfig struct {
 	Name            string
 	PluginType      consts.PluginType
+	Version         string
 	PluginSets      map[int]plugin.PluginSet
 	HandshakeConfig plugin.HandshakeConfig
 	Logger          log.Logger

sdk/helper/pluginutil/runner.go

@@ -6,6 +6,7 @@ import (
 
 	log "github.com/hashicorp/go-hclog"
 	plugin "github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/wrapping"
 	"google.golang.org/grpc"
@@ -45,6 +46,7 @@ const MultiplexingCtxKey string = "multiplex_id"
 type PluginRunner struct {
 	Name           string                      `json:"name" structs:"name"`
 	Type           consts.PluginType           `json:"type" structs:"type"`
+	Version        string                      `json:"version" structs:"version"`
 	Command        string                      `json:"command" structs:"command"`
 	Args           []string                    `json:"args" structs:"args"`
 	Env            []string                    `json:"env" structs:"env"`
@@ -81,6 +83,19 @@ func (r *PluginRunner) RunMetadataMode(ctx context.Context, wrapper RunnerUtil,
 	)
 }
 
+// VersionedPlugin holds any versioning information stored about a plugin in the
+// plugin catalog.
+type VersionedPlugin struct {
+	Type    string `json:"type"` // string instead of consts.PluginType so that we get the string form in API responses.
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	SHA256  string `json:"sha256,omitempty"`
+	Builtin bool   `json:"builtin"`
+
+	// Pre-parsed semver struct of the Version field
+	SemanticVersion *version.Version `json:"-"`
+}
+
 // CtxCancelIfCanceled takes a context cancel func and a context. If the context is
 // shutdown the cancelfunc is called. This is useful for merging two cancel
 // functions.

vault/auth.go

@@ -787,7 +787,7 @@ func (c *Core) setupCredentials(ctx context.Context) error {
 		backend, err = c.newCredentialBackend(ctx, entry, sysView, view)
 		if err != nil {
 			c.logger.Error("failed to create credential entry", "path", entry.Path, "error", err)
-			if plug, plugerr := c.pluginCatalog.Get(ctx, entry.Type, consts.PluginTypeCredential); plugerr == nil && !plug.Builtin {
+			if plug, plugerr := c.pluginCatalog.Get(ctx, entry.Type, consts.PluginTypeCredential, ""); plugerr == nil && !plug.Builtin {
 				// If we encounter an error instantiating the backend due to an error,
 				// skip backend initialization but register the entry to the mount table
 				// to preserve storage and path.
@@ -911,7 +911,7 @@ func (c *Core) newCredentialBackend(ctx context.Context, entry *MountEntry, sysV
 
 	f, ok := c.credentialBackends[t]
 	if !ok {
-		plug, err := c.pluginCatalog.Get(ctx, t, consts.PluginTypeCredential)
+		plug, err := c.pluginCatalog.Get(ctx, t, consts.PluginTypeCredential, "")
 		if err != nil {
 			return nil, err
 		}

vault/dynamic_system_view.go

@@ -240,7 +240,7 @@ func (d dynamicSystemView) LookupPlugin(ctx context.Context, name string, plugin
 	if d.core.pluginCatalog == nil {
 		return nil, fmt.Errorf("system view core plugin catalog is nil")
 	}
-	r, err := d.core.pluginCatalog.Get(ctx, name, pluginType)
+	r, err := d.core.pluginCatalog.Get(ctx, name, pluginType, "")
 	if err != nil {
 		return nil, err
 	}

vault/logical_system.go

@@ -29,6 +29,7 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
+	semver "github.com/hashicorp/go-version"
 	"github.com/hashicorp/vault/helper/hostutil"
 	"github.com/hashicorp/vault/helper/identity"
 	"github.com/hashicorp/vault/helper/metricsutil"
@@ -38,6 +39,7 @@ import (
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
+	"github.com/hashicorp/vault/sdk/helper/pluginutil"
 	"github.com/hashicorp/vault/sdk/helper/wrapping"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/version"
@@ -399,27 +401,61 @@ func (b *SystemBackend) handlePluginCatalogTypedList(ctx context.Context, req *l
 	if err != nil {
 		return nil, err
 	}
+	sort.Strings(plugins)
 	return logical.ListResponse(plugins), nil
 }
 
-func (b *SystemBackend) handlePluginCatalogUntypedList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
-	pluginsByType := make(map[string]interface{})
+func (b *SystemBackend) handlePluginCatalogUntypedList(ctx context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) {
+	data := make(map[string]interface{})
+	var versionedPlugins []pluginutil.VersionedPlugin
 	for _, pluginType := range consts.PluginTypes {
 		plugins, err := b.Core.pluginCatalog.List(ctx, pluginType)
 		if err != nil {
 			return nil, err
 		}
 		if len(plugins) > 0 {
 			sort.Strings(plugins)
-			pluginsByType[pluginType.String()] = plugins
+			data[pluginType.String()] = plugins
 		}
+
+		versioned, err := b.Core.pluginCatalog.ListVersionedPlugins(ctx, pluginType)
+		if err != nil {
+			return nil, err
+		}
+
+		// Sort for consistent ordering
+		sortVersionedPlugins(versionedPlugins)
+
+		versionedPlugins = append(versionedPlugins, versioned...)
+	}
+
+	if len(versionedPlugins) != 0 {
+		data["detailed"] = versionedPlugins
 	}
+
 	return &logical.Response{
-		Data: pluginsByType,
+		Data: data,
 	}, nil
 }
 
-func (b *SystemBackend) handlePluginCatalogUpdate(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+func sortVersionedPlugins(versionedPlugins []pluginutil.VersionedPlugin) {
+	sort.SliceStable(versionedPlugins, func(i, j int) bool {
+		left, right := versionedPlugins[i], versionedPlugins[j]
+		if left.Type != right.Type {
+			return left.Type < right.Type
+		}
+		if left.Name != right.Name {
+			return left.Name < right.Name
+		}
+		if left.Version != right.Version {
+			return right.SemanticVersion.GreaterThan(left.SemanticVersion)
+		}
+
+		return false
+	})
+}
+
+func (b *SystemBackend) handlePluginCatalogUpdate(ctx context.Context, _ *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	pluginName := d.Get("name").(string)
 	if pluginName == "" {
 		return logical.ErrorResponse("missing plugin name"), nil
@@ -436,6 +472,11 @@ func (b *SystemBackend) handlePluginCatalogUpdate(ctx context.Context, req *logi
 		return nil, err
 	}
 
+	pluginVersion, err := getVersion(d)
+	if err != nil {
+		return logical.ErrorResponse(err.Error()), nil
+	}
+
 	sha256 := d.Get("sha256").(string)
 	if sha256 == "" {
 		sha256 = d.Get("sha_256").(string)
@@ -472,15 +513,15 @@ func (b *SystemBackend) handlePluginCatalogUpdate(ctx context.Context, req *logi
 		return logical.ErrorResponse("Could not decode SHA-256 value from Hex"), err
 	}
 
-	err = b.Core.pluginCatalog.Set(ctx, pluginName, pluginType, parts[0], args, env, sha256Bytes)
+	err = b.Core.pluginCatalog.Set(ctx, pluginName, pluginType, pluginVersion, parts[0], args, env, sha256Bytes)
 	if err != nil {
 		return nil, err
 	}
 
 	return nil, nil
 }
 
-func (b *SystemBackend) handlePluginCatalogRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+func (b *SystemBackend) handlePluginCatalogRead(ctx context.Context, _ *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	pluginName := d.Get("name").(string)
 	if pluginName == "" {
 		return logical.ErrorResponse("missing plugin name"), nil
@@ -501,7 +542,12 @@ func (b *SystemBackend) handlePluginCatalogRead(ctx context.Context, req *logica
 		return nil, err
 	}
 
-	plugin, err := b.Core.pluginCatalog.Get(ctx, pluginName, pluginType)
+	pluginVersion, err := getVersion(d)
+	if err != nil {
+		return logical.ErrorResponse(err.Error()), nil
+	}
+
+	plugin, err := b.Core.pluginCatalog.Get(ctx, pluginName, pluginType, pluginVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -523,19 +569,25 @@ func (b *SystemBackend) handlePluginCatalogRead(ctx context.Context, req *logica
 		"command": command,
 		"sha256":  hex.EncodeToString(plugin.Sha256),
 		"builtin": plugin.Builtin,
+		"version": plugin.Version,
 	}
 
 	return &logical.Response{
 		Data: data,
 	}, nil
 }
 
-func (b *SystemBackend) handlePluginCatalogDelete(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+func (b *SystemBackend) handlePluginCatalogDelete(ctx context.Context, _ *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	pluginName := d.Get("name").(string)
 	if pluginName == "" {
 		return logical.ErrorResponse("missing plugin name"), nil
 	}
 
+	pluginVersion, err := getVersion(d)
+	if err != nil {
+		return logical.ErrorResponse(err.Error()), nil
+	}
+
 	var resp *logical.Response
 	pluginTypeStr := d.Get("type").(string)
 	if pluginTypeStr == "" {
@@ -552,13 +604,28 @@ func (b *SystemBackend) handlePluginCatalogDelete(ctx context.Context, req *logi
 	if err != nil {
 		return nil, err
 	}
-	if err := b.Core.pluginCatalog.Delete(ctx, pluginName, pluginType); err != nil {
+	if err := b.Core.pluginCatalog.Delete(ctx, pluginName, pluginType, pluginVersion); err != nil {
 		return nil, err
 	}
 
 	return resp, nil
 }
 
+func getVersion(d *framework.FieldData) (string, error) {
+	version := d.Get("version").(string)
+	if version != "" {
+		semanticVersion, err := semver.NewSemver(version)
+		if err != nil {
+			return "", fmt.Errorf("version %q is not a valid semantic version: %w", version, err)
+		}
+
+		// Canonicalize the version string.
+		version = semanticVersion.String()
+	}
+
+	return version, nil
+}
+
 func (b *SystemBackend) handlePluginReloadUpdate(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	pluginName := d.Get("plugin").(string)
 	pluginMounts := d.Get("mounts").([]string)
@@ -5221,6 +5288,10 @@ plugin directory.`,
 Each entry is of the form "key=value".`,
 		"",
 	},
+	"plugin-catalog_version": {
+		"The semantic version of the plugin to use.",
+		"",
+	},
 	"leases": {
 		`View or list lease metadata.`,
 		`

vault/logical_system_paths.go

@@ -766,6 +766,10 @@ func (b *SystemBackend) pluginsCatalogCRUDPath() *framework.Path {
 				Type:        framework.TypeStringSlice,
 				Description: strings.TrimSpace(sysHelp["plugin-catalog_env"][0]),
 			},
+			"version": {
+				Type:        framework.TypeString,
+				Description: strings.TrimSpace(sysHelp["plugin-catalog_version"][0]),
+			},
 		},
 
 		Operations: map[logical.Operation]framework.OperationHandler{