csi: support Secrets parameter in CSI RPCs (#7923)

CSI plugins can require credentials for some publishing and
unpublishing workflow RPCs. Secrets are configured at the time of
volume registration, stored in the volume struct, and then passed
around as an opaque map by Nomad to the plugins.

api/csi.go

@@ -87,6 +87,8 @@ type CSIMountOptions struct {
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` // report unexpected keys
 }
 
+type CSISecrets map[string]string
+
 // CSIVolume is used for serialization, see also nomad/structs/csi.go
 type CSIVolume struct {
 	ID             string
@@ -97,6 +99,7 @@ type CSIVolume struct {
 	AccessMode     CSIVolumeAccessMode     `hcl:"access_mode"`
 	AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"`
 	MountOptions   *CSIMountOptions        `hcl:"mount_options"`
+	Secrets        CSISecrets              `hcl:"secrets"`
 
 	// Allocations, tracking claim status
 	ReadAllocs  map[string]*Allocation
@@ -162,7 +165,6 @@ type CSIVolumeListStub struct {
 	Topologies          []*CSITopology
 	AccessMode          CSIVolumeAccessMode
 	AttachmentMode      CSIVolumeAttachmentMode
-	MountOptions        *CSIMountOptions
 	Schedulable         bool
 	PluginID            string
 	Provider            string

client/csi_endpoint.go

@@ -61,6 +61,7 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV
 	// CSI ValidateVolumeCapabilities errors for timeout, codes.Unavailable and
 	// codes.ResourceExhausted are retried; all other errors are fatal.
 	return plugin.ControllerValidateCapabilities(ctx, req.VolumeID, caps,
+		req.Secrets,
 		grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout),
 		grpc_retry.WithMax(3),
 		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)))

client/pluginmanager/csimanager/volume.go

@@ -170,6 +170,7 @@ func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume,
 		publishContext,
 		pluginStagingPath,
 		capability,
+		vol.Secrets,
 		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
 		grpc_retry.WithMax(3),
 		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
@@ -208,6 +209,7 @@ func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolum
 		TargetPath:        pluginTargetPath,
 		VolumeCapability:  capabilities,
 		Readonly:          usage.ReadOnly,
+		Secrets:           vol.Secrets,
 	},
 		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
 		grpc_retry.WithMax(3),

client/structs/csi.go

@@ -35,6 +35,8 @@ type ClientCSIControllerValidateVolumeRequest struct {
 
 	AttachmentMode structs.CSIVolumeAttachmentMode
 	AccessMode     structs.CSIVolumeAccessMode
+	Secrets        structs.CSISecrets
+	// Parameters map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
 
 	CSIControllerQuery
 }
@@ -66,6 +68,15 @@ type ClientCSIControllerAttachVolumeRequest struct {
 	// only works when the Controller has the PublishReadonly capability.
 	ReadOnly bool
 
+	// Secrets required by plugin to complete the controller publish
+	// volume request. This field is OPTIONAL.
+	Secrets structs.CSISecrets
+
+	// TODO https://github.com/hashicorp/nomad/issues/7771
+	// Volume context as returned by storage provider in CreateVolumeResponse.
+	// This field is optional.
+	// VolumeContext map[string]string
+
 	CSIControllerQuery
 }
 
@@ -82,8 +93,10 @@ func (c *ClientCSIControllerAttachVolumeRequest) ToCSIRequest() (*csi.Controller
 	return &csi.ControllerPublishVolumeRequest{
 		VolumeID:         c.VolumeID,
 		NodeID:           c.ClientCSINodeID,
-		ReadOnly:         c.ReadOnly,
 		VolumeCapability: caps,
+		ReadOnly:         c.ReadOnly,
+		Secrets:          c.Secrets,
+		// VolumeContext:    c.VolumeContext, TODO: https://github.com/hashicorp/nomad/issues/7771
 	}, nil
 }
 
@@ -117,6 +130,10 @@ type ClientCSIControllerDetachVolumeRequest struct {
 	// by the target node for this plugin name.
 	ClientCSINodeID string
 
+	// Secrets required by plugin to complete the controller unpublish
+	// volume request. This field is OPTIONAL.
+	Secrets structs.CSISecrets
+
 	CSIControllerQuery
 }
 

command/agent/csi_endpoint.go

@@ -85,6 +85,10 @@ func (s *HTTPServer) csiVolumeGet(id string, resp http.ResponseWriter, req *http
 		return nil, CodedError(404, "volume not found")
 	}
 
+	// remove sensitive fields, as our redaction mechanism doesn't
+	// help serializing here
+	out.Volume.Secrets = nil
+	out.Volume.MountOptions = nil
 	return out.Volume, nil
 }
 

command/node_status.go

@@ -569,19 +569,17 @@ func (c *NodeStatusCommand) outputNodeCSIVolumeInfo(client *api.Client, node *ap
 
 	// Output the volumes in name order
 	output := make([]string, 0, len(names)+1)
-	output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode|Mount Options")
+	output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode")
 	for _, name := range names {
 		v := volumes[name]
-		r := requests[v.ID]
 		output = append(output, fmt.Sprintf(
-			"%s|%s|%s|%t|%s|%s|%s",
+			"%s|%s|%s|%t|%s|%s",
 			v.ID,
 			name,
 			v.PluginID,
 			v.Schedulable,
 			v.Provider,
 			v.AccessMode,
-			csiVolMountOption(v.MountOptions, r.MountOptions),
 		))
 	}
 

command/volume_register_test.go

@@ -57,13 +57,17 @@ namespace = "n"
 access_mode = "single-node-writer"
 attachment_mode = "file-system"
 plugin_id = "p"
+secrets {
+  mysecret = "secretvalue"
+}
 `,
 		q: &api.CSIVolume{
 			ID:             "foo",
 			Namespace:      "n",
 			AccessMode:     "single-node-writer",
 			AttachmentMode: "file-system",
 			PluginID:       "p",
+			Secrets:        api.CSISecrets{"mysecret": "secretvalue"},
 		},
 		err: "",
 	}, {

nomad/csi_endpoint.go

@@ -237,6 +237,8 @@ func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterReque
 		VolumeID:       vol.RemoteID(),
 		AttachmentMode: vol.AttachmentMode,
 		AccessMode:     vol.AccessMode,
+		Secrets:        vol.Secrets,
+		// Parameters: TODO: https://github.com/hashicorp/nomad/issues/7670
 	}
 	cReq.PluginID = plugin.ID
 	cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{}
@@ -440,6 +442,8 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest,
 		AttachmentMode:  vol.AttachmentMode,
 		AccessMode:      vol.AccessMode,
 		ReadOnly:        req.Claim == structs.CSIVolumeClaimRead,
+		Secrets:         vol.Secrets,
+		// VolumeContext: TODO https://github.com/hashicorp/nomad/issues/7771
 	}
 	cReq.PluginID = plug.ID
 	cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{}

nomad/csi_endpoint_test.go

@@ -37,6 +37,7 @@ func TestCSIVolumeEndpoint_Get(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err := state.CSIVolumeRegister(999, vols)
 	require.NoError(t, err)
@@ -84,6 +85,7 @@ func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err := state.CSIVolumeRegister(999, vols)
 	require.NoError(t, err)
@@ -139,6 +141,7 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) {
 		PluginID:       "minnie",
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 
 	// Create the register request
@@ -255,6 +258,7 @@ func TestCSIVolumeEndpoint_Claim(t *testing.T) {
 		Topologies: []*structs.CSITopology{{
 			Segments: map[string]string{"foo": "bar"},
 		}},
+		Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	index++
 	err = state.CSIVolumeRegister(index, vols)
@@ -373,6 +377,7 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) {
 		ControllerRequired: true,
 		AccessMode:         structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode:     structs.CSIVolumeAttachmentModeFilesystem,
+		Secrets:            structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err = state.CSIVolumeRegister(1003, vols)
 
@@ -439,6 +444,7 @@ func TestCSIVolumeEndpoint_List(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}, {
 		ID:             id1,
 		Namespace:      structs.DefaultNamespace,

nomad/mock/mock.go

@@ -1311,6 +1311,7 @@ func CSIVolume(plugin *structs.CSIPlugin) *structs.CSIVolume {
 		AccessMode:          structs.CSIVolumeAccessModeSingleNodeWriter,
 		AttachmentMode:      structs.CSIVolumeAttachmentModeFilesystem,
 		MountOptions:        &structs.CSIMountOptions{},
+		Secrets:             structs.CSISecrets{},
 		ReadAllocs:          map[string]*structs.Allocation{},
 		WriteAllocs:         map[string]*structs.Allocation{},
 		ReadClaims:          map[string]*structs.CSIVolumeClaim{},

nomad/structs/csi.go

@@ -185,6 +185,27 @@ func (v *CSIMountOptions) GoString() string {
 	return v.String()
 }
 
+// CSISecrets contain optional additional configuration that can be used
+// when specifying that a Volume should be used with VolumeAccessTypeMount.
+type CSISecrets map[string]string
+
+// CSISecrets implements the Stringer and GoStringer interfaces to prevent
+// accidental leakage of secrets via logs.
+var _ fmt.Stringer = &CSISecrets{}
+var _ fmt.GoStringer = &CSISecrets{}
+
+func (s *CSISecrets) String() string {
+	redacted := map[string]string{}
+	for k := range *s {
+		redacted[k] = "[REDACTED]"
+	}
+	return fmt.Sprintf("csi.CSISecrets(%v)", redacted)
+}
+
+func (s *CSISecrets) GoString() string {
+	return s.String()
+}
+
 type CSIVolumeClaim struct {
 	AllocationID string
 	NodeID       string
@@ -214,6 +235,7 @@ type CSIVolume struct {
 	AccessMode     CSIVolumeAccessMode
 	AttachmentMode CSIVolumeAttachmentMode
 	MountOptions   *CSIMountOptions
+	Secrets        CSISecrets
 
 	// Allocations, tracking claim status
 	ReadAllocs  map[string]*Allocation // AllocID -> Allocation
@@ -249,7 +271,6 @@ type CSIVolListStub struct {
 	Topologies          []*CSITopology
 	AccessMode          CSIVolumeAccessMode
 	AttachmentMode      CSIVolumeAttachmentMode
-	MountOptions        *CSIMountOptions
 	CurrentReaders      int
 	CurrentWriters      int
 	Schedulable         bool
@@ -279,6 +300,9 @@ func (v *CSIVolume) newStructs() {
 	if v.Topologies == nil {
 		v.Topologies = []*CSITopology{}
 	}
+	if v.Secrets == nil {
+		v.Secrets = CSISecrets{}
+	}
 
 	v.ReadAllocs = map[string]*Allocation{}
 	v.WriteAllocs = map[string]*Allocation{}
@@ -304,7 +328,6 @@ func (v *CSIVolume) Stub() *CSIVolListStub {
 		Topologies:          v.Topologies,
 		AccessMode:          v.AccessMode,
 		AttachmentMode:      v.AttachmentMode,
-		MountOptions:        v.MountOptions,
 		CurrentReaders:      len(v.ReadAllocs),
 		CurrentWriters:      len(v.WriteAllocs),
 		Schedulable:         v.Schedulable,
@@ -365,6 +388,9 @@ func (v *CSIVolume) Copy() *CSIVolume {
 	copy := *v
 	out := &copy
 	out.newStructs()
+	for k, v := range v.Secrets {
+		out.Secrets[k] = v
+	}
 
 	for k, v := range v.ReadAllocs {
 		out.ReadAllocs[k] = v

nomad/volumewatcher/volume_watcher.go

@@ -350,6 +350,7 @@ func (vw *volumeWatcher) controllerDetach(vol *structs.CSIVolume, claim *structs
 	cReq := &cstructs.ClientCSIControllerDetachVolumeRequest{
 		VolumeID:        vol.RemoteID(),
 		ClientCSINodeID: targetCSIInfo.NodeInfo.ID,
+		Secrets:         vol.Secrets,
 	}
 	cReq.PluginID = plug.ID
 	err = vw.rpc.ControllerDetachVolume(cReq,

plugins/csi/client.go

@@ -12,6 +12,7 @@ import (
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/helper/grpc-middleware/logging"
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
 	"google.golang.org/grpc"
@@ -293,7 +294,7 @@ func (c *client) ControllerUnpublishVolume(ctx context.Context, req *ControllerU
 	return &ControllerUnpublishVolumeResponse{}, nil
 }
 
-func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
+func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	if c == nil {
 		return fmt.Errorf("Client not initialized")
 	}
@@ -314,6 +315,9 @@ func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID st
 		VolumeCapabilities: []*csipbv1.VolumeCapability{
 			capabilities.ToCSIRepresentation(),
 		},
+		// VolumeContext: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771
+		// Parameters: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
+		Secrets: secrets,
 	}
 
 	resp, err := c.controllerClient.ValidateVolumeCapabilities(ctx, req, opts...)
@@ -461,7 +465,7 @@ func (c *client) NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)
 	return result, nil
 }
 
-func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
+func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	if c == nil {
 		return fmt.Errorf("Client not initialized")
 	}
@@ -483,6 +487,7 @@ func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishCo
 		PublishContext:    publishContext,
 		StagingTargetPath: stagingTargetPath,
 		VolumeCapability:  capabilities.ToCSIRepresentation(),
+		Secrets:           secrets,
 	}
 
 	// NodeStageVolume's response contains no extra data. If err == nil, we were

plugins/csi/client_test.go

@@ -578,7 +578,7 @@ func TestClient_RPC_ControllerValidateVolume(t *testing.T) {
 			cc.NextErr = c.ResponseErr
 
 			err := client.ControllerValidateCapabilities(
-				context.TODO(), "volumeID", requestedCaps)
+				context.TODO(), "volumeID", requestedCaps, structs.CSISecrets{})
 			if c.ExpectedErr != nil {
 				require.Error(t, c.ExpectedErr, err, c.Name)
 			} else {
@@ -616,7 +616,8 @@ func TestClient_RPC_NodeStageVolume(t *testing.T) {
 			nc.NextErr = c.ResponseErr
 			nc.NextStageVolumeResponse = c.Response
 
-			err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo", &VolumeCapability{})
+			err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo",
+				&VolumeCapability{}, structs.CSISecrets{})
 			if c.ExpectedErr != nil {
 				require.Error(t, c.ExpectedErr, err)
 			} else {

plugins/csi/fake/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/csi"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
@@ -159,7 +160,7 @@ func (c *Client) ControllerUnpublishVolume(ctx context.Context, req *csi.Control
 	return c.NextControllerUnpublishVolumeResponse, c.NextControllerUnpublishVolumeErr
 }
 
-func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
+func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	c.Mu.Lock()
 	defer c.Mu.Unlock()
 
@@ -191,7 +192,7 @@ func (c *Client) NodeGetInfo(ctx context.Context) (*csi.NodeGetInfoResponse, err
 // NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
 // to prepare a volume for usage on a host. If err == nil, the response should
 // be assumed to be successful.
-func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
+func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	c.Mu.Lock()
 	defer c.Mu.Unlock()