csi: support Secrets parameter in CSI RPCs

api/csi.go

@@ -87,6 +87,8 @@ type CSIMountOptions struct {
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` // report unexpected keys
 }
 
+type CSISecrets map[string]string
+
 // CSIVolume is used for serialization, see also nomad/structs/csi.go
 type CSIVolume struct {
 	ID             string
@@ -97,6 +99,7 @@ type CSIVolume struct {
 	AccessMode     CSIVolumeAccessMode     `hcl:"access_mode"`
 	AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"`
 	MountOptions   *CSIMountOptions        `hcl:"mount_options"`
+	Secrets        CSISecrets              `hcl:"secrets"`
 
 	// Allocations, tracking claim status
 	ReadAllocs  map[string]*Allocation
@@ -162,7 +165,6 @@ type CSIVolumeListStub struct {
 	Topologies          []*CSITopology
 	AccessMode          CSIVolumeAccessMode
 	AttachmentMode      CSIVolumeAttachmentMode
-	MountOptions        *CSIMountOptions
 	Schedulable         bool
 	PluginID            string
 	Provider            string

client/csi_endpoint.go

@@ -61,6 +61,7 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV
 	// CSI ValidateVolumeCapabilities errors for timeout, codes.Unavailable and
 	// codes.ResourceExhausted are retried; all other errors are fatal.
 	return plugin.ControllerValidateCapabilities(ctx, req.VolumeID, caps,
+		req.Secrets,
 		grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout),
 		grpc_retry.WithMax(3),
 		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)))

client/pluginmanager/csimanager/volume.go

@@ -170,6 +170,7 @@ func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume,
 		publishContext,
 		pluginStagingPath,
 		capability,
+		vol.Secrets,
 		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
 		grpc_retry.WithMax(3),
 		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
@@ -208,6 +209,7 @@ func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolum
 		TargetPath:        pluginTargetPath,
 		VolumeCapability:  capabilities,
 		Readonly:          usage.ReadOnly,
+		Secrets:           vol.Secrets,
 	},
 		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
 		grpc_retry.WithMax(3),

client/structs/csi.go

@@ -35,6 +35,8 @@ type ClientCSIControllerValidateVolumeRequest struct {
 
 	AttachmentMode structs.CSIVolumeAttachmentMode
 	AccessMode     structs.CSIVolumeAccessMode
+	Secrets        structs.CSISecrets
+	// Parameters map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
 
 	CSIControllerQuery
 }
@@ -66,6 +68,15 @@ type ClientCSIControllerAttachVolumeRequest struct {
 	// only works when the Controller has the PublishReadonly capability.
 	ReadOnly bool
 
+	// Secrets required by plugin to complete the controller publish
+	// volume request. This field is OPTIONAL.
+	Secrets structs.CSISecrets
+
+	// TODO https://github.com/hashicorp/nomad/issues/7771
+	// Volume context as returned by storage provider in CreateVolumeResponse.
+	// This field is optional.
+	// VolumeContext map[string]string
+
 	CSIControllerQuery
 }
 
@@ -82,8 +93,10 @@ func (c *ClientCSIControllerAttachVolumeRequest) ToCSIRequest() (*csi.Controller
 	return &csi.ControllerPublishVolumeRequest{
 		VolumeID:         c.VolumeID,
 		NodeID:           c.ClientCSINodeID,
-		ReadOnly:         c.ReadOnly,
 		VolumeCapability: caps,
+		ReadOnly:         c.ReadOnly,
+		Secrets:          c.Secrets,
+		// VolumeContext:    c.VolumeContext, TODO: https://github.com/hashicorp/nomad/issues/7771
 	}, nil
 }
 
@@ -117,6 +130,10 @@ type ClientCSIControllerDetachVolumeRequest struct {
 	// by the target node for this plugin name.
 	ClientCSINodeID string
 
+	// Secrets required by plugin to complete the controller unpublish
+	// volume request. This field is OPTIONAL.
+	Secrets structs.CSISecrets
+
 	CSIControllerQuery
 }
 

command/agent/csi_endpoint.go

@@ -85,6 +85,10 @@ func (s *HTTPServer) csiVolumeGet(id string, resp http.ResponseWriter, req *http
 		return nil, CodedError(404, "volume not found")
 	}
 
+	// remove sensitive fields, as our redaction mechanism doesn't
+	// help serializing here
+	out.Volume.Secrets = nil
+	out.Volume.MountOptions = nil
 	return out.Volume, nil
 }
 

command/node_status.go

@@ -569,19 +569,17 @@ func (c *NodeStatusCommand) outputNodeCSIVolumeInfo(client *api.Client, node *ap
 
 	// Output the volumes in name order
 	output := make([]string, 0, len(names)+1)
-	output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode|Mount Options")
+	output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode")
 	for _, name := range names {
 		v := volumes[name]
-		r := requests[v.ID]
 		output = append(output, fmt.Sprintf(
-			"%s|%s|%s|%t|%s|%s|%s",
+			"%s|%s|%s|%t|%s|%s",
 			v.ID,
 			name,
 			v.PluginID,
 			v.Schedulable,
 			v.Provider,
 			v.AccessMode,
-			csiVolMountOption(v.MountOptions, r.MountOptions),
 		))
 	}
 

command/volume_register_test.go

@@ -57,13 +57,17 @@ namespace = "n"
 access_mode = "single-node-writer"
 attachment_mode = "file-system"
 plugin_id = "p"
+secrets {
+  mysecret = "secretvalue"
+}
 `,
 		q: &api.CSIVolume{
 			ID:             "foo",
 			Namespace:      "n",
 			AccessMode:     "single-node-writer",
 			AttachmentMode: "file-system",
 			PluginID:       "p",
+			Secrets:        api.CSISecrets{"mysecret": "secretvalue"},
 		},
 		err: "",
 	}, {

nomad/csi_endpoint.go

@@ -237,6 +237,8 @@ func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterReque
 		VolumeID:       vol.RemoteID(),
 		AttachmentMode: vol.AttachmentMode,
 		AccessMode:     vol.AccessMode,
+		Secrets:        vol.Secrets,
+		// Parameters: TODO: https://github.com/hashicorp/nomad/issues/7670
 	}
 	cReq.PluginID = plugin.ID
 	cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{}
@@ -440,6 +442,8 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest,
 		AttachmentMode:  vol.AttachmentMode,
 		AccessMode:      vol.AccessMode,
 		ReadOnly:        req.Claim == structs.CSIVolumeClaimRead,
+		Secrets:         vol.Secrets,
+		// VolumeContext: TODO https://github.com/hashicorp/nomad/issues/7771
 	}
 	cReq.PluginID = plug.ID
 	cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{}

nomad/csi_endpoint_test.go

@@ -37,6 +37,7 @@ func TestCSIVolumeEndpoint_Get(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err := state.CSIVolumeRegister(999, vols)
 	require.NoError(t, err)
@@ -84,6 +85,7 @@ func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err := state.CSIVolumeRegister(999, vols)
 	require.NoError(t, err)
@@ -139,6 +141,7 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) {
 		PluginID:       "minnie",
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 
 	// Create the register request
@@ -255,6 +258,7 @@ func TestCSIVolumeEndpoint_Claim(t *testing.T) {
 		Topologies: []*structs.CSITopology{{
 			Segments: map[string]string{"foo": "bar"},
 		}},
+		Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	index++
 	err = state.CSIVolumeRegister(index, vols)
@@ -373,6 +377,7 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) {
 		ControllerRequired: true,
 		AccessMode:         structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode:     structs.CSIVolumeAttachmentModeFilesystem,
+		Secrets:            structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err = state.CSIVolumeRegister(1003, vols)
 
@@ -439,6 +444,7 @@ func TestCSIVolumeEndpoint_List(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}, {
 		ID:             id1,
 		Namespace:      structs.DefaultNamespace,

nomad/mock/mock.go

@@ -1311,6 +1311,7 @@ func CSIVolume(plugin *structs.CSIPlugin) *structs.CSIVolume {
 		AccessMode:          structs.CSIVolumeAccessModeSingleNodeWriter,
 		AttachmentMode:      structs.CSIVolumeAttachmentModeFilesystem,
 		MountOptions:        &structs.CSIMountOptions{},
+		Secrets:             structs.CSISecrets{},
 		ReadAllocs:          map[string]*structs.Allocation{},
 		WriteAllocs:         map[string]*structs.Allocation{},
 		ReadClaims:          map[string]*structs.CSIVolumeClaim{},

nomad/structs/csi.go

@@ -185,6 +185,27 @@ func (v *CSIMountOptions) GoString() string {
 	return v.String()
 }
 
+// CSISecrets contain optional additional configuration that can be used
+// when specifying that a Volume should be used with VolumeAccessTypeMount.
+type CSISecrets map[string]string
+
+// CSISecrets implements the Stringer and GoStringer interfaces to prevent
+// accidental leakage of secrets via logs.
+var _ fmt.Stringer = &CSISecrets{}
+var _ fmt.GoStringer = &CSISecrets{}
+
+func (s *CSISecrets) String() string {
+	redacted := map[string]string{}
+	for k := range *s {
+		redacted[k] = "[REDACTED]"
+	}
+	return fmt.Sprintf("csi.CSISecrets(%v)", redacted)
+}
+
+func (s *CSISecrets) GoString() string {
+	return s.String()
+}
+
 type CSIVolumeClaim struct {
 	AllocationID string
 	NodeID       string
@@ -214,6 +235,7 @@ type CSIVolume struct {
 	AccessMode     CSIVolumeAccessMode
 	AttachmentMode CSIVolumeAttachmentMode
 	MountOptions   *CSIMountOptions
+	Secrets        CSISecrets
 
 	// Allocations, tracking claim status
 	ReadAllocs  map[string]*Allocation // AllocID -> Allocation
@@ -249,7 +271,6 @@ type CSIVolListStub struct {
 	Topologies          []*CSITopology
 	AccessMode          CSIVolumeAccessMode
 	AttachmentMode      CSIVolumeAttachmentMode
-	MountOptions        *CSIMountOptions
 	CurrentReaders      int
 	CurrentWriters      int
 	Schedulable         bool
@@ -279,6 +300,9 @@ func (v *CSIVolume) newStructs() {
 	if v.Topologies == nil {
 		v.Topologies = []*CSITopology{}
 	}
+	if v.Secrets == nil {
+		v.Secrets = CSISecrets{}
+	}
 
 	v.ReadAllocs = map[string]*Allocation{}
 	v.WriteAllocs = map[string]*Allocation{}
@@ -304,7 +328,6 @@ func (v *CSIVolume) Stub() *CSIVolListStub {
 		Topologies:          v.Topologies,
 		AccessMode:          v.AccessMode,
 		AttachmentMode:      v.AttachmentMode,
-		MountOptions:        v.MountOptions,
 		CurrentReaders:      len(v.ReadAllocs),
 		CurrentWriters:      len(v.WriteAllocs),
 		Schedulable:         v.Schedulable,
@@ -365,6 +388,9 @@ func (v *CSIVolume) Copy() *CSIVolume {
 	copy := *v
 	out := &copy
 	out.newStructs()
+	for k, v := range v.Secrets {
+		out.Secrets[k] = v
+	}
 
 	for k, v := range v.ReadAllocs {
 		out.ReadAllocs[k] = v

nomad/volumewatcher/volume_watcher.go

@@ -350,6 +350,7 @@ func (vw *volumeWatcher) controllerDetach(vol *structs.CSIVolume, claim *structs
 	cReq := &cstructs.ClientCSIControllerDetachVolumeRequest{
 		VolumeID:        vol.RemoteID(),
 		ClientCSINodeID: targetCSIInfo.NodeInfo.ID,
+		Secrets:         vol.Secrets,
 	}
 	cReq.PluginID = plug.ID
 	err = vw.rpc.ControllerDetachVolume(cReq,

plugins/csi/client.go

@@ -12,6 +12,7 @@ import (
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/helper/grpc-middleware/logging"
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
 	"google.golang.org/grpc"
@@ -293,7 +294,7 @@ func (c *client) ControllerUnpublishVolume(ctx context.Context, req *ControllerU
 	return &ControllerUnpublishVolumeResponse{}, nil
 }
 
-func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
+func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	if c == nil {
 		return fmt.Errorf("Client not initialized")
 	}
@@ -314,6 +315,9 @@ func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID st
 		VolumeCapabilities: []*csipbv1.VolumeCapability{
 			capabilities.ToCSIRepresentation(),
 		},
+		// VolumeContext: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771
+		// Parameters: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
+		Secrets: secrets,
 	}
 
 	resp, err := c.controllerClient.ValidateVolumeCapabilities(ctx, req, opts...)
@@ -461,7 +465,7 @@ func (c *client) NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)
 	return result, nil
 }
 
-func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
+func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	if c == nil {
 		return fmt.Errorf("Client not initialized")
 	}
@@ -483,6 +487,7 @@ func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishCo
 		PublishContext:    publishContext,
 		StagingTargetPath: stagingTargetPath,
 		VolumeCapability:  capabilities.ToCSIRepresentation(),
+		Secrets:           secrets,
 	}
 
 	// NodeStageVolume's response contains no extra data. If err == nil, we were

plugins/csi/client_test.go

@@ -578,7 +578,7 @@ func TestClient_RPC_ControllerValidateVolume(t *testing.T) {
 			cc.NextErr = c.ResponseErr
 
 			err := client.ControllerValidateCapabilities(
-				context.TODO(), "volumeID", requestedCaps)
+				context.TODO(), "volumeID", requestedCaps, structs.CSISecrets{})
 			if c.ExpectedErr != nil {
 				require.Error(t, c.ExpectedErr, err, c.Name)
 			} else {
@@ -616,7 +616,8 @@ func TestClient_RPC_NodeStageVolume(t *testing.T) {
 			nc.NextErr = c.ResponseErr
 			nc.NextStageVolumeResponse = c.Response
 
-			err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo", &VolumeCapability{})
+			err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo",
+				&VolumeCapability{}, structs.CSISecrets{})
 			if c.ExpectedErr != nil {
 				require.Error(t, c.ExpectedErr, err)
 			} else {

plugins/csi/fake/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/csi"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
@@ -159,7 +160,7 @@ func (c *Client) ControllerUnpublishVolume(ctx context.Context, req *csi.Control
 	return c.NextControllerUnpublishVolumeResponse, c.NextControllerUnpublishVolumeErr
 }
 
-func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
+func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	c.Mu.Lock()
 	defer c.Mu.Unlock()
 
@@ -191,7 +192,7 @@ func (c *Client) NodeGetInfo(ctx context.Context) (*csi.NodeGetInfoResponse, err
 // NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
 // to prepare a volume for usage on a host. If err == nil, the response should
 // be assumed to be successful.
-func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
+func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	c.Mu.Lock()
 	defer c.Mu.Unlock()