Chap fallback

Add fallback logic to retrieve CHAP info from controller

frontend/csi/rest.go → frontend/csi/controller_api/rest.go

@@ -1,6 +1,6 @@
 // Copyright 2022 NetApp, Inc. All Rights Reserved.
 
-package csi
+package controllerAPI
 
 import (
 	"bytes"
@@ -20,12 +20,12 @@ import (
 
 const HTTPClientTimeout = time.Second * 30
 
-type RestClient struct {
+type ControllerRestClient struct {
 	url        string
 	httpClient http.Client
 }
 
-func CreateTLSRestClient(url, caFile, certFile, keyFile string) (*RestClient, error) {
+func CreateTLSRestClient(url, caFile, certFile, keyFile string) (TridentController, error) {
 	tlsConfig := &tls.Config{MinVersion: config.MinClientTLSVersion}
 	if "" != caFile {
 		caCert, err := ioutil.ReadFile(caFile)
@@ -46,7 +46,7 @@ func CreateTLSRestClient(url, caFile, certFile, keyFile string) (*RestClient, er
 		}
 		tlsConfig.Certificates = []tls.Certificate{cert}
 	}
-	return &RestClient{
+	return &ControllerRestClient{
 		url: url,
 		httpClient: http.Client{
 			Transport: &http.Transport{
@@ -60,8 +60,9 @@ func CreateTLSRestClient(url, caFile, certFile, keyFile string) (*RestClient, er
 // InvokeAPI makes a REST call to the CSI Controller REST endpoint. The body must be a marshaled JSON byte array (
 // or nil). The method is the HTTP verb (i.e. GET, POST, ...).  The resource path is appended to the base URL to
 // identify the desired server resource; it should start with '/'.
-func (c *RestClient) InvokeAPI(
-	ctx context.Context, requestBody []byte, method string, resourcePath string,
+func (c *ControllerRestClient) InvokeAPI(
+	ctx context.Context, requestBody []byte, method string, resourcePath string, redactRequestBody,
+	redactResponseBody bool,
 ) (*http.Response, []byte, error) {
 
 	// Build URL
@@ -91,7 +92,8 @@ func (c *RestClient) InvokeAPI(
 			return nil, nil, fmt.Errorf("error formating request body; %v", err)
 		}
 	}
-	utils.LogHTTPRequest(request, prettyRequestBuffer.Bytes())
+
+	utils.LogHTTPRequest(request, prettyRequestBuffer.Bytes(), redactRequestBody)
 
 	response, err := c.httpClient.Do(request)
 	if err != nil {
@@ -110,7 +112,7 @@ func (c *RestClient) InvokeAPI(
 			return nil, nil, fmt.Errorf("error formating response body; %v", err)
 		}
 	}
-	utils.LogHTTPResponse(ctx, response, prettyResponseBuffer.Bytes())
+	utils.LogHTTPResponse(ctx, response, prettyResponseBuffer.Bytes(), redactResponseBody)
 
 	return response, responseBody, err
 }
@@ -120,12 +122,12 @@ type CreateNodeResponse struct {
 }
 
 // CreateNode registers the node with the CSI controller server
-func (c *RestClient) CreateNode(ctx context.Context, node *utils.Node) (CreateNodeResponse, error) {
+func (c *ControllerRestClient) CreateNode(ctx context.Context, node *utils.Node) (CreateNodeResponse, error) {
 	nodeData, err := json.MarshalIndent(node, "", " ")
 	if err != nil {
 		return CreateNodeResponse{}, fmt.Errorf("error parsing create node request; %v", err)
 	}
-	resp, respBody, err := c.InvokeAPI(ctx, nodeData, "PUT", config.NodeURL+"/"+node.Name)
+	resp, respBody, err := c.InvokeAPI(ctx, nodeData, "PUT", config.NodeURL+"/"+node.Name, false, false)
 	if err != nil {
 		return CreateNodeResponse{}, fmt.Errorf("could not log into the Trident CSI Controller: %v", err)
 	}
@@ -146,8 +148,8 @@ type ListNodesResponse struct {
 }
 
 // GetNodes returns a list of nodes registered with the controller
-func (c *RestClient) GetNodes(ctx context.Context) ([]string, error) {
-	resp, respBody, err := c.InvokeAPI(ctx, nil, "GET", config.NodeURL)
+func (c *ControllerRestClient) GetNodes(ctx context.Context) ([]string, error) {
+	resp, respBody, err := c.InvokeAPI(ctx, nil, "GET", config.NodeURL, false, false)
 	if err != nil {
 		return nil, fmt.Errorf("could not log into the Trident CSI Controller: %v", err)
 	}
@@ -166,8 +168,8 @@ func (c *RestClient) GetNodes(ctx context.Context) ([]string, error) {
 }
 
 // DeleteNode deregisters the node with the CSI controller server
-func (c *RestClient) DeleteNode(ctx context.Context, name string) error {
-	resp, _, err := c.InvokeAPI(ctx, nil, "DELETE", config.NodeURL+"/"+name)
+func (c *ControllerRestClient) DeleteNode(ctx context.Context, name string) error {
+	resp, _, err := c.InvokeAPI(ctx, nil, "DELETE", config.NodeURL+"/"+name, false, false)
 	if err != nil {
 		return fmt.Errorf("could not log into the Trident CSI Controller: %v", err)
 	}
@@ -184,3 +186,27 @@ func (c *RestClient) DeleteNode(ctx context.Context, name string) error {
 	}
 	return nil
 }
+
+type GetCHAPResponse struct {
+	CHAP  *utils.IscsiChapInfo `json:"chap"`
+	Error string               `json:"error,omitempty"`
+}
+
+// GetChap requests the current CHAP credentials for a given volume/node pair from the Trident controller
+func (c *ControllerRestClient) GetChap(ctx context.Context, volume, node string) (*utils.IscsiChapInfo, error) {
+	resp, respBody, err := c.InvokeAPI(ctx, nil, "GET", config.ChapURL+"/"+volume+"/"+node, false, true)
+	if err != nil {
+		return &utils.IscsiChapInfo{}, fmt.Errorf("could not communicate with the Trident CSI Controller: %v", err)
+	}
+	createResponse := GetCHAPResponse{}
+	if err := json.Unmarshal(respBody, &createResponse); err != nil {
+		return nil, fmt.Errorf("could not parse CHAP info : %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		msg := "could not add get CHAP info"
+		Logc(ctx).WithError(fmt.Errorf(createResponse.Error)).Errorf(msg)
+		return nil, fmt.Errorf(msg)
+	}
+	return createResponse.CHAP, nil
+}

frontend/csi/controller_api/types.go

@@ -0,0 +1,23 @@
+// Copyright 2022 NetApp, Inc. All Rights Reserved.
+
+package controllerAPI
+
+//go:generate mockgen -destination=../../../mocks/mock_frontend/mock_csi/mock_controller_api/mock_controller_api.go github.com/netapp/trident/frontend/csi/controller_api TridentController
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/netapp/trident/utils"
+)
+
+type TridentController interface {
+	InvokeAPI(
+		ctx context.Context, requestBody []byte, method string, resourcePath string, redactRequestBody,
+		redactResponseBody bool,
+	) (*http.Response, []byte, error)
+	CreateNode(ctx context.Context, node *utils.Node) (CreateNodeResponse, error)
+	GetNodes(ctx context.Context) ([]string, error)
+	DeleteNode(ctx context.Context, name string) error
+	GetChap(ctx context.Context, volume, node string) (*utils.IscsiChapInfo, error)
+}

frontend/csi/node_server.go

@@ -947,19 +947,36 @@ func (p *Plugin) nodeStageISCSIVolume(
 
 		if p.aesKey != nil && encryptedCHAP {
 			if err = decryptCHAPPublishInfo(ctx, publishInfo, req.PublishContext, p.aesKey); err != nil {
-				return nil, status.Error(codes.Internal, err.Error())
+				Logc(ctx).WithError(err).Warn(
+					"Could not decrypt CHAP credentials; will retrieve from Trident controller.")
+				if err = p.updateChapInfoFromController(ctx, req, publishInfo); err != nil {
+					return nil, status.Error(codes.Internal, err.Error())
+				}
 			}
 		} else if encryptedCHAP {
 			// Only error if the key is not set and the publishContext includes encrypted fields
-			msg := "encryption key not set; cannot decrypt CHAP credentials"
-			Logc(ctx).Error(msg)
-			return nil, status.Error(codes.Internal, msg)
+			Logc(ctx).Warn(
+				"Encryption key not set; cannot decrypt CHAP credentials; will retrieve from Trident controller.")
+			if err = p.updateChapInfoFromController(ctx, req, publishInfo); err != nil {
+				return nil, status.Error(codes.Internal, err.Error())
+			}
 		}
 	}
 
 	// Perform the login/rescan/discovery/(optionally)format, mount & get the device back in the publish info
 	if err := utils.AttachISCSIVolume(ctx, req.VolumeContext["internalName"], "", publishInfo); err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
+		// Did we fail to log in?
+		if utils.IsAuthError(err) {
+			// Update CHAP info from the controller and try one more time
+			Logc(ctx).Warn("iSCSI login failed; will retrieve CHAP credentials from Trident controller and try again.")
+			if err = p.updateChapInfoFromController(ctx, req, publishInfo); err != nil {
+				return nil, status.Error(codes.Internal, err.Error())
+			}
+			if err = utils.AttachISCSIVolume(ctx, req.VolumeContext["internalName"], "", publishInfo); err != nil {
+				// Bail out no matter what as we've now tried with updated credentials
+				return nil, status.Error(codes.Internal, err.Error())
+			}
+		}
 	}
 
 	volumeId, stagingTargetPath, err := p.getVolumeIdAndStagingPath(req)
@@ -975,6 +992,23 @@ func (p *Plugin) nodeStageISCSIVolume(
 	return &csi.NodeStageVolumeResponse{}, nil
 }
 
+func (p *Plugin) updateChapInfoFromController(
+	ctx context.Context, req *csi.NodeStageVolumeRequest, publishInfo *utils.VolumePublishInfo,
+) error {
+	chapInfo, err := p.restClient.GetChap(ctx, req.GetVolumeId(), p.nodeName)
+	if err != nil {
+		msg := "could not retrieve CHAP credentials from Trident controller"
+		Logc(ctx).WithError(err).Error(msg)
+		return fmt.Errorf(msg)
+	}
+	publishInfo.UseCHAP = chapInfo.UseCHAP
+	publishInfo.IscsiUsername = chapInfo.IscsiUsername
+	publishInfo.IscsiInitiatorSecret = chapInfo.IscsiInitiatorSecret
+	publishInfo.IscsiTargetUsername = chapInfo.IscsiTargetUsername
+	publishInfo.IscsiTargetSecret = chapInfo.IscsiTargetSecret
+	return nil
+}
+
 func (p *Plugin) nodeUnstageISCSIVolume(
 	ctx context.Context, req *csi.NodeUnstageVolumeRequest, publishInfo *utils.VolumePublishInfo,
 ) (*csi.NodeUnstageVolumeResponse, error) {

frontend/csi/node_server_test.go

@@ -0,0 +1,75 @@
+// Copyright 2022 NetApp, Inc. All Rights Reserved.
+
+package csi
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+
+	mockControllerAPI "github.com/netapp/trident/mocks/mock_frontend/mock_csi/mock_controller_api"
+	"github.com/netapp/trident/utils"
+)
+
+func TestUpdateChapInfoFromController_Success(t *testing.T) {
+	testCtx := context.Background()
+	volumeName := "foo"
+	nodeName := "bar"
+	expectedChapInfo := utils.IscsiChapInfo{
+		UseCHAP:              true,
+		IscsiUsername:        "user",
+		IscsiInitiatorSecret: "pass",
+		IscsiTargetUsername:  "user2",
+		IscsiTargetSecret:    "pass2",
+	}
+
+	mockCtrl := gomock.NewController(t)
+	mockClient := mockControllerAPI.NewMockTridentController(mockCtrl)
+	mockClient.EXPECT().GetChap(testCtx, volumeName, nodeName).Return(&expectedChapInfo, nil)
+	nodeServer := &Plugin{
+		nodeName:   nodeName,
+		role:       CSINode,
+		restClient: mockClient,
+	}
+
+	fakeRequest := &csi.NodeStageVolumeRequest{VolumeId: volumeName}
+	testPublishInfo := &utils.VolumePublishInfo{}
+
+	err := nodeServer.updateChapInfoFromController(testCtx, fakeRequest, testPublishInfo)
+	assert.Nil(t, err, "Unexpected error")
+	assert.EqualValues(t, expectedChapInfo, testPublishInfo.IscsiAccessInfo.IscsiChapInfo)
+}
+
+func TestUpdateChapInfoFromController_Error(t *testing.T) {
+	testCtx := context.Background()
+	volumeName := "foo"
+	nodeName := "bar"
+	expectedChapInfo := utils.IscsiChapInfo{
+		UseCHAP:              true,
+		IscsiUsername:        "user",
+		IscsiInitiatorSecret: "pass",
+		IscsiTargetUsername:  "user2",
+		IscsiTargetSecret:    "pass2",
+	}
+
+	mockCtrl := gomock.NewController(t)
+	mockClient := mockControllerAPI.NewMockTridentController(mockCtrl)
+	mockClient.EXPECT().GetChap(testCtx, volumeName, nodeName).Return(&expectedChapInfo, fmt.Errorf("some error"))
+	nodeServer := &Plugin{
+		nodeName:   nodeName,
+		role:       CSINode,
+		restClient: mockClient,
+	}
+
+	fakeRequest := &csi.NodeStageVolumeRequest{VolumeId: volumeName}
+	testPublishInfo := &utils.VolumePublishInfo{}
+
+	err := nodeServer.updateChapInfoFromController(testCtx, fakeRequest, testPublishInfo)
+	assert.NotNil(t, err, "Unexpected success")
+	assert.NotEqualValues(t, expectedChapInfo, testPublishInfo.IscsiAccessInfo.IscsiChapInfo)
+	assert.EqualValues(t, utils.IscsiChapInfo{}, testPublishInfo.IscsiAccessInfo.IscsiChapInfo)
+}

frontend/csi/plugin.go

@@ -1,4 +1,4 @@
-// Copyright 2021 NetApp, Inc. All Rights Reserved.
+// Copyright 2022 NetApp, Inc. All Rights Reserved.
 
 package csi
 
@@ -16,6 +16,7 @@ import (
 
 	tridentconfig "github.com/netapp/trident/config"
 	"github.com/netapp/trident/core"
+	controllerAPI "github.com/netapp/trident/frontend/csi/controller_api"
 	"github.com/netapp/trident/frontend/csi/helpers"
 	. "github.com/netapp/trident/logger"
 	"github.com/netapp/trident/utils"
@@ -41,7 +42,7 @@ type Plugin struct {
 	hostInfo *utils.HostSystem
 	nodePrep *utils.NodePrep
 
-	restClient *RestClient
+	restClient controllerAPI.TridentController
 	helper     helpers.HybridPlugin
 
 	aesKey []byte
@@ -146,7 +147,7 @@ func NewNodePlugin(
 
 	restURL := "https://" + hostname + ":" + port
 	var err error
-	p.restClient, err = CreateTLSRestClient(restURL, caCert, clientCert, clientKey)
+	p.restClient, err = controllerAPI.CreateTLSRestClient(restURL, caCert, clientCert, clientKey)
 	if err != nil {
 		return nil, err
 	}
@@ -223,7 +224,7 @@ func NewAllInOnePlugin(
 	}
 	restURL := "https://" + tridentconfig.ServerCertName + ":" + port
 	var err error
-	p.restClient, err = CreateTLSRestClient(restURL, caCert, clientCert, clientKey)
+	p.restClient, err = controllerAPI.CreateTLSRestClient(restURL, caCert, clientCert, clientKey)
 	if err != nil {
 		return nil, err
 	}