e2e: restart rbd-nbd process after nodeplugin reboot

Bringup the rbd-nbd map/attach process on the rbd node plugin and expect the
IO to continue uninterrupted.

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

e2e/rbd.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"regexp"
 	"strings"
 	"sync"
 
@@ -412,6 +413,103 @@ var _ = Describe("RBD", func() {
 				validateRBDImageCount(f, 0)
 			})
 
+			By("restart rbd-nbd process on nodeplugin and continue IO after nodeplugin restart", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass with error %v", err)
+				}
+				// Storage class with rbd-nbd mounter and reattach options
+				scOpts := map[string]string{
+					"mounter":    "rbd-nbd",
+					"mapOptions": "try-netlink,reattach-timeout=180",
+				}
+				err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC with error %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+
+				app, err := loadApp(appPath)
+				if err != nil {
+					e2elog.Failf("failed to  load application with error %v", err)
+				}
+
+				app.Namespace = f.UniqueName
+				label := map[string]string{
+					"app": app.Name,
+				}
+				app.Labels = label
+				app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+				app.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
+				err = createPVCAndApp("", f, pvc, app, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application with error %v", err)
+				}
+
+				// validate created backend rbd images
+				validateRBDImageCount(f, 1)
+
+				rbdNodeKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.csi-rbd-node", cephCSINamespace)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("error getting admin key err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				// # ps -eo  'cmd' | grep [r]bd-nbd
+				// /usr/bin/rbd-nbd --id csi-rbd-node -m 10.99.247.176:6789 --keyfile=/tmp/csi/keys/keyfile-733475537 attach --device /dev/nbd0 replicapool/csi-vol-93f8a346-76cc-11eb-b207-b632446b7b60 --try-netlink --reattach-timeout=120
+				opt1 := metav1.ListOptions{
+					LabelSelector: "app=csi-rbdplugin",
+				}
+				mapCmd, stdErr, err := execCommandInPod(f, "ps -eo  'cmd' | grep [r]bd-nbd", app.Namespace, &opt1)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to write key to a file err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				attachCmd := strings.ReplaceAll(mapCmd, "map", "attach --device /dev/nbd0")
+				m1 := regexp.MustCompile(`keyfile-* `)
+				m1.ReplaceAllString(attachCmd, "keyfile-test")
+				err = deletePodWithLabel("app=csi-rbdplugin", cephCSINamespace, false)
+				if err != nil {
+					e2elog.Failf("fail to delete pod with error %v", err)
+				}
+
+				// wait for nodeplugin pods to come up
+				err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
+				if err != nil {
+					e2elog.Failf("timeout waiting for daemonset pods with error %v", err)
+				}
+
+				_, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), app.Namespace, &opt1)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to write key to a file err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				_, stdErr, err = execCommandInPod(f, attachCmd, app.Namespace, &opt1)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to run attach cmd err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				opt2 := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("app=%s", app.Name),
+				}
+				// Write something to mountpoint and expect a failure
+				filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+				_, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &opt2)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to run IO err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				err = deletePVCAndApp("", f, pvc, app)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC and application with error %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0)
+			})
+
 			By("create a PVC and bind it to an app with encrypted RBD volume", func() {
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {