From c0b8f078c5013aae892340fb8b273e75c1ad6c9e Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 25 Feb 2021 12:53:22 +0530 Subject: [PATCH] e2e: restart rbd-nbd process after nodeplugin reboot Bringup the rbd-nbd map/attach process on the rbd node plugin and expect the IO to continue uninterrupted. Signed-off-by: Prasanna Kumar Kalever --- e2e/rbd.go | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/e2e/rbd.go b/e2e/rbd.go index 9b54f1ec175b..e27bc38edc59 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "os" + "regexp" "strings" . "github.com/onsi/ginkgo" // nolint @@ -460,6 +461,139 @@ var _ = Describe("RBD", func() { } }) + By("restart rbd-nbd process on nodeplugin and continue IO after nodeplugin restart", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + // Tweak Storageclass to add netlink,reattach rbd-nbd mounter options + scOpts := map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": "try-netlink,reattach-timeout=180", + } + err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + + pvc, err := loadPVC(pvcPath) + if err != nil { + e2elog.Failf("failed to load PVC with error %v", err) + } + pvc.Namespace = f.UniqueName + + app, err := loadApp(appPath) + if err != nil { + e2elog.Failf("failed to load application with error %v", err) + } + app.Namespace = f.UniqueName + label := map[string]string{ + "app": app.Name, + } + app.Labels = label + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application with error %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 1) + + selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName) + if err != nil { + e2elog.Failf("failed to get the labels with error %v", err) + } + + opt := metav1.ListOptions{ + LabelSelector: selector, + } + + uname, stdErr, err := execCommandInContainer(f, "uname -a", cephCSINamespace, "csi-rbdplugin", &opt) + if err != nil || stdErr != "" { + e2elog.Failf("failed to run uname cmd : %v, stdErr: %v ", err, stdErr) + } + e2elog.Logf("uname -a: %v", uname) + rpmv, stdErr, err := execCommandInContainer(f, "rpm -qa | grep rbd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) + if err != nil || stdErr != "" { + e2elog.Failf("failed to run rpm -qa cmd : %v, stdErr: %v ", err, stdErr) + } + e2elog.Logf("rbd-nbd package version: %v", rpmv) + + // Get details of rbd-nbd process + // # ps -eo 'cmd' | grep [r]bd-nbd + // /usr/bin/rbd-nbd --id cephcsi-rbd-node -m svc-name:6789 --keyfile=/tmp/csi/keys/keyfile attach --device /dev/nbd0 pool-name/image-name --try-netlink --reattach-timeout=180 + mapCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) + if err != nil || stdErr != "" { + e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr) + } + e2elog.Logf("map command running before restart, mapCmd: %v", mapCmd) + + rbdNodeKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.cephcsi-rbd-node", rookNamespace) + if err != nil || stdErr != "" { + e2elog.Failf("error getting cephcsi-rbd-node key, err: %v, stdErr: %v ", err, stdErr) + } + + // restart the rbd node plugin + err = deletePodWithLabel(selector, cephCSINamespace, false) + if err != nil { + e2elog.Failf("fail to delete pod with error %v", err) + } + + // wait for nodeplugin pods to come up + err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for daemonset pods with error %v", err) + } + + // Prepare the rbd-nbd with command args + attachCmd := strings.ReplaceAll(mapCmd, "map", "attach --device /dev/nbd0") + m1 := regexp.MustCompile(`/keyfile-[0-9]* `) + attachCmd = m1.ReplaceAllString(attachCmd, "/keyfile-test ") + e2elog.Logf("attach command to run after restart, attachCmd: %v", attachCmd) + + // create the keyfile + _, stdErr, err = execCommandInContainer(f, fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), cephCSINamespace, "csi-rbdplugin", &opt) + if err != nil || stdErr != "" { + e2elog.Failf("failed to write key to a file, err: %v, stdErr: %v ", err, stdErr) + } + + _, stdErr, err = execCommandInContainer(f, attachCmd, cephCSINamespace, "csi-rbdplugin", &opt) + if err != nil || stdErr != "" { + e2elog.Failf("failed to run attach cmd err: %v, stdErr: %v ", err, stdErr) + } + + runningAttachCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) + if err != nil || stdErr != "" { + e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr) + } + e2elog.Logf("attach command running after restart, runningAttachCmd: %v", runningAttachCmd) + + appOpt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", app.Name), + } + // Write something to mountpoint and expect it to happen + filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + _, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &appOpt) + if err != nil || stdErr != "" { + e2elog.Failf("failed to write IO, err: %v, stdErr: %v ", err, stdErr) + } + + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + e2elog.Failf("failed to delete PVC and application with error %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0) + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + }) + By("create a PVC and bind it to an app with encrypted RBD volume", func() { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil {