diff --git a/e2e/rbd.go b/e2e/rbd.go index 45c49b6c65f7..4621f3b55610 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "regexp" "strings" "sync" @@ -412,6 +413,103 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 0) }) + By("restart rbd-nbd process on nodeplugin and continue IO after nodeplugin restart", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + // Storage class with rbd-nbd mounter and reattach options + scOpts := map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": "try-netlink,reattach-timeout=180", + } + err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + pvc, err := loadPVC(pvcPath) + if err != nil { + e2elog.Failf("failed to load PVC with error %v", err) + } + pvc.Namespace = f.UniqueName + + app, err := loadApp(appPath) + if err != nil { + e2elog.Failf("failed to load application with error %v", err) + } + + app.Namespace = f.UniqueName + label := map[string]string{ + "app": app.Name, + } + app.Labels = label + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + app.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application with error %v", err) + } + + // validate created backend rbd images + validateRBDImageCount(f, 1) + + rbdNodeKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.csi-rbd-node", cephCSINamespace) + if err != nil || stdErr != "" { + e2elog.Failf("error getting admin key err: %v, stdErr: %v ", err, stdErr) + } + + // # ps -eo 'cmd' | grep [r]bd-nbd + // /usr/bin/rbd-nbd --id csi-rbd-node -m 10.99.247.176:6789 --keyfile=/tmp/csi/keys/keyfile-733475537 attach --device /dev/nbd0 replicapool/csi-vol-93f8a346-76cc-11eb-b207-b632446b7b60 --try-netlink --reattach-timeout=120 + opt1 := metav1.ListOptions{ + LabelSelector: "app=csi-rbdplugin", + } + mapCmd, stdErr, err := execCommandInPod(f, "ps -eo 'cmd' | grep [r]bd-nbd", app.Namespace, &opt1) + if err != nil || stdErr != "" { + e2elog.Failf("failed to write key to a file err: %v, stdErr: %v ", err, stdErr) + } + + attachCmd := strings.ReplaceAll(mapCmd, "map", "attach --device /dev/nbd0") + m1 := regexp.MustCompile(`keyfile-* `) + m1.ReplaceAllString(attachCmd, "keyfile-test") + err = deletePodWithLabel("app=csi-rbdplugin", cephCSINamespace, false) + if err != nil { + e2elog.Failf("fail to delete pod with error %v", err) + } + + // wait for nodeplugin pods to come up + err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for daemonset pods with error %v", err) + } + + _, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), app.Namespace, &opt1) + if err != nil || stdErr != "" { + e2elog.Failf("failed to write key to a file err: %v, stdErr: %v ", err, stdErr) + } + + _, stdErr, err = execCommandInPod(f, attachCmd, app.Namespace, &opt1) + if err != nil || stdErr != "" { + e2elog.Failf("failed to run attach cmd err: %v, stdErr: %v ", err, stdErr) + } + + opt2 := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", app.Name), + } + // Write something to mountpoint and expect a failure + filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + _, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &opt2) + if err != nil || stdErr != "" { + e2elog.Failf("failed to run IO err: %v, stdErr: %v ", err, stdErr) + } + + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + e2elog.Failf("failed to delete PVC and application with error %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0) + }) + By("create a PVC and bind it to an app with encrypted RBD volume", func() { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil {