e2e: add a test case for rbd-nbd mounter #1839
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,6 +5,7 @@ import ( | |
| "fmt" | ||
| "io/ioutil" | ||
| "os" | ||
| "regexp" | ||
| "strings" | ||
|
|
||
| . "github.com/onsi/ginkgo" // nolint | ||
|
|
@@ -349,6 +350,250 @@ var _ = Describe("RBD", func() { | |
| } | ||
| }) | ||
|
|
||
| By("create a PVC and bind it to an app using rbd-nbd mounter", func() { | ||
|
There was a problem hiding this comment. can you please enable this E2E also https://github.com/ceph/ceph-csi/blob/devel/e2e/rbd.go#L485-L496? |
||
| err := deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"mounter": "rbd-nbd"}, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
| err = validatePVCAndAppBinding(pvcPath, appPath, f) | ||
| if err != nil { | ||
| e2elog.Failf("failed to validate pvc and application binding with error %v", err) | ||
| } | ||
| // validate created backend rbd images | ||
| validateRBDImageCount(f, 0) | ||
| err = deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
| }) | ||
|
|
||
| By("perform IO on rbd-nbd volume after nodeplugin restart and expect a failure", func() { | ||
| err := deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| // Storage class with rbd-nbd mounter | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"mounter": "rbd-nbd"}, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
| pvc, err := loadPVC(pvcPath) | ||
| if err != nil { | ||
| e2elog.Failf("failed to load PVC with error %v", err) | ||
| } | ||
| pvc.Namespace = f.UniqueName | ||
|
|
||
| app, err := loadApp(appPath) | ||
| if err != nil { | ||
| e2elog.Failf("failed to load application with error %v", err) | ||
| } | ||
|
|
||
| app.Namespace = f.UniqueName | ||
| label := map[string]string{ | ||
| "app": app.Name, | ||
| } | ||
| app.Labels = label | ||
| app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name | ||
| err = createPVCAndApp("", f, pvc, app, deployTimeout) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create PVC and application with error %v", err) | ||
| } | ||
|
|
||
| // validate created backend rbd images | ||
| validateRBDImageCount(f, 1) | ||
|
|
||
| selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName) | ||
| if err != nil { | ||
| e2elog.Failf("failed to get the labels with error %v", err) | ||
| } | ||
| // delete rbd nodeplugin pods | ||
| err = deletePodWithLabel(selector, cephCSINamespace, false) | ||
| if err != nil { | ||
| e2elog.Failf("fail to delete pod with error %v", err) | ||
| } | ||
|
|
||
| // wait for nodeplugin pods to come up | ||
| err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout) | ||
| if err != nil { | ||
| e2elog.Failf("timeout waiting for daemonset pods with error %v", err) | ||
| } | ||
|
|
||
| opt := metav1.ListOptions{ | ||
| LabelSelector: fmt.Sprintf("app=%s", app.Name), | ||
| } | ||
|
|
||
| // FIXME: Fix this behavior, i.e. when the nodeplugin is | ||
| // restarted, the rbd-nbd processes should be back to life | ||
| // as rbd-nbd processes are responsible for IO | ||
|
|
||
| // For now to prove this isn't working, write something to | ||
| // mountpoint and expect a failure as the processes are terminated. | ||
| filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" | ||
| _, stdErr := execCommandInPodAndAllowFail(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &opt) | ||
| IOErr := fmt.Sprintf("cannot create %s: Input/output error", filePath) | ||
|
There was a problem hiding this comment. Add a FIXME to fix this one as this wont be true when we implement logic to handle daemonset pod restart? |
||
| if !strings.Contains(stdErr, IOErr) { | ||
| e2elog.Failf(stdErr) | ||
| } else { | ||
| e2elog.Logf("failed IO as expected: %v", stdErr) | ||
| } | ||
|
|
||
| err = deletePVCAndApp("", f, pvc, app) | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete PVC and application with error %v", err) | ||
| } | ||
| // validate created backend rbd images | ||
| validateRBDImageCount(f, 0) | ||
|
There was a problem hiding this comment. delete the storageclass here as this is specific to |
||
| err = deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
| }) | ||
|
|
||
| By("restart rbd-nbd process on nodeplugin and continue IO after nodeplugin restart", func() { | ||
|
There was a problem hiding this comment. Just a question Is this whole test case required? as we are doing some hacks to remount the rbd image which is not the case in the real world and this is expected to handled automatically by cephcsi or other sidecars in daemonset pod? There was a problem hiding this comment. The subject of the tests remains the same after the automation as well, just that we might remove the lines which are trying to manually bring the process online and override them with the steps to bring the process back to online in the background. In anyway having this in the CI till then will help us understand the regressions and true failures within rbd-nbd. There was a problem hiding this comment. currently this PR adds two tests There was a problem hiding this comment. These are the POC level tests, for someone who is new to rbd-nbd mounter can understand the problem we are dealing here very clearly with the help of these test cases. In the future when the real story is working we can clear these bits and makeup more useful test cases. Till then the intention of these tests is to catch some regressions or failures at rbd-nbd only. |
||
| err := deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| // Tweak Storageclass to add netlink,reattach rbd-nbd mounter options | ||
| scOpts := map[string]string{ | ||
| "mounter": "rbd-nbd", | ||
| "mapOptions": "try-netlink,reattach-timeout=180", | ||
| } | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
|
|
||
| pvc, err := loadPVC(pvcPath) | ||
| if err != nil { | ||
| e2elog.Failf("failed to load PVC with error %v", err) | ||
| } | ||
| pvc.Namespace = f.UniqueName | ||
|
|
||
| app, err := loadApp(appPath) | ||
| if err != nil { | ||
| e2elog.Failf("failed to load application with error %v", err) | ||
| } | ||
| app.Namespace = f.UniqueName | ||
| label := map[string]string{ | ||
| "app": app.Name, | ||
| } | ||
| app.Labels = label | ||
| app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name | ||
| err = createPVCAndApp("", f, pvc, app, deployTimeout) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create PVC and application with error %v", err) | ||
| } | ||
| // validate created backend rbd images | ||
| validateRBDImageCount(f, 1) | ||
|
|
||
| selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName) | ||
| if err != nil { | ||
| e2elog.Failf("failed to get the labels with error %v", err) | ||
| } | ||
|
|
||
| opt := metav1.ListOptions{ | ||
| LabelSelector: selector, | ||
| } | ||
|
|
||
| uname, stdErr, err := execCommandInContainer(f, "uname -a", cephCSINamespace, "csi-rbdplugin", &opt) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("failed to run uname cmd : %v, stdErr: %v ", err, stdErr) | ||
| } | ||
| e2elog.Logf("uname -a: %v", uname) | ||
| rpmv, stdErr, err := execCommandInContainer(f, "rpm -qa | grep rbd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("failed to run rpm -qa cmd : %v, stdErr: %v ", err, stdErr) | ||
| } | ||
| e2elog.Logf("rbd-nbd package version: %v", rpmv) | ||
|
|
||
| // Get details of rbd-nbd process | ||
| // # ps -eo 'cmd' | grep [r]bd-nbd | ||
| // /usr/bin/rbd-nbd --id cephcsi-rbd-node -m svc-name:6789 --keyfile=/tmp/csi/keys/keyfile attach --device /dev/nbd0 pool-name/image-name --try-netlink --reattach-timeout=180 | ||
| mapCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr) | ||
| } | ||
| e2elog.Logf("map command running before restart, mapCmd: %v", mapCmd) | ||
|
|
||
| rbdNodeKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.cephcsi-rbd-node", rookNamespace) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("error getting cephcsi-rbd-node key, err: %v, stdErr: %v ", err, stdErr) | ||
| } | ||
|
|
||
| // restart the rbd node plugin | ||
| err = deletePodWithLabel(selector, cephCSINamespace, false) | ||
| if err != nil { | ||
| e2elog.Failf("fail to delete pod with error %v", err) | ||
| } | ||
|
|
||
| // wait for nodeplugin pods to come up | ||
| err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout) | ||
| if err != nil { | ||
| e2elog.Failf("timeout waiting for daemonset pods with error %v", err) | ||
| } | ||
|
|
||
| // Prepare the rbd-nbd with command args | ||
| attachCmd := strings.ReplaceAll(mapCmd, "map", "attach --device /dev/nbd0") | ||
| m1 := regexp.MustCompile(`/keyfile-[0-9]* `) | ||
| attachCmd = m1.ReplaceAllString(attachCmd, "/keyfile-test ") | ||
| e2elog.Logf("attach command to run after restart, attachCmd: %v", attachCmd) | ||
|
|
||
| // create the keyfile | ||
| _, stdErr, err = execCommandInContainer(f, fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), cephCSINamespace, "csi-rbdplugin", &opt) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("failed to write key to a file, err: %v, stdErr: %v ", err, stdErr) | ||
| } | ||
|
|
||
| _, stdErr, err = execCommandInContainer(f, attachCmd, cephCSINamespace, "csi-rbdplugin", &opt) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("failed to run attach cmd err: %v, stdErr: %v ", err, stdErr) | ||
| } | ||
|
|
||
| runningAttachCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr) | ||
| } | ||
| e2elog.Logf("attach command running after restart, runningAttachCmd: %v", runningAttachCmd) | ||
|
|
||
| appOpt := metav1.ListOptions{ | ||
| LabelSelector: fmt.Sprintf("app=%s", app.Name), | ||
| } | ||
| // Write something to mountpoint and expect it to happen | ||
| filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" | ||
| _, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &appOpt) | ||
| if err != nil || stdErr != "" { | ||
| e2elog.Failf("failed to write IO, err: %v, stdErr: %v ", err, stdErr) | ||
| } | ||
|
|
||
| err = deletePVCAndApp("", f, pvc, app) | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete PVC and application with error %v", err) | ||
| } | ||
| // validate created backend rbd images | ||
| validateRBDImageCount(f, 0) | ||
| err = deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
| }) | ||
|
|
||
| By("create a PVC and bind it to an app with encrypted RBD volume", func() { | ||
| err := deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
|
|
@@ -482,17 +727,28 @@ var _ = Describe("RBD", func() { | |
| } | ||
| }) | ||
|
|
||
| By("create a PVC and Bind it to an app with journaling/exclusive-lock image-features and rbd-nbd mounter", func() { | ||
| err := deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"imageFeatures": "layering,journaling,exclusive-lock", "mounter": "rbd-nbd"}, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
| err = validatePVCAndAppBinding(pvcPath, appPath, f) | ||
| if err != nil { | ||
| e2elog.Failf("failed to validate pvc and application binding with error %v", err) | ||
| } | ||
| err = deleteResource(rbdExamplePath + "storageclass.yaml") | ||
| if err != nil { | ||
| e2elog.Failf("failed to delete storageclass with error %v", err) | ||
| } | ||
| err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy) | ||
| if err != nil { | ||
| e2elog.Failf("failed to create storageclass with error %v", err) | ||
| } | ||
| }) | ||
|
|
||
| By("create a PVC clone and bind it to an app", func() { | ||
| // snapshot beta is only supported from v1.17+ | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
found is only using if the
cnis not empty can we move found:=false inside if check?