Update Disable CSI e2e test

aws · Dec 8, 2022 · 8328af9 · 8328af9
1 parent 6279d8a
commit 8328af9
Show file tree

Hide file tree

Showing 4 changed files with 100 additions and 45 deletions.
diff --git a/pkg/executables/kubectl.go b/pkg/executables/kubectl.go
@@ -1770,18 +1770,12 @@ func (k *Kubectl) GetBundles(ctx context.Context, kubeconfigFile, name, namespac
 }
 
 func (k *Kubectl) GetClusterResourceSet(ctx context.Context, kubeconfigFile, name, namespace string) (*addons.ClusterResourceSet, error) {
-	params := []string{"get", clusterResourceSetResourceType, name, "-o", "json", "--kubeconfig", kubeconfigFile, "--namespace", namespace}
-	stdOut, err := k.Execute(ctx, params...)
-	if err != nil {
-		return nil, fmt.Errorf("getting ClusterResourceSet with kubectl: %v", err)
-	}
-
-	response := &addons.ClusterResourceSet{}
-	if err = json.Unmarshal(stdOut.Bytes(), response); err != nil {
-		return nil, fmt.Errorf("parsing ClusterResourceSet response: %v", err)
+	obj := &addons.ClusterResourceSet{}
+	if err := k.GetObject(ctx, clusterResourceSetResourceType, name, namespace, kubeconfigFile, obj); err != nil {
+		return nil, err
 	}
 
-	return response, nil
+	return obj, nil
 }
 
 func (k *Kubectl) GetConfigMap(ctx context.Context, kubeconfigFile, name, namespace string) (*corev1.ConfigMap, error) {

diff --git a/pkg/executables/kubectl_test.go b/pkg/executables/kubectl_test.go
@@ -1971,42 +1971,38 @@ func TestKubectlGetBundles(t *testing.T) {
 }
 
 func TestKubectlGetClusterResourceSet(t *testing.T) {
-	tt := newKubectlTest(t)
-	resourceSetJson := test.ReadFile(t, "testdata/kubectl_clusterresourceset.json")
-	resourceSetName := "Bundle-name"
-	wantResourceSet := &addons.ClusterResourceSet{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "addons.cluster.x-k8s.io/v1beta1",
-			Kind:       "ClusterResourceSet",
-		},
-		Spec: addons.ClusterResourceSetSpec{
-			ClusterSelector: metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					"cluster.x-k8s.io/cluster-name": "cluster-1",
-				},
+	newKubectlGetterTest(t).withResourceType(
+		"clusterresourcesets.addons.cluster.x-k8s.io",
+	).withGetter(func(tt *kubectlGetterTest) (client.Object, error) {
+		return tt.k.GetClusterResourceSet(tt.ctx, tt.kubeconfig, tt.name, tt.namespace)
+	}).withJsonFromFile(
+		"testdata/kubectl_clusterresourceset.json",
+	).andWant(
+		&addons.ClusterResourceSet{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: "addons.cluster.x-k8s.io/v1beta1",
+				Kind:       "ClusterResourceSet",
 			},
-			Strategy: "ApplyOnce",
-			Resources: []addons.ResourceRef{
-				{
-					Kind: "Secret",
-					Name: "vsphere-csi-controller",
+			Spec: addons.ClusterResourceSetSpec{
+				ClusterSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"cluster.x-k8s.io/cluster-name": "cluster-1",
+					},
 				},
-				{
-					Kind: "ConfigMap",
-					Name: "vsphere-csi-controller-role",
+				Strategy: "ApplyOnce",
+				Resources: []addons.ResourceRef{
+					{
+						Kind: "Secret",
+						Name: "vsphere-csi-controller",
+					},
+					{
+						Kind: "ConfigMap",
+						Name: "vsphere-csi-controller-role",
+					},
 				},
 			},
 		},
-	}
-
-	tt.e.EXPECT().Execute(
-		tt.ctx,
-		"get", "clusterresourcesets.addons.cluster.x-k8s.io", resourceSetName, "-o", "json", "--kubeconfig", tt.cluster.KubeconfigFile, "--namespace", tt.namespace,
-	).Return(*bytes.NewBufferString(resourceSetJson), nil)
-
-	gotResourceSet, err := tt.k.GetClusterResourceSet(tt.ctx, tt.cluster.KubeconfigFile, resourceSetName, tt.namespace)
-	tt.Expect(err).To(BeNil())
-	tt.Expect(gotResourceSet).To(Equal(wantResourceSet))
+	).testSuccess()
 }
 
 func TestKubectlGetConfigMap(t *testing.T) {

diff --git a/test/e2e/vsphere_disable_csi_test.go b/test/e2e/vsphere_disable_csi_test.go
@@ -15,8 +15,8 @@ func runVSphereDisableCSIUpgradeFlow(test *framework.ClusterE2ETest, updateVersi
 	test.GenerateClusterConfig()
 	test.CreateCluster()
 	test.ValidateVSphereCSI(true)
-	test.DeleteVSphereCSI()
 	test.UpgradeCluster([]framework.ClusterE2ETestOpt{provider.WithProviderUpgrade(api.WithDisableCSI(true))})
+	test.DeleteVSphereCSI()
 	test.ValidateCluster(updateVersion)
 	test.ValidateVSphereCSI(false)
 	test.UpgradeCluster([]framework.ClusterE2ETestOpt{provider.WithProviderUpgrade(api.WithDisableCSI(false))})

diff --git a/test/framework/vspherecsi.go b/test/framework/vspherecsi.go
@@ -2,8 +2,15 @@ package framework
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"testing"
+	"time"
+
+	v1 "k8s.io/api/storage/v1"
+
+	"github.com/aws/eks-anywhere/pkg/constants"
+	"github.com/aws/eks-anywhere/pkg/retrier"
 )
 
 const (
@@ -14,24 +21,33 @@ const (
 	kubeSystemNameSpace        = "kube-system"
 )
 
+var maxRetries = 5
+
 // ValidateVSphereCSI checks whether vsphere csi exists as expected or not.
 func (e *ClusterE2ETest) ValidateVSphereCSI(installed bool) {
 	ctx := context.Background()
-	_, err := e.KubectlClient.GetDeployment(ctx, csiDeployment, kubeSystemNameSpace, e.cluster().KubeconfigFile)
+	if !installed {
+		maxRetries = 1
+	}
+	err := e.getDeployment(ctx, maxRetries)
 	if err != nil {
 		handleError(e.T, installed, err)
 	}
-	_, err = e.KubectlClient.GetDaemonSet(ctx, csiDaemonSet, kubeSystemNameSpace, e.cluster().KubeconfigFile)
+	err = e.getDaemonSet(ctx, maxRetries)
 	if err != nil {
 		handleError(e.T, installed, err)
 	}
-	storageclass, err := e.KubectlClient.GetStorageClass(ctx, csiStorageClassName, e.cluster().KubeconfigFile)
+	storageclass, err := e.getStorageClass(ctx, maxRetries)
 	if err != nil {
 		handleError(e.T, installed, err)
 	}
 	if installed && storageclass.Provisioner != csiStorageClassProvisioner {
 		e.T.Fatalf("provisioners don't match. got: %v, want: %v", storageclass.Provisioner, csiStorageClassProvisioner)
 	}
+	err = e.getClusterResourceSet(ctx, maxRetries)
+	if err != nil {
+		handleError(e.T, installed, err)
+	}
 }
 
 func handleError(t *testing.T, installed bool, err error) {
@@ -55,4 +71,53 @@ func (e *ClusterE2ETest) DeleteVSphereCSI() {
 	if err != nil {
 		e.T.Fatal(err)
 	}
+	csiClusterResourceSetName := fmt.Sprintf("%s-csi", e.ClusterName)
+	err = e.KubectlClient.Delete(ctx, "clusterresourceset", csiClusterResourceSetName, constants.EksaSystemNamespace, e.cluster().KubeconfigFile)
+	if err != nil {
+		e.T.Fatal(err)
+	}
+}
+
+func (e *ClusterE2ETest) getDeployment(ctx context.Context, retries int) error {
+	return retrier.Retry(retries, time.Second*5, func() error {
+		_, err := e.KubectlClient.GetDeployment(ctx, csiDeployment, kubeSystemNameSpace, e.cluster().KubeconfigFile)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func (e *ClusterE2ETest) getDaemonSet(ctx context.Context, retries int) error {
+	return retrier.Retry(retries, time.Second*5, func() error {
+		_, err := e.KubectlClient.GetDaemonSet(ctx, csiDaemonSet, kubeSystemNameSpace, e.cluster().KubeconfigFile)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func (e *ClusterE2ETest) getStorageClass(ctx context.Context, retries int) (*v1.StorageClass, error) {
+	var storageclass *v1.StorageClass
+	err := retrier.Retry(retries, time.Second*5, func() error {
+		s, err := e.KubectlClient.GetStorageClass(ctx, csiStorageClassName, e.cluster().KubeconfigFile)
+		if err != nil {
+			return err
+		}
+		storageclass = s
+		return nil
+	})
+	return storageclass, err
+}
+
+func (e *ClusterE2ETest) getClusterResourceSet(ctx context.Context, retries int) error {
+	return retrier.Retry(retries, time.Second*5, func() error {
+		csiClusterResourceSetName := fmt.Sprintf("%s-csi", e.ClusterName)
+		_, err := e.KubectlClient.GetClusterResourceSet(ctx, e.cluster().KubeconfigFile, csiClusterResourceSetName, constants.EksaSystemNamespace)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
 }