From 8328af950d9022a77884347a33e8e61af84359ba Mon Sep 17 00:00:00 2001 From: Taylor Neyland Date: Fri, 2 Dec 2022 10:42:36 -0600 Subject: [PATCH] Update Disable CSI e2e test --- pkg/executables/kubectl.go | 14 ++---- pkg/executables/kubectl_test.go | 58 +++++++++++------------ test/e2e/vsphere_disable_csi_test.go | 2 +- test/framework/vspherecsi.go | 71 ++++++++++++++++++++++++++-- 4 files changed, 100 insertions(+), 45 deletions(-) diff --git a/pkg/executables/kubectl.go b/pkg/executables/kubectl.go index e0fa1359a4077..613cfaf5fef4f 100644 --- a/pkg/executables/kubectl.go +++ b/pkg/executables/kubectl.go @@ -1770,18 +1770,12 @@ func (k *Kubectl) GetBundles(ctx context.Context, kubeconfigFile, name, namespac } func (k *Kubectl) GetClusterResourceSet(ctx context.Context, kubeconfigFile, name, namespace string) (*addons.ClusterResourceSet, error) { - params := []string{"get", clusterResourceSetResourceType, name, "-o", "json", "--kubeconfig", kubeconfigFile, "--namespace", namespace} - stdOut, err := k.Execute(ctx, params...) - if err != nil { - return nil, fmt.Errorf("getting ClusterResourceSet with kubectl: %v", err) - } - - response := &addons.ClusterResourceSet{} - if err = json.Unmarshal(stdOut.Bytes(), response); err != nil { - return nil, fmt.Errorf("parsing ClusterResourceSet response: %v", err) + obj := &addons.ClusterResourceSet{} + if err := k.GetObject(ctx, clusterResourceSetResourceType, name, namespace, kubeconfigFile, obj); err != nil { + return nil, err } - return response, nil + return obj, nil } func (k *Kubectl) GetConfigMap(ctx context.Context, kubeconfigFile, name, namespace string) (*corev1.ConfigMap, error) { diff --git a/pkg/executables/kubectl_test.go b/pkg/executables/kubectl_test.go index 8594100fb8c8a..d9b41d1f8d2cc 100644 --- a/pkg/executables/kubectl_test.go +++ b/pkg/executables/kubectl_test.go @@ -1971,42 +1971,38 @@ func TestKubectlGetBundles(t *testing.T) { } func TestKubectlGetClusterResourceSet(t *testing.T) { - tt := newKubectlTest(t) - resourceSetJson := test.ReadFile(t, "testdata/kubectl_clusterresourceset.json") - resourceSetName := "Bundle-name" - wantResourceSet := &addons.ClusterResourceSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "addons.cluster.x-k8s.io/v1beta1", - Kind: "ClusterResourceSet", - }, - Spec: addons.ClusterResourceSetSpec{ - ClusterSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "cluster.x-k8s.io/cluster-name": "cluster-1", - }, + newKubectlGetterTest(t).withResourceType( + "clusterresourcesets.addons.cluster.x-k8s.io", + ).withGetter(func(tt *kubectlGetterTest) (client.Object, error) { + return tt.k.GetClusterResourceSet(tt.ctx, tt.kubeconfig, tt.name, tt.namespace) + }).withJsonFromFile( + "testdata/kubectl_clusterresourceset.json", + ).andWant( + &addons.ClusterResourceSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "addons.cluster.x-k8s.io/v1beta1", + Kind: "ClusterResourceSet", }, - Strategy: "ApplyOnce", - Resources: []addons.ResourceRef{ - { - Kind: "Secret", - Name: "vsphere-csi-controller", + Spec: addons.ClusterResourceSetSpec{ + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cluster.x-k8s.io/cluster-name": "cluster-1", + }, }, - { - Kind: "ConfigMap", - Name: "vsphere-csi-controller-role", + Strategy: "ApplyOnce", + Resources: []addons.ResourceRef{ + { + Kind: "Secret", + Name: "vsphere-csi-controller", + }, + { + Kind: "ConfigMap", + Name: "vsphere-csi-controller-role", + }, }, }, }, - } - - tt.e.EXPECT().Execute( - tt.ctx, - "get", "clusterresourcesets.addons.cluster.x-k8s.io", resourceSetName, "-o", "json", "--kubeconfig", tt.cluster.KubeconfigFile, "--namespace", tt.namespace, - ).Return(*bytes.NewBufferString(resourceSetJson), nil) - - gotResourceSet, err := tt.k.GetClusterResourceSet(tt.ctx, tt.cluster.KubeconfigFile, resourceSetName, tt.namespace) - tt.Expect(err).To(BeNil()) - tt.Expect(gotResourceSet).To(Equal(wantResourceSet)) + ).testSuccess() } func TestKubectlGetConfigMap(t *testing.T) { diff --git a/test/e2e/vsphere_disable_csi_test.go b/test/e2e/vsphere_disable_csi_test.go index f6e371dbb39e0..4b42f0826d025 100644 --- a/test/e2e/vsphere_disable_csi_test.go +++ b/test/e2e/vsphere_disable_csi_test.go @@ -15,8 +15,8 @@ func runVSphereDisableCSIUpgradeFlow(test *framework.ClusterE2ETest, updateVersi test.GenerateClusterConfig() test.CreateCluster() test.ValidateVSphereCSI(true) - test.DeleteVSphereCSI() test.UpgradeCluster([]framework.ClusterE2ETestOpt{provider.WithProviderUpgrade(api.WithDisableCSI(true))}) + test.DeleteVSphereCSI() test.ValidateCluster(updateVersion) test.ValidateVSphereCSI(false) test.UpgradeCluster([]framework.ClusterE2ETestOpt{provider.WithProviderUpgrade(api.WithDisableCSI(false))}) diff --git a/test/framework/vspherecsi.go b/test/framework/vspherecsi.go index 83b44299c81d7..d27ed48a68efd 100644 --- a/test/framework/vspherecsi.go +++ b/test/framework/vspherecsi.go @@ -2,8 +2,15 @@ package framework import ( "context" + "fmt" "strings" "testing" + "time" + + v1 "k8s.io/api/storage/v1" + + "github.com/aws/eks-anywhere/pkg/constants" + "github.com/aws/eks-anywhere/pkg/retrier" ) const ( @@ -14,24 +21,33 @@ const ( kubeSystemNameSpace = "kube-system" ) +var maxRetries = 5 + // ValidateVSphereCSI checks whether vsphere csi exists as expected or not. func (e *ClusterE2ETest) ValidateVSphereCSI(installed bool) { ctx := context.Background() - _, err := e.KubectlClient.GetDeployment(ctx, csiDeployment, kubeSystemNameSpace, e.cluster().KubeconfigFile) + if !installed { + maxRetries = 1 + } + err := e.getDeployment(ctx, maxRetries) if err != nil { handleError(e.T, installed, err) } - _, err = e.KubectlClient.GetDaemonSet(ctx, csiDaemonSet, kubeSystemNameSpace, e.cluster().KubeconfigFile) + err = e.getDaemonSet(ctx, maxRetries) if err != nil { handleError(e.T, installed, err) } - storageclass, err := e.KubectlClient.GetStorageClass(ctx, csiStorageClassName, e.cluster().KubeconfigFile) + storageclass, err := e.getStorageClass(ctx, maxRetries) if err != nil { handleError(e.T, installed, err) } if installed && storageclass.Provisioner != csiStorageClassProvisioner { e.T.Fatalf("provisioners don't match. got: %v, want: %v", storageclass.Provisioner, csiStorageClassProvisioner) } + err = e.getClusterResourceSet(ctx, maxRetries) + if err != nil { + handleError(e.T, installed, err) + } } func handleError(t *testing.T, installed bool, err error) { @@ -55,4 +71,53 @@ func (e *ClusterE2ETest) DeleteVSphereCSI() { if err != nil { e.T.Fatal(err) } + csiClusterResourceSetName := fmt.Sprintf("%s-csi", e.ClusterName) + err = e.KubectlClient.Delete(ctx, "clusterresourceset", csiClusterResourceSetName, constants.EksaSystemNamespace, e.cluster().KubeconfigFile) + if err != nil { + e.T.Fatal(err) + } +} + +func (e *ClusterE2ETest) getDeployment(ctx context.Context, retries int) error { + return retrier.Retry(retries, time.Second*5, func() error { + _, err := e.KubectlClient.GetDeployment(ctx, csiDeployment, kubeSystemNameSpace, e.cluster().KubeconfigFile) + if err != nil { + return err + } + return nil + }) +} + +func (e *ClusterE2ETest) getDaemonSet(ctx context.Context, retries int) error { + return retrier.Retry(retries, time.Second*5, func() error { + _, err := e.KubectlClient.GetDaemonSet(ctx, csiDaemonSet, kubeSystemNameSpace, e.cluster().KubeconfigFile) + if err != nil { + return err + } + return nil + }) +} + +func (e *ClusterE2ETest) getStorageClass(ctx context.Context, retries int) (*v1.StorageClass, error) { + var storageclass *v1.StorageClass + err := retrier.Retry(retries, time.Second*5, func() error { + s, err := e.KubectlClient.GetStorageClass(ctx, csiStorageClassName, e.cluster().KubeconfigFile) + if err != nil { + return err + } + storageclass = s + return nil + }) + return storageclass, err +} + +func (e *ClusterE2ETest) getClusterResourceSet(ctx context.Context, retries int) error { + return retrier.Retry(retries, time.Second*5, func() error { + csiClusterResourceSetName := fmt.Sprintf("%s-csi", e.ClusterName) + _, err := e.KubectlClient.GetClusterResourceSet(ctx, e.cluster().KubeconfigFile, csiClusterResourceSetName, constants.EksaSystemNamespace) + if err != nil { + return err + } + return nil + }) }