Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize multicluster e2e tests #403

Merged
merged 1 commit into from
Oct 10, 2024
Merged

Optimize multicluster e2e tests #403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 17 additions & 21 deletions tests/e2e/multicluster/multicluster_multiprimary_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,14 @@ var _ = Describe("Multicluster deployment models", Ordered, func() {
Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)).
To(Succeed(), "Operator failed to be deployed in Cluster #1")

Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)).
To(Succeed(), "Operator failed to be deployed in Cluster #2")

Eventually(common.GetObject).
WithArguments(ctx, clPrimary, kube.Key(deploymentName, namespace), &appsv1.Deployment{}).
Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD")
Success("Operator is deployed in the Cluster #1 namespace and Running")

Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)).
To(Succeed(), "Operator failed to be deployed in Cluster #2")

Eventually(common.GetObject).
WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}).
Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD")
Expand Down Expand Up @@ -254,20 +254,18 @@ spec:
})

AfterAll(func(ctx SpecContext) {
// Delete namespace to ensure clean up for new tests iteration
Expect(k1.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1")
Expect(k2.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2")

common.CheckNamespaceEmpty(ctx, clPrimary, controlPlaneNamespace)
common.CheckNamespaceEmpty(ctx, clRemote, controlPlaneNamespace)
// Delete namespaces to ensure clean up for new tests iteration
Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1")
Expect(k2.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2")
Expect(k1.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #1")
Expect(k2.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #2")

Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed())
Expect(k2.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed())
Success("ControlPlane Namespaces are empty")

// Delete the entire sample namespace in both clusters
Expect(k1.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #1")
Expect(k2.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #2")

common.CheckNamespaceEmpty(ctx, clPrimary, "sample")
common.CheckNamespaceEmpty(ctx, clRemote, "sample")
Expect(k1.WaitNamespaceDeleted("sample")).To(Succeed())
Expect(k2.WaitNamespaceDeleted("sample")).To(Succeed())
Success("Sample app is deleted in both clusters")
})
})
Expand All @@ -276,12 +274,10 @@ spec:

AfterAll(func(ctx SpecContext) {
// Delete the Sail Operator from both clusters
Expect(k1.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1")
Expect(k2.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2")

// Delete the intermediate CA from both clusters
common.CheckNamespaceEmpty(ctx, clPrimary, namespace)
common.CheckNamespaceEmpty(ctx, clRemote, namespace)
Expect(k1.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1")
Expect(k2.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2")
Expect(k1.WaitNamespaceDeleted(namespace)).To(Succeed())
Expect(k2.WaitNamespaceDeleted(namespace)).To(Succeed())
})
})

Expand Down
42 changes: 19 additions & 23 deletions tests/e2e/multicluster/multicluster_primaryremote_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,14 @@ var _ = Describe("Multicluster deployment models", Ordered, func() {
Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)).
To(Succeed(), "Operator failed to be deployed in Primary Cluster")

Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)).
To(Succeed(), "Operator failed to be deployed in Remote Cluster")

Eventually(common.GetObject).
WithArguments(ctx, clPrimary, kube.Key(deploymentName, namespace), &appsv1.Deployment{}).
Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD")
Success("Operator is deployed in the Primary namespace and Running")

Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)).
To(Succeed(), "Operator failed to be deployed in Remote Cluster")

Eventually(common.GetObject).
WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}).
Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD")
Expand Down Expand Up @@ -296,20 +296,18 @@ spec:
})

AfterAll(func(ctx SpecContext) {
// Delete namespace to ensure clean up for new tests iteration
Expect(k1.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster")
Expect(k2.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster")

common.CheckNamespaceEmpty(ctx, clPrimary, controlPlaneNamespace)
common.CheckNamespaceEmpty(ctx, clRemote, controlPlaneNamespace)
Success("ControlPlane Namespaces are empty")

// Delete the entire sample namespace in both clusters
Expect(k1.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Primary Cluster")
Expect(k2.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Remote Cluster")

common.CheckNamespaceEmpty(ctx, clPrimary, "sample")
common.CheckNamespaceEmpty(ctx, clRemote, "sample")
// Delete namespaces to ensure clean up for new tests iteration
Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster")
Expect(k2.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster")
Expect(k1.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Primary Cluster")
Expect(k2.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Remote Cluster")

Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed())
Expect(k2.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed())
Success("ControlPlane Namespaces were deleted")

Expect(k1.WaitNamespaceDeleted("sample")).To(Succeed())
Expect(k2.WaitNamespaceDeleted("sample")).To(Succeed())
Success("Sample app is deleted in both clusters")
})
})
Expand All @@ -318,11 +316,9 @@ spec:

AfterAll(func(ctx SpecContext) {
// Delete the Sail Operator from both clusters
Expect(k1.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster")
Expect(k2.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster")

// Check that the namespace is empty
common.CheckNamespaceEmpty(ctx, clPrimary, namespace)
common.CheckNamespaceEmpty(ctx, clRemote, namespace)
Expect(k1.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster")
Expect(k2.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster")
Expect(k1.WaitNamespaceDeleted(namespace)).To(Succeed())
Expect(k2.WaitNamespaceDeleted(namespace)).To(Succeed())
})
})
27 changes: 24 additions & 3 deletions tests/e2e/util/kubectl/kubectl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package kubectl
import (
"fmt"
"os"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -122,14 +123,22 @@ func (k Kubectl) DeleteCRDs(crds []string) error {
return nil
}

// DeleteNamespace deletes a namespace
// DeleteNamespaceNoWait deletes a namespace and returns immediately (without waiting for the namespace to be removed).
func (k Kubectl) DeleteNamespaceNoWait(ns string) error {
return k.deleteNamespace(ns, false)
}
Comment on lines +127 to +129
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is overkill right now but having something like Delete(obj, name, ...opts) where you can pass Delete("namespace", "myns", DeleteWithNoWait()) or Delete("namespace", "myns", DeleteWithTimeout(time.Minute)) would be nice if we keep adding opts. Maybe we're just rewriting the go client at that point though...


// DeleteNamespace deletes a namespace and waits for it to be removed completely.
func (k Kubectl) DeleteNamespace(ns string) error {
cmd := k.build(" delete namespace " + ns)
return k.deleteNamespace(ns, true)
}

func (k Kubectl) deleteNamespace(ns string, wait bool) error {
cmd := k.build(" delete namespace " + ns + " --wait=" + strconv.FormatBool(wait))
_, err := k.executeCommand(cmd)
if err != nil {
return fmt.Errorf("error deleting namespace: %w", err)
}

return nil
}

Expand Down Expand Up @@ -183,6 +192,13 @@ func (k Kubectl) Delete(kind, name string) error {
return nil
}

// Wait waits for a specific condition on one or many resources
func (k Kubectl) Wait(waitFor, resource string, timeout time.Duration) error {
cmd := k.build(fmt.Sprintf("wait --for %s %s --timeout %s", waitFor, resource, timeout.String()))
_, err := k.executeCommand(cmd)
return err
}

// Patch patches a resource
func (k Kubectl) Patch(kind, name, patchType, patch string) error {
cmd := k.build(fmt.Sprintf(" patch %s %s --type=%s -p=%q", kind, name, patchType, patch))
Expand Down Expand Up @@ -281,6 +297,11 @@ func (k Kubectl) executeCommand(cmd string) (string, error) {
return shell.ExecuteCommand(cmd)
}

// WaitNamespaceDeleted waits for a namespace to be deleted
func (k Kubectl) WaitNamespaceDeleted(ns string) error {
return k.Wait("delete", "namespace/"+ns, 2*time.Minute)
}

func sinceFlag(since *time.Duration) string {
if since == nil {
return ""
Expand Down