Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix flakiness in multi-cluster tests #400

Merged
merged 1 commit into from
Oct 9, 2024
Merged

Fix flakiness in multi-cluster tests #400

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions tests/e2e/multicluster/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package multicluster

import (
"fmt"
"time"

"github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl"
. "github.com/onsi/gomega"
)

// verifyResponsesAreReceivedFromBothClusters checks that when the sleep pod in the sample namespace
// sends a request to the helloworld service, it receives responses from both v1 and v2 versions,
// which are deployed in different clusters
func verifyResponsesAreReceivedFromBothClusters(k kubectl.Kubectl, clusterName string) {
expectedVersions := []string{"v1", "v2"}
for _, v := range expectedVersions {
Eventually(k.WithNamespace("sample").Exec, 10*time.Second, 10*time.Millisecond).
WithArguments("deploy/sleep", "sleep", "curl -sS helloworld.sample:5000/hello").
Should(ContainSubstring(fmt.Sprintf("Hello version: %s", v)),
fmt.Sprintf("sleep pod in %s did not receive any response from %s", clusterName, v))
}
}
36 changes: 4 additions & 32 deletions tests/e2e/multicluster/multicluster_multiprimary_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"fmt"
"path/filepath"
"strings"
"time"

"github.com/istio-ecosystem/sail-operator/api/v1alpha1"
Expand All @@ -33,7 +32,6 @@ import (
. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
"github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm"
"github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl"
"github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -61,7 +59,7 @@ var _ = Describe("Multicluster deployment models", Ordered, func() {
Success("Operator is deployed in the Cluster #1 namespace and Running")

Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)).
To(Succeed(), "Operator failed to be deployed in Cluster #2")
To(Succeed(), "Operator failed to be deployed in Cluster #2")

Eventually(common.GetObject).
WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}).
Expand Down Expand Up @@ -174,7 +172,7 @@ spec:
Expect(internalIPCluster1).NotTo(BeEmpty(), "Internal IP is empty for Cluster #1")

internalIPCluster2, err := k2.GetInternalIP("node-role.kubernetes.io/control-plane")
Expect(internalIPCluster2).NotTo(BeEmpty(), "Internal IP is empty for Cluster #2")
Expect(internalIPCluster2).NotTo(BeEmpty(), "Internal IP is empty for Cluster #2")
Expect(err).NotTo(HaveOccurred())

// Install a remote secret in Cluster #1 that provides access to the Cluster #2 API server.
Expand Down Expand Up @@ -232,23 +230,8 @@ spec:
})

It("can access the sample app from both clusters", func(ctx SpecContext) {
sleepPodNameCluster1, err := common.GetPodNameByLabel(ctx, clPrimary, "sample", "app", "sleep")
Expect(sleepPodNameCluster1).NotTo(BeEmpty(), "Sleep pod not found on Cluster #1")
Expect(err).NotTo(HaveOccurred(), "Error getting sleep pod name on Cluster #1")

sleepPodNameCluster2, err := common.GetPodNameByLabel(ctx, clRemote, "sample", "app", "sleep")
Expect(sleepPodNameCluster2).NotTo(BeEmpty(), "Sleep pod not found on Cluster #2")
Expect(err).NotTo(HaveOccurred(), "Error getting sleep pod name on Cluster #2")

// Run the curl command from the sleep pod in the Cluster #2 and get response list to validate that we get responses from both clusters
Cluster2Responses := strings.Join(getListCurlResponses(k2, sleepPodNameCluster2), "\n")
Expect(Cluster2Responses).To(ContainSubstring("Hello version: v1"), "Responses from Cluster #2 are not the expected")
Expect(Cluster2Responses).To(ContainSubstring("Hello version: v2"), "Responses from Cluster #2 are not the expected")

// Run the curl command from the sleep pod in the Cluster #1 and get response list to validate that we get responses from both clusters
Cluster1Responses := strings.Join(getListCurlResponses(k1, sleepPodNameCluster1), "\n")
Expect(Cluster1Responses).To(ContainSubstring("Hello version: v1"), "Responses from Cluster #1 are not the expected")
Expect(Cluster1Responses).To(ContainSubstring("Hello version: v2"), "Responses from Cluster #1 are not the expected")
verifyResponsesAreReceivedFromBothClusters(k1, "Cluster #1")
verifyResponsesAreReceivedFromBothClusters(k2, "Cluster #2")
Success("Sample app is accessible from both clusters")
})
})
Expand Down Expand Up @@ -330,14 +313,3 @@ func deploySampleApp(ns string, istioVersion supportedversion.VersionInfo) {
Expect(k1.WithNamespace(ns).Apply(sleepURL)).To(Succeed(), "Sample sleep deploy failed on Cluster #1")
Expect(k2.WithNamespace(ns).Apply(sleepURL)).To(Succeed(), "Sample sleep deploy failed on Cluster #2")
}

// getListCurlResponses runs the curl command 10 times from the sleep pod in the given cluster and get response list
func getListCurlResponses(k kubectl.Kubectl, podName string) []string {
var responses []string
for i := 0; i < 10; i++ {
response, err := k.WithNamespace("sample").Exec(podName, "sleep", "curl -sS helloworld.sample:5000/hello")
Expect(err).NotTo(HaveOccurred())
responses = append(responses, response)
}
return responses
}
20 changes: 2 additions & 18 deletions tests/e2e/multicluster/multicluster_primaryremote_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"fmt"
"path/filepath"
"strings"
"time"

"github.com/istio-ecosystem/sail-operator/api/v1alpha1"
Expand Down Expand Up @@ -276,23 +275,8 @@ spec:
})

It("can access the sample app from both clusters", func(ctx SpecContext) {
sleepPodNamePrimary, err := common.GetPodNameByLabel(ctx, clPrimary, "sample", "app", "sleep")
Expect(sleepPodNamePrimary).NotTo(BeEmpty(), "Sleep pod not found on Primary Cluster")
Expect(err).NotTo(HaveOccurred(), "Error getting sleep pod name on Primary Cluster")

sleepPodNameRemote, err := common.GetPodNameByLabel(ctx, clRemote, "sample", "app", "sleep")
Expect(sleepPodNameRemote).NotTo(BeEmpty(), "Sleep pod not found on Remote Cluster")
Expect(err).NotTo(HaveOccurred(), "Error getting sleep pod name on Remote Cluster")

// Run the curl command from the sleep pod in the Remote Cluster and get response list to validate that we get responses from both clusters
remoteResponses := strings.Join(getListCurlResponses(k2, sleepPodNameRemote), "\n")
Expect(remoteResponses).To(ContainSubstring("Hello version: v1"), "Responses from Remote Cluster are not the expected")
Expect(remoteResponses).To(ContainSubstring("Hello version: v2"), "Responses from Remote Cluster are not the expected")

// Run the curl command from the sleep pod in the Primary Cluster and get response list to validate that we get responses from both clusters
primaryResponses := strings.Join(getListCurlResponses(k1, sleepPodNamePrimary), "\n")
Expect(primaryResponses).To(ContainSubstring("Hello version: v1"), "Responses from Primary Cluster are not the expected")
Expect(primaryResponses).To(ContainSubstring("Hello version: v2"), "Responses from Primary Cluster are not the expected")
verifyResponsesAreReceivedFromBothClusters(k1, "Cluster #1")
verifyResponsesAreReceivedFromBothClusters(k2, "Cluster #2")
Success("Sample app is accessible from both clusters")
})
})
Expand Down