diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go index 872a0578a..627edfc5d 100644 --- a/tests/e2e/controlplane/control_plane_suite_test.go +++ b/tests/e2e/controlplane/control_plane_suite_test.go @@ -21,6 +21,7 @@ import ( k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" env "github.com/istio-ecosystem/sail-operator/tests/e2e/util/env" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,6 +42,8 @@ var ( expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") bookinfoNamespace = env.Get("BOOKINFO_NAMESPACE", "bookinfo") multicluster = env.GetBool("MULTICLUSTER", false) + + k *kubectl.KubectlBuilder ) func TestInstall(t *testing.T) { @@ -58,4 +61,6 @@ func setup() { GinkgoWriter.Println("Initializing k8s client") cl, err = k8sclient.InitK8sClient("") Expect(err).NotTo(HaveOccurred()) + + k = kubectl.NewKubectlBuilder() } diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go index ec7a6574a..8beb63065 100644 --- a/tests/e2e/controlplane/control_plane_test.go +++ b/tests/e2e/controlplane/control_plane_test.go @@ -30,7 +30,6 @@ import ( common "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" @@ -45,11 +44,10 @@ import ( var _ = Describe("Control Plane Installation", Ordered, func() { SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) - debugInfoLogged := false BeforeAll(func(ctx SpecContext) { - Expect(kubectl.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") + Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") extraArg := "" if ocp { @@ -79,7 +77,7 @@ kind: IstioCNI metadata: name: default ` + spec - Expect(kubectl.CreateFromString(yaml)).To(Succeed(), "IstioCNI creation failed") + Expect(k.CreateFromString(yaml)).To(Succeed(), "IstioCNI creation failed") Success("IstioCNI created") cni := &v1alpha1.IstioCNI{} @@ -103,7 +101,7 @@ kind: Istio metadata: name: default ` + spec - Expect(kubectl.CreateFromString(yaml)).To(Succeed(), "Istio creation failed") + Expect(k.CreateFromString(yaml)).To(Succeed(), "Istio creation failed") Success("Istio created") istio := &v1alpha1.Istio{} @@ -126,8 +124,8 @@ metadata: Context(version.Name, func() { BeforeAll(func() { - Expect(kubectl.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(kubectl.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") + Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") + Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") }) When("the IstioCNI CR is created", func() { @@ -142,7 +140,7 @@ spec: namespace: %s` yaml = fmt.Sprintf(yaml, version.Name, istioCniNamespace) Log("IstioCNI YAML:", indent(2, yaml)) - Expect(kubectl.CreateFromString(yaml)).To(Succeed(), "IstioCNI creation failed") + Expect(k.CreateFromString(yaml)).To(Succeed(), "IstioCNI creation failed") Success("IstioCNI created") }) @@ -174,9 +172,10 @@ spec: }) It("doesn't continuously reconcile the IstioCNI CR", func() { - Eventually(kubectl.Logs).WithArguments(namespace, "deploy/"+deploymentName, ptr.Of(30*time.Second)). - ShouldNot(ContainSubstring("Reconciliation done"), "Istio Operator is continuously reconciling") - Success("Istio Operator stopped reconciling") + Eventually(k.SetNamespace(namespace).Logs).WithArguments("deploy/"+deploymentName, ptr.Of(30*time.Second)). + ShouldNot(ContainSubstring("Reconciliation done"), "IstioCNI is continuously reconciling") + k.ResetNamespace() + Success("IstioCNI stopped reconciling") }) }) @@ -192,7 +191,7 @@ spec: namespace: %s` istioYAML = fmt.Sprintf(istioYAML, version.Name, controlPlaneNamespace) Log("Istio YAML:", indent(2, istioYAML)) - Expect(kubectl.CreateFromString(istioYAML)). + Expect(k.CreateFromString(istioYAML)). To(Succeed(), "Istio CR failed to be created") Success("Istio CR created") }) @@ -222,16 +221,17 @@ spec: }) It("doesn't continuously reconcile the Istio CR", func() { - Eventually(kubectl.Logs).WithArguments(namespace, "deploy/"+deploymentName, ptr.Of(30*time.Second)). - ShouldNot(ContainSubstring("Reconciliation done"), "Istio Operator is continuously reconciling") - Success("Istio Operator stopped reconciling") + Eventually(k.SetNamespace(namespace).Logs).WithArguments("deploy/"+deploymentName, ptr.Of(30*time.Second)). + ShouldNot(ContainSubstring("Reconciliation done"), "Istio CR is continuously reconciling") + k.ResetNamespace() + Success("Istio CR stopped reconciling") }) }) When("bookinfo is deployed", func() { BeforeAll(func() { - Expect(kubectl.CreateNamespace(bookinfoNamespace)).To(Succeed(), "Bookinfo namespace failed to be created") - Expect(kubectl.Patch("", "namespace", bookinfoNamespace, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). + Expect(k.CreateNamespace(bookinfoNamespace)).To(Succeed(), "Bookinfo namespace failed to be created") + Expect(k.Patch("namespace", bookinfoNamespace, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). To(Succeed(), "Error patching bookinfo namespace") Expect(deployBookinfo(version)).To(Succeed(), "Error deploying bookinfo") Success("Bookinfo deployed") @@ -261,14 +261,14 @@ spec: AfterAll(func(ctx SpecContext) { By("Deleting bookinfo") - Expect(kubectl.DeleteNamespace(bookinfoNamespace)).To(Succeed(), "Bookinfo namespace failed to be deleted") + Expect(k.DeleteNamespace(bookinfoNamespace)).To(Succeed(), "Bookinfo namespace failed to be deleted") Success("Bookinfo deleted") }) }) When("the Istio CR is deleted", func() { BeforeEach(func() { - Expect(kubectl.Delete(controlPlaneNamespace, "istio", istioName)).To(Succeed(), "Istio CR failed to be deleted") + Expect(k.SetNamespace(controlPlaneNamespace).Delete("istio", istioName)).To(Succeed(), "Istio CR failed to be deleted") Success("Istio CR deleted") }) @@ -282,7 +282,7 @@ spec: When("the IstioCNI CR is deleted", func() { BeforeEach(func() { - Expect(kubectl.Delete(istioCniNamespace, "istiocni", istioCniName)).To(Succeed(), "IstioCNI CR failed to be deleted") + Expect(k.SetNamespace(istioCniNamespace).Delete("istiocni", istioCniName)).To(Succeed(), "IstioCNI CR failed to be deleted") Success("IstioCNI deleted") }) @@ -336,7 +336,7 @@ spec: Success("Skipping deletion of operator namespace to avoid removal of operator container image from internal registry") return } - Expect(kubectl.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted") + Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted") Success("Namespace deleted") }) }) @@ -357,17 +357,17 @@ func indent(level int, str string) string { func forceDeleteIstioResources() error { // This is a workaround to delete the Istio CRs that are left in the cluster // This will be improved by splitting the tests into different Nodes with their independent setups and cleanups - err := kubectl.ForceDelete("", "istio", istioName) + err := k.ForceDelete("istio", istioName) if err != nil && !strings.Contains(err.Error(), "not found") { return fmt.Errorf("failed to delete %s CR: %w", "istio", err) } - err = kubectl.ForceDelete("", "istiorevision", "default") + err = k.ForceDelete("istiorevision", "default") if err != nil && !strings.Contains(err.Error(), "not found") { return fmt.Errorf("failed to delete %s CR: %w", "istiorevision", err) } - err = kubectl.Delete("", "istiocni", istioCniName) + err = k.Delete("istiocni", istioCniName) if err != nil && !strings.Contains(err.Error(), "not found") { return fmt.Errorf("failed to delete %s CR: %w", "istiocni", err) } @@ -388,7 +388,7 @@ func getBookinfoURL(version supportedversion.VersionInfo) string { func deployBookinfo(version supportedversion.VersionInfo) error { bookinfoURL := getBookinfoURL(version) - kubectl.Apply(bookinfoNamespace, bookinfoURL) + k.SetNamespace(bookinfoNamespace).Apply(bookinfoURL) if err != nil { return fmt.Errorf("error deploying bookinfo: %w", err) } @@ -397,7 +397,7 @@ func deployBookinfo(version supportedversion.VersionInfo) error { } func getProxyVersion(podName, namespace string) (string, error) { - proxyVersion, err := kubectl.Exec(namespace, + proxyVersion, err := k.SetNamespace(namespace).Exec( podName, "istio-proxy", `curl -s http://localhost:15000/server_info | grep "ISTIO_VERSION" | awk -F '"' '{print $4}'`) diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index 97c404c52..687e763f7 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -49,8 +49,8 @@ var _ = Describe("Multicluster deployment models", Ordered, func() { BeforeAll(func(ctx SpecContext) { if !skipDeploy { // Deploy the Sail Operator on both clusters - Expect(kubectl.CreateNamespace(namespace, kubeconfig)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(kubectl.CreateNamespace(namespace, kubeconfig2)).To(Succeed(), "Namespace failed to be created on Cluster #2") + Expect(kubectlClient1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") + Expect(kubectlClient2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)). To(Succeed(), "Operator failed to be deployed in Cluster #1") @@ -76,8 +76,8 @@ var _ = Describe("Multicluster deployment models", Ordered, func() { Context("Istio version is: "+version.Version, func() { When("Istio resources are created in both clusters with multicluster configuration", func() { BeforeAll(func(ctx SpecContext) { - Expect(kubectl.CreateNamespace(controlPlaneNamespace, kubeconfig)).To(Succeed(), "Namespace failed to be created") - Expect(kubectl.CreateNamespace(controlPlaneNamespace, kubeconfig2)).To(Succeed(), "Namespace failed to be created") + Expect(kubectlClient1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") + Expect(kubectlClient2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") // Push the intermediate CA to both clusters certs.PushIntermediateCA(controlPlaneNamespace, kubeconfig, "east", "network1", artifacts, clPrimary) @@ -110,11 +110,11 @@ spec: network: %s` multiclusterCluster1YAML := fmt.Sprintf(multiclusterYAML, version.Name, controlPlaneNamespace, "mesh1", "cluster1", "network1") Log("Istio CR Cluster #1: ", multiclusterCluster1YAML) - Expect(kubectl.CreateFromString(multiclusterCluster1YAML, kubeconfig)).To(Succeed(), "Istio Resource creation failed on Cluster #1") + Expect(kubectlClient1.CreateFromString(multiclusterCluster1YAML)).To(Succeed(), "Istio Resource creation failed on Cluster #1") multiclusterCluster2YAML := fmt.Sprintf(multiclusterYAML, version.Name, controlPlaneNamespace, "mesh1", "cluster2", "network2") Log("Istio CR Cluster #2: ", multiclusterCluster2YAML) - Expect(kubectl.CreateFromString(multiclusterCluster2YAML, kubeconfig2)).To(Succeed(), "Istio Resource creation failed on Cluster #2") + Expect(kubectlClient2.CreateFromString(multiclusterCluster2YAML)).To(Succeed(), "Istio Resource creation failed on Cluster #2") }) It("updates both Istio CR status to Ready", func(ctx SpecContext) { @@ -146,13 +146,13 @@ spec: When("Gateway is created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(kubectl.Apply(controlPlaneNamespace, eastGatewayYAML, kubeconfig)).To(Succeed(), "Gateway creation failed on Cluster #1") + Expect(kubectlClient1.SetNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") - Expect(kubectl.Apply(controlPlaneNamespace, westGatewayYAML, kubeconfig2)).To(Succeed(), "Gateway creation failed on Cluster #2") + Expect(kubectlClient2.SetNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #2") // Expose the Gateway service in both clusters - Expect(kubectl.Apply(controlPlaneNamespace, exposeServiceYAML, kubeconfig)).To(Succeed(), "Expose Service creation failed on Cluster #1") - Expect(kubectl.Apply(controlPlaneNamespace, exposeServiceYAML, kubeconfig2)).To(Succeed(), "Expose Service creation failed on Cluster #2") + Expect(kubectlClient1.SetNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #1") + Expect(kubectlClient2.SetNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #2") }) It("updates both Gateway status to Available", func(ctx SpecContext) { @@ -170,23 +170,23 @@ spec: When("are installed remote secrets on each cluster", func() { BeforeAll(func(ctx SpecContext) { // Get the internal IP of the control plane node in both clusters - internalIPCluster1, err := kubectl.GetInternalIP("node-role.kubernetes.io/control-plane", kubeconfig) + internalIPCluster1, err := kubectlClient1.GetInternalIP("node-role.kubernetes.io/control-plane") Expect(err).NotTo(HaveOccurred()) Expect(internalIPCluster1).NotTo(BeEmpty(), "Internal IP is empty for Cluster #1") - internalIPCluster2, err := kubectl.GetInternalIP("node-role.kubernetes.io/control-plane", kubeconfig2) + internalIPCluster2, err := kubectlClient2.GetInternalIP("node-role.kubernetes.io/control-plane") Expect(internalIPCluster2).NotTo(BeEmpty(), "Internal IP is empty for Cluster #2") Expect(err).NotTo(HaveOccurred()) // Install a remote secret in Cluster #1 that provides access to the Cluster #2 API server. secret, err := istioctl.CreateRemoteSecret(kubeconfig2, "cluster2", internalIPCluster2) Expect(err).NotTo(HaveOccurred()) - Expect(kubectl.ApplyString("", secret, kubeconfig)).To(Succeed(), "Remote secret creation failed on Cluster #1") + Expect(kubectlClient1.ApplyString(secret)).To(Succeed(), "Remote secret creation failed on Cluster #1") // Install a remote secret in Cluster #2 that provides access to the Cluster #1 API server. secret, err = istioctl.CreateRemoteSecret(kubeconfig, "cluster1", internalIPCluster1) Expect(err).NotTo(HaveOccurred()) - Expect(kubectl.ApplyString("", secret, kubeconfig2)).To(Succeed(), "Remote secret creation failed on Cluster #1") + Expect(kubectlClient2.ApplyString(secret)).To(Succeed(), "Remote secret creation failed on Cluster #1") }) It("remote secrets are created", func(ctx SpecContext) { @@ -204,7 +204,7 @@ spec: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { // Deploy the sample app in both clusters - deploySampleApp("sample", version, kubeconfig, kubeconfig2) + deploySampleApp("sample", version) Success("Sample app is deployed in both clusters") }) @@ -242,12 +242,12 @@ spec: Expect(err).NotTo(HaveOccurred(), "Error getting sleep pod name on Cluster #2") // Run the curl command from the sleep pod in the Cluster #2 and get response list to validate that we get responses from both clusters - Cluster2Responses := strings.Join(getListCurlResponses(sleepPodNameCluster2, kubeconfig2), "\n") + Cluster2Responses := strings.Join(getListCurlResponses(kubectlClient2, sleepPodNameCluster2), "\n") Expect(Cluster2Responses).To(ContainSubstring("Hello version: v1"), "Responses from Cluster #2 are not the expected") Expect(Cluster2Responses).To(ContainSubstring("Hello version: v2"), "Responses from Cluster #2 are not the expected") // Run the curl command from the sleep pod in the Cluster #1 and get response list to validate that we get responses from both clusters - Cluster1Responses := strings.Join(getListCurlResponses(sleepPodNameCluster1, kubeconfig), "\n") + Cluster1Responses := strings.Join(getListCurlResponses(kubectlClient1, sleepPodNameCluster1), "\n") Expect(Cluster1Responses).To(ContainSubstring("Hello version: v1"), "Responses from Cluster #1 are not the expected") Expect(Cluster1Responses).To(ContainSubstring("Hello version: v2"), "Responses from Cluster #1 are not the expected") Success("Sample app is accessible from both clusters") @@ -257,8 +257,8 @@ spec: When("istio CR is deleted in both clusters", func() { BeforeEach(func() { // Delete the Istio CR in both clusters - Expect(kubectl.Delete(controlPlaneNamespace, "istio", istioName, kubeconfig)).To(Succeed(), "Istio CR failed to be deleted") - Expect(kubectl.Delete(controlPlaneNamespace, "istio", istioName, kubeconfig2)).To(Succeed(), "Istio CR failed to be deleted") + Expect(kubectlClient1.SetNamespace(controlPlaneNamespace).Delete("istio", istioName)).To(Succeed(), "Istio CR failed to be deleted") + Expect(kubectlClient2.SetNamespace(controlPlaneNamespace).Delete("istio", istioName)).To(Succeed(), "Istio CR failed to be deleted") Success("Istio CR is deleted in both clusters") }) @@ -273,16 +273,16 @@ spec: AfterAll(func(ctx SpecContext) { // Delete namespace to ensure clean up for new tests iteration - Expect(kubectl.DeleteNamespace(controlPlaneNamespace, kubeconfig)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(kubectl.DeleteNamespace(controlPlaneNamespace, kubeconfig2)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") + Expect(kubectlClient1.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") + Expect(kubectlClient2.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") common.CheckNamespaceEmpty(ctx, clPrimary, controlPlaneNamespace) common.CheckNamespaceEmpty(ctx, clRemote, controlPlaneNamespace) Success("ControlPlane Namespaces are empty") // Delete the entire sample namespace in both clusters - Expect(kubectl.DeleteNamespace("sample", kubeconfig)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(kubectl.DeleteNamespace("sample", kubeconfig2)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") + Expect(kubectlClient1.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #1") + Expect(kubectlClient2.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #2") common.CheckNamespaceEmpty(ctx, clPrimary, "sample") common.CheckNamespaceEmpty(ctx, clRemote, "sample") @@ -294,8 +294,8 @@ spec: AfterAll(func(ctx SpecContext) { // Delete the Sail Operator from both clusters - Expect(kubectl.DeleteNamespace(namespace, kubeconfig)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(kubectl.DeleteNamespace(namespace, kubeconfig2)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") + Expect(kubectlClient1.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") + Expect(kubectlClient2.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") // Delete the intermediate CA from both clusters common.CheckNamespaceEmpty(ctx, clPrimary, namespace) @@ -304,15 +304,15 @@ spec: }) // deploySampleApp deploys the sample app in the given cluster -func deploySampleApp(ns string, istioVersion supportedversion.VersionInfo, kubeconfig string, kubeconfig2 string) { +func deploySampleApp(ns string, istioVersion supportedversion.VersionInfo) { // Create the namespace - Expect(kubectl.CreateNamespace(ns, kubeconfig)).To(Succeed(), "Namespace failed to be created") - Expect(kubectl.CreateNamespace(ns, kubeconfig2)).To(Succeed(), "Namespace failed to be created") + Expect(kubectlClient1.CreateNamespace(ns)).To(Succeed(), "Namespace failed to be created") + Expect(kubectlClient2.CreateNamespace(ns)).To(Succeed(), "Namespace failed to be created") // Label the namespace - Expect(kubectl.Patch("", "namespace", ns, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). + Expect(kubectlClient1.Patch("namespace", ns, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). To(Succeed(), "Error patching sample namespace") - Expect(kubectl.Patch("", "namespace", ns, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`, kubeconfig2)). + Expect(kubectlClient2.Patch("namespace", ns, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). To(Succeed(), "Error patching sample namespace") version := istioVersion.Version @@ -321,22 +321,22 @@ func deploySampleApp(ns string, istioVersion supportedversion.VersionInfo, kubec version = "master" } helloWorldURL := fmt.Sprintf("https://raw.githubusercontent.com/istio/istio/%s/samples/helloworld/helloworld.yaml", version) - Expect(kubectl.ApplyWithLabels(ns, helloWorldURL, "service=helloworld", kubeconfig)).To(Succeed(), "Sample service deploy failed on Cluster #1") - Expect(kubectl.ApplyWithLabels(ns, helloWorldURL, "service=helloworld", kubeconfig2)).To(Succeed(), "Sample service deploy failed on Cluster #2") + Expect(kubectlClient1.SetNamespace(ns).ApplyWithLabels(helloWorldURL, "service=helloworld")).To(Succeed(), "Sample service deploy failed on Cluster #1") + Expect(kubectlClient2.SetNamespace(ns).ApplyWithLabels(helloWorldURL, "service=helloworld")).To(Succeed(), "Sample service deploy failed on Cluster #2") - Expect(kubectl.ApplyWithLabels(ns, helloWorldURL, "version=v1", kubeconfig)).To(Succeed(), "Sample service deploy failed on Cluster #1") - Expect(kubectl.ApplyWithLabels(ns, helloWorldURL, "version=v2", kubeconfig2)).To(Succeed(), "Sample service deploy failed on Cluster #2") + Expect(kubectlClient1.SetNamespace(ns).ApplyWithLabels(helloWorldURL, "version=v1")).To(Succeed(), "Sample service deploy failed on Cluster #1") + Expect(kubectlClient2.SetNamespace(ns).ApplyWithLabels(helloWorldURL, "version=v2")).To(Succeed(), "Sample service deploy failed on Cluster #2") sleepURL := fmt.Sprintf("https://raw.githubusercontent.com/istio/istio/%s/samples/sleep/sleep.yaml", version) - Expect(kubectl.Apply(ns, sleepURL, kubeconfig)).To(Succeed(), "Sample sleep deploy failed on Cluster #1") - Expect(kubectl.Apply(ns, sleepURL, kubeconfig2)).To(Succeed(), "Sample sleep deploy failed on Cluster #2") + Expect(kubectlClient1.SetNamespace(ns).Apply(sleepURL)).To(Succeed(), "Sample sleep deploy failed on Cluster #1") + Expect(kubectlClient2.SetNamespace(ns).Apply(sleepURL)).To(Succeed(), "Sample sleep deploy failed on Cluster #2") } // getListCurlResponses runs the curl command 10 times from the sleep pod in the given cluster and get response list -func getListCurlResponses(podName, kubeconfig string) []string { +func getListCurlResponses(k *kubectl.KubectlBuilder, podName string) []string { var responses []string for i := 0; i < 10; i++ { - response, err := kubectl.Exec("sample", podName, "sleep", "curl -sS helloworld.sample:5000/hello", kubeconfig) + response, err := k.SetNamespace("sample").Exec(podName, "sleep", "curl -sS helloworld.sample:5000/hello") Expect(err).NotTo(HaveOccurred()) responses = append(responses, response) } diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index 798db3c2c..d7a200c66 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -33,7 +33,6 @@ import ( . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" @@ -49,8 +48,8 @@ var _ = Describe("Multicluster deployment models", Ordered, func() { BeforeAll(func(ctx SpecContext) { if !skipDeploy { // Deploy the Sail Operator on both clusters - Expect(kubectl.CreateNamespace(namespace, kubeconfig)).To(Succeed(), "Namespace failed to be created on Primary Cluster") - Expect(kubectl.CreateNamespace(namespace, kubeconfig2)).To(Succeed(), "Namespace failed to be created on Remote Cluster") + Expect(kubectlClient1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Primary Cluster") + Expect(kubectlClient2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Remote Cluster") Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)). To(Succeed(), "Operator failed to be deployed in Primary Cluster") @@ -81,8 +80,8 @@ var _ = Describe("Multicluster deployment models", Ordered, func() { Context("Istio version is: "+version.Version, func() { When("Istio resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(kubectl.CreateNamespace(controlPlaneNamespace, kubeconfig)).To(Succeed(), "Namespace failed to be created") - Expect(kubectl.CreateNamespace(controlPlaneNamespace, kubeconfig2)).To(Succeed(), "Namespace failed to be created") + Expect(kubectlClient1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") + Expect(kubectlClient2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") // Push the intermediate CA to both clusters Expect(certs.PushIntermediateCA(controlPlaneNamespace, kubeconfig, "east", "network1", artifacts, clPrimary)). @@ -120,7 +119,7 @@ spec: network: %s` multiclusterPrimaryYAML := fmt.Sprintf(PrimaryYAML, version.Name, controlPlaneNamespace, "mesh1", "cluster1", "network1") Log("Istio CR Primary: ", multiclusterPrimaryYAML) - Expect(kubectl.CreateFromString(multiclusterPrimaryYAML, kubeconfig)).To(Succeed(), "Istio Resource creation failed on Primary Cluster") + Expect(kubectlClient1.CreateFromString(multiclusterPrimaryYAML)).To(Succeed(), "Istio Resource creation failed on Primary Cluster") }) It("updates Istio CR on Primary cluster status to Ready", func(ctx SpecContext) { @@ -141,13 +140,13 @@ spec: When("Gateway is created on Primary cluster ", func() { BeforeAll(func(ctx SpecContext) { - Expect(kubectl.Apply(controlPlaneNamespace, eastGatewayYAML, kubeconfig)).To(Succeed(), "Gateway creation failed on Primary Cluster") + Expect(kubectlClient1.SetNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Primary Cluster") // Expose istiod service in Primary cluster - Expect(kubectl.Apply(controlPlaneNamespace, exposeIstiodYAML, kubeconfig)).To(Succeed(), "Expose Istiod creation failed on Primary Cluster") + Expect(kubectlClient1.SetNamespace(controlPlaneNamespace).Apply(exposeIstiodYAML)).To(Succeed(), "Expose Istiod creation failed on Primary Cluster") // Expose the Gateway service in both clusters - Expect(kubectl.Apply(controlPlaneNamespace, exposeServiceYAML, kubeconfig)).To(Succeed(), "Expose Service creation failed on Primary Cluster") + Expect(kubectlClient1.SetNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Primary Cluster") }) It("updates Gateway status to Available", func(ctx SpecContext) { @@ -179,31 +178,29 @@ spec: remoteIstioYAML := fmt.Sprintf(RemoteYAML, version.Name, remotePilotAddress) Log("RemoteIstio CR: ", remoteIstioYAML) By("Creating RemoteIstio CR on Remote Cluster") - Expect(kubectl.CreateFromString(remoteIstioYAML, kubeconfig2)).To(Succeed(), "RemoteIstio Resource creation failed on Remote Cluster") + Expect(kubectlClient2.CreateFromString(remoteIstioYAML)).To(Succeed(), "RemoteIstio Resource creation failed on Remote Cluster") // Set the controlplane cluster and network for Remote namespace By("Patching the istio-system namespace on Remote Cluster") Expect( - kubectl.Patch("", + kubectlClient2.Patch( "namespace", controlPlaneNamespace, "merge", - `{"metadata":{"annotations":{"topology.istio.io/controlPlaneClusters":"cluster1"}}}`, - kubeconfig2)). + `{"metadata":{"annotations":{"topology.istio.io/controlPlaneClusters":"cluster1"}}}`)). To(Succeed(), "Error patching istio-system namespace") Expect( - kubectl.Patch("", + kubectlClient2.Patch( "namespace", controlPlaneNamespace, "merge", - `{"metadata":{"labels":{"topology.istio.io/network":"network2"}}}`, - kubeconfig2)). + `{"metadata":{"labels":{"topology.istio.io/network":"network2"}}}`)). To(Succeed(), "Error patching istio-system namespace") // To be able to access the remote cluster from the primary cluster, we need to create a secret in the primary cluster // RemoteIstio resource will not be Ready until the secret is created // Get the internal IP of the control plane node in Remote cluster - internalIPRemote, err := kubectl.GetInternalIP("node-role.kubernetes.io/control-plane", kubeconfig2) + internalIPRemote, err := kubectlClient2.GetInternalIP("node-role.kubernetes.io/control-plane") Expect(internalIPRemote).NotTo(BeEmpty(), "Internal IP is empty for Remote Cluster") Expect(err).NotTo(HaveOccurred()) @@ -214,7 +211,7 @@ spec: By("Creating Remote Secret on Primary Cluster") secret, err := istioctl.CreateRemoteSecret(kubeconfig2, "remote", internalIPRemote) Expect(err).NotTo(HaveOccurred()) - Expect(kubectl.ApplyString("", secret, kubeconfig)).To(Succeed(), "Remote secret creation failed on Primary Cluster") + Expect(kubectlClient1.ApplyString(secret)).To(Succeed(), "Remote secret creation failed on Primary Cluster") }) It("secret is created", func(ctx SpecContext) { @@ -234,7 +231,7 @@ spec: When("gateway is created in Remote cluster", func() { BeforeAll(func(ctx SpecContext) { - Expect(kubectl.Apply(controlPlaneNamespace, westGatewayYAML, kubeconfig2)).To(Succeed(), "Gateway creation failed on Remote Cluster") + Expect(kubectlClient2.SetNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Remote Cluster") Success("Gateway is created in Remote cluster") }) @@ -249,7 +246,7 @@ spec: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { // Deploy the sample app in both clusters - deploySampleApp("sample", version, kubeconfig, kubeconfig2) + deploySampleApp("sample", version) Success("Sample app is deployed in both clusters") }) @@ -287,12 +284,12 @@ spec: Expect(err).NotTo(HaveOccurred(), "Error getting sleep pod name on Remote Cluster") // Run the curl command from the sleep pod in the Remote Cluster and get response list to validate that we get responses from both clusters - remoteResponses := strings.Join(getListCurlResponses(sleepPodNameRemote, kubeconfig2), "\n") + remoteResponses := strings.Join(getListCurlResponses(kubectlClient2, sleepPodNameRemote), "\n") Expect(remoteResponses).To(ContainSubstring("Hello version: v1"), "Responses from Remote Cluster are not the expected") Expect(remoteResponses).To(ContainSubstring("Hello version: v2"), "Responses from Remote Cluster are not the expected") // Run the curl command from the sleep pod in the Primary Cluster and get response list to validate that we get responses from both clusters - primaryResponses := strings.Join(getListCurlResponses(sleepPodNamePrimary, kubeconfig), "\n") + primaryResponses := strings.Join(getListCurlResponses(kubectlClient1, sleepPodNamePrimary), "\n") Expect(primaryResponses).To(ContainSubstring("Hello version: v1"), "Responses from Primary Cluster are not the expected") Expect(primaryResponses).To(ContainSubstring("Hello version: v2"), "Responses from Primary Cluster are not the expected") Success("Sample app is accessible from both clusters") @@ -301,8 +298,8 @@ spec: When("Istio CR and RemoteIstio CR are deleted in both clusters", func() { BeforeEach(func() { - Expect(kubectl.Delete(controlPlaneNamespace, "istio", istioName, kubeconfig)).To(Succeed(), "Istio CR failed to be deleted") - Expect(kubectl.Delete(controlPlaneNamespace, "remoteistio", istioName, kubeconfig2)).To(Succeed(), "RemoteIstio CR failed to be deleted") + Expect(kubectlClient1.SetNamespace(controlPlaneNamespace).Delete("istio", istioName)).To(Succeed(), "Istio CR failed to be deleted") + Expect(kubectlClient2.SetNamespace(controlPlaneNamespace).Delete("remoteistio", istioName)).To(Succeed(), "RemoteIstio CR failed to be deleted") Success("Istio and RemoteIstio are deleted") }) @@ -315,16 +312,16 @@ spec: AfterAll(func(ctx SpecContext) { // Delete namespace to ensure clean up for new tests iteration - Expect(kubectl.DeleteNamespace(controlPlaneNamespace, kubeconfig)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(kubectl.DeleteNamespace(controlPlaneNamespace, kubeconfig2)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") + Expect(kubectlClient1.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") + Expect(kubectlClient2.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") common.CheckNamespaceEmpty(ctx, clPrimary, controlPlaneNamespace) common.CheckNamespaceEmpty(ctx, clRemote, controlPlaneNamespace) Success("ControlPlane Namespaces are empty") // Delete the entire sample namespace in both clusters - Expect(kubectl.DeleteNamespace("sample", kubeconfig)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(kubectl.DeleteNamespace("sample", kubeconfig2)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") + Expect(kubectlClient1.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") + Expect(kubectlClient2.DeleteNamespace("sample")).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") common.CheckNamespaceEmpty(ctx, clPrimary, "sample") common.CheckNamespaceEmpty(ctx, clRemote, "sample") @@ -336,8 +333,8 @@ spec: AfterAll(func(ctx SpecContext) { // Delete the Sail Operator from both clusters - Expect(kubectl.DeleteNamespace(namespace, kubeconfig)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(kubectl.DeleteNamespace(namespace, kubeconfig2)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") + Expect(kubectlClient1.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") + Expect(kubectlClient2.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") // Check that the namespace is empty common.CheckNamespaceEmpty(ctx, clPrimary, namespace) diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index 5c0cd061a..77515b124 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -25,6 +25,7 @@ import ( "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" env "github.com/istio-ecosystem/sail-operator/tests/e2e/util/env" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" @@ -50,6 +51,9 @@ var ( westGatewayYAML string exposeServiceYAML string exposeIstiodYAML string + + kubectlClient1 *kubectl.KubectlBuilder + kubectlClient2 *kubectl.KubectlBuilder ) func TestInstall(t *testing.T) { @@ -59,6 +63,7 @@ func TestInstall(t *testing.T) { if ocp { t.Skip("Skipping test. Not valid for OCP") // TODO: Implement the steps to run the test on OCP + // https://github.com/istio-ecosystem/sail-operator/issues/365 } RegisterFailHandler(Fail) setup(t) @@ -92,4 +97,8 @@ func setup(t *testing.T) { westGatewayYAML = fmt.Sprintf("%s/docs/multicluster/east-west-gateway-net2.yaml", baseRepoDir) exposeServiceYAML = fmt.Sprintf("%s/docs/multicluster/expose-services.yaml", baseRepoDir) exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir) + + // Initialize kubectl utilities, one for each cluster + kubectlClient1 = kubectl.NewKubectlBuilder().SetKubeconfig(kubeconfig) + kubectlClient2 = kubectl.NewKubectlBuilder().SetKubeconfig(kubeconfig2) } diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go index cc6777d6c..cbd051feb 100644 --- a/tests/e2e/operator/operator_install_test.go +++ b/tests/e2e/operator/operator_install_test.go @@ -26,7 +26,6 @@ import ( common "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" @@ -60,7 +59,7 @@ var _ = Describe("Operator", Ordered, func() { Describe("installation", func() { BeforeAll(func() { - Expect(kubectl.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") + Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") extraArg := "" if ocp { @@ -125,7 +124,7 @@ var _ = Describe("Operator", Ordered, func() { Success("Operator uninstalled") By("Deleting the CRDs") - Expect(kubectl.DeleteCRDs(sailCRDs)).To(Succeed(), "CRDs failed to be deleted") + Expect(k.DeleteCRDs(sailCRDs)).To(Succeed(), "CRDs failed to be deleted") Success("CRDs deleted") }) }) diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go index 32b737594..f0bd1261f 100644 --- a/tests/e2e/operator/operator_suite_test.go +++ b/tests/e2e/operator/operator_suite_test.go @@ -21,6 +21,7 @@ import ( k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" env "github.com/istio-ecosystem/sail-operator/tests/e2e/util/env" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,6 +35,8 @@ var ( namespace = env.Get("NAMESPACE", "sail-operator") deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") multicluster = env.GetBool("MULTICLUSTER", false) + + k *kubectl.KubectlBuilder ) func TestInstall(t *testing.T) { @@ -58,4 +61,6 @@ func setup() { } else { GinkgoWriter.Println("Running on Kubernetes") } + + k = kubectl.NewKubectlBuilder() } diff --git a/tests/e2e/util/certs/certs.go b/tests/e2e/util/certs/certs.go index 78aaaa004..daddf56d9 100644 --- a/tests/e2e/util/certs/certs.go +++ b/tests/e2e/util/certs/certs.go @@ -227,7 +227,9 @@ func PushIntermediateCA(ns, kubeconfig, zone, network, basePath string, cl clien _, err := common.GetObject(context.Background(), cl, kube.Key("cacerts", ns), &corev1.Secret{}) if err != nil { // Label the namespace with the network - err = kubectl.Patch("", "namespace", ns, "merge", `{"metadata":{"labels":{"topology.istio.io/network":"`+network+`"}}}`, kubeconfig) + k := kubectl.NewKubectlBuilder() + k.SetKubeconfig(kubeconfig) + err = k.Patch("namespace", ns, "merge", `{"metadata":{"labels":{"topology.istio.io/network":"`+network+`"}}}`) if err != nil { return fmt.Errorf("failed to label namespace: %w", err) } diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index e17e07686..50cbc0c7f 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -47,6 +47,8 @@ var ( // - 1.23.0-rc.1 // - 1.24-alpha istiodVersionRegex = regexp.MustCompile(`Version:"(\d+\.\d+(\.\d+)?(-\w+(\.\d+)?)?)`) + + k = kubectl.NewKubectlBuilder() ) // getObject returns the object with the given key @@ -135,52 +137,54 @@ func LogDebugInfo() { } func logOperatorDebugInfo() { - operator, err := kubectl.GetYAML(namespace, "deployment", deploymentName) + operator, err := k.SetNamespace(namespace).GetYAML("deployment", deploymentName) logDebugElement("Operator Deployment YAML", operator, err) - logs, err := kubectl.Logs(namespace, "deploy/"+deploymentName, ptr.Of(120*time.Second)) + logs, err := k.SetNamespace(namespace).Logs("deploy/"+deploymentName, ptr.Of(120*time.Second)) + k.ResetNamespace() logDebugElement("Operator logs", logs, err) - events, err := kubectl.GetEvents(namespace) + events, err := k.SetNamespace(namespace).GetEvents() logDebugElement("Events in "+namespace, events, err) // Temporaty information to gather more details about failure - pods, err := kubectl.GetPods(namespace, "", "-o wide") + pods, err := k.SetNamespace(namespace).GetPods("", "-o wide") logDebugElement("Pods in "+namespace, pods, err) - describe, err := kubectl.Describe(namespace, "deployment", deploymentName) + describe, err := k.SetNamespace(namespace).Describe("deployment", deploymentName) logDebugElement("Operator Deployment describe", describe, err) } func logIstioDebugInfo() { - resource, err := kubectl.GetYAML("", "istio", istioName) + resource, err := k.GetYAML("istio", istioName) logDebugElement("Istio YAML", resource, err) - output, err := kubectl.GetPods(controlPlaneNamespace, "", "-o wide") + output, err := k.SetNamespace(controlPlaneNamespace).GetPods("", "-o wide") logDebugElement("Pods in "+controlPlaneNamespace, output, err) - logs, err := kubectl.Logs(controlPlaneNamespace, "deploy/istiod", ptr.Of(120*time.Second)) + logs, err := k.SetNamespace(controlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) + k.ResetNamespace() logDebugElement("Istiod logs", logs, err) - events, err := kubectl.GetEvents(controlPlaneNamespace) + events, err := k.SetNamespace(controlPlaneNamespace).GetEvents() logDebugElement("Events in "+controlPlaneNamespace, events, err) } func logCNIDebugInfo() { - resource, err := kubectl.GetYAML("", "istiocni", istioCniName) + resource, err := k.GetYAML("istiocni", istioCniName) logDebugElement("IstioCNI YAML", resource, err) - ds, err := kubectl.GetYAML(istioCniNamespace, "daemonset", "istio-cni-node") + ds, err := k.SetNamespace(istioCniNamespace).GetYAML("daemonset", "istio-cni-node") logDebugElement("Istio CNI DaemonSet YAML", ds, err) - events, err := kubectl.GetEvents(istioCniNamespace) + events, err := k.SetNamespace(istioCniNamespace).GetEvents() logDebugElement("Events in "+istioCniNamespace, events, err) // Temporaty information to gather more details about failure - pods, err := kubectl.GetPods(istioCniNamespace, "", "-o wide") + pods, err := k.SetNamespace(istioCniNamespace).GetPods("", "-o wide") logDebugElement("Pods in "+istioCniNamespace, pods, err) - describe, err := kubectl.Describe(istioCniNamespace, "daemonset", "istio-cni-node") + describe, err := k.SetNamespace(istioCniNamespace).Describe("daemonset", "istio-cni-node") logDebugElement("Istio CNI DaemonSet describe", describe, err) } @@ -195,7 +199,8 @@ func logDebugElement(caption string, info string, err error) { } func GetVersionFromIstiod() (string, error) { - output, err := kubectl.Exec(controlPlaneNamespace, "deploy/istiod", "", "pilot-discovery version") + k := kubectl.NewKubectlBuilder() + output, err := k.SetNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") if err != nil { return "", fmt.Errorf("error getting version from istiod: %w", err) } diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index 931470851..f07035513 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -23,181 +23,208 @@ import ( "github.com/istio-ecosystem/sail-operator/tests/e2e/util/shell" ) +type KubectlBuilder struct { + binary string + namespace string + kubeconfig string +} + const DefaultBinary = "kubectl" -// optionalKubeconfig add the flag --kubeconfig if the kubeconfig is set -func optionalKubeconfig(kubeconfig []string) string { - if len(kubeconfig) > 0 && kubeconfig[0] != "" { - return fmt.Sprintf("--kubeconfig %s", kubeconfig[0]) - } - return "" +func newKubectlBuilder() *KubectlBuilder { + return &KubectlBuilder{} } -// kubectl return the kubectl command -// If the environment variable COMMAND is set, it will return the value of COMMAND -// Otherwise, it will return the default value "kubectl" as default -// Arguments: -// - format: format of the command without kubeclt or oc -// - args: arguments of the command -func kubectl(format string, args ...interface{}) string { +func (k *KubectlBuilder) setBinary() { binary := DefaultBinary if cmd := os.Getenv("COMMAND"); cmd != "" { binary = cmd } - return binary + " " + fmt.Sprintf(format, args...) + k.binary = binary } -// CreateFromString creates a resource from the given yaml string -func CreateFromString(yamlString string, kubeconfig ...string) error { - cmd := kubectl("create %s -f -", optionalKubeconfig(kubeconfig)) - _, err := shell.ExecuteCommandWithInput(cmd, yamlString) - if err != nil { - return fmt.Errorf("error creating resource from yaml: %w", err) +func (k *KubectlBuilder) build(cmd string) string { + args := []string{k.binary} + + // Only append namespace if it's set + if k.namespace != "" { + args = append(args, k.namespace) } - return nil + + // Only append kubeconfig if it's set + if k.kubeconfig != "" { + args = append(args, k.kubeconfig) + } + + args = append(args, cmd) + + // Join all the arguments with a space + return strings.Join(args, " ") } -// ApplyString applies the given yaml string to the cluster -func ApplyString(ns, yamlString string, kubeconfig ...string) error { - nsflag := nsflag(ns) - // If the namespace is empty, we need to remove the flag because it will fail - // TODO: improve the nsflag function to handle this case +// NewKubectlBuilder creates a new KubectlBuilder +func NewKubectlBuilder() *KubectlBuilder { + k := newKubectlBuilder() + k.setBinary() + return k +} + +// SetNamespace sets the namespace +func (k *KubectlBuilder) SetNamespace(ns string) *KubectlBuilder { if ns == "" { - nsflag = "" + k.namespace = "--all-namespaces" + } else { + k.namespace = fmt.Sprintf("-n %s", ns) } + return k +} - cmd := kubectl("apply %s %s --server-side -f -", nsflag, optionalKubeconfig(kubeconfig)) - _, err := shell.ExecuteCommandWithInput(cmd, yamlString) +// SetKubeconfig sets the kubeconfig +func (k *KubectlBuilder) SetKubeconfig(kubeconfig string) *KubectlBuilder { + if kubeconfig != "" { + k.kubeconfig = fmt.Sprintf("--kubeconfig %s", kubeconfig) + } + return k +} + +// CreateNamespace creates a namespace +// If the namespace already exists, it will return nil +func (k *KubectlBuilder) CreateNamespace(ns string) error { + cmd := k.build(" create namespace " + ns) + output, err := k.executeCommand(cmd) if err != nil { - return fmt.Errorf("error applying yaml: %w", err) + if strings.Contains(output, "AlreadyExists") { + return nil + } + + return fmt.Errorf("error creating namespace: %w, output: %s", err, output) } return nil } -// Apply applies the given yaml file to the cluster -func Apply(ns, yamlFile string, kubeconfig ...string) error { - err := ApplyWithLabels(ns, yamlFile, "", kubeconfig...) - return err +// CreateFromString creates a resource from the given yaml string +func (k *KubectlBuilder) CreateFromString(yamlString string) error { + cmd := k.build(" create -f -") + _, err := shell.ExecuteCommandWithInput(cmd, yamlString) + k.ResetNamespace() + if err != nil { + return fmt.Errorf("error creating resource from yaml: %w", err) + } + return nil } -// ApplyWithLabels applies the given yaml file to the cluster with the given labels -func ApplyWithLabels(ns, yamlFile string, label string, kubeconfig ...string) error { - cmd := kubectl("apply -n %s %s -f %s %s", ns, labelFlag(label), yamlFile, optionalKubeconfig(kubeconfig)) - _, err := shell.ExecuteCommand(cmd) - if err != nil { - return fmt.Errorf("error applying yaml: %w", err) +// DeleteCRDs deletes the CRDs by given list of crds names +func (k *KubectlBuilder) DeleteCRDs(crds []string) error { + for _, crd := range crds { + cmd := k.build(" delete crd " + crd) + _, err := shell.ExecuteCommand(cmd) + if err != nil { + k.ResetNamespace() + return fmt.Errorf("error deleting crd %s: %w", crd, err) + } } + k.ResetNamespace() return nil } -// DeleteFromFile deletes a resource from the given yaml file -func DeleteFromFile(yamlFile string, kubeconfig ...string) error { - cmd := kubectl("delete -f %s %s", yamlFile, optionalKubeconfig(kubeconfig)) - _, err := shell.ExecuteCommand(cmd) +// DeleteNamespace deletes a namespace +func (k *KubectlBuilder) DeleteNamespace(ns string) error { + cmd := k.build(" delete namespace " + ns) + _, err := k.executeCommand(cmd) if err != nil { - return fmt.Errorf("error deleting resource from yaml: %w", err) + return fmt.Errorf("error deleting namespace: %w", err) } return nil } -// CreateNamespace creates a namespace -// If the namespace already exists, it will return nil -// Arguments: -// - ns: namespace -// - kubeconfig: optional kubeconfig to set the target file -func CreateNamespace(ns string, kubeconfig ...string) error { - cmd := kubectl("create namespace %s %s", ns, optionalKubeconfig(kubeconfig)) - output, err := shell.ExecuteCommand(cmd) +// ApplyString applies the given yaml string to the cluster +func (k *KubectlBuilder) ApplyString(yamlString string) error { + cmd := k.build(" apply --server-side -f -") + _, err := shell.ExecuteCommandWithInput(cmd, yamlString) + k.ResetNamespace() if err != nil { - if strings.Contains(output, "AlreadyExists") { - return nil - } - - return fmt.Errorf("error creating namespace: %w, output: %s", err, output) + return fmt.Errorf("error applying yaml: %w", err) } return nil } -// DeleteNamespace deletes a namespace -// Arguments: -// - ns: namespace -// - kubeconfig: optional kubeconfig to set the target file -func DeleteNamespace(ns string, kubeconfig ...string) error { - cmd := kubectl("delete namespace %s %s", ns, optionalKubeconfig(kubeconfig)) - _, err := shell.ExecuteCommand(cmd) +// Apply applies the given yaml file to the cluster +func (k *KubectlBuilder) Apply(yamlFile string) error { + err := k.ApplyWithLabels(yamlFile, "") + return err +} + +// ApplyWithLabels applies the given yaml file to the cluster with the given labels +func (k *KubectlBuilder) ApplyWithLabels(yamlFile, label string) error { + cmd := k.build(" apply " + labelFlag(label) + " -f " + yamlFile) + _, err := k.executeCommand(cmd) if err != nil { - return fmt.Errorf("error deleting namespace: %w", err) + return fmt.Errorf("error applying yaml: %w", err) } return nil } -// Delete deletes a resource based on the namespace, kind and the name. Optionally, you can provide a kubeconfig -func Delete(ns, kind, name string, kubeconfig ...string) error { - cmd := kubectl("delete %s %s %s %s", kind, name, nsflag(ns), optionalKubeconfig(kubeconfig)) - _, err := shell.ExecuteCommand(cmd) +// DeleteFromFile deletes a resource from the given yaml file +func (k *KubectlBuilder) DeleteFromFile(yamlFile string) error { + cmd := k.build(" delete -f " + yamlFile) + _, err := k.executeCommand(cmd) if err != nil { - return fmt.Errorf("error deleting deployment: %w", err) + return fmt.Errorf("error deleting resource from yaml: %w", err) } return nil } -// DeleteCRDs deletes the CRDs by given list of crds names -func DeleteCRDs(crds []string) error { - for _, crd := range crds { - cmd := kubectl("delete crd %s", crd) - _, err := shell.ExecuteCommand(cmd) - if err != nil { - return fmt.Errorf("error deleting crd %s: %w", crd, err) - } +// Delete deletes a resource based on the namespace, kind and the name +func (k *KubectlBuilder) Delete(kind, name string) error { + cmd := k.build(" delete " + kind + " " + name) + _, err := k.executeCommand(cmd) + if err != nil { + return fmt.Errorf("error deleting deployment: %w", err) } return nil } -// Patch patches a resource. -func Patch(ns, kind, name, patchType, patch string, kubeconfig ...string) error { - cmd := kubectl(`patch %s %s %s %s --type=%s -p=%q`, kind, name, prepend("-n", ns), optionalKubeconfig(kubeconfig), patchType, patch) - _, err := shell.ExecuteCommand(cmd) +// Patch patches a resource +func (k *KubectlBuilder) Patch(kind, name, patchType, patch string) error { + cmd := k.build(fmt.Sprintf(" patch %s %s --type=%s -p=%q", kind, name, patchType, patch)) + _, err := k.executeCommand(cmd) if err != nil { return fmt.Errorf("error patching resource: %w", err) } return nil } -// ForceDelete deletes a resource by removing its finalizers. -func ForceDelete(ns, kind, name string) error { +// ForceDelete deletes a resource by removing its finalizers +func (k *KubectlBuilder) ForceDelete(kind, name string) error { // Not all resources have finalizers, trying to remove them returns an error here. // We explicitly ignore the error and attempt to delete the resource anyway. - _ = Patch(ns, kind, name, "json", `[{"op": "remove", "path": "/metadata/finalizers"}]`) - return Delete(ns, kind, name) + _ = k.Patch(kind, name, "json", `[{"op": "remove", "path": "/metadata/finalizers"}]`) + return k.Delete(kind, name) } // GetYAML returns the yaml of a resource -// Arguments: -// - ns: namespace -// - kind: type of the resource -// - name: name of the resource -func GetYAML(ns, kind, name string) (string, error) { - cmd := kubectl("get %s %s %s -o yaml", kind, name, nsflag(ns)) - return shell.ExecuteCommand(cmd) +func (k *KubectlBuilder) GetYAML(kind, name string) (string, error) { + cmd := k.build(fmt.Sprintf(" get %s %s -o yaml", kind, name)) + output, err := k.executeCommand(cmd) + if err != nil { + return "", fmt.Errorf("error getting yaml: %w, output: %s", err, output) + } + + return output, nil } // GetPods returns the pods of a namespace -func GetPods(ns string, kubeconfig string, args ...string) (string, error) { - kubeconfigFlag := "" - if kubeconfig != "" { - kubeconfigFlag = fmt.Sprintf("--kubeconfig %s", kubeconfig) - } - - cmd := kubectl("get pods %s %s %s", nsflag(ns), strings.Join(args, " "), kubeconfigFlag) - output, err := shell.ExecuteCommand(cmd) +func (k *KubectlBuilder) GetPods(args ...string) (string, error) { + cmd := k.build(fmt.Sprintf(" get pods %s", strings.Join(args, " "))) + output, err := k.executeCommand(cmd) if err != nil { return "", fmt.Errorf("error getting pods: %w, output: %s", err, output) } @@ -205,70 +232,52 @@ func GetPods(ns string, kubeconfig string, args ...string) (string, error) { return output, nil } -// GetEvents returns the events of a namespace -func GetEvents(ns string) (string, error) { - cmd := kubectl("get events %s", nsflag(ns)) - output, err := shell.ExecuteCommand(cmd) +// GetInternalIP returns the internal IP of a node +func (k *KubectlBuilder) GetInternalIP(label string) (string, error) { + cmd := k.build(fmt.Sprintf(" get nodes -l %s -o jsonpath='{.items[0].status.addresses[?(@.type==\"InternalIP\")].address}'", label)) + output, err := k.executeCommand(cmd) if err != nil { - return "", fmt.Errorf("error getting events: %w, output: %s", err, output) + return "", fmt.Errorf("error getting internal IP: %w, output: %s", err, output) } return output, nil } -// Describe returns the description of a resource -// Arguments: -// - ns: namespace -// - kind: type of the resource -// - name: name of the resource -func Describe(ns, kind, name string) (string, error) { - cmd := kubectl("describe %s %s %s", kind, name, nsflag(ns)) - output, err := shell.ExecuteCommand(cmd) +// Exec executes a command in the pod or specific container +func (k *KubectlBuilder) Exec(pod, container, command string) (string, error) { + cmd := k.build(fmt.Sprintf(" exec %s %s -- %s", pod, containerflag(container), command)) + output, err := k.executeCommand(cmd) if err != nil { - return "", fmt.Errorf("error describing resource: %w, output: %s", err, output) + return "", err } - return output, nil } -// GetInternalIP returns the internal IP of a node -// Arguments: -// - label: label of the node -// - kubeconfig: optional kubeconfig to set the target file -func GetInternalIP(label string, kubeconfig ...string) (string, error) { - cmd := kubectl("get nodes -l %s -o jsonpath='{.items[0].status.addresses[?(@.type==\"InternalIP\")].address}' %s", label, optionalKubeconfig(kubeconfig)) - output, err := shell.ExecuteCommand(cmd) +// GetEvents returns the events of a namespace +func (k *KubectlBuilder) GetEvents() (string, error) { + cmd := k.build(" get events") + output, err := k.executeCommand(cmd) if err != nil { - return "", fmt.Errorf("error getting internal IP: %w, output: %s", err, output) + return "", fmt.Errorf("error getting events: %w, output: %s", err, output) } return output, nil } -// Logs returns the logs of a deployment -// Arguments: -// - ns: namespace -// - pod: the pod name, "kind/name", or "-l labelselector" -// - Since: time range -func Logs(ns, pod string, since *time.Duration) (string, error) { - cmd := kubectl("logs %s %s %s", pod, nsflag(ns), sinceFlag(since)) - output, err := shell.ExecuteCommand(cmd) +// Describe returns the description of a resource +func (k *KubectlBuilder) Describe(kind, name string) (string, error) { + cmd := k.build(fmt.Sprintf(" describe %s %s", kind, name)) + output, err := k.executeCommand(cmd) if err != nil { - return "", err + return "", fmt.Errorf("error describing resource: %w, output: %s", err, output) } - return output, nil -} -func sinceFlag(since *time.Duration) string { - if since == nil { - return "" - } - return "--since=" + since.String() + return output, nil } -// Exec executes a command in the pod or specific container -func Exec(ns, pod, container, command string, kubeconfig ...string) (string, error) { - cmd := kubectl("exec %s %s %s %s -- %s", pod, containerflag(container), nsflag(ns), optionalKubeconfig(kubeconfig), command) +// Logs returns the logs of a deployment +func (k *KubectlBuilder) Logs(pod string, since *time.Duration) (string, error) { + cmd := k.build(fmt.Sprintf(" logs %s %s", pod, sinceFlag(since))) output, err := shell.ExecuteCommand(cmd) if err != nil { return "", err @@ -276,19 +285,23 @@ func Exec(ns, pod, container, command string, kubeconfig ...string) (string, err return output, nil } -// prepend prepends the prefix, but only if str is not empty -func prepend(prefix, str string) string { - if str == "" { - return str - } - return prefix + str +// executeCommand handles running the command and then resets the namespace automatically +func (k *KubectlBuilder) executeCommand(cmd string) (string, error) { + result, err := shell.ExecuteCommand(cmd) + k.ResetNamespace() + return result, err } -func nsflag(ns string) string { - if ns == "" { - return "--all-namespaces" +// ResetNamespace resets the namespace +func (k *KubectlBuilder) ResetNamespace() { + k.namespace = "" +} + +func sinceFlag(since *time.Duration) string { + if since == nil { + return "" } - return "-n " + ns + return "--since=" + since.String() } func labelFlag(label string) string {