Adding E2e multicluster test (#342)

* Adding E2e multicluster test Signed-off-by: frherrer <frherrer@redhat.com> * Add istioctl util go build Signed-off-by: frherrer <frherrer@redhat.com> * Delete kind config breaking the test Signed-off-by: frherrer <frherrer@redhat.com> * Removing references to raw.githubusercontent on multicluster related YAML Signed-off-by: frherrer <frherrer@redhat.com> * Updates from review: improve cleanup, minor fixes Signed-off-by: frherrer <frherrer@redhat.com> --------- Signed-off-by: frherrer <frherrer@redhat.com>
istio-ecosystem · Sep 23, 2024 · ab57cdf · ab57cdf
1 parent bd6e048
commit ab57cdf
Show file tree

Hide file tree

Showing 20 changed files with 1,416 additions and 74 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,6 +1,6 @@
 {
   "name": "istio build-tools",
-  "image": "gcr.io/istio-testing/build-tools:master-0aa2afb4bac9a4fd1bfe50a929c077a643066b3a",
+  "image": "gcr.io/istio-testing/build-tools:master-8584ca511549c1cd96d9cb8b900297de83f4cb64",
   "privileged": true,
   "remoteEnv": {
     "USE_GKE_GCLOUD_AUTH_PLUGIN": "True",

diff --git a/Makefile.core.mk b/Makefile.core.mk
@@ -165,8 +165,8 @@ test.e2e.ocp: ## Run the end-to-end tests against an existing OCP cluster.
 	GINKGO_FLAGS="$(GINKGO_FLAGS)" ${SOURCE_DIR}/tests/e2e/integ-suite-ocp.sh
 
 .PHONY: test.e2e.kind
-test.e2e.kind: ## Deploy a KinD cluster and run the end-to-end tests against it.
-	GINKGO_FLAGS="$(GINKGO_FLAGS)" ${SOURCE_DIR}/tests/e2e/integ-suite-kind.sh
+test.e2e.kind: istioctl ## Deploy a KinD cluster and run the end-to-end tests against it.
+	GINKGO_FLAGS="$(GINKGO_FLAGS)" ISTIOCTL="$(ISTIOCTL)" ${SOURCE_DIR}/tests/e2e/integ-suite-kind.sh
 
 .PHONY: test.e2e.describe
 test.e2e.describe: ## Runs ginkgo outline -format indent over the e2e test to show in BDD style the steps and test structure
@@ -450,13 +450,15 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GITLEAKS ?= $(LOCALBIN)/gitleaks
 OPM ?= $(LOCALBIN)/opm
+ISTIOCTL ?= $(LOCALBIN)/istioctl
 
 ## Tool Versions
 OPERATOR_SDK_VERSION ?= v1.36.1
 HELM_VERSION ?= v3.15.3
 CONTROLLER_TOOLS_VERSION ?= v0.16.0
 OPM_VERSION ?= v1.45.0
 GITLEAKS_VERSION ?= v8.18.4
+ISTIOCTL_VERSION ?= 1.23.0
 
 # GENERATE_RELATED_IMAGES defines whether `spec.relatedImages` is going to be generated or not
 # To disable set flag to false
@@ -483,6 +485,28 @@ $(OPERATOR_SDK): $(LOCALBIN)
 	curl -sSLfo $(LOCALBIN)/operator-sdk https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$(OS)_$(ARCH) && \
 	chmod +x $(LOCALBIN)/operator-sdk;
 
+.PHONY: istioctl $(ISTIOCTL)
+istioctl: $(ISTIOCTL) ## Download istioctl to bin directory.
+istioctl: TARGET_OS=$(shell go env GOOS)
+istioctl: TARGET_ARCH=$(shell go env GOARCH)
+$(ISTIOCTL): $(LOCALBIN)
+	@test -s $(LOCALBIN)/istioctl || { \
+		OSEXT=$(if $(filter $(TARGET_OS),Darwin),osx,linux); \
+		URL="https://github.com/istio/istio/releases/download/$(ISTIOCTL_VERSION)/istioctl-$(ISTIOCTL_VERSION)-$$OSEXT-$(TARGET_ARCH).tar.gz"; \
+		echo "Fetching istioctl from $$URL"; \
+		curl -fsL $$URL -o /tmp/istioctl.tar.gz || { \
+			echo "Download failed! Please check the URL and ISTIO_VERSION."; \
+			exit 1; \
+		}; \
+		tar -xzf /tmp/istioctl.tar.gz -C /tmp || { \
+			echo "Extraction failed!"; \
+			exit 1; \
+		}; \
+		mv /tmp/istioctl $(LOCALBIN)/istioctl; \
+		rm -f /tmp/istioctl.tar.gz; \
+		echo "istioctl has been downloaded and placed in $(LOCALBIN)"; \
+	}
+
 .PHONY: controller-gen
 controller-gen: $(LOCALBIN) ## Download controller-gen to bin directory. If wrong version is installed, it will be overwritten.
 	@test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \

diff --git a/common/scripts/kind_provisioner.sh b/common/scripts/kind_provisioner.sh
@@ -34,6 +34,9 @@ set -x
 # DEFAULT_KIND_IMAGE is used to set the Kubernetes version for KinD unless overridden in params to setup_kind_cluster(s)
 DEFAULT_KIND_IMAGE="gcr.io/istio-testing/kind-node:v1.28.4"
 
+# the default kind cluster should be ipv4 if not otherwise specified
+IP_FAMILY="${IP_FAMILY:-ipv4}"
+
 # COMMON_SCRIPTS contains the directory this file is in.
 COMMON_SCRIPTS=$(dirname "${BASH_SOURCE:-$0}")
 
@@ -174,11 +177,6 @@ function setup_kind_cluster() {
     CONFIG=${DEFAULT_CLUSTER_YAML}
   fi
 
-  # Configure the ipFamily of the cluster
-  if [ -n "${IP_FAMILY}" ]; then
-      yq eval ".networking.ipFamily = \"${IP_FAMILY}\"" -i "${CONFIG}"
-  fi
-
   KIND_WAIT_FLAG="--wait=180s"
   KIND_DISABLE_CNI="false"
   if [[ -n "${KUBERNETES_CNI:-}" ]]; then
@@ -187,7 +185,8 @@ function setup_kind_cluster() {
   fi
 
   # Create KinD cluster
-  if ! (yq eval "${CONFIG}" --expression ".networking.disableDefaultCNI = ${KIND_DISABLE_CNI}" | \
+  if ! (yq eval "${CONFIG}" --expression ".networking.disableDefaultCNI = ${KIND_DISABLE_CNI}" \
+    --expression ".networking.ipFamily = \"${IP_FAMILY}\"" | \
     kind create cluster --name="${NAME}" -v4 --retain --image "${IMAGE}" ${KIND_WAIT_FLAG:+"$KIND_WAIT_FLAG"} --config -); then
     echo "Could not setup KinD environment. Something wrong with KinD setup. Exporting logs."
     return 9
@@ -478,4 +477,4 @@ function ips_to_cidrs() {
 from ipaddress import summarize_address_range, IPv4Address
 [ print(n.compressed) for n in summarize_address_range(IPv4Address(u'$IP_RANGE_START'), IPv4Address(u'$IP_RANGE_END')) ]
 EOF
-}
+}
diff --git a/common/scripts/setup_env.sh b/common/scripts/setup_env.sh
@@ -75,7 +75,7 @@ fi
 TOOLS_REGISTRY_PROVIDER=${TOOLS_REGISTRY_PROVIDER:-gcr.io}
 PROJECT_ID=${PROJECT_ID:-istio-testing}
 if [[ "${IMAGE_VERSION:-}" == "" ]]; then
-  IMAGE_VERSION=master-0aa2afb4bac9a4fd1bfe50a929c077a643066b3a
+  IMAGE_VERSION=master-8584ca511549c1cd96d9cb8b900297de83f4cb64
 fi
 if [[ "${IMAGE_NAME:-}" == "" ]]; then
   IMAGE_NAME=build-tools

diff --git a/pkg/test/util/supportedversion/supportedversion.go b/pkg/test/util/supportedversion/supportedversion.go
@@ -17,6 +17,8 @@ package supportedversion
 import (
 	"os"
 	"path/filepath"
+	"regexp"
+	"strconv"
 
 	"github.com/istio-ecosystem/sail-operator/pkg/test/project"
 	"gopkg.in/yaml.v3"
@@ -47,6 +49,12 @@ func init() {
 		panic(err)
 	}
 
+	// Major, Minor and Patch needs to be set from parsing the version string
+	for i := range versions.Versions {
+		v := &versions.Versions[i]
+		v.Major, v.Minor, v.Patch = parseVersion(v.Version)
+	}
+
 	List = versions.Versions
 	Default = List[0].Name
 	if len(List) > 1 {
@@ -55,13 +63,32 @@ func init() {
 	New = List[0].Name
 }
 
+func parseVersion(version string) (int, int, int) {
+	// The version can have this formats: "1.22.2", "1.23.0-rc.1", "1.24-alpha"
+	re := regexp.MustCompile(`^(\d+)\.(\d+)\.?(\d*)`)
+
+	matches := re.FindStringSubmatch(version)
+	if len(matches) < 4 {
+		return 0, 0, 0
+	}
+
+	major, _ := strconv.Atoi(matches[1])
+	minor, _ := strconv.Atoi(matches[2])
+	patch, _ := strconv.Atoi(matches[3])
+
+	return major, minor, patch
+}
+
 type Versions struct {
 	Versions []VersionInfo `json:"versions"`
 }
 
 type VersionInfo struct {
 	Name    string   `json:"name"`
 	Version string   `json:"version"`
+	Major   int      `json:"major"`
+	Minor   int      `json:"minor"`
+	Patch   int      `json:"patch"`
 	Repo    string   `json:"repo"`
 	Branch  string   `json:"branch,omitempty"`
 	Commit  string   `json:"commit"`

diff --git a/tests/e2e/common-operator-integ-suite.sh b/tests/e2e/common-operator-integ-suite.sh
@@ -29,6 +29,7 @@ parse_flags() {
   SKIP_DEPLOY=${SKIP_DEPLOY:-false}
   OLM=${OLM:-false}
   DESCRIBE=false
+  MULTICLUSTER=false
   while [ $# -gt 0 ]; do
     case "$1" in
       --ocp)
@@ -39,6 +40,10 @@ parse_flags() {
         shift
         OCP=false
         ;;
+      --multicluster)
+        shift
+        MULTICLUSTER=true
+        ;;
       --skip-build)
         shift
         SKIP_BUILD=true
@@ -80,6 +85,10 @@ parse_flags() {
     echo "Running on kind"
   fi
 
+  if [ "${MULTICLUSTER}" == "true" ]; then
+    echo "Running on multicluster"
+  fi
+
   if [ "${SKIP_BUILD}" == "true" ]; then
     echo "Skipping build"
   fi
@@ -108,6 +117,7 @@ initialize_variables() {
   COMMAND="kubectl"
   ARTIFACTS="${ARTIFACTS:-$(mktemp -d)}"
   KUBECONFIG="${KUBECONFIG:-"${ARTIFACTS}/config"}"
+  ISTIOCTL="${ISTIOCTL:-"istioctl"}"
   LOCALBIN="${LOCALBIN:-${HOME}/bin}"
   OPERATOR_SDK=${LOCALBIN}/operator-sdk
 
@@ -258,6 +268,6 @@ fi
 # Run the go test passing the env variables defined that are going to be used in the operator tests
 # shellcheck disable=SC2086
 IMAGE="${HUB}/${IMAGE_BASE}:${TAG}" SKIP_DEPLOY="${SKIP_DEPLOY}" OCP="${OCP}" ISTIO_MANIFEST="${ISTIO_MANIFEST}" \
-NAMESPACE="${NAMESPACE}" CONTROL_PLANE_NS="${CONTROL_PLANE_NS}" DEPLOYMENT_NAME="${DEPLOYMENT_NAME}" \
-ISTIO_NAME="${ISTIO_NAME}" COMMAND="${COMMAND}" VERSIONS_YAML_FILE="${VERSIONS_YAML_FILE}" KUBECONFIG="${KUBECONFIG}" \
+NAMESPACE="${NAMESPACE}" CONTROL_PLANE_NS="${CONTROL_PLANE_NS}" DEPLOYMENT_NAME="${DEPLOYMENT_NAME}" MULTICLUSTER="${MULTICLUSTER}" ARTIFACTS="${ARTIFACTS}" \
+ISTIO_NAME="${ISTIO_NAME}" COMMAND="${COMMAND}" VERSIONS_YAML_FILE="${VERSIONS_YAML_FILE}" KUBECONFIG="${KUBECONFIG}" ISTIOCTL_PATH="${ISTIOCTL}" \
 go run github.com/onsi/ginkgo/v2/ginkgo -tags e2e --timeout 30m --junit-report=report.xml ${GINKGO_FLAGS} "${WD}"/...
diff --git a/tests/e2e/config/default.yaml b/tests/e2e/config/default.yaml
@@ -27,9 +27,4 @@ containerdConfigPatches:
   - |-
     [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
       endpoint = ["http://kind-registry:5000"]
-networking:
-  # MAISTRA specific:
-  # our prow cluster uses serviceSubnet 10.96.0.0/12, so the kind cluster must use other subnet to correctly route traffic;
-  # in this case, address 10.224.0.0 is chosen randomly from available set of subnets.
-  serviceSubnet: "10.224.0.0/12"
-  ipFamily: ipv4
+
diff --git a/tests/e2e/config/multicluster.json b/tests/e2e/config/multicluster.json
@@ -0,0 +1,14 @@
+[
+    {
+      "cluster_name": "primary",
+      "pod_subnet": "10.10.0.0/16",
+      "svc_subnet": "10.255.10.0/24",
+      "network_id": "0"
+    },
+    {
+      "cluster_name": "remote",
+      "pod_subnet": "10.20.0.0/16",
+      "svc_subnet": "10.255.20.0/24",
+      "network_id": "1"
+    }
+]
diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go
@@ -40,9 +40,13 @@ var (
 	skipDeploy            = env.GetBool("SKIP_DEPLOY", false)
 	expectedRegistry      = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io")
 	bookinfoNamespace     = env.Get("BOOKINFO_NAMESPACE", "bookinfo")
+	multicluster          = env.GetBool("MULTICLUSTER", false)
 )
 
 func TestInstall(t *testing.T) {
+	if multicluster {
+		t.Skip("Skipping test for multicluster")
+	}
 	RegisterFailHandler(Fail)
 	setup()
 	RunSpecs(t, "Control Plane Suite")
@@ -52,6 +56,6 @@ func setup() {
 	GinkgoWriter.Println("************ Running Setup ************")
 
 	GinkgoWriter.Println("Initializing k8s client")
-	cl, err = k8sclient.InitK8sClient()
+	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 }
diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go
@@ -19,7 +19,6 @@ package controlplane
 import (
 	"fmt"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"time"
 
@@ -43,12 +42,6 @@ import (
 	"istio.io/istio/pkg/ptr"
 )
 
-// version can have one of the following formats:
-// - 1.22.2
-// - 1.23.0-rc.1
-// - 1.24-alpha
-var istiodVersionRegex = regexp.MustCompile(`Version:"(\d+\.\d+(\.\d+)?(-\w+(\.\d+)?)?)`)
-
 var _ = Describe("Control Plane Installation", Ordered, func() {
 	SetDefaultEventuallyTimeout(180 * time.Second)
 	SetDefaultEventuallyPollingInterval(time.Second)
@@ -219,7 +212,7 @@ spec:
 					It("deploys istiod", func(ctx SpecContext) {
 						Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}).
 							Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition")
-						Expect(getVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version")
+						Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version")
 						Success("Istiod is deployed in the namespace and Running")
 					})
 
@@ -356,19 +349,6 @@ func ImageFromRegistry(regexp string) types.GomegaMatcher {
 	return HaveField("Image", MatchRegexp(regexp))
 }
 
-func getVersionFromIstiod() (string, error) {
-	output, err := kubectl.Exec(controlPlaneNamespace, "deploy/istiod", "", "pilot-discovery version")
-	if err != nil {
-		return "", fmt.Errorf("error getting version from istiod: %w", err)
-	}
-
-	matches := istiodVersionRegex.FindStringSubmatch(output)
-	if len(matches) > 1 && matches[1] != "" {
-		return matches[1], nil
-	}
-	return "", fmt.Errorf("error getting version from istiod: version not found in output: %s", output)
-}
-
 func indent(level int, str string) string {
 	indent := strings.Repeat(" ", level)
 	return indent + strings.ReplaceAll(str, "\n", "\n"+indent)

diff --git a/tests/e2e/integ-suite-kind.sh b/tests/e2e/integ-suite-kind.sh
@@ -29,9 +29,19 @@ export KIND_REGISTRY="localhost:${KIND_REGISTRY_PORT}"
 export DEFAULT_CLUSTER_YAML="${SCRIPTPATH}/config/default.yaml"
 export IP_FAMILY="${IP_FAMILY:-ipv4}"
 export ARTIFACTS="${ARTIFACTS:-$(mktemp -d)}"
+export MULTICLUSTER="${MULTICLUSTER:-false}"
+# Set variable to exclude kind clusters from kubectl annotations. 
+# You need to set kind clusters names separated by comma
+export KIND_EXCLUDE_CLUSTERS="${KIND_EXCLUDE_CLUSTERS:-}"
+export ISTIOCTL="${ISTIOCTL:-${ROOT}/bin/istioctl}"
+
 
 # Set variable for cluster kind name
 export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-operator-integration-tests}"
+if [ "${MULTICLUSTER}" == "true" ]; then
+  export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME}-1"
+  export KIND_CLUSTER_NAME_2="${KIND_CLUSTER_NAME}-2"
+fi
 
 # Use the local registry instead of the default HUB
 export HUB="${KIND_REGISTRY}"
@@ -52,16 +62,44 @@ function setup_kind_registry() {
   fi
 
   # https://docs.tilt.dev/choosing_clusters.html#discovering-the-registry
-  # TODO get context/config from existing variables
-  kind export kubeconfig --name="${KIND_CLUSTER_NAME}"
-  for node in $(kind get nodes --name="${KIND_CLUSTER_NAME}"); do
-    kubectl annotate node "${node}" "kind.x-k8s.io/registry=localhost:${KIND_REGISTRY_PORT}" --overwrite;
+  for cluster in $(kind get clusters); do
+    # TODO get context/config from existing variables
+    # Avoid adding the registry to excluded clusters. Use when you have multiple clusters running.
+    if [[ "${KIND_EXCLUDE_CLUSTERS}" == *"${cluster}"* ]]; then
+      continue
+    fi
+
+    kind export kubeconfig --name="${cluster}"
+    for node in $(kind get nodes --name="${cluster}"); do
+      kubectl annotate node "${node}" "kind.x-k8s.io/registry=localhost:${KIND_REGISTRY_PORT}" --overwrite;
+    done
   done
 }
 
-KUBECONFIG="${ARTIFACTS}/config" setup_kind_cluster "${KIND_CLUSTER_NAME}" "" "" "true" "true"
-setup_kind_registry
+if [ "${MULTICLUSTER}" == "true" ]; then
+    CLUSTER_TOPOLOGY_CONFIG_FILE="${SCRIPTPATH}/config/multicluster.json"
+    load_cluster_topology "${CLUSTER_TOPOLOGY_CONFIG_FILE}"
+    setup_kind_clusters "" ""
+    setup_kind_registry
+
+    export KUBECONFIG="${KUBECONFIGS[0]}"
+    export KUBECONFIG2="${KUBECONFIGS[1]}"
+else
+  KUBECONFIG="${ARTIFACTS}/config" setup_kind_cluster "${KIND_CLUSTER_NAME}" "" "" "true" "true"
+  setup_kind_registry
+fi
+
+
+# Check that istioctl is present using ${ISTIOCTL}
+if ! command -v "${ISTIOCTL}" &> /dev/null; then
+  echo "istioctl not found. Please set the ISTIOCTL environment variable to the path of the istioctl binary"
+  exit 1
+fi
 
 # Run the integration tests
 echo "Running integration tests"
+if [ "${MULTICLUSTER}" == "true" ]; then
+  ARTIFACTS="${ARTIFACTS}" ISTIOCTL="${ISTIOCTL}" ./tests/e2e/common-operator-integ-suite.sh --kind --multicluster
+else
 ARTIFACTS="${ARTIFACTS}" ./tests/e2e/common-operator-integ-suite.sh --kind
+fi