Backport: E2E test improvements (#1269)

* Update kubectl ValidateWorkerNodes arg to pass in kubeconfig (#1149) * Refactor GitOps e2e tests (#1176) * Refactor GitOps e2e test to pass in testOpts, add tests for workload cluster * Remove unused arg * Rename upgradeGitops methods * Remove unnecessary method * Generate GitOpsConfig name every time when running e2e flux test * Only generate lowercase alphanumeric string for K8s object names * Update newly added GitOps e2e test name to include "Flux" (#1193) * (#743) support passing multiple ips to cluster tests (#931) * Improvements for 'upgrades from latest release' e2e tests (#966) * Improvements for 'upgrades from latest release' e2e tests * Increas the timeout for kubeadm control plane ready in e2e tests * Increased timeout for cilium retrier * E2E tests for multiple worker node groups (#1194) Expands current e2e testing framework to support multiple worker node groups. It allows to add/remove/update worker node groups to the initial spec and during upgrades. Adds one simple vSphere tests that starts with two worker node groups. During the upgrade, it scales down one of them to 1 replica, it deletes the other one and creates a new one. * Update e2e test flow so cluster ip pool is not required (#1213) Co-authored-by: Joey Wang <jiayiwang7@yahoo.com> Co-authored-by: Greg Westerfield, Jr <gwesterf@amazon.com> Co-authored-by: Guillermo Gaston <gaslor@amazon.com> Co-authored-by: Greg Westerfield, Jr <gwesterfieldjr@gmail.com>
aws · Feb 23, 2022 · 45dec22 · 45dec22
1 parent db1a3fc
commit 45dec22
Show file tree

Hide file tree

Showing 36 changed files with 907 additions and 332 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,3 +15,5 @@ cluster.yaml
 eksa-test
 generated/
 eks-anywhere-downloads*
+hardware-manifests/
+*.env
diff --git a/Makefile b/Makefile
@@ -6,6 +6,15 @@ export INTEGRATION_TEST_SUBNET_ID?=integration_test_subnet_id
 export INTEGRATION_TEST_INSTANCE_TAG?=integration_test_instance_tag
 export JOB_ID?=${PROW_JOB_ID}
 
+LOCAL_ENV=.env
+
+# Include local .env file for exporting e2e test env vars locally
+# FYI, This file is git ignored
+ifneq ("$(wildcard $(LOCAL_ENV))","")
+include $(LOCAL_ENV)
+export $(shell sed 's/=.*//' $(LOCAL_ENV))
+endif
+
 SHELL := /bin/bash
 
 ARTIFACTS_BUCKET?=my-s3-bucket
@@ -109,6 +118,7 @@ EKS_A_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-cross-platf
 EKS_A_RELEASE_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-release-cross-platform-$(platform))
 
 DOCKER_E2E_TEST := TestDockerKubernetes121SimpleFlow
+LOCAL_E2E_TESTS ?= $(DOCKER_E2E_TEST)
 
 export KUBEBUILDER_ENVTEST_KUBERNETES_VERSION ?= 1.21.x
 
@@ -343,9 +353,13 @@ unit-test: KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path
 unit-test:
 	KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" $(GO_TEST) ./... -cover -tags "$(BUILD_TAGS)"
 
+.PHONY: local-e2e
+local-e2e: e2e ## Run e2e test's locally
+	./bin/e2e.test -test.v -test.run $(LOCAL_E2E_TESTS)
+
 .PHONY: capd-test
-capd-test: e2e ## Run default e2e capd test locally
-	./bin/e2e.test -test.v -test.run $(DOCKER_E2E_TEST)
+capd-test: ## Run default e2e capd test locally
+	$(MAKE) local-e2e LOCAL_E2E_TESTS=$(DOCKER_E2E_TEST)
 
 .PHONY: docker-e2e-test
 docker-e2e-test: e2e ## Run docker integration test in new ec2 instance

diff --git a/cmd/eks-a-tool/cmd/validatecluster.go b/cmd/eks-a-tool/cmd/validatecluster.go
@@ -59,7 +59,7 @@ func validateCluster(ctx context.Context, cluster *types.Cluster, clusterName st
 	if err != nil {
 		return err
 	}
-	err = kubectl.ValidateWorkerNodes(ctx, cluster, clusterName)
+	err = kubectl.ValidateWorkerNodes(ctx, clusterName, cluster.KubeconfigFile)
 	if err != nil {
 		return err
 	}

diff --git a/internal/pkg/api/cluster.go b/internal/pkg/api/cluster.go
@@ -2,18 +2,29 @@ package api
 
 import (
 	"fmt"
+	"io/ioutil"
 
 	"sigs.k8s.io/yaml"
 
-	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/logger"
 )
 
-type ClusterFiller func(c *v1alpha1.Cluster)
+type ClusterFiller func(c *anywherev1.Cluster)
 
-func AutoFillCluster(filename string, fillers ...ClusterFiller) ([]byte, error) {
-	clusterConfig, err := v1alpha1.GetAndValidateClusterConfig(filename)
+func AutoFillClusterFromFile(filename string, fillers ...ClusterFiller) ([]byte, error) {
+	content, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return nil, fmt.Errorf("unable to get cluster config from file: %v", err)
+		return nil, fmt.Errorf("unable to read file due to: %v", err)
+	}
+
+	return AutoFillClusterFromYaml(content, fillers...)
+}
+
+func AutoFillClusterFromYaml(yamlContent []byte, fillers ...ClusterFiller) ([]byte, error) {
+	clusterConfig, err := anywherev1.GetClusterConfigFromContent(yamlContent)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get cluster config from content: %v", err)
 	}
 
 	for _, f := range fillers {
@@ -28,83 +39,83 @@ func AutoFillCluster(filename string, fillers ...ClusterFiller) ([]byte, error)
 	return clusterOutput, nil
 }
 
-func WithKubernetesVersion(v v1alpha1.KubernetesVersion) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+func WithKubernetesVersion(v anywherev1.KubernetesVersion) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.KubernetesVersion = v
 	}
 }
 
 func WithClusterNamespace(ns string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Namespace = ns
 	}
 }
 
 func WithControlPlaneCount(r int) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ControlPlaneConfiguration.Count = r
 	}
 }
 
 func WithControlPlaneEndpointIP(value string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ControlPlaneConfiguration.Endpoint.Host = value
 	}
 }
 
 func WithPodCidr(podCidr string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ClusterNetwork.Pods.CidrBlocks = []string{podCidr}
 	}
 }
 
 func WithServiceCidr(svcCidr string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ClusterNetwork.Services.CidrBlocks = []string{svcCidr}
 	}
 }
 
 func WithWorkerNodeCount(r int) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if len(c.Spec.WorkerNodeGroupConfigurations) == 0 {
-			c.Spec.WorkerNodeGroupConfigurations = []v1alpha1.WorkerNodeGroupConfiguration{{Count: 0}}
+			c.Spec.WorkerNodeGroupConfigurations = []anywherev1.WorkerNodeGroupConfiguration{{Count: 0}}
 		}
 		c.Spec.WorkerNodeGroupConfigurations[0].Count = r
 	}
 }
 
 func WithOIDCIdentityProviderRef(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.IdentityProviderRefs = append(c.Spec.IdentityProviderRefs,
-			v1alpha1.Ref{Name: name, Kind: v1alpha1.OIDCConfigKind})
+			anywherev1.Ref{Name: name, Kind: anywherev1.OIDCConfigKind})
 	}
 }
 
 func WithGitOpsRef(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
-		c.Spec.GitOpsRef = &v1alpha1.Ref{Name: name, Kind: v1alpha1.GitOpsConfigKind}
+	return func(c *anywherev1.Cluster) {
+		c.Spec.GitOpsRef = &anywherev1.Ref{Name: name, Kind: anywherev1.GitOpsConfigKind}
 	}
 }
 
 func WithExternalEtcdTopology(count int) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if c.Spec.ExternalEtcdConfiguration == nil {
-			c.Spec.ExternalEtcdConfiguration = &v1alpha1.ExternalEtcdConfiguration{}
+			c.Spec.ExternalEtcdConfiguration = &anywherev1.ExternalEtcdConfiguration{}
 		}
 		c.Spec.ExternalEtcdConfiguration.Count = count
 	}
 }
 
 func WithStackedEtcdTopology() ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ExternalEtcdConfiguration = nil
 	}
 }
 
 func WithProxyConfig(httpProxy, httpsProxy string, noProxy []string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if c.Spec.ProxyConfiguration == nil {
-			c.Spec.ProxyConfiguration = &v1alpha1.ProxyConfiguration{}
+			c.Spec.ProxyConfiguration = &anywherev1.ProxyConfiguration{}
 		}
 		c.Spec.ProxyConfiguration.HttpProxy = httpProxy
 		c.Spec.ProxyConfiguration.HttpsProxy = httpProxy
@@ -113,24 +124,69 @@ func WithProxyConfig(httpProxy, httpsProxy string, noProxy []string) ClusterFill
 }
 
 func WithRegistryMirror(endpoint, caCert string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if c.Spec.RegistryMirrorConfiguration == nil {
-			c.Spec.RegistryMirrorConfiguration = &v1alpha1.RegistryMirrorConfiguration{}
+			c.Spec.RegistryMirrorConfiguration = &anywherev1.RegistryMirrorConfiguration{}
 		}
 		c.Spec.RegistryMirrorConfiguration.Endpoint = endpoint
 		c.Spec.RegistryMirrorConfiguration.CACertContent = caCert
 	}
 }
 
 func WithManagementCluster(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ManagementCluster.Name = name
 	}
 }
 
 func WithAWSIamIdentityProviderRef(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.IdentityProviderRefs = append(c.Spec.IdentityProviderRefs,
-			v1alpha1.Ref{Name: name, Kind: v1alpha1.AWSIamConfigKind})
+			anywherev1.Ref{Name: name, Kind: anywherev1.AWSIamConfigKind})
+	}
+}
+
+func RemoveAllWorkerNodeGroups() ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		c.Spec.WorkerNodeGroupConfigurations = make([]anywherev1.WorkerNodeGroupConfiguration, 0)
+	}
+}
+
+func RemoveWorkerNodeGroup(name string) ClusterFiller {
+	logger.Info("removing", "name", name)
+	return func(c *anywherev1.Cluster) {
+		logger.Info("before deleting", "w", c.Spec.WorkerNodeGroupConfigurations)
+		for i, w := range c.Spec.WorkerNodeGroupConfigurations {
+			if w.Name == name {
+				copy(c.Spec.WorkerNodeGroupConfigurations[i:], c.Spec.WorkerNodeGroupConfigurations[i+1:])
+				c.Spec.WorkerNodeGroupConfigurations[len(c.Spec.WorkerNodeGroupConfigurations)-1] = anywherev1.WorkerNodeGroupConfiguration{}
+				c.Spec.WorkerNodeGroupConfigurations = c.Spec.WorkerNodeGroupConfigurations[:len(c.Spec.WorkerNodeGroupConfigurations)-1]
+				logger.Info("after deleting", "w", c.Spec.WorkerNodeGroupConfigurations)
+				return
+			}
+		}
+	}
+}
+
+func WithWorkerNodeGroup(name string, fillers ...WorkerNodeGroupFiller) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		logger.Info("adding or updating", "name", name)
+		var nodeGroup *anywherev1.WorkerNodeGroupConfiguration
+		position := -1
+		for i, w := range c.Spec.WorkerNodeGroupConfigurations {
+			if w.Name == name {
+				nodeGroup = &w
+				position = i
+			}
+		}
+
+		if nodeGroup == nil {
+			nodeGroup = &anywherev1.WorkerNodeGroupConfiguration{Name: name}
+			c.Spec.WorkerNodeGroupConfigurations = append(c.Spec.WorkerNodeGroupConfigurations, *nodeGroup)
+			position = len(c.Spec.WorkerNodeGroupConfigurations) - 1
+		}
+
+		FillWorkerNodeGroup(nodeGroup, fillers...)
+		c.Spec.WorkerNodeGroupConfigurations[position] = *nodeGroup
 	}
 }
diff --git a/internal/pkg/api/nodegroups.go b/internal/pkg/api/nodegroups.go
@@ -0,0 +1,42 @@
+package api
+
+import (
+	corev1 "k8s.io/api/core/v1"
+
+	anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+)
+
+type WorkerNodeGroupFiller func(w *anywherev1.WorkerNodeGroupConfiguration)
+
+func FillWorkerNodeGroup(w *anywherev1.WorkerNodeGroupConfiguration, fillers ...WorkerNodeGroupFiller) {
+	for _, f := range fillers {
+		f(w)
+	}
+}
+
+func WithTaint(taint corev1.Taint) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.Taints = append(w.Taints, taint)
+	}
+}
+
+func WithLabel(key, value string) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.Labels[key] = value
+	}
+}
+
+func WithCount(count int) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.Count = count
+	}
+}
+
+func WithMachineGroupRef(name, kind string) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.MachineGroupRef = &anywherev1.Ref{
+			Name: name,
+			Kind: kind,
+		}
+	}
+}