Backport: E2E test improvements

.gitignore

@@ -15,3 +15,5 @@ cluster.yaml
 eksa-test
 generated/
 eks-anywhere-downloads*
+hardware-manifests/
+*.env

Makefile

@@ -6,6 +6,15 @@ export INTEGRATION_TEST_SUBNET_ID?=integration_test_subnet_id
 export INTEGRATION_TEST_INSTANCE_TAG?=integration_test_instance_tag
 export JOB_ID?=${PROW_JOB_ID}
 
+LOCAL_ENV=.env
+
+# Include local .env file for exporting e2e test env vars locally
+# FYI, This file is git ignored
+ifneq ("$(wildcard $(LOCAL_ENV))","")
+include $(LOCAL_ENV)
+export $(shell sed 's/=.*//' $(LOCAL_ENV))
+endif
+
 SHELL := /bin/bash
 
 ARTIFACTS_BUCKET?=my-s3-bucket
@@ -109,6 +118,7 @@ EKS_A_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-cross-platf
 EKS_A_RELEASE_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-release-cross-platform-$(platform))
 
 DOCKER_E2E_TEST := TestDockerKubernetes121SimpleFlow
+LOCAL_E2E_TESTS ?= $(DOCKER_E2E_TEST)
 
 export KUBEBUILDER_ENVTEST_KUBERNETES_VERSION ?= 1.21.x
 
@@ -343,9 +353,13 @@ unit-test: KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path
 unit-test:
 	KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" $(GO_TEST) ./... -cover -tags "$(BUILD_TAGS)"
 
+.PHONY: local-e2e
+local-e2e: e2e ## Run e2e test's locally
+	./bin/e2e.test -test.v -test.run $(LOCAL_E2E_TESTS)
+
 .PHONY: capd-test
-capd-test: e2e ## Run default e2e capd test locally
-	./bin/e2e.test -test.v -test.run $(DOCKER_E2E_TEST)
+capd-test: ## Run default e2e capd test locally
+	$(MAKE) local-e2e LOCAL_E2E_TESTS=$(DOCKER_E2E_TEST)
 
 .PHONY: docker-e2e-test
 docker-e2e-test: e2e ## Run docker integration test in new ec2 instance

cmd/eks-a-tool/cmd/validatecluster.go

@@ -59,7 +59,7 @@ func validateCluster(ctx context.Context, cluster *types.Cluster, clusterName st
 	if err != nil {
 		return err
 	}
-	err = kubectl.ValidateWorkerNodes(ctx, cluster, clusterName)
+	err = kubectl.ValidateWorkerNodes(ctx, clusterName, cluster.KubeconfigFile)
 	if err != nil {
 		return err
 	}

internal/pkg/api/cluster.go

@@ -2,18 +2,29 @@ package api
 
 import (
 	"fmt"
+	"io/ioutil"
 
 	"sigs.k8s.io/yaml"
 
-	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/logger"
 )
 
-type ClusterFiller func(c *v1alpha1.Cluster)
+type ClusterFiller func(c *anywherev1.Cluster)
 
-func AutoFillCluster(filename string, fillers ...ClusterFiller) ([]byte, error) {
-	clusterConfig, err := v1alpha1.GetAndValidateClusterConfig(filename)
+func AutoFillClusterFromFile(filename string, fillers ...ClusterFiller) ([]byte, error) {
+	content, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return nil, fmt.Errorf("unable to get cluster config from file: %v", err)
+		return nil, fmt.Errorf("unable to read file due to: %v", err)
+	}
+
+	return AutoFillClusterFromYaml(content, fillers...)
+}
+
+func AutoFillClusterFromYaml(yamlContent []byte, fillers ...ClusterFiller) ([]byte, error) {
+	clusterConfig, err := anywherev1.GetClusterConfigFromContent(yamlContent)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get cluster config from content: %v", err)
 	}
 
 	for _, f := range fillers {
@@ -28,83 +39,83 @@ func AutoFillCluster(filename string, fillers ...ClusterFiller) ([]byte, error)
 	return clusterOutput, nil
 }
 
-func WithKubernetesVersion(v v1alpha1.KubernetesVersion) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+func WithKubernetesVersion(v anywherev1.KubernetesVersion) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.KubernetesVersion = v
 	}
 }
 
 func WithClusterNamespace(ns string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Namespace = ns
 	}
 }
 
 func WithControlPlaneCount(r int) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ControlPlaneConfiguration.Count = r
 	}
 }
 
 func WithControlPlaneEndpointIP(value string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ControlPlaneConfiguration.Endpoint.Host = value
 	}
 }
 
 func WithPodCidr(podCidr string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ClusterNetwork.Pods.CidrBlocks = []string{podCidr}
 	}
 }
 
 func WithServiceCidr(svcCidr string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ClusterNetwork.Services.CidrBlocks = []string{svcCidr}
 	}
 }
 
 func WithWorkerNodeCount(r int) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if len(c.Spec.WorkerNodeGroupConfigurations) == 0 {
-			c.Spec.WorkerNodeGroupConfigurations = []v1alpha1.WorkerNodeGroupConfiguration{{Count: 0}}
+			c.Spec.WorkerNodeGroupConfigurations = []anywherev1.WorkerNodeGroupConfiguration{{Count: 0}}
 		}
 		c.Spec.WorkerNodeGroupConfigurations[0].Count = r
 	}
 }
 
 func WithOIDCIdentityProviderRef(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.IdentityProviderRefs = append(c.Spec.IdentityProviderRefs,
-			v1alpha1.Ref{Name: name, Kind: v1alpha1.OIDCConfigKind})
+			anywherev1.Ref{Name: name, Kind: anywherev1.OIDCConfigKind})
 	}
 }
 
 func WithGitOpsRef(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
-		c.Spec.GitOpsRef = &v1alpha1.Ref{Name: name, Kind: v1alpha1.GitOpsConfigKind}
+	return func(c *anywherev1.Cluster) {
+		c.Spec.GitOpsRef = &anywherev1.Ref{Name: name, Kind: anywherev1.GitOpsConfigKind}
 	}
 }
 
 func WithExternalEtcdTopology(count int) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if c.Spec.ExternalEtcdConfiguration == nil {
-			c.Spec.ExternalEtcdConfiguration = &v1alpha1.ExternalEtcdConfiguration{}
+			c.Spec.ExternalEtcdConfiguration = &anywherev1.ExternalEtcdConfiguration{}
 		}
 		c.Spec.ExternalEtcdConfiguration.Count = count
 	}
 }
 
 func WithStackedEtcdTopology() ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ExternalEtcdConfiguration = nil
 	}
 }
 
 func WithProxyConfig(httpProxy, httpsProxy string, noProxy []string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if c.Spec.ProxyConfiguration == nil {
-			c.Spec.ProxyConfiguration = &v1alpha1.ProxyConfiguration{}
+			c.Spec.ProxyConfiguration = &anywherev1.ProxyConfiguration{}
 		}
 		c.Spec.ProxyConfiguration.HttpProxy = httpProxy
 		c.Spec.ProxyConfiguration.HttpsProxy = httpProxy
@@ -113,24 +124,69 @@ func WithProxyConfig(httpProxy, httpsProxy string, noProxy []string) ClusterFill
 }
 
 func WithRegistryMirror(endpoint, caCert string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		if c.Spec.RegistryMirrorConfiguration == nil {
-			c.Spec.RegistryMirrorConfiguration = &v1alpha1.RegistryMirrorConfiguration{}
+			c.Spec.RegistryMirrorConfiguration = &anywherev1.RegistryMirrorConfiguration{}
 		}
 		c.Spec.RegistryMirrorConfiguration.Endpoint = endpoint
 		c.Spec.RegistryMirrorConfiguration.CACertContent = caCert
 	}
 }
 
 func WithManagementCluster(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.ManagementCluster.Name = name
 	}
 }
 
 func WithAWSIamIdentityProviderRef(name string) ClusterFiller {
-	return func(c *v1alpha1.Cluster) {
+	return func(c *anywherev1.Cluster) {
 		c.Spec.IdentityProviderRefs = append(c.Spec.IdentityProviderRefs,
-			v1alpha1.Ref{Name: name, Kind: v1alpha1.AWSIamConfigKind})
+			anywherev1.Ref{Name: name, Kind: anywherev1.AWSIamConfigKind})
+	}
+}
+
+func RemoveAllWorkerNodeGroups() ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		c.Spec.WorkerNodeGroupConfigurations = make([]anywherev1.WorkerNodeGroupConfiguration, 0)
+	}
+}
+
+func RemoveWorkerNodeGroup(name string) ClusterFiller {
+	logger.Info("removing", "name", name)
+	return func(c *anywherev1.Cluster) {
+		logger.Info("before deleting", "w", c.Spec.WorkerNodeGroupConfigurations)
+		for i, w := range c.Spec.WorkerNodeGroupConfigurations {
+			if w.Name == name {
+				copy(c.Spec.WorkerNodeGroupConfigurations[i:], c.Spec.WorkerNodeGroupConfigurations[i+1:])
+				c.Spec.WorkerNodeGroupConfigurations[len(c.Spec.WorkerNodeGroupConfigurations)-1] = anywherev1.WorkerNodeGroupConfiguration{}
+				c.Spec.WorkerNodeGroupConfigurations = c.Spec.WorkerNodeGroupConfigurations[:len(c.Spec.WorkerNodeGroupConfigurations)-1]
+				logger.Info("after deleting", "w", c.Spec.WorkerNodeGroupConfigurations)
+				return
+			}
+		}
+	}
+}
+
+func WithWorkerNodeGroup(name string, fillers ...WorkerNodeGroupFiller) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		logger.Info("adding or updating", "name", name)
+		var nodeGroup *anywherev1.WorkerNodeGroupConfiguration
+		position := -1
+		for i, w := range c.Spec.WorkerNodeGroupConfigurations {
+			if w.Name == name {
+				nodeGroup = &w
+				position = i
+			}
+		}
+
+		if nodeGroup == nil {
+			nodeGroup = &anywherev1.WorkerNodeGroupConfiguration{Name: name}
+			c.Spec.WorkerNodeGroupConfigurations = append(c.Spec.WorkerNodeGroupConfigurations, *nodeGroup)
+			position = len(c.Spec.WorkerNodeGroupConfigurations) - 1
+		}
+
+		FillWorkerNodeGroup(nodeGroup, fillers...)
+		c.Spec.WorkerNodeGroupConfigurations[position] = *nodeGroup
 	}
 }

internal/pkg/api/nodegroups.go

@@ -0,0 +1,42 @@
+package api
+
+import (
+	corev1 "k8s.io/api/core/v1"
+
+	anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+)
+
+type WorkerNodeGroupFiller func(w *anywherev1.WorkerNodeGroupConfiguration)
+
+func FillWorkerNodeGroup(w *anywherev1.WorkerNodeGroupConfiguration, fillers ...WorkerNodeGroupFiller) {
+	for _, f := range fillers {
+		f(w)
+	}
+}
+
+func WithTaint(taint corev1.Taint) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.Taints = append(w.Taints, taint)
+	}
+}
+
+func WithLabel(key, value string) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.Labels[key] = value
+	}
+}
+
+func WithCount(count int) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.Count = count
+	}
+}
+
+func WithMachineGroupRef(name, kind string) WorkerNodeGroupFiller {
+	return func(w *anywherev1.WorkerNodeGroupConfiguration) {
+		w.MachineGroupRef = &anywherev1.Ref{
+			Name: name,
+			Kind: kind,
+		}
+	}
+}